From 836b47cb7e99a977c5a23b059ca1d0b5065d310e Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 24 Jul 2024 11:54:23 +0200 Subject: Merging upstream version 1.46.3. Signed-off-by: Daniel Baumann --- fluent-bit/plugins/CMakeLists.txt | 416 --- fluent-bit/plugins/custom_calyptia/CMakeLists.txt | 4 - fluent-bit/plugins/custom_calyptia/calyptia.c | 615 ----- .../plugins/filter_alter_size/CMakeLists.txt | 4 - fluent-bit/plugins/filter_alter_size/alter_size.c | 212 -- fluent-bit/plugins/filter_alter_size/alter_size.h | 0 fluent-bit/plugins/filter_aws/CMakeLists.txt | 5 - fluent-bit/plugins/filter_aws/aws.c | 1062 -------- fluent-bit/plugins/filter_aws/aws.h | 131 - fluent-bit/plugins/filter_checklist/CMakeLists.txt | 4 - fluent-bit/plugins/filter_checklist/checklist.c | 656 ----- fluent-bit/plugins/filter_checklist/checklist.h | 69 - fluent-bit/plugins/filter_ecs/CMakeLists.txt | 5 - fluent-bit/plugins/filter_ecs/ecs.c | 1760 ------------ fluent-bit/plugins/filter_ecs/ecs.h | 152 -- fluent-bit/plugins/filter_expect/CMakeLists.txt | 4 - fluent-bit/plugins/filter_expect/expect.c | 614 ----- fluent-bit/plugins/filter_expect/expect.h | 53 - fluent-bit/plugins/filter_geoip2/.gitignore | 1 - fluent-bit/plugins/filter_geoip2/CMakeLists.txt | 19 - fluent-bit/plugins/filter_geoip2/geoip2.c | 519 ---- fluent-bit/plugins/filter_geoip2/geoip2.h | 46 - .../filter_geoip2/libmaxminddb/CMakeLists.txt | 101 - .../plugins/filter_geoip2/libmaxminddb/LICENSE | 202 -- .../plugins/filter_geoip2/libmaxminddb/NOTICE | 13 - .../plugins/filter_geoip2/libmaxminddb/VERSION | 1 - .../filter_geoip2/libmaxminddb/bin/CMakeLists.txt | 13 - .../filter_geoip2/libmaxminddb/bin/Makefile.am | 10 - .../filter_geoip2/libmaxminddb/bin/mmdblookup.c | 762 ------ .../filter_geoip2/libmaxminddb/include/maxminddb.h | 255 -- .../include/maxminddb_config.h.cmake.in | 14 - .../libmaxminddb/include/maxminddb_config.h.in | 14 - .../filter_geoip2/libmaxminddb/src/Makefile.am | 25 - .../filter_geoip2/libmaxminddb/src/data-pool.c | 180 -- .../filter_geoip2/libmaxminddb/src/data-pool.h | 52 - .../libmaxminddb/src/libmaxminddb.pc.in | 11 - .../libmaxminddb/src/maxminddb-compat-util.h | 167 -- .../filter_geoip2/libmaxminddb/src/maxminddb.c | 2157 --------------- fluent-bit/plugins/filter_grep/CMakeLists.txt | 4 - fluent-bit/plugins/filter_grep/grep.c | 434 --- fluent-bit/plugins/filter_grep/grep.h | 58 - .../plugins/filter_kubernetes/CMakeLists.txt | 14 - fluent-bit/plugins/filter_kubernetes/kube_conf.c | 232 -- fluent-bit/plugins/filter_kubernetes/kube_conf.h | 174 -- fluent-bit/plugins/filter_kubernetes/kube_meta.c | 1650 ----------- fluent-bit/plugins/filter_kubernetes/kube_meta.h | 69 - .../plugins/filter_kubernetes/kube_property.c | 360 --- .../plugins/filter_kubernetes/kube_property.h | 40 - fluent-bit/plugins/filter_kubernetes/kube_props.h | 44 - fluent-bit/plugins/filter_kubernetes/kube_regex.c | 43 - fluent-bit/plugins/filter_kubernetes/kube_regex.h | 31 - fluent-bit/plugins/filter_kubernetes/kubernetes.c | 1000 ------- .../plugins/filter_log_to_metrics/CMakeLists.txt | 8 - .../plugins/filter_log_to_metrics/log_to_metrics.c | 965 ------- .../plugins/filter_log_to_metrics/log_to_metrics.h | 85 - fluent-bit/plugins/filter_lua/CMakeLists.txt | 13 - fluent-bit/plugins/filter_lua/lua.c | 713 ----- fluent-bit/plugins/filter_lua/lua_config.c | 206 -- fluent-bit/plugins/filter_lua/lua_config.h | 49 - fluent-bit/plugins/filter_modify/CMakeLists.txt | 4 - fluent-bit/plugins/filter_modify/modify.c | 1659 ----------- fluent-bit/plugins/filter_modify/modify.h | 96 - fluent-bit/plugins/filter_multiline/CMakeLists.txt | 5 - fluent-bit/plugins/filter_multiline/ml.c | 931 ------- fluent-bit/plugins/filter_multiline/ml.h | 87 - fluent-bit/plugins/filter_multiline/ml_concat.c | 473 ---- fluent-bit/plugins/filter_multiline/ml_concat.h | 84 - fluent-bit/plugins/filter_nest/CMakeLists.txt | 4 - fluent-bit/plugins/filter_nest/nest.c | 761 ------ fluent-bit/plugins/filter_nest/nest.h | 55 - fluent-bit/plugins/filter_nightfall/CMakeLists.txt | 6 - fluent-bit/plugins/filter_nightfall/nightfall.c | 654 ----- fluent-bit/plugins/filter_nightfall/nightfall.h | 57 - .../plugins/filter_nightfall/nightfall_api.c | 536 ---- .../plugins/filter_nightfall/nightfall_api.h | 31 - fluent-bit/plugins/filter_parser/CMakeLists.txt | 4 - fluent-bit/plugins/filter_parser/filter_parser.c | 452 --- fluent-bit/plugins/filter_parser/filter_parser.h | 42 - .../plugins/filter_record_modifier/CMakeLists.txt | 4 - .../filter_record_modifier/filter_modifier.c | 531 ---- .../filter_record_modifier/filter_modifier.h | 68 - .../plugins/filter_rewrite_tag/CMakeLists.txt | 4 - .../plugins/filter_rewrite_tag/rewrite_tag.c | 621 ----- .../plugins/filter_rewrite_tag/rewrite_tag.h | 64 - fluent-bit/plugins/filter_stdout/CMakeLists.txt | 4 - fluent-bit/plugins/filter_stdout/stdout.c | 107 - .../plugins/filter_tensorflow/CMakeLists.txt | 6 - fluent-bit/plugins/filter_tensorflow/tensorflow.c | 540 ---- fluent-bit/plugins/filter_tensorflow/tensorflow.h | 46 - fluent-bit/plugins/filter_throttle/CMakeLists.txt | 6 - fluent-bit/plugins/filter_throttle/throttle.c | 337 --- fluent-bit/plugins/filter_throttle/throttle.h | 56 - fluent-bit/plugins/filter_throttle/window.c | 97 - fluent-bit/plugins/filter_throttle/window.h | 37 - .../plugins/filter_throttle_size/CMakeLists.txt | 3 - .../plugins/filter_throttle_size/size_window.c | 226 -- .../plugins/filter_throttle_size/size_window.h | 140 - .../plugins/filter_throttle_size/throttle_size.c | 774 ------ .../plugins/filter_throttle_size/throttle_size.h | 60 - .../plugins/filter_type_converter/CMakeLists.txt | 5 - .../plugins/filter_type_converter/type_converter.c | 399 --- .../plugins/filter_type_converter/type_converter.h | 46 - fluent-bit/plugins/filter_wasm/CMakeLists.txt | 11 - fluent-bit/plugins/filter_wasm/filter_wasm.c | 318 --- fluent-bit/plugins/filter_wasm/filter_wasm.h | 41 - .../plugins/in_calyptia_fleet/CMakeLists.txt | 4 - .../plugins/in_calyptia_fleet/in_calyptia_fleet.c | 1269 --------- fluent-bit/plugins/in_collectd/CMakeLists.txt | 7 - fluent-bit/plugins/in_collectd/in_collectd.c | 226 -- fluent-bit/plugins/in_collectd/in_collectd.h | 46 - fluent-bit/plugins/in_collectd/netprot.c | 308 --- fluent-bit/plugins/in_collectd/netprot.h | 22 - fluent-bit/plugins/in_collectd/typesdb.c | 223 -- fluent-bit/plugins/in_collectd/typesdb.h | 45 - fluent-bit/plugins/in_collectd/typesdb_parser.c | 214 -- fluent-bit/plugins/in_collectd/typesdb_parser.h | 20 - fluent-bit/plugins/in_cpu/CMakeLists.txt | 4 - fluent-bit/plugins/in_cpu/cpu.c | 672 ----- fluent-bit/plugins/in_cpu/cpu.h | 129 - fluent-bit/plugins/in_disk/CMakeLists.txt | 4 - fluent-bit/plugins/in_disk/in_disk.c | 387 --- fluent-bit/plugins/in_disk/in_disk.h | 48 - fluent-bit/plugins/in_docker/CMakeLists.txt | 6 - fluent-bit/plugins/in_docker/cgroup_v1.c | 397 --- fluent-bit/plugins/in_docker/docker.c | 560 ---- fluent-bit/plugins/in_docker/docker.h | 94 - fluent-bit/plugins/in_docker_events/CMakeLists.txt | 5 - .../plugins/in_docker_events/docker_events.c | 476 ---- .../plugins/in_docker_events/docker_events.h | 56 - .../in_docker_events/docker_events_config.c | 106 - .../in_docker_events/docker_events_config.h | 29 - fluent-bit/plugins/in_dummy/CMakeLists.txt | 4 - fluent-bit/plugins/in_dummy/in_dummy.c | 438 --- fluent-bit/plugins/in_dummy/in_dummy.h | 58 - fluent-bit/plugins/in_elasticsearch/CMakeLists.txt | 12 - .../plugins/in_elasticsearch/in_elasticsearch.c | 245 -- .../plugins/in_elasticsearch/in_elasticsearch.h | 59 - .../in_elasticsearch/in_elasticsearch_bulk_conn.c | 307 --- .../in_elasticsearch/in_elasticsearch_bulk_conn.h | 55 - .../in_elasticsearch/in_elasticsearch_bulk_prot.c | 922 ------- .../in_elasticsearch/in_elasticsearch_bulk_prot.h | 40 - .../in_elasticsearch/in_elasticsearch_config.c | 105 - .../in_elasticsearch/in_elasticsearch_config.h | 29 - fluent-bit/plugins/in_emitter/CMakeLists.txt | 5 - fluent-bit/plugins/in_emitter/emitter.c | 321 --- fluent-bit/plugins/in_event_test/CMakeLists.txt | 4 - fluent-bit/plugins/in_event_test/event_test.c | 407 --- fluent-bit/plugins/in_event_type/CMakeLists.txt | 4 - fluent-bit/plugins/in_event_type/event_type.c | 482 ---- fluent-bit/plugins/in_exec/CMakeLists.txt | 4 - fluent-bit/plugins/in_exec/in_exec.c | 491 ---- fluent-bit/plugins/in_exec/in_exec.h | 52 - fluent-bit/plugins/in_exec/in_exec_win32_compat.h | 94 - fluent-bit/plugins/in_exec_wasi/CMakeLists.txt | 11 - fluent-bit/plugins/in_exec_wasi/in_exec_wasi.c | 451 --- fluent-bit/plugins/in_exec_wasi/in_exec_wasi.h | 55 - .../plugins/in_fluentbit_metrics/CMakeLists.txt | 5 - fluent-bit/plugins/in_fluentbit_metrics/metrics.c | 201 -- fluent-bit/plugins/in_forward/CMakeLists.txt | 7 - fluent-bit/plugins/in_forward/fw.c | 325 --- fluent-bit/plugins/in_forward/fw.h | 52 - fluent-bit/plugins/in_forward/fw_config.c | 120 - fluent-bit/plugins/in_forward/fw_config.h | 28 - fluent-bit/plugins/in_forward/fw_conn.c | 199 -- fluent-bit/plugins/in_forward/fw_conn.h | 57 - fluent-bit/plugins/in_forward/fw_prot.c | 846 ------ fluent-bit/plugins/in_forward/fw_prot.h | 28 - fluent-bit/plugins/in_head/CMakeLists.txt | 4 - fluent-bit/plugins/in_head/in_head.c | 473 ---- fluent-bit/plugins/in_head/in_head.h | 59 - fluent-bit/plugins/in_health/CMakeLists.txt | 4 - fluent-bit/plugins/in_health/health.c | 293 -- fluent-bit/plugins/in_http/CMakeLists.txt | 12 - fluent-bit/plugins/in_http/http.c | 204 -- fluent-bit/plugins/in_http/http.h | 58 - fluent-bit/plugins/in_http/http_config.c | 157 -- fluent-bit/plugins/in_http/http_config.h | 29 - fluent-bit/plugins/in_http/http_conn.c | 306 --- fluent-bit/plugins/in_http/http_conn.h | 54 - fluent-bit/plugins/in_http/http_prot.c | 665 ----- fluent-bit/plugins/in_http/http_prot.h | 31 - fluent-bit/plugins/in_kafka/CMakeLists.txt | 6 - fluent-bit/plugins/in_kafka/in_kafka.c | 382 --- fluent-bit/plugins/in_kafka/in_kafka.h | 48 - fluent-bit/plugins/in_kmsg/CMakeLists.txt | 4 - fluent-bit/plugins/in_kmsg/in_kmsg.c | 390 --- fluent-bit/plugins/in_kmsg/in_kmsg.h | 68 - .../plugins/in_kubernetes_events/CMakeLists.txt | 5 - .../in_kubernetes_events/kubernetes_events.c | 921 ------- .../in_kubernetes_events/kubernetes_events.h | 106 - .../in_kubernetes_events/kubernetes_events_conf.c | 326 --- .../in_kubernetes_events/kubernetes_events_conf.h | 47 - .../in_kubernetes_events/kubernetes_events_sql.h | 60 - fluent-bit/plugins/in_lib/CMakeLists.txt | 10 - fluent-bit/plugins/in_lib/in_lib.c | 279 -- fluent-bit/plugins/in_lib/in_lib.h | 45 - fluent-bit/plugins/in_mem/CMakeLists.txt | 5 - fluent-bit/plugins/in_mem/mem.c | 320 --- fluent-bit/plugins/in_mem/mem.h | 51 - fluent-bit/plugins/in_mem/proc.c | 185 -- fluent-bit/plugins/in_mem/proc.h | 68 - fluent-bit/plugins/in_mqtt/CMakeLists.txt | 7 - fluent-bit/plugins/in_mqtt/mqtt.c | 162 -- fluent-bit/plugins/in_mqtt/mqtt.h | 45 - fluent-bit/plugins/in_mqtt/mqtt_config.c | 82 - fluent-bit/plugins/in_mqtt/mqtt_config.h | 29 - fluent-bit/plugins/in_mqtt/mqtt_conn.c | 157 -- fluent-bit/plugins/in_mqtt/mqtt_conn.h | 49 - fluent-bit/plugins/in_mqtt/mqtt_prot.c | 465 ---- fluent-bit/plugins/in_mqtt/mqtt_prot.h | 62 - fluent-bit/plugins/in_netif/CMakeLists.txt | 4 - fluent-bit/plugins/in_netif/in_netif.c | 392 --- fluent-bit/plugins/in_netif/in_netif.h | 70 - .../in_nginx_exporter_metrics/CMakeLists.txt | 4 - .../plugins/in_nginx_exporter_metrics/nginx.c | 2363 ---------------- .../plugins/in_nginx_exporter_metrics/nginx.h | 150 - .../in_node_exporter_metrics/CMakeLists.txt | 26 - fluent-bit/plugins/in_node_exporter_metrics/ne.c | 1107 -------- fluent-bit/plugins/in_node_exporter_metrics/ne.h | 191 -- .../plugins/in_node_exporter_metrics/ne_config.c | 69 - .../plugins/in_node_exporter_metrics/ne_config.h | 31 - .../plugins/in_node_exporter_metrics/ne_cpu.c | 23 - .../plugins/in_node_exporter_metrics/ne_cpu.h | 28 - .../in_node_exporter_metrics/ne_cpu_linux.c | 396 --- .../plugins/in_node_exporter_metrics/ne_cpufreq.h | 28 - .../in_node_exporter_metrics/ne_cpufreq_linux.c | 196 -- .../in_node_exporter_metrics/ne_diskstats.c | 22 - .../in_node_exporter_metrics/ne_diskstats.h | 29 - .../in_node_exporter_metrics/ne_diskstats_linux.c | 449 --- .../in_node_exporter_metrics/ne_filefd_linux.c | 115 - .../in_node_exporter_metrics/ne_filefd_linux.h | 28 - .../in_node_exporter_metrics/ne_filesystem.c | 39 - .../in_node_exporter_metrics/ne_filesystem.h | 29 - .../in_node_exporter_metrics/ne_filesystem_linux.c | 404 --- .../plugins/in_node_exporter_metrics/ne_loadavg.c | 22 - .../plugins/in_node_exporter_metrics/ne_loadavg.h | 29 - .../in_node_exporter_metrics/ne_loadavg_linux.c | 126 - .../plugins/in_node_exporter_metrics/ne_meminfo.c | 23 - .../plugins/in_node_exporter_metrics/ne_meminfo.h | 29 - .../in_node_exporter_metrics/ne_meminfo_linux.c | 283 -- .../plugins/in_node_exporter_metrics/ne_netdev.c | 22 - .../plugins/in_node_exporter_metrics/ne_netdev.h | 29 - .../in_node_exporter_metrics/ne_netdev_linux.c | 363 --- .../in_node_exporter_metrics/ne_stat_linux.c | 152 -- .../in_node_exporter_metrics/ne_stat_linux.h | 28 - .../plugins/in_node_exporter_metrics/ne_systemd.c | 807 ------ .../plugins/in_node_exporter_metrics/ne_systemd.h | 127 - .../plugins/in_node_exporter_metrics/ne_textfile.c | 22 - .../plugins/in_node_exporter_metrics/ne_textfile.h | 28 - .../in_node_exporter_metrics/ne_textfile_linux.c | 204 -- .../plugins/in_node_exporter_metrics/ne_time.c | 59 - .../plugins/in_node_exporter_metrics/ne_time.h | 28 - .../plugins/in_node_exporter_metrics/ne_uname.c | 22 - .../plugins/in_node_exporter_metrics/ne_uname.h | 28 - .../in_node_exporter_metrics/ne_uname_linux.c | 84 - .../plugins/in_node_exporter_metrics/ne_utils.c | 256 -- .../plugins/in_node_exporter_metrics/ne_utils.h | 39 - .../in_node_exporter_metrics/ne_vmstat_linux.c | 216 -- .../in_node_exporter_metrics/ne_vmstat_linux.h | 29 - fluent-bit/plugins/in_opentelemetry/CMakeLists.txt | 12 - fluent-bit/plugins/in_opentelemetry/http_conn.c | 301 -- fluent-bit/plugins/in_opentelemetry/http_conn.h | 57 - .../plugins/in_opentelemetry/opentelemetry.c | 200 -- .../plugins/in_opentelemetry/opentelemetry.h | 51 - .../in_opentelemetry/opentelemetry_config.c | 92 - .../in_opentelemetry/opentelemetry_config.h | 29 - .../plugins/in_opentelemetry/opentelemetry_prot.c | 1674 ------------ .../plugins/in_opentelemetry/opentelemetry_prot.h | 31 - .../plugins/in_podman_metrics/CMakeLists.txt | 6 - .../plugins/in_podman_metrics/podman_metrics.c | 515 ---- .../plugins/in_podman_metrics/podman_metrics.h | 98 - .../in_podman_metrics/podman_metrics_config.h | 211 -- .../in_podman_metrics/podman_metrics_data.c | 407 --- .../in_podman_metrics/podman_metrics_data.h | 51 - fluent-bit/plugins/in_proc/CMakeLists.txt | 4 - fluent-bit/plugins/in_proc/in_proc.c | 534 ---- fluent-bit/plugins/in_proc/in_proc.h | 78 - .../plugins/in_prometheus_scrape/CMakeLists.txt | 4 - .../plugins/in_prometheus_scrape/prom_scrape.c | 261 -- .../plugins/in_prometheus_scrape/prom_scrape.h | 45 - fluent-bit/plugins/in_random/CMakeLists.txt | 4 - fluent-bit/plugins/in_random/random.c | 245 -- fluent-bit/plugins/in_serial/CMakeLists.txt | 4 - fluent-bit/plugins/in_serial/in_serial.c | 443 --- fluent-bit/plugins/in_serial/in_serial.h | 63 - fluent-bit/plugins/in_serial/in_serial_config.c | 82 - fluent-bit/plugins/in_serial/in_serial_config.h | 77 - fluent-bit/plugins/in_splunk/CMakeLists.txt | 12 - fluent-bit/plugins/in_splunk/splunk.c | 213 -- fluent-bit/plugins/in_splunk/splunk.h | 60 - fluent-bit/plugins/in_splunk/splunk_config.c | 184 -- fluent-bit/plugins/in_splunk/splunk_config.h | 29 - fluent-bit/plugins/in_splunk/splunk_conn.c | 306 --- fluent-bit/plugins/in_splunk/splunk_conn.h | 54 - fluent-bit/plugins/in_splunk/splunk_prot.c | 779 ------ fluent-bit/plugins/in_splunk/splunk_prot.h | 36 - fluent-bit/plugins/in_statsd/CMakeLists.txt | 4 - fluent-bit/plugins/in_statsd/statsd.c | 386 --- fluent-bit/plugins/in_stdin/CMakeLists.txt | 10 - fluent-bit/plugins/in_stdin/in_stdin.c | 472 ---- fluent-bit/plugins/in_stdin/in_stdin.h | 48 - .../plugins/in_storage_backlog/CMakeLists.txt | 5 - fluent-bit/plugins/in_storage_backlog/sb.c | 713 ----- .../plugins/in_stream_processor/CMakeLists.txt | 5 - fluent-bit/plugins/in_stream_processor/sp.c | 173 -- fluent-bit/plugins/in_syslog/CMakeLists.txt | 8 - fluent-bit/plugins/in_syslog/syslog.c | 263 -- fluent-bit/plugins/in_syslog/syslog.h | 82 - fluent-bit/plugins/in_syslog/syslog_conf.c | 193 -- fluent-bit/plugins/in_syslog/syslog_conf.h | 32 - fluent-bit/plugins/in_syslog/syslog_conn.c | 247 -- fluent-bit/plugins/in_syslog/syslog_conn.h | 53 - fluent-bit/plugins/in_syslog/syslog_prot.c | 324 --- fluent-bit/plugins/in_syslog/syslog_prot.h | 35 - fluent-bit/plugins/in_syslog/syslog_server.c | 235 -- fluent-bit/plugins/in_syslog/syslog_server.h | 31 - fluent-bit/plugins/in_systemd/CMakeLists.txt | 11 - fluent-bit/plugins/in_systemd/systemd.c | 555 ---- fluent-bit/plugins/in_systemd/systemd_config.c | 314 --- fluent-bit/plugins/in_systemd/systemd_config.h | 82 - fluent-bit/plugins/in_systemd/systemd_db.c | 197 -- fluent-bit/plugins/in_systemd/systemd_db.h | 64 - fluent-bit/plugins/in_tail/CMakeLists.txt | 37 - fluent-bit/plugins/in_tail/tail.c | 783 ------ fluent-bit/plugins/in_tail/tail.h | 45 - fluent-bit/plugins/in_tail/tail_config.c | 472 ---- fluent-bit/plugins/in_tail/tail_config.h | 168 -- fluent-bit/plugins/in_tail/tail_db.c | 277 -- fluent-bit/plugins/in_tail/tail_db.h | 43 - fluent-bit/plugins/in_tail/tail_dockermode.c | 459 ---- fluent-bit/plugins/in_tail/tail_dockermode.h | 38 - fluent-bit/plugins/in_tail/tail_file.c | 1860 ------------- fluent-bit/plugins/in_tail/tail_file.h | 137 - fluent-bit/plugins/in_tail/tail_file_internal.h | 130 - fluent-bit/plugins/in_tail/tail_fs.h | 96 - fluent-bit/plugins/in_tail/tail_fs_inotify.c | 433 --- fluent-bit/plugins/in_tail/tail_fs_inotify.h | 37 - fluent-bit/plugins/in_tail/tail_fs_stat.c | 253 -- fluent-bit/plugins/in_tail/tail_fs_stat.h | 37 - fluent-bit/plugins/in_tail/tail_multiline.c | 606 ----- fluent-bit/plugins/in_tail/tail_multiline.h | 57 - fluent-bit/plugins/in_tail/tail_scan.c | 71 - fluent-bit/plugins/in_tail/tail_scan.h | 29 - fluent-bit/plugins/in_tail/tail_scan_glob.c | 278 -- fluent-bit/plugins/in_tail/tail_scan_win32.c | 245 -- fluent-bit/plugins/in_tail/tail_signal.h | 98 - fluent-bit/plugins/in_tail/tail_sql.h | 65 - fluent-bit/plugins/in_tail/win32.h | 67 - fluent-bit/plugins/in_tail/win32/interface.h | 44 - fluent-bit/plugins/in_tail/win32/io.c | 47 - fluent-bit/plugins/in_tail/win32/stat.c | 332 --- fluent-bit/plugins/in_tcp/CMakeLists.txt | 6 - fluent-bit/plugins/in_tcp/tcp.c | 184 -- fluent-bit/plugins/in_tcp/tcp.h | 50 - fluent-bit/plugins/in_tcp/tcp_config.c | 155 -- fluent-bit/plugins/in_tcp/tcp_config.h | 28 - fluent-bit/plugins/in_tcp/tcp_conn.c | 412 --- fluent-bit/plugins/in_tcp/tcp_conn.h | 59 - fluent-bit/plugins/in_thermal/CMakeLists.txt | 4 - fluent-bit/plugins/in_thermal/in_thermal.c | 372 --- fluent-bit/plugins/in_thermal/in_thermal.h | 55 - fluent-bit/plugins/in_udp/CMakeLists.txt | 6 - fluent-bit/plugins/in_udp/udp.c | 197 -- fluent-bit/plugins/in_udp/udp.h | 54 - fluent-bit/plugins/in_udp/udp_config.c | 155 -- fluent-bit/plugins/in_udp/udp_config.h | 28 - fluent-bit/plugins/in_udp/udp_conn.c | 500 ---- fluent-bit/plugins/in_udp/udp_conn.h | 57 - fluent-bit/plugins/in_unix_socket/CMakeLists.txt | 6 - fluent-bit/plugins/in_unix_socket/unix_socket.c | 320 --- fluent-bit/plugins/in_unix_socket/unix_socket.h | 55 - .../plugins/in_unix_socket/unix_socket_config.c | 153 -- .../plugins/in_unix_socket/unix_socket_config.h | 28 - .../plugins/in_unix_socket/unix_socket_conn.c | 433 --- .../plugins/in_unix_socket/unix_socket_conn.h | 60 - .../in_windows_exporter_metrics/CMakeLists.txt | 28 - .../plugins/in_windows_exporter_metrics/we.c | 1144 -------- .../plugins/in_windows_exporter_metrics/we.h | 332 --- .../in_windows_exporter_metrics/we_config.c | 154 -- .../in_windows_exporter_metrics/we_config.h | 32 - .../plugins/in_windows_exporter_metrics/we_cpu.c | 304 --- .../plugins/in_windows_exporter_metrics/we_cpu.h | 30 - .../plugins/in_windows_exporter_metrics/we_cs.c | 112 - .../plugins/in_windows_exporter_metrics/we_cs.h | 29 - .../in_windows_exporter_metrics/we_logical_disk.c | 272 -- .../in_windows_exporter_metrics/we_logical_disk.h | 29 - .../in_windows_exporter_metrics/we_metric.c | 368 --- .../in_windows_exporter_metrics/we_metric.h | 98 - .../plugins/in_windows_exporter_metrics/we_net.c | 253 -- .../plugins/in_windows_exporter_metrics/we_net.h | 29 - .../plugins/in_windows_exporter_metrics/we_os.c | 268 -- .../plugins/in_windows_exporter_metrics/we_os.h | 32 - .../in_windows_exporter_metrics/we_perflib.c | 1048 ------- .../in_windows_exporter_metrics/we_perflib.h | 72 - .../plugins/in_windows_exporter_metrics/we_util.c | 167 -- .../plugins/in_windows_exporter_metrics/we_util.h | 37 - .../plugins/in_windows_exporter_metrics/we_wmi.c | 572 ---- .../plugins/in_windows_exporter_metrics/we_wmi.h | 59 - .../in_windows_exporter_metrics/we_wmi_cpu_info.c | 116 - .../in_windows_exporter_metrics/we_wmi_cpu_info.h | 29 - .../in_windows_exporter_metrics/we_wmi_logon.c | 198 -- .../in_windows_exporter_metrics/we_wmi_logon.h | 29 - .../in_windows_exporter_metrics/we_wmi_memory.c | 557 ---- .../in_windows_exporter_metrics/we_wmi_memory.h | 29 - .../we_wmi_paging_file.c | 156 -- .../we_wmi_paging_file.h | 29 - .../in_windows_exporter_metrics/we_wmi_process.c | 417 --- .../in_windows_exporter_metrics/we_wmi_process.h | 29 - .../in_windows_exporter_metrics/we_wmi_service.c | 493 ---- .../in_windows_exporter_metrics/we_wmi_service.h | 29 - .../in_windows_exporter_metrics/we_wmi_system.c | 190 -- .../in_windows_exporter_metrics/we_wmi_system.h | 29 - .../we_wmi_thermalzone.c | 171 -- .../we_wmi_thermalzone.h | 29 - fluent-bit/plugins/in_winevtlog/CMakeLists.txt | 6 - fluent-bit/plugins/in_winevtlog/in_winevtlog.c | 279 -- fluent-bit/plugins/in_winevtlog/pack.c | 625 ----- fluent-bit/plugins/in_winevtlog/winevtlog.c | 840 ------ fluent-bit/plugins/in_winevtlog/winevtlog.h | 134 - fluent-bit/plugins/in_winlog/CMakeLists.txt | 6 - fluent-bit/plugins/in_winlog/in_winlog.c | 267 -- fluent-bit/plugins/in_winlog/pack.c | 451 --- fluent-bit/plugins/in_winlog/winlog.c | 300 -- fluent-bit/plugins/in_winlog/winlog.h | 110 - fluent-bit/plugins/in_winstat/CMakeLists.txt | 4 - fluent-bit/plugins/in_winstat/winstat.c | 340 --- fluent-bit/plugins/out_azure/CMakeLists.txt | 6 - fluent-bit/plugins/out_azure/azure.c | 452 --- fluent-bit/plugins/out_azure/azure.h | 62 - fluent-bit/plugins/out_azure/azure_conf.c | 219 -- fluent-bit/plugins/out_azure/azure_conf.h | 29 - fluent-bit/plugins/out_azure_blob/CMakeLists.txt | 10 - fluent-bit/plugins/out_azure_blob/azure_blob.c | 594 ---- fluent-bit/plugins/out_azure_blob/azure_blob.h | 74 - .../plugins/out_azure_blob/azure_blob_appendblob.c | 44 - .../plugins/out_azure_blob/azure_blob_appendblob.h | 28 - .../plugins/out_azure_blob/azure_blob_blockblob.c | 238 -- .../plugins/out_azure_blob/azure_blob_blockblob.h | 32 - .../plugins/out_azure_blob/azure_blob_conf.c | 245 -- .../plugins/out_azure_blob/azure_blob_conf.h | 29 - .../plugins/out_azure_blob/azure_blob_http.c | 361 --- .../plugins/out_azure_blob/azure_blob_http.h | 36 - fluent-bit/plugins/out_azure_blob/azure_blob_uri.c | 150 - fluent-bit/plugins/out_azure_blob/azure_blob_uri.h | 34 - fluent-bit/plugins/out_azure_kusto/CMakeLists.txt | 7 - fluent-bit/plugins/out_azure_kusto/azure_kusto.c | 477 ---- fluent-bit/plugins/out_azure_kusto/azure_kusto.h | 110 - .../plugins/out_azure_kusto/azure_kusto_conf.c | 665 ----- .../plugins/out_azure_kusto/azure_kusto_conf.h | 31 - .../plugins/out_azure_kusto/azure_kusto_ingest.c | 496 ---- .../plugins/out_azure_kusto/azure_kusto_ingest.h | 28 - .../out_azure_logs_ingestion/CMakeLists.txt | 6 - .../azure_logs_ingestion.c | 445 --- .../azure_logs_ingestion.h | 74 - .../azure_logs_ingestion_conf.c | 172 -- .../azure_logs_ingestion_conf.h | 29 - fluent-bit/plugins/out_bigquery/CMakeLists.txt | 6 - fluent-bit/plugins/out_bigquery/bigquery.c | 1159 -------- fluent-bit/plugins/out_bigquery/bigquery.h | 132 - fluent-bit/plugins/out_bigquery/bigquery_conf.c | 435 --- fluent-bit/plugins/out_bigquery/bigquery_conf.h | 29 - fluent-bit/plugins/out_calyptia/CMakeLists.txt | 4 - fluent-bit/plugins/out_calyptia/calyptia.c | 1025 ------- fluent-bit/plugins/out_calyptia/calyptia.h | 85 - fluent-bit/plugins/out_chronicle/CMakeLists.txt | 6 - fluent-bit/plugins/out_chronicle/chronicle.c | 962 ------- fluent-bit/plugins/out_chronicle/chronicle.h | 96 - fluent-bit/plugins/out_chronicle/chronicle_conf.c | 421 --- fluent-bit/plugins/out_chronicle/chronicle_conf.h | 29 - .../plugins/out_cloudwatch_logs/CMakeLists.txt | 5 - .../plugins/out_cloudwatch_logs/cloudwatch_api.c | 1564 ----------- .../plugins/out_cloudwatch_logs/cloudwatch_api.h | 57 - .../plugins/out_cloudwatch_logs/cloudwatch_logs.c | 670 ----- .../plugins/out_cloudwatch_logs/cloudwatch_logs.h | 158 -- fluent-bit/plugins/out_counter/CMakeLists.txt | 4 - fluent-bit/plugins/out_counter/counter.c | 106 - fluent-bit/plugins/out_datadog/CMakeLists.txt | 6 - fluent-bit/plugins/out_datadog/datadog.c | 568 ---- fluent-bit/plugins/out_datadog/datadog.h | 81 - fluent-bit/plugins/out_datadog/datadog_conf.c | 223 -- fluent-bit/plugins/out_datadog/datadog_conf.h | 33 - fluent-bit/plugins/out_datadog/datadog_remap.c | 277 -- fluent-bit/plugins/out_datadog/datadog_remap.h | 37 - fluent-bit/plugins/out_es/CMakeLists.txt | 8 - fluent-bit/plugins/out_es/es.c | 1230 --------- fluent-bit/plugins/out_es/es.h | 140 - fluent-bit/plugins/out_es/es_bulk.c | 113 - fluent-bit/plugins/out_es/es_bulk.h | 46 - fluent-bit/plugins/out_es/es_conf.c | 537 ---- fluent-bit/plugins/out_es/es_conf.h | 33 - fluent-bit/plugins/out_es/murmur3.c | 314 --- fluent-bit/plugins/out_es/murmur3.h | 29 - fluent-bit/plugins/out_exit/CMakeLists.txt | 4 - fluent-bit/plugins/out_exit/exit.c | 108 - fluent-bit/plugins/out_file/CMakeLists.txt | 4 - fluent-bit/plugins/out_file/file.c | 705 ----- fluent-bit/plugins/out_file/file.h | 32 - fluent-bit/plugins/out_flowcounter/CMakeLists.txt | 4 - .../plugins/out_flowcounter/out_flowcounter.c | 297 -- .../plugins/out_flowcounter/out_flowcounter.h | 49 - fluent-bit/plugins/out_forward/CMakeLists.txt | 6 - fluent-bit/plugins/out_forward/README.md | 12 - fluent-bit/plugins/out_forward/forward.c | 1832 ------------- fluent-bit/plugins/out_forward/forward.h | 146 - fluent-bit/plugins/out_forward/forward_format.c | 640 ----- fluent-bit/plugins/out_forward/forward_format.h | 48 - fluent-bit/plugins/out_gelf/CMakeLists.txt | 5 - fluent-bit/plugins/out_gelf/gelf.c | 556 ---- fluent-bit/plugins/out_gelf/gelf.h | 47 - fluent-bit/plugins/out_http/CMakeLists.txt | 6 - fluent-bit/plugins/out_http/http.c | 774 ------ fluent-bit/plugins/out_http/http.h | 103 - fluent-bit/plugins/out_http/http_conf.c | 298 -- fluent-bit/plugins/out_http/http_conf.h | 32 - fluent-bit/plugins/out_influxdb/CMakeLists.txt | 5 - fluent-bit/plugins/out_influxdb/influxdb.c | 682 ----- fluent-bit/plugins/out_influxdb/influxdb.h | 78 - fluent-bit/plugins/out_influxdb/influxdb_bulk.c | 233 -- fluent-bit/plugins/out_influxdb/influxdb_bulk.h | 54 - fluent-bit/plugins/out_kafka/CMakeLists.txt | 8 - fluent-bit/plugins/out_kafka/kafka.c | 658 ----- fluent-bit/plugins/out_kafka/kafka_callbacks.h | 31 - fluent-bit/plugins/out_kafka/kafka_config.c | 253 -- fluent-bit/plugins/out_kafka/kafka_config.h | 129 - fluent-bit/plugins/out_kafka/kafka_topic.c | 120 - fluent-bit/plugins/out_kafka/kafka_topic.h | 34 - fluent-bit/plugins/out_kafka_rest/CMakeLists.txt | 5 - fluent-bit/plugins/out_kafka_rest/kafka.c | 351 --- fluent-bit/plugins/out_kafka_rest/kafka.h | 66 - fluent-bit/plugins/out_kafka_rest/kafka_conf.c | 223 -- fluent-bit/plugins/out_kafka_rest/kafka_conf.h | 33 - .../plugins/out_kinesis_firehose/CMakeLists.txt | 5 - fluent-bit/plugins/out_kinesis_firehose/firehose.c | 503 ---- fluent-bit/plugins/out_kinesis_firehose/firehose.h | 104 - .../plugins/out_kinesis_firehose/firehose_api.c | 959 ------- .../plugins/out_kinesis_firehose/firehose_api.h | 45 - .../plugins/out_kinesis_streams/CMakeLists.txt | 5 - fluent-bit/plugins/out_kinesis_streams/kinesis.c | 499 ---- fluent-bit/plugins/out_kinesis_streams/kinesis.h | 109 - .../plugins/out_kinesis_streams/kinesis_api.c | 987 ------- .../plugins/out_kinesis_streams/kinesis_api.h | 44 - fluent-bit/plugins/out_lib/CMakeLists.txt | 5 - fluent-bit/plugins/out_lib/out_lib.c | 222 -- fluent-bit/plugins/out_lib/out_lib.h | 42 - fluent-bit/plugins/out_logdna/CMakeLists.txt | 5 - fluent-bit/plugins/out_logdna/logdna.c | 591 ---- fluent-bit/plugins/out_logdna/logdna.h | 51 - fluent-bit/plugins/out_loki/CMakeLists.txt | 5 - fluent-bit/plugins/out_loki/loki.c | 1868 ------------- fluent-bit/plugins/out_loki/loki.h | 98 - fluent-bit/plugins/out_nats/CMakeLists.txt | 4 - fluent-bit/plugins/out_nats/nats.c | 252 -- fluent-bit/plugins/out_nats/nats.h | 33 - fluent-bit/plugins/out_nrlogs/CMakeLists.txt | 5 - fluent-bit/plugins/out_nrlogs/newrelic.c | 566 ---- fluent-bit/plugins/out_nrlogs/newrelic.h | 52 - fluent-bit/plugins/out_null/CMakeLists.txt | 4 - fluent-bit/plugins/out_null/null.c | 178 -- fluent-bit/plugins/out_opensearch/CMakeLists.txt | 6 - fluent-bit/plugins/out_opensearch/opensearch.c | 1291 --------- fluent-bit/plugins/out_opensearch/opensearch.h | 155 -- fluent-bit/plugins/out_opensearch/os_conf.c | 411 --- fluent-bit/plugins/out_opensearch/os_conf.h | 33 - .../plugins/out_opentelemetry/CMakeLists.txt | 6 - .../plugins/out_opentelemetry/opentelemetry.c | 1207 -------- .../plugins/out_opentelemetry/opentelemetry.h | 80 - .../plugins/out_opentelemetry/opentelemetry_conf.c | 262 -- .../plugins/out_opentelemetry/opentelemetry_conf.h | 33 - .../out_oracle_log_analytics/CMakeLists.txt | 6 - .../plugins/out_oracle_log_analytics/oci_logan.c | 1313 --------- .../plugins/out_oracle_log_analytics/oci_logan.h | 215 -- .../out_oracle_log_analytics/oci_logan_conf.c | 493 ---- .../out_oracle_log_analytics/oci_logan_conf.h | 34 - fluent-bit/plugins/out_pgsql/CMakeLists.txt | 8 - fluent-bit/plugins/out_pgsql/pgsql.c | 389 --- fluent-bit/plugins/out_pgsql/pgsql.h | 91 - fluent-bit/plugins/out_pgsql/pgsql_connections.c | 193 -- fluent-bit/plugins/out_pgsql/pgsql_connections.h | 27 - fluent-bit/plugins/out_plot/CMakeLists.txt | 4 - fluent-bit/plugins/out_plot/plot.c | 242 -- .../plugins/out_prometheus_exporter/CMakeLists.txt | 14 - fluent-bit/plugins/out_prometheus_exporter/prom.c | 298 -- fluent-bit/plugins/out_prometheus_exporter/prom.h | 46 - .../plugins/out_prometheus_exporter/prom_http.c | 268 -- .../plugins/out_prometheus_exporter/prom_http.h | 56 - .../out_prometheus_remote_write/CMakeLists.txt | 6 - .../out_prometheus_remote_write/remote_write.c | 466 ---- .../out_prometheus_remote_write/remote_write.h | 83 - .../remote_write_conf.c | 254 -- .../remote_write_conf.h | 33 - fluent-bit/plugins/out_retry/CMakeLists.txt | 4 - fluent-bit/plugins/out_retry/retry.c | 116 - fluent-bit/plugins/out_s3/CMakeLists.txt | 6 - fluent-bit/plugins/out_s3/s3.c | 2500 ----------------- fluent-bit/plugins/out_s3/s3.h | 203 -- fluent-bit/plugins/out_s3/s3_multipart.c | 707 ----- fluent-bit/plugins/out_s3/s3_store.c | 543 ---- fluent-bit/plugins/out_s3/s3_store.h | 68 - fluent-bit/plugins/out_skywalking/CMakeLists.txt | 4 - fluent-bit/plugins/out_skywalking/skywalking.c | 427 --- fluent-bit/plugins/out_skywalking/skywalking.h | 41 - fluent-bit/plugins/out_slack/CMakeLists.txt | 5 - fluent-bit/plugins/out_slack/slack.c | 336 --- fluent-bit/plugins/out_slack/slack.h | 43 - fluent-bit/plugins/out_splunk/CMakeLists.txt | 6 - fluent-bit/plugins/out_splunk/splunk.c | 873 ------ fluent-bit/plugins/out_splunk/splunk.h | 119 - fluent-bit/plugins/out_splunk/splunk_conf.c | 313 --- fluent-bit/plugins/out_splunk/splunk_conf.h | 29 - fluent-bit/plugins/out_stackdriver/CMakeLists.txt | 13 - fluent-bit/plugins/out_stackdriver/gce_metadata.c | 222 -- fluent-bit/plugins/out_stackdriver/gce_metadata.h | 48 - fluent-bit/plugins/out_stackdriver/stackdriver.c | 2867 -------------------- fluent-bit/plugins/out_stackdriver/stackdriver.h | 241 -- .../plugins/out_stackdriver/stackdriver_conf.c | 667 ----- .../plugins/out_stackdriver/stackdriver_conf.h | 29 - .../plugins/out_stackdriver/stackdriver_helper.c | 63 - .../plugins/out_stackdriver/stackdriver_helper.h | 51 - .../out_stackdriver/stackdriver_http_request.c | 393 --- .../out_stackdriver/stackdriver_http_request.h | 120 - .../out_stackdriver/stackdriver_operation.c | 147 - .../out_stackdriver/stackdriver_operation.h | 82 - .../out_stackdriver/stackdriver_resource_types.c | 143 - .../out_stackdriver/stackdriver_resource_types.h | 41 - .../out_stackdriver/stackdriver_source_location.c | 139 - .../out_stackdriver/stackdriver_source_location.h | 80 - .../out_stackdriver/stackdriver_timestamp.c | 180 -- .../out_stackdriver/stackdriver_timestamp.h | 47 - fluent-bit/plugins/out_stdout/CMakeLists.txt | 4 - fluent-bit/plugins/out_stdout/stdout.c | 301 -- fluent-bit/plugins/out_stdout/stdout.h | 34 - fluent-bit/plugins/out_syslog/CMakeLists.txt | 5 - fluent-bit/plugins/out_syslog/syslog.c | 1170 -------- fluent-bit/plugins/out_syslog/syslog_conf.c | 162 -- fluent-bit/plugins/out_syslog/syslog_conf.h | 70 - fluent-bit/plugins/out_tcp/CMakeLists.txt | 6 - fluent-bit/plugins/out_tcp/tcp.c | 269 -- fluent-bit/plugins/out_tcp/tcp.h | 46 - fluent-bit/plugins/out_tcp/tcp_conf.c | 154 -- fluent-bit/plugins/out_tcp/tcp_conf.h | 32 - fluent-bit/plugins/out_td/CMakeLists.txt | 7 - fluent-bit/plugins/out_td/td.c | 271 -- fluent-bit/plugins/out_td/td.h | 24 - fluent-bit/plugins/out_td/td_config.c | 86 - fluent-bit/plugins/out_td/td_config.h | 41 - fluent-bit/plugins/out_td/td_http.c | 94 - fluent-bit/plugins/out_td/td_http.h | 35 - fluent-bit/plugins/out_udp/CMakeLists.txt | 6 - fluent-bit/plugins/out_udp/udp.c | 351 --- fluent-bit/plugins/out_udp/udp.h | 47 - fluent-bit/plugins/out_udp/udp_conf.c | 135 - fluent-bit/plugins/out_udp/udp_conf.h | 32 - .../plugins/out_vivo_exporter/CMakeLists.txt | 15 - fluent-bit/plugins/out_vivo_exporter/vivo.c | 343 --- fluent-bit/plugins/out_vivo_exporter/vivo.h | 45 - fluent-bit/plugins/out_vivo_exporter/vivo_http.c | 266 -- fluent-bit/plugins/out_vivo_exporter/vivo_http.h | 56 - fluent-bit/plugins/out_vivo_exporter/vivo_stream.c | 239 -- fluent-bit/plugins/out_vivo_exporter/vivo_stream.h | 59 - fluent-bit/plugins/out_websocket/CMakeLists.txt | 5 - fluent-bit/plugins/out_websocket/websocket.c | 331 --- fluent-bit/plugins/out_websocket/websocket.h | 54 - fluent-bit/plugins/out_websocket/websocket_conf.c | 159 -- fluent-bit/plugins/out_websocket/websocket_conf.h | 32 - .../plugins/processor_attributes/CMakeLists.txt | 4 - .../plugins/processor_attributes/attributes.c | 1408 ---------- .../plugins/processor_attributes/variant_utils.h | 626 ----- fluent-bit/plugins/processor_labels/CMakeLists.txt | 4 - fluent-bit/plugins/processor_labels/labels.c | 1784 ------------ 669 files changed, 149123 deletions(-) delete mode 100644 fluent-bit/plugins/CMakeLists.txt delete mode 100644 fluent-bit/plugins/custom_calyptia/CMakeLists.txt delete mode 100644 fluent-bit/plugins/custom_calyptia/calyptia.c delete mode 100644 fluent-bit/plugins/filter_alter_size/CMakeLists.txt delete mode 100644 fluent-bit/plugins/filter_alter_size/alter_size.c delete mode 100644 fluent-bit/plugins/filter_alter_size/alter_size.h delete mode 100644 fluent-bit/plugins/filter_aws/CMakeLists.txt delete mode 100644 fluent-bit/plugins/filter_aws/aws.c delete mode 100644 fluent-bit/plugins/filter_aws/aws.h delete mode 100644 fluent-bit/plugins/filter_checklist/CMakeLists.txt delete mode 100644 fluent-bit/plugins/filter_checklist/checklist.c delete mode 100644 fluent-bit/plugins/filter_checklist/checklist.h delete mode 100644 fluent-bit/plugins/filter_ecs/CMakeLists.txt delete mode 100644 fluent-bit/plugins/filter_ecs/ecs.c delete mode 100644 fluent-bit/plugins/filter_ecs/ecs.h delete mode 100644 fluent-bit/plugins/filter_expect/CMakeLists.txt delete mode 100644 fluent-bit/plugins/filter_expect/expect.c delete mode 100644 fluent-bit/plugins/filter_expect/expect.h delete mode 100644 fluent-bit/plugins/filter_geoip2/.gitignore delete mode 100644 fluent-bit/plugins/filter_geoip2/CMakeLists.txt delete mode 100644 fluent-bit/plugins/filter_geoip2/geoip2.c delete mode 100644 fluent-bit/plugins/filter_geoip2/geoip2.h delete mode 100644 fluent-bit/plugins/filter_geoip2/libmaxminddb/CMakeLists.txt delete mode 100644 fluent-bit/plugins/filter_geoip2/libmaxminddb/LICENSE delete mode 100644 fluent-bit/plugins/filter_geoip2/libmaxminddb/NOTICE delete mode 100644 fluent-bit/plugins/filter_geoip2/libmaxminddb/VERSION delete mode 100644 fluent-bit/plugins/filter_geoip2/libmaxminddb/bin/CMakeLists.txt delete mode 100644 fluent-bit/plugins/filter_geoip2/libmaxminddb/bin/Makefile.am delete mode 100644 fluent-bit/plugins/filter_geoip2/libmaxminddb/bin/mmdblookup.c delete mode 100644 fluent-bit/plugins/filter_geoip2/libmaxminddb/include/maxminddb.h delete mode 100644 fluent-bit/plugins/filter_geoip2/libmaxminddb/include/maxminddb_config.h.cmake.in delete mode 100644 fluent-bit/plugins/filter_geoip2/libmaxminddb/include/maxminddb_config.h.in delete mode 100644 fluent-bit/plugins/filter_geoip2/libmaxminddb/src/Makefile.am delete mode 100644 fluent-bit/plugins/filter_geoip2/libmaxminddb/src/data-pool.c delete mode 100644 fluent-bit/plugins/filter_geoip2/libmaxminddb/src/data-pool.h delete mode 100644 fluent-bit/plugins/filter_geoip2/libmaxminddb/src/libmaxminddb.pc.in delete mode 100644 fluent-bit/plugins/filter_geoip2/libmaxminddb/src/maxminddb-compat-util.h delete mode 100644 fluent-bit/plugins/filter_geoip2/libmaxminddb/src/maxminddb.c delete mode 100644 fluent-bit/plugins/filter_grep/CMakeLists.txt delete mode 100644 fluent-bit/plugins/filter_grep/grep.c delete mode 100644 fluent-bit/plugins/filter_grep/grep.h delete mode 100644 fluent-bit/plugins/filter_kubernetes/CMakeLists.txt delete mode 100644 fluent-bit/plugins/filter_kubernetes/kube_conf.c delete mode 100644 fluent-bit/plugins/filter_kubernetes/kube_conf.h delete mode 100644 fluent-bit/plugins/filter_kubernetes/kube_meta.c delete mode 100644 fluent-bit/plugins/filter_kubernetes/kube_meta.h delete mode 100644 fluent-bit/plugins/filter_kubernetes/kube_property.c delete mode 100644 fluent-bit/plugins/filter_kubernetes/kube_property.h delete mode 100644 fluent-bit/plugins/filter_kubernetes/kube_props.h delete mode 100644 fluent-bit/plugins/filter_kubernetes/kube_regex.c delete mode 100644 fluent-bit/plugins/filter_kubernetes/kube_regex.h delete mode 100644 fluent-bit/plugins/filter_kubernetes/kubernetes.c delete mode 100644 fluent-bit/plugins/filter_log_to_metrics/CMakeLists.txt delete mode 100644 fluent-bit/plugins/filter_log_to_metrics/log_to_metrics.c delete mode 100644 fluent-bit/plugins/filter_log_to_metrics/log_to_metrics.h delete mode 100644 fluent-bit/plugins/filter_lua/CMakeLists.txt delete mode 100644 fluent-bit/plugins/filter_lua/lua.c delete mode 100644 fluent-bit/plugins/filter_lua/lua_config.c delete mode 100644 fluent-bit/plugins/filter_lua/lua_config.h delete mode 100644 fluent-bit/plugins/filter_modify/CMakeLists.txt delete mode 100644 fluent-bit/plugins/filter_modify/modify.c delete mode 100644 fluent-bit/plugins/filter_modify/modify.h delete mode 100644 fluent-bit/plugins/filter_multiline/CMakeLists.txt delete mode 100644 fluent-bit/plugins/filter_multiline/ml.c delete mode 100644 fluent-bit/plugins/filter_multiline/ml.h delete mode 100644 fluent-bit/plugins/filter_multiline/ml_concat.c delete mode 100644 fluent-bit/plugins/filter_multiline/ml_concat.h delete mode 100644 fluent-bit/plugins/filter_nest/CMakeLists.txt delete mode 100644 fluent-bit/plugins/filter_nest/nest.c delete mode 100644 fluent-bit/plugins/filter_nest/nest.h delete mode 100644 fluent-bit/plugins/filter_nightfall/CMakeLists.txt delete mode 100644 fluent-bit/plugins/filter_nightfall/nightfall.c delete mode 100644 fluent-bit/plugins/filter_nightfall/nightfall.h delete mode 100644 fluent-bit/plugins/filter_nightfall/nightfall_api.c delete mode 100644 fluent-bit/plugins/filter_nightfall/nightfall_api.h delete mode 100644 fluent-bit/plugins/filter_parser/CMakeLists.txt delete mode 100644 fluent-bit/plugins/filter_parser/filter_parser.c delete mode 100644 fluent-bit/plugins/filter_parser/filter_parser.h delete mode 100644 fluent-bit/plugins/filter_record_modifier/CMakeLists.txt delete mode 100644 fluent-bit/plugins/filter_record_modifier/filter_modifier.c delete mode 100644 fluent-bit/plugins/filter_record_modifier/filter_modifier.h delete mode 100644 fluent-bit/plugins/filter_rewrite_tag/CMakeLists.txt delete mode 100644 fluent-bit/plugins/filter_rewrite_tag/rewrite_tag.c delete mode 100644 fluent-bit/plugins/filter_rewrite_tag/rewrite_tag.h delete mode 100644 fluent-bit/plugins/filter_stdout/CMakeLists.txt delete mode 100644 fluent-bit/plugins/filter_stdout/stdout.c delete mode 100644 fluent-bit/plugins/filter_tensorflow/CMakeLists.txt delete mode 100644 fluent-bit/plugins/filter_tensorflow/tensorflow.c delete mode 100644 fluent-bit/plugins/filter_tensorflow/tensorflow.h delete mode 100644 fluent-bit/plugins/filter_throttle/CMakeLists.txt delete mode 100644 fluent-bit/plugins/filter_throttle/throttle.c delete mode 100644 fluent-bit/plugins/filter_throttle/throttle.h delete mode 100644 fluent-bit/plugins/filter_throttle/window.c delete mode 100644 fluent-bit/plugins/filter_throttle/window.h delete mode 100644 fluent-bit/plugins/filter_throttle_size/CMakeLists.txt delete mode 100644 fluent-bit/plugins/filter_throttle_size/size_window.c delete mode 100644 fluent-bit/plugins/filter_throttle_size/size_window.h delete mode 100644 fluent-bit/plugins/filter_throttle_size/throttle_size.c delete mode 100644 fluent-bit/plugins/filter_throttle_size/throttle_size.h delete mode 100644 fluent-bit/plugins/filter_type_converter/CMakeLists.txt delete mode 100644 fluent-bit/plugins/filter_type_converter/type_converter.c delete mode 100644 fluent-bit/plugins/filter_type_converter/type_converter.h delete mode 100644 fluent-bit/plugins/filter_wasm/CMakeLists.txt delete mode 100644 fluent-bit/plugins/filter_wasm/filter_wasm.c delete mode 100644 fluent-bit/plugins/filter_wasm/filter_wasm.h delete mode 100644 fluent-bit/plugins/in_calyptia_fleet/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_calyptia_fleet/in_calyptia_fleet.c delete mode 100644 fluent-bit/plugins/in_collectd/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_collectd/in_collectd.c delete mode 100644 fluent-bit/plugins/in_collectd/in_collectd.h delete mode 100644 fluent-bit/plugins/in_collectd/netprot.c delete mode 100644 fluent-bit/plugins/in_collectd/netprot.h delete mode 100644 fluent-bit/plugins/in_collectd/typesdb.c delete mode 100644 fluent-bit/plugins/in_collectd/typesdb.h delete mode 100644 fluent-bit/plugins/in_collectd/typesdb_parser.c delete mode 100644 fluent-bit/plugins/in_collectd/typesdb_parser.h delete mode 100644 fluent-bit/plugins/in_cpu/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_cpu/cpu.c delete mode 100644 fluent-bit/plugins/in_cpu/cpu.h delete mode 100644 fluent-bit/plugins/in_disk/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_disk/in_disk.c delete mode 100644 fluent-bit/plugins/in_disk/in_disk.h delete mode 100644 fluent-bit/plugins/in_docker/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_docker/cgroup_v1.c delete mode 100644 fluent-bit/plugins/in_docker/docker.c delete mode 100644 fluent-bit/plugins/in_docker/docker.h delete mode 100644 fluent-bit/plugins/in_docker_events/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_docker_events/docker_events.c delete mode 100644 fluent-bit/plugins/in_docker_events/docker_events.h delete mode 100644 fluent-bit/plugins/in_docker_events/docker_events_config.c delete mode 100644 fluent-bit/plugins/in_docker_events/docker_events_config.h delete mode 100644 fluent-bit/plugins/in_dummy/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_dummy/in_dummy.c delete mode 100644 fluent-bit/plugins/in_dummy/in_dummy.h delete mode 100644 fluent-bit/plugins/in_elasticsearch/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_elasticsearch/in_elasticsearch.c delete mode 100644 fluent-bit/plugins/in_elasticsearch/in_elasticsearch.h delete mode 100644 fluent-bit/plugins/in_elasticsearch/in_elasticsearch_bulk_conn.c delete mode 100644 fluent-bit/plugins/in_elasticsearch/in_elasticsearch_bulk_conn.h delete mode 100644 fluent-bit/plugins/in_elasticsearch/in_elasticsearch_bulk_prot.c delete mode 100644 fluent-bit/plugins/in_elasticsearch/in_elasticsearch_bulk_prot.h delete mode 100644 fluent-bit/plugins/in_elasticsearch/in_elasticsearch_config.c delete mode 100644 fluent-bit/plugins/in_elasticsearch/in_elasticsearch_config.h delete mode 100644 fluent-bit/plugins/in_emitter/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_emitter/emitter.c delete mode 100644 fluent-bit/plugins/in_event_test/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_event_test/event_test.c delete mode 100644 fluent-bit/plugins/in_event_type/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_event_type/event_type.c delete mode 100644 fluent-bit/plugins/in_exec/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_exec/in_exec.c delete mode 100644 fluent-bit/plugins/in_exec/in_exec.h delete mode 100644 fluent-bit/plugins/in_exec/in_exec_win32_compat.h delete mode 100644 fluent-bit/plugins/in_exec_wasi/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_exec_wasi/in_exec_wasi.c delete mode 100644 fluent-bit/plugins/in_exec_wasi/in_exec_wasi.h delete mode 100644 fluent-bit/plugins/in_fluentbit_metrics/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_fluentbit_metrics/metrics.c delete mode 100644 fluent-bit/plugins/in_forward/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_forward/fw.c delete mode 100644 fluent-bit/plugins/in_forward/fw.h delete mode 100644 fluent-bit/plugins/in_forward/fw_config.c delete mode 100644 fluent-bit/plugins/in_forward/fw_config.h delete mode 100644 fluent-bit/plugins/in_forward/fw_conn.c delete mode 100644 fluent-bit/plugins/in_forward/fw_conn.h delete mode 100644 fluent-bit/plugins/in_forward/fw_prot.c delete mode 100644 fluent-bit/plugins/in_forward/fw_prot.h delete mode 100644 fluent-bit/plugins/in_head/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_head/in_head.c delete mode 100644 fluent-bit/plugins/in_head/in_head.h delete mode 100644 fluent-bit/plugins/in_health/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_health/health.c delete mode 100644 fluent-bit/plugins/in_http/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_http/http.c delete mode 100644 fluent-bit/plugins/in_http/http.h delete mode 100644 fluent-bit/plugins/in_http/http_config.c delete mode 100644 fluent-bit/plugins/in_http/http_config.h delete mode 100644 fluent-bit/plugins/in_http/http_conn.c delete mode 100644 fluent-bit/plugins/in_http/http_conn.h delete mode 100644 fluent-bit/plugins/in_http/http_prot.c delete mode 100644 fluent-bit/plugins/in_http/http_prot.h delete mode 100644 fluent-bit/plugins/in_kafka/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_kafka/in_kafka.c delete mode 100644 fluent-bit/plugins/in_kafka/in_kafka.h delete mode 100644 fluent-bit/plugins/in_kmsg/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_kmsg/in_kmsg.c delete mode 100644 fluent-bit/plugins/in_kmsg/in_kmsg.h delete mode 100644 fluent-bit/plugins/in_kubernetes_events/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_kubernetes_events/kubernetes_events.c delete mode 100644 fluent-bit/plugins/in_kubernetes_events/kubernetes_events.h delete mode 100644 fluent-bit/plugins/in_kubernetes_events/kubernetes_events_conf.c delete mode 100644 fluent-bit/plugins/in_kubernetes_events/kubernetes_events_conf.h delete mode 100644 fluent-bit/plugins/in_kubernetes_events/kubernetes_events_sql.h delete mode 100644 fluent-bit/plugins/in_lib/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_lib/in_lib.c delete mode 100644 fluent-bit/plugins/in_lib/in_lib.h delete mode 100644 fluent-bit/plugins/in_mem/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_mem/mem.c delete mode 100644 fluent-bit/plugins/in_mem/mem.h delete mode 100644 fluent-bit/plugins/in_mem/proc.c delete mode 100644 fluent-bit/plugins/in_mem/proc.h delete mode 100644 fluent-bit/plugins/in_mqtt/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_mqtt/mqtt.c delete mode 100644 fluent-bit/plugins/in_mqtt/mqtt.h delete mode 100644 fluent-bit/plugins/in_mqtt/mqtt_config.c delete mode 100644 fluent-bit/plugins/in_mqtt/mqtt_config.h delete mode 100644 fluent-bit/plugins/in_mqtt/mqtt_conn.c delete mode 100644 fluent-bit/plugins/in_mqtt/mqtt_conn.h delete mode 100644 fluent-bit/plugins/in_mqtt/mqtt_prot.c delete mode 100644 fluent-bit/plugins/in_mqtt/mqtt_prot.h delete mode 100644 fluent-bit/plugins/in_netif/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_netif/in_netif.c delete mode 100644 fluent-bit/plugins/in_netif/in_netif.h delete mode 100644 fluent-bit/plugins/in_nginx_exporter_metrics/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_nginx_exporter_metrics/nginx.c delete mode 100644 fluent-bit/plugins/in_nginx_exporter_metrics/nginx.h delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne.c delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne.h delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_config.c delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_config.h delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_cpu.c delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_cpu.h delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_cpu_linux.c delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_cpufreq.h delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_cpufreq_linux.c delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_diskstats.c delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_diskstats.h delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_diskstats_linux.c delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_filefd_linux.c delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_filefd_linux.h delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_filesystem.c delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_filesystem.h delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_filesystem_linux.c delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_loadavg.c delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_loadavg.h delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_loadavg_linux.c delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_meminfo.c delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_meminfo.h delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_meminfo_linux.c delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_netdev.c delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_netdev.h delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_netdev_linux.c delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_stat_linux.c delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_stat_linux.h delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_systemd.c delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_systemd.h delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_textfile.c delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_textfile.h delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_textfile_linux.c delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_time.c delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_time.h delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_uname.c delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_uname.h delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_uname_linux.c delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_utils.c delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_utils.h delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_vmstat_linux.c delete mode 100644 fluent-bit/plugins/in_node_exporter_metrics/ne_vmstat_linux.h delete mode 100644 fluent-bit/plugins/in_opentelemetry/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_opentelemetry/http_conn.c delete mode 100644 fluent-bit/plugins/in_opentelemetry/http_conn.h delete mode 100644 fluent-bit/plugins/in_opentelemetry/opentelemetry.c delete mode 100644 fluent-bit/plugins/in_opentelemetry/opentelemetry.h delete mode 100644 fluent-bit/plugins/in_opentelemetry/opentelemetry_config.c delete mode 100644 fluent-bit/plugins/in_opentelemetry/opentelemetry_config.h delete mode 100644 fluent-bit/plugins/in_opentelemetry/opentelemetry_prot.c delete mode 100644 fluent-bit/plugins/in_opentelemetry/opentelemetry_prot.h delete mode 100644 fluent-bit/plugins/in_podman_metrics/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_podman_metrics/podman_metrics.c delete mode 100644 fluent-bit/plugins/in_podman_metrics/podman_metrics.h delete mode 100644 fluent-bit/plugins/in_podman_metrics/podman_metrics_config.h delete mode 100644 fluent-bit/plugins/in_podman_metrics/podman_metrics_data.c delete mode 100644 fluent-bit/plugins/in_podman_metrics/podman_metrics_data.h delete mode 100644 fluent-bit/plugins/in_proc/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_proc/in_proc.c delete mode 100644 fluent-bit/plugins/in_proc/in_proc.h delete mode 100644 fluent-bit/plugins/in_prometheus_scrape/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_prometheus_scrape/prom_scrape.c delete mode 100644 fluent-bit/plugins/in_prometheus_scrape/prom_scrape.h delete mode 100644 fluent-bit/plugins/in_random/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_random/random.c delete mode 100644 fluent-bit/plugins/in_serial/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_serial/in_serial.c delete mode 100644 fluent-bit/plugins/in_serial/in_serial.h delete mode 100644 fluent-bit/plugins/in_serial/in_serial_config.c delete mode 100644 fluent-bit/plugins/in_serial/in_serial_config.h delete mode 100644 fluent-bit/plugins/in_splunk/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_splunk/splunk.c delete mode 100644 fluent-bit/plugins/in_splunk/splunk.h delete mode 100644 fluent-bit/plugins/in_splunk/splunk_config.c delete mode 100644 fluent-bit/plugins/in_splunk/splunk_config.h delete mode 100644 fluent-bit/plugins/in_splunk/splunk_conn.c delete mode 100644 fluent-bit/plugins/in_splunk/splunk_conn.h delete mode 100644 fluent-bit/plugins/in_splunk/splunk_prot.c delete mode 100644 fluent-bit/plugins/in_splunk/splunk_prot.h delete mode 100644 fluent-bit/plugins/in_statsd/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_statsd/statsd.c delete mode 100644 fluent-bit/plugins/in_stdin/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_stdin/in_stdin.c delete mode 100644 fluent-bit/plugins/in_stdin/in_stdin.h delete mode 100644 fluent-bit/plugins/in_storage_backlog/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_storage_backlog/sb.c delete mode 100644 fluent-bit/plugins/in_stream_processor/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_stream_processor/sp.c delete mode 100644 fluent-bit/plugins/in_syslog/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_syslog/syslog.c delete mode 100644 fluent-bit/plugins/in_syslog/syslog.h delete mode 100644 fluent-bit/plugins/in_syslog/syslog_conf.c delete mode 100644 fluent-bit/plugins/in_syslog/syslog_conf.h delete mode 100644 fluent-bit/plugins/in_syslog/syslog_conn.c delete mode 100644 fluent-bit/plugins/in_syslog/syslog_conn.h delete mode 100644 fluent-bit/plugins/in_syslog/syslog_prot.c delete mode 100644 fluent-bit/plugins/in_syslog/syslog_prot.h delete mode 100644 fluent-bit/plugins/in_syslog/syslog_server.c delete mode 100644 fluent-bit/plugins/in_syslog/syslog_server.h delete mode 100644 fluent-bit/plugins/in_systemd/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_systemd/systemd.c delete mode 100644 fluent-bit/plugins/in_systemd/systemd_config.c delete mode 100644 fluent-bit/plugins/in_systemd/systemd_config.h delete mode 100644 fluent-bit/plugins/in_systemd/systemd_db.c delete mode 100644 fluent-bit/plugins/in_systemd/systemd_db.h delete mode 100644 fluent-bit/plugins/in_tail/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_tail/tail.c delete mode 100644 fluent-bit/plugins/in_tail/tail.h delete mode 100644 fluent-bit/plugins/in_tail/tail_config.c delete mode 100644 fluent-bit/plugins/in_tail/tail_config.h delete mode 100644 fluent-bit/plugins/in_tail/tail_db.c delete mode 100644 fluent-bit/plugins/in_tail/tail_db.h delete mode 100644 fluent-bit/plugins/in_tail/tail_dockermode.c delete mode 100644 fluent-bit/plugins/in_tail/tail_dockermode.h delete mode 100644 fluent-bit/plugins/in_tail/tail_file.c delete mode 100644 fluent-bit/plugins/in_tail/tail_file.h delete mode 100644 fluent-bit/plugins/in_tail/tail_file_internal.h delete mode 100644 fluent-bit/plugins/in_tail/tail_fs.h delete mode 100644 fluent-bit/plugins/in_tail/tail_fs_inotify.c delete mode 100644 fluent-bit/plugins/in_tail/tail_fs_inotify.h delete mode 100644 fluent-bit/plugins/in_tail/tail_fs_stat.c delete mode 100644 fluent-bit/plugins/in_tail/tail_fs_stat.h delete mode 100644 fluent-bit/plugins/in_tail/tail_multiline.c delete mode 100644 fluent-bit/plugins/in_tail/tail_multiline.h delete mode 100644 fluent-bit/plugins/in_tail/tail_scan.c delete mode 100644 fluent-bit/plugins/in_tail/tail_scan.h delete mode 100644 fluent-bit/plugins/in_tail/tail_scan_glob.c delete mode 100644 fluent-bit/plugins/in_tail/tail_scan_win32.c delete mode 100644 fluent-bit/plugins/in_tail/tail_signal.h delete mode 100644 fluent-bit/plugins/in_tail/tail_sql.h delete mode 100644 fluent-bit/plugins/in_tail/win32.h delete mode 100644 fluent-bit/plugins/in_tail/win32/interface.h delete mode 100644 fluent-bit/plugins/in_tail/win32/io.c delete mode 100644 fluent-bit/plugins/in_tail/win32/stat.c delete mode 100644 fluent-bit/plugins/in_tcp/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_tcp/tcp.c delete mode 100644 fluent-bit/plugins/in_tcp/tcp.h delete mode 100644 fluent-bit/plugins/in_tcp/tcp_config.c delete mode 100644 fluent-bit/plugins/in_tcp/tcp_config.h delete mode 100644 fluent-bit/plugins/in_tcp/tcp_conn.c delete mode 100644 fluent-bit/plugins/in_tcp/tcp_conn.h delete mode 100644 fluent-bit/plugins/in_thermal/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_thermal/in_thermal.c delete mode 100644 fluent-bit/plugins/in_thermal/in_thermal.h delete mode 100644 fluent-bit/plugins/in_udp/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_udp/udp.c delete mode 100644 fluent-bit/plugins/in_udp/udp.h delete mode 100644 fluent-bit/plugins/in_udp/udp_config.c delete mode 100644 fluent-bit/plugins/in_udp/udp_config.h delete mode 100644 fluent-bit/plugins/in_udp/udp_conn.c delete mode 100644 fluent-bit/plugins/in_udp/udp_conn.h delete mode 100644 fluent-bit/plugins/in_unix_socket/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_unix_socket/unix_socket.c delete mode 100644 fluent-bit/plugins/in_unix_socket/unix_socket.h delete mode 100644 fluent-bit/plugins/in_unix_socket/unix_socket_config.c delete mode 100644 fluent-bit/plugins/in_unix_socket/unix_socket_config.h delete mode 100644 fluent-bit/plugins/in_unix_socket/unix_socket_conn.c delete mode 100644 fluent-bit/plugins/in_unix_socket/unix_socket_conn.h delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we.c delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we.h delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_config.c delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_config.h delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_cpu.c delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_cpu.h delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_cs.c delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_cs.h delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_logical_disk.c delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_logical_disk.h delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_metric.c delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_metric.h delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_net.c delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_net.h delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_os.c delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_os.h delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_perflib.c delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_perflib.h delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_util.c delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_util.h delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_wmi.c delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_wmi.h delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_cpu_info.c delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_cpu_info.h delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_logon.c delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_logon.h delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_memory.c delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_memory.h delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_paging_file.c delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_paging_file.h delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_process.c delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_process.h delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_service.c delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_service.h delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_system.c delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_system.h delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_thermalzone.c delete mode 100644 fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_thermalzone.h delete mode 100644 fluent-bit/plugins/in_winevtlog/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_winevtlog/in_winevtlog.c delete mode 100644 fluent-bit/plugins/in_winevtlog/pack.c delete mode 100644 fluent-bit/plugins/in_winevtlog/winevtlog.c delete mode 100644 fluent-bit/plugins/in_winevtlog/winevtlog.h delete mode 100644 fluent-bit/plugins/in_winlog/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_winlog/in_winlog.c delete mode 100644 fluent-bit/plugins/in_winlog/pack.c delete mode 100644 fluent-bit/plugins/in_winlog/winlog.c delete mode 100644 fluent-bit/plugins/in_winlog/winlog.h delete mode 100644 fluent-bit/plugins/in_winstat/CMakeLists.txt delete mode 100644 fluent-bit/plugins/in_winstat/winstat.c delete mode 100644 fluent-bit/plugins/out_azure/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_azure/azure.c delete mode 100644 fluent-bit/plugins/out_azure/azure.h delete mode 100644 fluent-bit/plugins/out_azure/azure_conf.c delete mode 100644 fluent-bit/plugins/out_azure/azure_conf.h delete mode 100644 fluent-bit/plugins/out_azure_blob/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_azure_blob/azure_blob.c delete mode 100644 fluent-bit/plugins/out_azure_blob/azure_blob.h delete mode 100644 fluent-bit/plugins/out_azure_blob/azure_blob_appendblob.c delete mode 100644 fluent-bit/plugins/out_azure_blob/azure_blob_appendblob.h delete mode 100644 fluent-bit/plugins/out_azure_blob/azure_blob_blockblob.c delete mode 100644 fluent-bit/plugins/out_azure_blob/azure_blob_blockblob.h delete mode 100644 fluent-bit/plugins/out_azure_blob/azure_blob_conf.c delete mode 100644 fluent-bit/plugins/out_azure_blob/azure_blob_conf.h delete mode 100644 fluent-bit/plugins/out_azure_blob/azure_blob_http.c delete mode 100644 fluent-bit/plugins/out_azure_blob/azure_blob_http.h delete mode 100644 fluent-bit/plugins/out_azure_blob/azure_blob_uri.c delete mode 100644 fluent-bit/plugins/out_azure_blob/azure_blob_uri.h delete mode 100644 fluent-bit/plugins/out_azure_kusto/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_azure_kusto/azure_kusto.c delete mode 100644 fluent-bit/plugins/out_azure_kusto/azure_kusto.h delete mode 100644 fluent-bit/plugins/out_azure_kusto/azure_kusto_conf.c delete mode 100644 fluent-bit/plugins/out_azure_kusto/azure_kusto_conf.h delete mode 100644 fluent-bit/plugins/out_azure_kusto/azure_kusto_ingest.c delete mode 100644 fluent-bit/plugins/out_azure_kusto/azure_kusto_ingest.h delete mode 100644 fluent-bit/plugins/out_azure_logs_ingestion/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_azure_logs_ingestion/azure_logs_ingestion.c delete mode 100644 fluent-bit/plugins/out_azure_logs_ingestion/azure_logs_ingestion.h delete mode 100644 fluent-bit/plugins/out_azure_logs_ingestion/azure_logs_ingestion_conf.c delete mode 100644 fluent-bit/plugins/out_azure_logs_ingestion/azure_logs_ingestion_conf.h delete mode 100644 fluent-bit/plugins/out_bigquery/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_bigquery/bigquery.c delete mode 100644 fluent-bit/plugins/out_bigquery/bigquery.h delete mode 100644 fluent-bit/plugins/out_bigquery/bigquery_conf.c delete mode 100644 fluent-bit/plugins/out_bigquery/bigquery_conf.h delete mode 100644 fluent-bit/plugins/out_calyptia/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_calyptia/calyptia.c delete mode 100644 fluent-bit/plugins/out_calyptia/calyptia.h delete mode 100644 fluent-bit/plugins/out_chronicle/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_chronicle/chronicle.c delete mode 100644 fluent-bit/plugins/out_chronicle/chronicle.h delete mode 100644 fluent-bit/plugins/out_chronicle/chronicle_conf.c delete mode 100644 fluent-bit/plugins/out_chronicle/chronicle_conf.h delete mode 100644 fluent-bit/plugins/out_cloudwatch_logs/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_cloudwatch_logs/cloudwatch_api.c delete mode 100644 fluent-bit/plugins/out_cloudwatch_logs/cloudwatch_api.h delete mode 100644 fluent-bit/plugins/out_cloudwatch_logs/cloudwatch_logs.c delete mode 100644 fluent-bit/plugins/out_cloudwatch_logs/cloudwatch_logs.h delete mode 100644 fluent-bit/plugins/out_counter/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_counter/counter.c delete mode 100644 fluent-bit/plugins/out_datadog/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_datadog/datadog.c delete mode 100644 fluent-bit/plugins/out_datadog/datadog.h delete mode 100644 fluent-bit/plugins/out_datadog/datadog_conf.c delete mode 100644 fluent-bit/plugins/out_datadog/datadog_conf.h delete mode 100644 fluent-bit/plugins/out_datadog/datadog_remap.c delete mode 100644 fluent-bit/plugins/out_datadog/datadog_remap.h delete mode 100644 fluent-bit/plugins/out_es/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_es/es.c delete mode 100644 fluent-bit/plugins/out_es/es.h delete mode 100644 fluent-bit/plugins/out_es/es_bulk.c delete mode 100644 fluent-bit/plugins/out_es/es_bulk.h delete mode 100644 fluent-bit/plugins/out_es/es_conf.c delete mode 100644 fluent-bit/plugins/out_es/es_conf.h delete mode 100644 fluent-bit/plugins/out_es/murmur3.c delete mode 100644 fluent-bit/plugins/out_es/murmur3.h delete mode 100644 fluent-bit/plugins/out_exit/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_exit/exit.c delete mode 100644 fluent-bit/plugins/out_file/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_file/file.c delete mode 100644 fluent-bit/plugins/out_file/file.h delete mode 100644 fluent-bit/plugins/out_flowcounter/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_flowcounter/out_flowcounter.c delete mode 100644 fluent-bit/plugins/out_flowcounter/out_flowcounter.h delete mode 100644 fluent-bit/plugins/out_forward/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_forward/README.md delete mode 100644 fluent-bit/plugins/out_forward/forward.c delete mode 100644 fluent-bit/plugins/out_forward/forward.h delete mode 100644 fluent-bit/plugins/out_forward/forward_format.c delete mode 100644 fluent-bit/plugins/out_forward/forward_format.h delete mode 100644 fluent-bit/plugins/out_gelf/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_gelf/gelf.c delete mode 100644 fluent-bit/plugins/out_gelf/gelf.h delete mode 100644 fluent-bit/plugins/out_http/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_http/http.c delete mode 100644 fluent-bit/plugins/out_http/http.h delete mode 100644 fluent-bit/plugins/out_http/http_conf.c delete mode 100644 fluent-bit/plugins/out_http/http_conf.h delete mode 100644 fluent-bit/plugins/out_influxdb/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_influxdb/influxdb.c delete mode 100644 fluent-bit/plugins/out_influxdb/influxdb.h delete mode 100644 fluent-bit/plugins/out_influxdb/influxdb_bulk.c delete mode 100644 fluent-bit/plugins/out_influxdb/influxdb_bulk.h delete mode 100644 fluent-bit/plugins/out_kafka/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_kafka/kafka.c delete mode 100644 fluent-bit/plugins/out_kafka/kafka_callbacks.h delete mode 100644 fluent-bit/plugins/out_kafka/kafka_config.c delete mode 100644 fluent-bit/plugins/out_kafka/kafka_config.h delete mode 100644 fluent-bit/plugins/out_kafka/kafka_topic.c delete mode 100644 fluent-bit/plugins/out_kafka/kafka_topic.h delete mode 100644 fluent-bit/plugins/out_kafka_rest/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_kafka_rest/kafka.c delete mode 100644 fluent-bit/plugins/out_kafka_rest/kafka.h delete mode 100644 fluent-bit/plugins/out_kafka_rest/kafka_conf.c delete mode 100644 fluent-bit/plugins/out_kafka_rest/kafka_conf.h delete mode 100644 fluent-bit/plugins/out_kinesis_firehose/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_kinesis_firehose/firehose.c delete mode 100644 fluent-bit/plugins/out_kinesis_firehose/firehose.h delete mode 100644 fluent-bit/plugins/out_kinesis_firehose/firehose_api.c delete mode 100644 fluent-bit/plugins/out_kinesis_firehose/firehose_api.h delete mode 100644 fluent-bit/plugins/out_kinesis_streams/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_kinesis_streams/kinesis.c delete mode 100644 fluent-bit/plugins/out_kinesis_streams/kinesis.h delete mode 100644 fluent-bit/plugins/out_kinesis_streams/kinesis_api.c delete mode 100644 fluent-bit/plugins/out_kinesis_streams/kinesis_api.h delete mode 100644 fluent-bit/plugins/out_lib/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_lib/out_lib.c delete mode 100644 fluent-bit/plugins/out_lib/out_lib.h delete mode 100644 fluent-bit/plugins/out_logdna/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_logdna/logdna.c delete mode 100644 fluent-bit/plugins/out_logdna/logdna.h delete mode 100644 fluent-bit/plugins/out_loki/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_loki/loki.c delete mode 100644 fluent-bit/plugins/out_loki/loki.h delete mode 100644 fluent-bit/plugins/out_nats/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_nats/nats.c delete mode 100644 fluent-bit/plugins/out_nats/nats.h delete mode 100644 fluent-bit/plugins/out_nrlogs/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_nrlogs/newrelic.c delete mode 100644 fluent-bit/plugins/out_nrlogs/newrelic.h delete mode 100644 fluent-bit/plugins/out_null/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_null/null.c delete mode 100644 fluent-bit/plugins/out_opensearch/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_opensearch/opensearch.c delete mode 100644 fluent-bit/plugins/out_opensearch/opensearch.h delete mode 100644 fluent-bit/plugins/out_opensearch/os_conf.c delete mode 100644 fluent-bit/plugins/out_opensearch/os_conf.h delete mode 100644 fluent-bit/plugins/out_opentelemetry/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_opentelemetry/opentelemetry.c delete mode 100644 fluent-bit/plugins/out_opentelemetry/opentelemetry.h delete mode 100644 fluent-bit/plugins/out_opentelemetry/opentelemetry_conf.c delete mode 100644 fluent-bit/plugins/out_opentelemetry/opentelemetry_conf.h delete mode 100644 fluent-bit/plugins/out_oracle_log_analytics/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_oracle_log_analytics/oci_logan.c delete mode 100644 fluent-bit/plugins/out_oracle_log_analytics/oci_logan.h delete mode 100644 fluent-bit/plugins/out_oracle_log_analytics/oci_logan_conf.c delete mode 100644 fluent-bit/plugins/out_oracle_log_analytics/oci_logan_conf.h delete mode 100644 fluent-bit/plugins/out_pgsql/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_pgsql/pgsql.c delete mode 100644 fluent-bit/plugins/out_pgsql/pgsql.h delete mode 100644 fluent-bit/plugins/out_pgsql/pgsql_connections.c delete mode 100644 fluent-bit/plugins/out_pgsql/pgsql_connections.h delete mode 100644 fluent-bit/plugins/out_plot/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_plot/plot.c delete mode 100644 fluent-bit/plugins/out_prometheus_exporter/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_prometheus_exporter/prom.c delete mode 100644 fluent-bit/plugins/out_prometheus_exporter/prom.h delete mode 100644 fluent-bit/plugins/out_prometheus_exporter/prom_http.c delete mode 100644 fluent-bit/plugins/out_prometheus_exporter/prom_http.h delete mode 100644 fluent-bit/plugins/out_prometheus_remote_write/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_prometheus_remote_write/remote_write.c delete mode 100644 fluent-bit/plugins/out_prometheus_remote_write/remote_write.h delete mode 100644 fluent-bit/plugins/out_prometheus_remote_write/remote_write_conf.c delete mode 100644 fluent-bit/plugins/out_prometheus_remote_write/remote_write_conf.h delete mode 100644 fluent-bit/plugins/out_retry/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_retry/retry.c delete mode 100644 fluent-bit/plugins/out_s3/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_s3/s3.c delete mode 100644 fluent-bit/plugins/out_s3/s3.h delete mode 100644 fluent-bit/plugins/out_s3/s3_multipart.c delete mode 100644 fluent-bit/plugins/out_s3/s3_store.c delete mode 100644 fluent-bit/plugins/out_s3/s3_store.h delete mode 100644 fluent-bit/plugins/out_skywalking/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_skywalking/skywalking.c delete mode 100644 fluent-bit/plugins/out_skywalking/skywalking.h delete mode 100644 fluent-bit/plugins/out_slack/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_slack/slack.c delete mode 100644 fluent-bit/plugins/out_slack/slack.h delete mode 100644 fluent-bit/plugins/out_splunk/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_splunk/splunk.c delete mode 100644 fluent-bit/plugins/out_splunk/splunk.h delete mode 100644 fluent-bit/plugins/out_splunk/splunk_conf.c delete mode 100644 fluent-bit/plugins/out_splunk/splunk_conf.h delete mode 100644 fluent-bit/plugins/out_stackdriver/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_stackdriver/gce_metadata.c delete mode 100644 fluent-bit/plugins/out_stackdriver/gce_metadata.h delete mode 100644 fluent-bit/plugins/out_stackdriver/stackdriver.c delete mode 100644 fluent-bit/plugins/out_stackdriver/stackdriver.h delete mode 100644 fluent-bit/plugins/out_stackdriver/stackdriver_conf.c delete mode 100644 fluent-bit/plugins/out_stackdriver/stackdriver_conf.h delete mode 100644 fluent-bit/plugins/out_stackdriver/stackdriver_helper.c delete mode 100644 fluent-bit/plugins/out_stackdriver/stackdriver_helper.h delete mode 100644 fluent-bit/plugins/out_stackdriver/stackdriver_http_request.c delete mode 100644 fluent-bit/plugins/out_stackdriver/stackdriver_http_request.h delete mode 100644 fluent-bit/plugins/out_stackdriver/stackdriver_operation.c delete mode 100644 fluent-bit/plugins/out_stackdriver/stackdriver_operation.h delete mode 100644 fluent-bit/plugins/out_stackdriver/stackdriver_resource_types.c delete mode 100644 fluent-bit/plugins/out_stackdriver/stackdriver_resource_types.h delete mode 100644 fluent-bit/plugins/out_stackdriver/stackdriver_source_location.c delete mode 100644 fluent-bit/plugins/out_stackdriver/stackdriver_source_location.h delete mode 100644 fluent-bit/plugins/out_stackdriver/stackdriver_timestamp.c delete mode 100644 fluent-bit/plugins/out_stackdriver/stackdriver_timestamp.h delete mode 100644 fluent-bit/plugins/out_stdout/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_stdout/stdout.c delete mode 100644 fluent-bit/plugins/out_stdout/stdout.h delete mode 100644 fluent-bit/plugins/out_syslog/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_syslog/syslog.c delete mode 100644 fluent-bit/plugins/out_syslog/syslog_conf.c delete mode 100644 fluent-bit/plugins/out_syslog/syslog_conf.h delete mode 100644 fluent-bit/plugins/out_tcp/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_tcp/tcp.c delete mode 100644 fluent-bit/plugins/out_tcp/tcp.h delete mode 100644 fluent-bit/plugins/out_tcp/tcp_conf.c delete mode 100644 fluent-bit/plugins/out_tcp/tcp_conf.h delete mode 100644 fluent-bit/plugins/out_td/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_td/td.c delete mode 100644 fluent-bit/plugins/out_td/td.h delete mode 100644 fluent-bit/plugins/out_td/td_config.c delete mode 100644 fluent-bit/plugins/out_td/td_config.h delete mode 100644 fluent-bit/plugins/out_td/td_http.c delete mode 100644 fluent-bit/plugins/out_td/td_http.h delete mode 100644 fluent-bit/plugins/out_udp/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_udp/udp.c delete mode 100644 fluent-bit/plugins/out_udp/udp.h delete mode 100644 fluent-bit/plugins/out_udp/udp_conf.c delete mode 100644 fluent-bit/plugins/out_udp/udp_conf.h delete mode 100644 fluent-bit/plugins/out_vivo_exporter/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_vivo_exporter/vivo.c delete mode 100644 fluent-bit/plugins/out_vivo_exporter/vivo.h delete mode 100644 fluent-bit/plugins/out_vivo_exporter/vivo_http.c delete mode 100644 fluent-bit/plugins/out_vivo_exporter/vivo_http.h delete mode 100644 fluent-bit/plugins/out_vivo_exporter/vivo_stream.c delete mode 100644 fluent-bit/plugins/out_vivo_exporter/vivo_stream.h delete mode 100644 fluent-bit/plugins/out_websocket/CMakeLists.txt delete mode 100644 fluent-bit/plugins/out_websocket/websocket.c delete mode 100644 fluent-bit/plugins/out_websocket/websocket.h delete mode 100644 fluent-bit/plugins/out_websocket/websocket_conf.c delete mode 100644 fluent-bit/plugins/out_websocket/websocket_conf.h delete mode 100644 fluent-bit/plugins/processor_attributes/CMakeLists.txt delete mode 100644 fluent-bit/plugins/processor_attributes/attributes.c delete mode 100644 fluent-bit/plugins/processor_attributes/variant_utils.h delete mode 100644 fluent-bit/plugins/processor_labels/CMakeLists.txt delete mode 100644 fluent-bit/plugins/processor_labels/labels.c (limited to 'fluent-bit/plugins') diff --git a/fluent-bit/plugins/CMakeLists.txt b/fluent-bit/plugins/CMakeLists.txt deleted file mode 100644 index 36299074a..000000000 --- a/fluent-bit/plugins/CMakeLists.txt +++ /dev/null @@ -1,416 +0,0 @@ -set(flb_plugins "" CACHE INTERNAL "flb_plugins") - -# REGISTER_CUSTOM_PLUGIN -macro(REGISTER_CUSTOM_PLUGIN name) - string(FIND ${name} "=" pos) - if(pos GREATER -1) - string(REPLACE "=" ";" list ${name}) - list(GET list 0 p_name) - list(GET list 1 p_path) - message(STATUS "EXTERNAL CUSTOM PLUGIN name='${p_name}' path='${p_path}'") - else() - set(p_name ${name}) - endif() - - string(TOUPPER ${p_name} NAME) - if(FLB_${NAME} OR p_path) - set(FLB_IN_PLUGINS_DECL "${FLB_CUSTOM_PLUGINS_DECL}extern struct flb_custom_plugin ${p_name}_plugin;\n") - - # C code - set(C_CODE " custom = flb_malloc(sizeof(struct flb_custom_plugin));\n") - set(C_CODE "${C_CODE} if (!custom) {\n") - set(C_CODE "${C_CODE} flb_errno();\n") - set(C_CODE "${C_CODE} return -1;\n") - set(C_CODE "${C_CODE} }\n") - set(C_CODE "${C_CODE} memcpy(custom, &${p_name}_plugin, sizeof(struct flb_custom_plugin));\n") - set(C_CODE "${C_CODE} mk_list_add(&custom->_head, &config->custom_plugins);\n\n") - - set(FLB_IN_PLUGINS_ADD "${FLB_CUSTOM_PLUGINS_ADD}${C_CODE}") - - if (p_path) - add_subdirectory(${p_path} ${p_path}) - else() - add_subdirectory(${p_name}) - endif() - set(flb_plugins "${flb_plugins}flb-plugin-${p_name};") - endif() -endmacro() - -# REGISTER_IN_PLUGIN -macro(REGISTER_IN_PLUGIN name) - string(FIND ${name} "=" pos) - if(pos GREATER -1) - string(REPLACE "=" ";" list ${name}) - list(GET list 0 p_name) - list(GET list 1 p_path) - message(STATUS "EXTERNAL IN PLUGIN name='${p_name}' path='${p_path}'") - else() - set(p_name ${name}) - endif() - - string(TOUPPER ${p_name} NAME) - if(FLB_${NAME} OR p_path) - set(FLB_IN_PLUGINS_DECL "${FLB_IN_PLUGINS_DECL}extern struct flb_input_plugin ${p_name}_plugin;\n") - - # C code - set(C_CODE " in = flb_malloc(sizeof(struct flb_input_plugin));\n") - set(C_CODE "${C_CODE} if (!in) {\n") - set(C_CODE "${C_CODE} flb_errno();\n") - set(C_CODE "${C_CODE} return -1;\n") - set(C_CODE "${C_CODE} }\n") - set(C_CODE "${C_CODE} memcpy(in, &${p_name}_plugin, sizeof(struct flb_input_plugin));\n") - set(C_CODE "${C_CODE} mk_list_add(&in->_head, &config->in_plugins);\n\n") - - set(FLB_IN_PLUGINS_ADD "${FLB_IN_PLUGINS_ADD}${C_CODE}") - - if (p_path) - add_subdirectory(${p_path} ${p_path}) - else() - add_subdirectory(${p_name}) - endif() - set(flb_plugins "${flb_plugins}flb-plugin-${p_name};") - endif() -endmacro() - -# REGISTER_OUT_PLUGIN -macro(REGISTER_OUT_PLUGIN name) - string(FIND ${name} "=" pos) - if(pos GREATER -1) - string(REPLACE "=" ";" list ${name}) - list(GET list 0 p_name) - list(GET list 1 p_path) - message(STATUS "EXTERNAL OUT PLUGIN name='${p_name}' path='${p_path}'") - else() - set(p_name ${name}) - endif() - - string(TOUPPER ${p_name} NAME) - if(FLB_${NAME} OR p_path) - set(FLB_OUT_PLUGINS_DECL "${FLB_OUT_PLUGINS_DECL}extern struct flb_output_plugin ${p_name}_plugin;\n") - - # C code - set(C_CODE " out = flb_malloc(sizeof(struct flb_output_plugin));\n") - set(C_CODE "${C_CODE} if (!out) {\n") - set(C_CODE "${C_CODE} flb_errno();\n") - set(C_CODE "${C_CODE} return -1;\n") - set(C_CODE "${C_CODE} }\n") - set(C_CODE "${C_CODE} memcpy(out, &${p_name}_plugin, sizeof(struct flb_output_plugin));\n") - set(C_CODE "${C_CODE} mk_list_add(&out->_head, &config->out_plugins);\n\n") - - set(FLB_OUT_PLUGINS_ADD "${FLB_OUT_PLUGINS_ADD}${C_CODE}") - if (p_path) - add_subdirectory(${p_path} ${p_path}) - else() - add_subdirectory(${p_name}) - endif() - set(flb_plugins "${flb_plugins}flb-plugin-${p_name};") - endif() -endmacro() - - -# REGISTER_PROCESSOR_PLUGIN -macro(REGISTER_PROCESSOR_PLUGIN name) - string(FIND ${name} "=" pos) - if(pos GREATER -1) - string(REPLACE "=" ";" list ${name}) - list(GET list 0 p_name) - list(GET list 1 p_path) - message(STATUS "EXTERNAL PROCESSOR PLUGIN name='${p_name}' path='${p_path}'") - else() - set(p_name ${name}) - endif() - - string(TOUPPER ${p_name} NAME) - if(FLB_${NAME} OR p_path) - set(FLB_PROCESSOR_PLUGINS_DECL "${FLB_PROCESSOR_PLUGINS_DECL}extern struct flb_processor_plugin ${p_name}_plugin;\n") - - # C code - set(C_CODE " processor = flb_malloc(sizeof(struct flb_processor_plugin));\n") - set(C_CODE "${C_CODE} if (!processor) {\n") - set(C_CODE "${C_CODE} flb_errno();\n") - set(C_CODE "${C_CODE} return -1;\n") - set(C_CODE "${C_CODE} }\n") - set(C_CODE "${C_CODE} memcpy(processor, &${p_name}_plugin, sizeof(struct flb_processor_plugin));\n") - set(C_CODE "${C_CODE} mk_list_add(&processor->_head, &config->processor_plugins);\n\n") - - set(FLB_PROCESSOR_PLUGINS_ADD "${FLB_PROCESSOR_PLUGINS_ADD}${C_CODE}") - if (p_path) - add_subdirectory(${p_path} ${p_path}) - else() - add_subdirectory(${p_name}) - endif() - set(flb_plugins "${flb_plugins}flb-plugin-${p_name};") - endif() -endmacro() - -# REGISTER_FILTER_PLUGIN -macro(REGISTER_FILTER_PLUGIN name) - string(FIND ${name} "=" pos) - if(pos GREATER -1) - string(REPLACE "=" ";" list ${name}) - list(GET list 0 p_name) - list(GET list 1 p_path) - message(STATUS "EXTERNAL FILTER PLUGIN name='${p_name}' path='${p_path}'") - else() - set(p_name ${name}) - endif() - - string(TOUPPER ${p_name} NAME) - if(FLB_${NAME} OR p_path) - set(FLB_FILTER_PLUGINS_DECL "${FLB_FILTER_PLUGINS_DECL}extern struct flb_filter_plugin ${p_name}_plugin;\n") - - # C code - set(C_CODE " filter = flb_malloc(sizeof(struct flb_filter_plugin));\n") - set(C_CODE "${C_CODE} if (!filter) {\n") - set(C_CODE "${C_CODE} flb_errno();\n") - set(C_CODE "${C_CODE} return -1;\n") - set(C_CODE "${C_CODE} }\n") - set(C_CODE "${C_CODE} memcpy(filter, &${p_name}_plugin, sizeof(struct flb_filter_plugin));\n") - set(C_CODE "${C_CODE} mk_list_add(&filter->_head, &config->filter_plugins);\n\n") - - set(FLB_FILTER_PLUGINS_ADD "${FLB_FILTER_PLUGINS_ADD}${C_CODE}") - if (p_path) - add_subdirectory(${p_path} ${p_path}) - else() - add_subdirectory(${p_name}) - endif() - set(flb_plugins "${flb_plugins}flb-plugin-${p_name};") - endif() -endmacro() - -# FLB_PLUGIN: used by plugins to perform registration and linking -macro(FLB_PLUGIN name src deps) - add_library(flb-plugin-${name} STATIC ${src}) - add_sanitizers(flb-plugin-${name}) - target_link_libraries(flb-plugin-${name} fluent-bit-static msgpack-c-static ${deps}) -endmacro() - - -# ====================== -# Plugins Registration -# ====================== - -# Custom Plugins -REGISTER_CUSTOM_PLUGIN("custom_calyptia") - -# These plugins works only on Linux -if(${CMAKE_SYSTEM_NAME} MATCHES "Linux") - REGISTER_IN_PLUGIN("in_cpu") - REGISTER_IN_PLUGIN("in_mem") - REGISTER_IN_PLUGIN("in_thermal") - REGISTER_IN_PLUGIN("in_kmsg") - REGISTER_IN_PLUGIN("in_proc") - REGISTER_IN_PLUGIN("in_disk") - REGISTER_IN_PLUGIN("in_systemd") - REGISTER_IN_PLUGIN("in_netif") - REGISTER_IN_PLUGIN("in_docker") - REGISTER_IN_PLUGIN("in_docker_events") - REGISTER_IN_PLUGIN("in_node_exporter_metrics") - REGISTER_IN_PLUGIN("in_podman_metrics") -endif() - -REGISTER_IN_PLUGIN("in_kubernetes_events") -REGISTER_IN_PLUGIN("in_kafka") -REGISTER_IN_PLUGIN("in_fluentbit_metrics") -REGISTER_IN_PLUGIN("in_prometheus_scrape") -REGISTER_IN_PLUGIN("in_emitter") -REGISTER_IN_PLUGIN("in_tail") -REGISTER_IN_PLUGIN("in_dummy") -REGISTER_IN_PLUGIN("in_head") -REGISTER_IN_PLUGIN("in_health") -REGISTER_IN_PLUGIN("in_http") -REGISTER_IN_PLUGIN("in_collectd") -REGISTER_IN_PLUGIN("in_statsd") -REGISTER_IN_PLUGIN("in_opentelemetry") -REGISTER_IN_PLUGIN("in_elasticsearch") -REGISTER_IN_PLUGIN("in_calyptia_fleet") -REGISTER_IN_PLUGIN("in_splunk") - -# Test the event loop messaging when used in threaded mode -REGISTER_IN_PLUGIN("in_event_test") - -# Send different event types: logs, metrics and traces -REGISTER_IN_PLUGIN("in_event_type") - - -if (FLB_IN_STORAGE_BACKLOG) - REGISTER_IN_PLUGIN("in_storage_backlog") -endif() - -REGISTER_IN_PLUGIN("in_nginx_exporter_metrics") - -if (FLB_STREAM_PROCESSOR) - REGISTER_IN_PLUGIN("in_stream_processor") -endif() - -if (FLB_SYSTEM_WINDOWS) - REGISTER_IN_PLUGIN("in_winlog") - REGISTER_IN_PLUGIN("in_winstat") - REGISTER_IN_PLUGIN("in_winevtlog") - REGISTER_IN_PLUGIN("in_windows_exporter_metrics") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W2") -else() - REGISTER_IN_PLUGIN("in_serial") -endif() - -if(FLB_REGEX) - REGISTER_IN_PLUGIN("in_stdin") -endif() - -if(FLB_PARSER) - REGISTER_IN_PLUGIN("in_syslog") - REGISTER_IN_PLUGIN("in_exec") -endif() - -REGISTER_IN_PLUGIN("in_udp") - -if(FLB_WASM) - REGISTER_IN_PLUGIN("in_exec_wasi") -endif() - -REGISTER_IN_PLUGIN("in_tcp") -REGISTER_IN_PLUGIN("in_unix_socket") -REGISTER_IN_PLUGIN("in_mqtt") -REGISTER_IN_PLUGIN("in_lib") -REGISTER_IN_PLUGIN("in_forward") -REGISTER_IN_PLUGIN("in_random") - -# PROCESSORS -# ========== -REGISTER_PROCESSOR_PLUGIN("processor_labels") -REGISTER_PROCESSOR_PLUGIN("processor_attributes") - -# OUTPUTS -# ======= -REGISTER_OUT_PLUGIN("out_azure") -REGISTER_OUT_PLUGIN("out_azure_blob") -REGISTER_OUT_PLUGIN("out_azure_logs_ingestion") -REGISTER_OUT_PLUGIN("out_azure_kusto") -REGISTER_OUT_PLUGIN("out_bigquery") -REGISTER_OUT_PLUGIN("out_calyptia") -REGISTER_OUT_PLUGIN("out_counter") -REGISTER_OUT_PLUGIN("out_datadog") -REGISTER_OUT_PLUGIN("out_es") -REGISTER_OUT_PLUGIN("out_exit") -REGISTER_OUT_PLUGIN("out_file") -REGISTER_OUT_PLUGIN("out_forward") -REGISTER_OUT_PLUGIN("out_http") -REGISTER_OUT_PLUGIN("out_influxdb") -REGISTER_OUT_PLUGIN("out_logdna") -REGISTER_OUT_PLUGIN("out_loki") -REGISTER_OUT_PLUGIN("out_kafka") -REGISTER_OUT_PLUGIN("out_kafka_rest") -REGISTER_OUT_PLUGIN("out_nats") -REGISTER_OUT_PLUGIN("out_nrlogs") -REGISTER_OUT_PLUGIN("out_null") -REGISTER_OUT_PLUGIN("out_opensearch") -REGISTER_OUT_PLUGIN("out_oracle_log_analytics") - -if (NOT CMAKE_SYSTEM_NAME MATCHES "Windows") - REGISTER_OUT_PLUGIN("out_plot") -endif() - -REGISTER_OUT_PLUGIN("out_pgsql") -REGISTER_OUT_PLUGIN("out_retry") -REGISTER_OUT_PLUGIN("out_skywalking") -REGISTER_OUT_PLUGIN("out_slack") -REGISTER_OUT_PLUGIN("out_splunk") -REGISTER_OUT_PLUGIN("out_stackdriver") -REGISTER_OUT_PLUGIN("out_stdout") -REGISTER_OUT_PLUGIN("out_syslog") -REGISTER_OUT_PLUGIN("out_tcp") -REGISTER_OUT_PLUGIN("out_udp") -REGISTER_OUT_PLUGIN("out_td") -REGISTER_OUT_PLUGIN("out_lib") -REGISTER_OUT_PLUGIN("out_flowcounter") -REGISTER_OUT_PLUGIN("out_gelf") -REGISTER_OUT_PLUGIN("out_websocket") -REGISTER_OUT_PLUGIN("out_cloudwatch_logs") -REGISTER_OUT_PLUGIN("out_kinesis_firehose") -REGISTER_OUT_PLUGIN("out_kinesis_streams") -REGISTER_OUT_PLUGIN("out_opentelemetry") -REGISTER_OUT_PLUGIN("out_prometheus_exporter") -REGISTER_OUT_PLUGIN("out_prometheus_remote_write") -REGISTER_OUT_PLUGIN("out_s3") -REGISTER_OUT_PLUGIN("out_vivo_exporter") -REGISTER_OUT_PLUGIN("out_chronicle") - -# FILTERS -# ======= -REGISTER_FILTER_PLUGIN("filter_alter_size") -REGISTER_FILTER_PLUGIN("filter_aws") -REGISTER_FILTER_PLUGIN("filter_checklist") -REGISTER_FILTER_PLUGIN("filter_ecs") -REGISTER_FILTER_PLUGIN("filter_record_modifier") -REGISTER_FILTER_PLUGIN("filter_throttle") -REGISTER_FILTER_PLUGIN("filter_throttle_size") -REGISTER_FILTER_PLUGIN("filter_tensorflow") -REGISTER_FILTER_PLUGIN("filter_type_converter") - -if(FLB_REGEX) - REGISTER_FILTER_PLUGIN("filter_kubernetes") - REGISTER_FILTER_PLUGIN("filter_modify") - REGISTER_FILTER_PLUGIN("filter_multiline") - REGISTER_FILTER_PLUGIN("filter_nest") - REGISTER_FILTER_PLUGIN("filter_parser") -endif() - -if(FLB_RECORD_ACCESSOR) - REGISTER_FILTER_PLUGIN("filter_expect") - REGISTER_FILTER_PLUGIN("filter_grep") - REGISTER_FILTER_PLUGIN("filter_rewrite_tag") -endif() - -if(FLB_METRICS) - REGISTER_FILTER_PLUGIN("filter_log_to_metrics") -endif() - -if(FLB_LUAJIT) - REGISTER_FILTER_PLUGIN("filter_lua") -endif() - -REGISTER_FILTER_PLUGIN("filter_stdout") - -REGISTER_FILTER_PLUGIN("filter_geoip2") - -REGISTER_FILTER_PLUGIN("filter_nightfall") -if (FLB_WASM) - REGISTER_FILTER_PLUGIN("filter_wasm") -endif () - -# Register external input and output plugins -if(EXT_IN_PLUGINS) - string(REPLACE "," ";" plugins ${EXT_IN_PLUGINS}) - foreach(entry ${plugins}) - REGISTER_IN_PLUGIN(${entry}) - endforeach() -endif() - -if(EXT_OUT_PLUGINS) - string(REPLACE "," ";" plugins ${EXT_OUT_PLUGINS}) - foreach(entry ${plugins}) - REGISTER_OUT_PLUGIN(${entry}) - endforeach() -endif() - -if(EXT_PROCESSOR_PLUGINS) - string(REPLACE "," ";" plugins ${EXT_PROCESSOR_PLUGINS}) - foreach(entry ${plugins}) - REGISTER_PROCESSOR_PLUGIN(${entry}) - endforeach() -endif() - -if(EXT_FILTER_PLUGINS) - string(REPLACE "," ";" plugins ${EXT_FILTER_PLUGINS}) - foreach(entry ${plugins}) - REGISTER_FILTER_PLUGIN(${entry}) - endforeach() -endif() - -# Generate the header from the template -configure_file( - "${PROJECT_SOURCE_DIR}/include/fluent-bit/flb_plugins.h.in" - "${PROJECT_SOURCE_DIR}/include/fluent-bit/flb_plugins.h" - ) - -set(FLB_PLUGINS "${flb_plugins}" PARENT_SCOPE) diff --git a/fluent-bit/plugins/custom_calyptia/CMakeLists.txt b/fluent-bit/plugins/custom_calyptia/CMakeLists.txt deleted file mode 100644 index 6449c74e3..000000000 --- a/fluent-bit/plugins/custom_calyptia/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - calyptia.c) - -FLB_PLUGIN(custom_calyptia "${src}" "") diff --git a/fluent-bit/plugins/custom_calyptia/calyptia.c b/fluent-bit/plugins/custom_calyptia/calyptia.c deleted file mode 100644 index 1cfbbd5ce..000000000 --- a/fluent-bit/plugins/custom_calyptia/calyptia.c +++ /dev/null @@ -1,615 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -/* pipeline plugins */ -#include -#include -#include - -#include - -struct calyptia { - /* config map options */ - flb_sds_t api_key; - flb_sds_t store_path; - flb_sds_t cloud_host; - flb_sds_t cloud_port; - flb_sds_t machine_id; - -/* used for reporting chunk trace records. */ -#ifdef FLB_HAVE_CHUNK_TRACE - flb_sds_t pipeline_id; -#endif /* FLB_HAVE_CHUNK_TRACE */ - - int cloud_tls; - int cloud_tls_verify; - - /* config reader for 'add_label' */ - struct mk_list *add_labels; - - /* instances */ - struct flb_input_instance *i; - struct flb_output_instance *o; - struct flb_input_instance *fleet; - struct flb_custom_instance *ins; - - /* Fleet configuration */ - flb_sds_t fleet_id; /* fleet-id */ - flb_sds_t fleet_name; - flb_sds_t fleet_config_dir; /* fleet configuration directory */ - int fleet_interval_sec; - int fleet_interval_nsec; -}; - -/* - * Check if the key belongs to a sensitive data field, if so report it. We never - * share any sensitive data. - */ -static int is_sensitive_property(char *key) -{ - - if (strcasecmp(key, "password") == 0 || - strcasecmp(key, "passwd") == 0 || - strcasecmp(key, "user") == 0 || - strcasecmp(key, "http_user") == 0 || - strcasecmp(key, "http_passwd") == 0 || - strcasecmp(key, "shared_key") == 0 || - strcasecmp(key, "endpoint") == 0 || - strcasecmp(key, "apikey") == 0 || - strcasecmp(key, "private_key") == 0 || - strcasecmp(key, "service_account_secret") == 0 || - strcasecmp(key, "splunk_token") == 0 || - strcasecmp(key, "logdna_host") == 0 || - strcasecmp(key, "api_key") == 0 || - strcasecmp(key, "hostname") == 0 || - strcasecmp(key, "license_key") == 0 || - strcasecmp(key, "base_uri") == 0 || - strcasecmp(key, "api") == 0) { - - return FLB_TRUE; - } - - return FLB_FALSE; -} - -static void pipeline_config_add_properties(flb_sds_t *buf, struct mk_list *props) -{ - struct mk_list *head; - struct flb_kv *kv; - - mk_list_foreach(head, props) { - kv = mk_list_entry(head, struct flb_kv, _head); - - if (kv->key != NULL && kv->val != NULL) { - flb_sds_printf(buf, " %s ", kv->key); - - if (is_sensitive_property(kv->key)) { - flb_sds_cat_safe(buf, "--redacted--", strlen("--redacted--")); - } - else { - flb_sds_cat_safe(buf, kv->val, strlen(kv->val)); - } - - flb_sds_cat_safe(buf, "\n", 1); - } - } -} - -flb_sds_t custom_calyptia_pipeline_config_get(struct flb_config *ctx) -{ - char tmp[32]; - flb_sds_t buf; - struct mk_list *head; - struct flb_input_instance *i_ins; - struct flb_filter_instance *f_ins; - struct flb_output_instance *o_ins; - - buf = flb_sds_create_size(2048); - - if (!buf) { - return NULL; - } - - /* [INPUT] */ - mk_list_foreach(head, &ctx->inputs) { - i_ins = mk_list_entry(head, struct flb_input_instance, _head); - flb_sds_printf(&buf, "[INPUT]\n"); - flb_sds_printf(&buf, " name %s\n", i_ins->name); - - if (i_ins->alias) { - flb_sds_printf(&buf, " alias %s\n", i_ins->alias); - } - - if (i_ins->tag) { - flb_sds_printf(&buf, " tag %s\n", i_ins->tag); - } - - if (i_ins->mem_buf_limit > 0) { - flb_utils_bytes_to_human_readable_size(i_ins->mem_buf_limit, - tmp, sizeof(tmp) - 1); - flb_sds_printf(&buf, " mem_buf_limit %s\n", tmp); - } - - pipeline_config_add_properties(&buf, &i_ins->properties); - } - flb_sds_printf(&buf, "\n"); - - /* Config: [FILTER] */ - mk_list_foreach(head, &ctx->filters) { - f_ins = mk_list_entry(head, struct flb_filter_instance, _head); - - flb_sds_printf(&buf, "[FILTER]\n"); - flb_sds_printf(&buf, " name %s\n", f_ins->name); - flb_sds_printf(&buf, " match %s\n", f_ins->match); - - pipeline_config_add_properties(&buf, &f_ins->properties); - } - flb_sds_printf(&buf, "\n"); - - /* Config: [OUTPUT] */ - mk_list_foreach(head, &ctx->outputs) { - o_ins = mk_list_entry(head, struct flb_output_instance, _head); - - flb_sds_printf(&buf, "[OUTPUT]\n"); - flb_sds_printf(&buf, " name %s\n", o_ins->name); - - if (o_ins->match) { - flb_sds_printf(&buf, " match %s\n", o_ins->match); - } - else { - flb_sds_printf(&buf, " match *\n"); - } - -#ifdef FLB_HAVE_TLS - if (o_ins->use_tls == FLB_TRUE) { - flb_sds_printf(&buf, " tls %s\n", o_ins->use_tls ? "on" : "off"); - flb_sds_printf(&buf, " tls.verify %s\n", - o_ins->tls_verify ? "on": "off"); - - if (o_ins->tls_ca_file) { - flb_sds_printf(&buf, " tls.ca_file %s\n", - o_ins->tls_ca_file); - } - - if (o_ins->tls_crt_file) { - flb_sds_printf(&buf, " tls.crt_file %s\n", - o_ins->tls_crt_file); - } - - if (o_ins->tls_key_file) { - flb_sds_printf(&buf, " tls.key_file %s\n", - o_ins->tls_key_file); - } - - if (o_ins->tls_key_passwd) { - flb_sds_printf(&buf, " tls.key_passwd --redacted--\n"); - } - } -#endif - - if (o_ins->retry_limit == FLB_OUT_RETRY_UNLIMITED) { - flb_sds_printf(&buf, " retry_limit no_limits\n"); - } - else if (o_ins->retry_limit == FLB_OUT_RETRY_NONE) { - flb_sds_printf(&buf, " retry_limit no_retries\n"); - } - else { - flb_sds_printf(&buf, " retry_limit %i\n", o_ins->retry_limit); - } - - if (o_ins->host.name) { - flb_sds_printf(&buf, " host --redacted--\n"); - } - - pipeline_config_add_properties(&buf, &o_ins->properties); - flb_sds_printf(&buf, "\n"); - } - - return buf; -} - -static struct flb_output_instance *setup_cloud_output(struct flb_config *config, struct calyptia *ctx) -{ - int ret; - struct flb_output_instance *cloud; - struct mk_list *head; - struct flb_slist_entry *key = NULL; - struct flb_slist_entry *val = NULL; - flb_sds_t label; - struct flb_config_map_val *mv; - - cloud = flb_output_new(config, "calyptia", ctx, FLB_FALSE); - - if (!cloud) { - flb_plg_error(ctx->ins, "could not load Calyptia Cloud connector"); - flb_free(ctx); - return NULL; - } - - /* direct connect / routing */ - ret = flb_router_connect_direct(ctx->i, cloud); - - if (ret != 0) { - flb_plg_error(ctx->ins, "could not load Calyptia Cloud connector"); - flb_free(ctx); - return NULL; - } - - if (ctx->add_labels && mk_list_size(ctx->add_labels) > 0) { - - /* iterate all 'add_label' definitions */ - flb_config_map_foreach(head, mv, ctx->add_labels) { - key = mk_list_entry_first(mv->val.list, struct flb_slist_entry, _head); - val = mk_list_entry_last(mv->val.list, struct flb_slist_entry, _head); - - label = flb_sds_create_size(strlen(key->str) + strlen(val->str) + 1); - - if (!label) { - flb_free(ctx); - return NULL; - } - - flb_sds_printf(&label, "%s %s", key->str, val->str); - flb_output_set_property(cloud, "add_label", label); - flb_sds_destroy(label); - } - } - - flb_output_set_property(cloud, "match", "_calyptia_cloud"); - flb_output_set_property(cloud, "api_key", ctx->api_key); - - if (ctx->store_path) { - flb_output_set_property(cloud, "store_path", ctx->store_path); - } - - if (ctx->machine_id) { - flb_output_set_property(cloud, "machine_id", ctx->machine_id); - } - - /* Override network details: development purposes only */ - if (ctx->cloud_host) { - flb_output_set_property(cloud, "cloud_host", ctx->cloud_host); - } - - if (ctx->cloud_port) { - flb_output_set_property(cloud, "cloud_port", ctx->cloud_port); - } - - if (ctx->cloud_tls) { - flb_output_set_property(cloud, "tls", "true"); - } - else { - flb_output_set_property(cloud, "tls", "false"); - } - - if (ctx->cloud_tls_verify) { - flb_output_set_property(cloud, "tls.verify", "true"); - } - else { - flb_output_set_property(cloud, "tls.verify", "false"); - } - - if (ctx->fleet_id) { - flb_output_set_property(cloud, "fleet_id", ctx->fleet_id); - label = flb_sds_create_size(strlen("fleet_id") + strlen(ctx->fleet_id) + 1); - - if (!label) { - flb_free(ctx); - return NULL; - } - - flb_sds_printf(&label, "fleet_id %s", ctx->fleet_id); - flb_output_set_property(cloud, "add_label", label); - flb_sds_destroy(label); - } - -#ifdef FLB_HAVE_CHUNK_TRACE - flb_output_set_property(cloud, "pipeline_id", ctx->pipeline_id); -#endif /* FLB_HAVE_CHUNK_TRACE */ - - return cloud; -} - -static flb_sds_t sha256_to_hex(unsigned char *sha256) -{ - int idx; - flb_sds_t hex; - flb_sds_t tmp; - - hex = flb_sds_create_size(64); - - if (!hex) { - return NULL; - } - - for (idx = 0; idx < 32; idx++) { - tmp = flb_sds_printf(&hex, "%02x", sha256[idx]); - - if (!tmp) { - flb_sds_destroy(hex); - return NULL; - } - - hex = tmp; - } - - flb_sds_len_set(hex, 64); - return hex; -} - -static flb_sds_t get_machine_id(struct calyptia *ctx) -{ - int ret; - char *buf; - size_t blen; - unsigned char sha256_buf[64] = {0}; - - /* retrieve raw machine id */ - ret = flb_utils_get_machine_id(&buf, &blen); - - if (ret == -1) { - flb_plg_error(ctx->ins, "could not obtain machine id"); - return NULL; - } - - ret = flb_hash_simple(FLB_HASH_SHA256, - (unsigned char *) buf, - blen, - sha256_buf, - sizeof(sha256_buf)); - flb_free(buf); - - if (ret != FLB_CRYPTO_SUCCESS) { - return NULL; - } - - /* convert to hex */ - return sha256_to_hex(sha256_buf); -} - -static int cb_calyptia_init(struct flb_custom_instance *ins, - struct flb_config *config, - void *data) -{ - int ret; - struct calyptia *ctx; - int is_fleet_mode; - (void) data; - - ctx = flb_calloc(1, sizeof(struct calyptia)); - - if (!ctx) { - flb_errno(); - return -1; - } - ctx->ins = ins; - - /* Load the config map */ - ret = flb_custom_config_map_set(ins, (void *) ctx); - - if (ret == -1) { - flb_free(ctx); - return -1; - } - - /* map instance and local context */ - flb_custom_set_context(ins, ctx); - - /* If no machine_id has been provided via a configuration option get it from the local machine-id. */ - if (!ctx->machine_id) { - /* machine id */ - ctx->machine_id = get_machine_id(ctx); - - if (ctx->machine_id == NULL) { - flb_plg_error(ctx->ins, "unable to retrieve machine_id"); - return -1; - } - } - - /* input collector */ - ctx->i = flb_input_new(config, "fluentbit_metrics", NULL, FLB_TRUE); - - if (!ctx->i) { - flb_plg_error(ctx->ins, "could not load metrics collector"); - return -1; - } - - flb_input_set_property(ctx->i, "tag", "_calyptia_cloud"); - flb_input_set_property(ctx->i, "scrape_on_start", "true"); - flb_input_set_property(ctx->i, "scrape_interval", "30"); - - if (ctx->fleet_name || ctx->fleet_id) { - is_fleet_mode = FLB_TRUE; - } - else { - is_fleet_mode = FLB_FALSE; - } - - /* output cloud connector */ - if ((is_fleet_mode == FLB_TRUE && ctx->fleet_id != NULL) || - (is_fleet_mode == FLB_FALSE)) { - ctx->o = setup_cloud_output(config, ctx); - - if (ctx->o == NULL) { - return -1; - } - } - - if (ctx->fleet_id || ctx->fleet_name) { - ctx->fleet = flb_input_new(config, "calyptia_fleet", NULL, FLB_FALSE); - - if (!ctx->fleet) { - flb_plg_error(ctx->ins, "could not load Calyptia Fleet plugin"); - return -1; - } - - if (ctx->fleet_name) { - // TODO: set this once the fleet_id has been retrieved... - // flb_output_set_property(ctx->o, "fleet_id", ctx->fleet_id); - flb_input_set_property(ctx->fleet, "fleet_name", ctx->fleet_name); - } - else { - flb_output_set_property(ctx->o, "fleet_id", ctx->fleet_id); - flb_input_set_property(ctx->fleet, "fleet_id", ctx->fleet_id); - } - - flb_input_set_property(ctx->fleet, "api_key", ctx->api_key); - flb_input_set_property(ctx->fleet, "host", ctx->cloud_host); - flb_input_set_property(ctx->fleet, "port", ctx->cloud_port); - - if (ctx->cloud_tls == 1) { - flb_input_set_property(ctx->fleet, "tls", "on"); - } - else { - flb_input_set_property(ctx->fleet, "tls", "off"); - } - - if (ctx->cloud_tls_verify == 1) { - flb_input_set_property(ctx->fleet, "tls.verify", "on"); - } - else { - flb_input_set_property(ctx->fleet, "tls.verify", "off"); - } - - if (ctx->fleet_config_dir) { - flb_input_set_property(ctx->fleet, "config_dir", ctx->fleet_config_dir); - } - - if (ctx->machine_id) { - flb_input_set_property(ctx->fleet, "machine_id", ctx->machine_id); - } - } - - if (ctx->o) { - flb_router_connect(ctx->i, ctx->o); - } - flb_plg_info(ins, "custom initialized!"); - return 0; -} - -static int cb_calyptia_exit(void *data, struct flb_config *config) -{ - struct calyptia *ctx = data; - - if (!ctx) { - return 0; - } - - flb_free(ctx); - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "api_key", NULL, - 0, FLB_TRUE, offsetof(struct calyptia, api_key), - "Calyptia Cloud API Key." - }, - - { - FLB_CONFIG_MAP_STR, "store_path", NULL, - 0, FLB_TRUE, offsetof(struct calyptia, store_path) - }, - - { - FLB_CONFIG_MAP_STR, "calyptia_host", "cloud-api.calyptia.com", - 0, FLB_TRUE, offsetof(struct calyptia, cloud_host), - "" - }, - - { - FLB_CONFIG_MAP_STR, "calyptia_port", "443", - 0, FLB_TRUE, offsetof(struct calyptia, cloud_port), - "" - }, - - { - FLB_CONFIG_MAP_BOOL, "calyptia_tls", "true", - 0, FLB_TRUE, offsetof(struct calyptia, cloud_tls), - "" - }, - - { - FLB_CONFIG_MAP_BOOL, "calyptia_tls.verify", "true", - 0, FLB_TRUE, offsetof(struct calyptia, cloud_tls_verify), - "" - }, - - { - FLB_CONFIG_MAP_SLIST_1, "add_label", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct calyptia, add_labels), - "Label to append to the generated metric." - }, - { - FLB_CONFIG_MAP_STR, "machine_id", NULL, - 0, FLB_TRUE, offsetof(struct calyptia, machine_id), - "Custom machine_id to be used when registering agent" - }, - { - FLB_CONFIG_MAP_STR, "fleet_id", NULL, - 0, FLB_TRUE, offsetof(struct calyptia, fleet_id), - "Fleet id to be used when registering agent in a fleet" - }, - { - FLB_CONFIG_MAP_STR, "fleet.config_dir", NULL, - 0, FLB_TRUE, offsetof(struct calyptia, fleet_config_dir), - "Base path for the configuration directory." - }, - { - FLB_CONFIG_MAP_INT, "fleet.interval_sec", "-1", - 0, FLB_TRUE, offsetof(struct calyptia, fleet_interval_sec), - "Set the collector interval" - }, - { - FLB_CONFIG_MAP_INT, "fleet.interval_nsec", "-1", - 0, FLB_TRUE, offsetof(struct calyptia, fleet_interval_nsec), - "Set the collector interval (nanoseconds)" - }, - { - FLB_CONFIG_MAP_STR, "fleet_name", NULL, - 0, FLB_TRUE, offsetof(struct calyptia, fleet_name), - "Fleet name to be used when registering agent in a fleet" - }, - -#ifdef FLB_HAVE_CHUNK_TRACE - { - FLB_CONFIG_MAP_STR, "pipeline_id", NULL, - 0, FLB_TRUE, offsetof(struct calyptia, pipeline_id), - "Pipeline ID for reporting to calyptia cloud." - }, -#endif /* FLB_HAVE_CHUNK_TRACE */ - - /* EOF */ - {0} -}; - -struct flb_custom_plugin custom_calyptia_plugin = { - .name = "calyptia", - .description = "Calyptia Cloud", - .config_map = config_map, - .cb_init = cb_calyptia_init, - .cb_exit = cb_calyptia_exit, -}; diff --git a/fluent-bit/plugins/filter_alter_size/CMakeLists.txt b/fluent-bit/plugins/filter_alter_size/CMakeLists.txt deleted file mode 100644 index c34d4f464..000000000 --- a/fluent-bit/plugins/filter_alter_size/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - alter_size.c) - -FLB_PLUGIN(filter_alter_size "${src}" "") diff --git a/fluent-bit/plugins/filter_alter_size/alter_size.c b/fluent-bit/plugins/filter_alter_size/alter_size.c deleted file mode 100644 index e20cccce7..000000000 --- a/fluent-bit/plugins/filter_alter_size/alter_size.c +++ /dev/null @@ -1,212 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include - -struct flb_alter_size { - int add; - int remove; - struct flb_log_event_decoder *log_decoder; - struct flb_log_event_encoder *log_encoder; -}; - -static int cb_alter_size_init(struct flb_filter_instance *ins, - struct flb_config *config, - void *data) -{ - int ret; - (void) data; - struct flb_alter_size *ctx; - - ctx = flb_calloc(1, sizeof(struct flb_alter_size)); - if (!ctx) { - flb_errno(); - return -1; - } - - ctx->log_decoder = flb_log_event_decoder_create(NULL, 0); - - if (ctx->log_decoder == NULL) { - flb_plg_error(ins, "could not initialize event decoder"); - - return -1; - } - - ctx->log_encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ctx->log_encoder == NULL) { - flb_plg_error(ins, "could not initialize event encoder"); - flb_log_event_decoder_destroy(ctx->log_decoder); - - return -1; - } - - ret = flb_filter_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_log_event_decoder_destroy(ctx->log_decoder); - flb_log_event_encoder_destroy(ctx->log_encoder); - flb_free(ctx); - return -1; - } - - if (ctx->add > 0 && ctx->remove > 0) { - flb_plg_error(ins, "cannot use 'add' and 'remove' at the same time"); - flb_log_event_decoder_destroy(ctx->log_decoder); - flb_log_event_encoder_destroy(ctx->log_encoder); - flb_free(ctx); - return -1; - } - - flb_filter_set_context(ins, ctx); - return 0; -} - -static int cb_alter_size_filter(const void *data, size_t bytes, - const char *tag, int tag_len, - void **out_buf, size_t *out_size, - struct flb_filter_instance *ins, - struct flb_input_instance *i_ins, - void *filter_context, - struct flb_config *config) -{ - int i; - int len; - int ret; - int total; - int count = 0; - char tmp[32]; - struct flb_log_event event; - struct flb_alter_size *ctx; - - (void) config; - (void) i_ins; - - ctx = (struct flb_alter_size *) filter_context; - - if (ctx->add > 0) { - flb_plg_debug(ins, "add %i records", ctx->add); - - /* append old data */ - ret = flb_log_event_encoder_emit_raw_record( - ctx->log_encoder, data, bytes); - - for (i = 0; i < ctx->add; i++) { - ret = flb_log_event_encoder_begin_record(ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp(ctx->log_encoder); - } - - len = snprintf(tmp, sizeof(tmp) - 1, "alter_size %i", i); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE("key"), - FLB_LOG_EVENT_STRING_VALUE(tmp, len)); - } - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(ctx->log_encoder); - } - } - else if (ctx->remove > 0) { - flb_plg_debug(ins, "remove %i records", ctx->remove); - count = 0; - - /* Count number of current items */ - total = flb_mp_count(data, bytes); - total -= ctx->remove; - if (total <= 0) { - /* zero records */ - goto exit; - } - - ret = flb_log_event_decoder_init(ctx->log_decoder, - (char *) data, - bytes); - - while (count < total && - flb_log_event_decoder_next( - ctx->log_decoder, &event) == FLB_EVENT_DECODER_SUCCESS) { - - ret = flb_log_event_encoder_emit_raw_record( - ctx->log_encoder, - ctx->log_decoder->record_base, - ctx->log_decoder->record_length); - - count++; - } - } - - exit: - /* link new buffers */ - *out_buf = ctx->log_encoder->output_buffer; - *out_size = ctx->log_encoder->output_length; - - flb_log_event_encoder_claim_internal_buffer_ownership( - ctx->log_encoder); - - return FLB_FILTER_MODIFIED; -} - -static int cb_alter_size_exit(void *data, struct flb_config *config) -{ - (void) config; - struct flb_alter_size *ctx = data; - - if (!ctx) { - return 0; - } - - flb_free(ctx); - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_INT, "add", "0", - FLB_FALSE, FLB_TRUE, offsetof(struct flb_alter_size, add), - "add N records to the chunk" - }, - { - FLB_CONFIG_MAP_INT, "remove", "0", - FLB_FALSE, FLB_TRUE, offsetof(struct flb_alter_size, remove), - "remove N records from the chunk" - }, - /* EOF */ - {0} -}; - -struct flb_filter_plugin filter_alter_size_plugin = { - .name = "alter_size", - .description = "Alter incoming chunk size", - .cb_init = cb_alter_size_init, - .cb_filter = cb_alter_size_filter, - .cb_exit = cb_alter_size_exit, - .config_map = config_map, - .flags = 0 -}; diff --git a/fluent-bit/plugins/filter_alter_size/alter_size.h b/fluent-bit/plugins/filter_alter_size/alter_size.h deleted file mode 100644 index e69de29bb..000000000 diff --git a/fluent-bit/plugins/filter_aws/CMakeLists.txt b/fluent-bit/plugins/filter_aws/CMakeLists.txt deleted file mode 100644 index ec0b0f703..000000000 --- a/fluent-bit/plugins/filter_aws/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -set(src - aws.c - ) - -FLB_PLUGIN(filter_aws "${src}" "") diff --git a/fluent-bit/plugins/filter_aws/aws.c b/fluent-bit/plugins/filter_aws/aws.c deleted file mode 100644 index 726b8b709..000000000 --- a/fluent-bit/plugins/filter_aws/aws.c +++ /dev/null @@ -1,1062 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include "aws.h" - -static int get_ec2_metadata(struct flb_filter_aws *ctx); - -static void expose_aws_meta(struct flb_filter_aws *ctx) -{ - struct flb_env *env; - struct flb_config *config = ctx->ins->config; - - env = config->env; - - flb_env_set(env, "aws", "enabled"); - - if (ctx->availability_zone_include) { - flb_env_set(env, - "aws." FLB_FILTER_AWS_AVAILABILITY_ZONE_KEY, - ctx->availability_zone); - } - - if (ctx->instance_id_include) { - flb_env_set(env, - "aws." FLB_FILTER_AWS_INSTANCE_ID_KEY, - ctx->instance_id); - } - - if (ctx->instance_type_include) { - flb_env_set(env, - "aws." FLB_FILTER_AWS_INSTANCE_TYPE_KEY, - ctx->instance_type); - } - - if (ctx->private_ip_include) { - flb_env_set(env, - "aws." FLB_FILTER_AWS_PRIVATE_IP_KEY, - ctx->private_ip); - } - - if (ctx->vpc_id_include) { - flb_env_set(env, - "aws." FLB_FILTER_AWS_VPC_ID_KEY, - ctx->vpc_id); - } - - if (ctx->ami_id_include) { - flb_env_set(env, - "aws." FLB_FILTER_AWS_AMI_ID_KEY, - ctx->ami_id); - } - - if (ctx->account_id_include) { - flb_env_set(env, - "aws." FLB_FILTER_AWS_ACCOUNT_ID_KEY, - ctx->account_id); - } - - if (ctx->hostname_include) { - flb_env_set(env, - "aws." FLB_FILTER_AWS_HOSTNAME_KEY, - ctx->hostname); - } -} - -static int cb_aws_init(struct flb_filter_instance *f_ins, - struct flb_config *config, - void *data) -{ - int imds_version = FLB_AWS_IMDS_VERSION_2; - int ret; - struct flb_filter_aws *ctx = NULL; - struct flb_filter_aws_init_options *options = data; - const char *tmp = NULL; - - /* Create context */ - ctx = flb_calloc(1, sizeof(struct flb_filter_aws)); - if (!ctx) { - flb_errno(); - return -1; - } - - ctx->options = options; - ctx->ins = f_ins; - - tmp = flb_filter_get_property("imds_version", f_ins); - if (tmp != NULL) { - if (strcasecmp(tmp, "v1") == 0) { - imds_version = FLB_AWS_IMDS_VERSION_1; - } - else if (strcasecmp(tmp, "v2") != 0) { - flb_plg_error(ctx->ins, "Invalid value %s for config option " - "'imds_version'. Valid values are 'v1' and 'v2'", - tmp); - flb_free(ctx); - return -1; - } - } - - struct flb_aws_client_generator *generator; - if (options && options->client_generator) { - generator = options->client_generator; - } else { - generator = flb_aws_client_generator(); - } - ctx->aws_ec2_filter_client = generator->create(); - ctx->aws_ec2_filter_client->name = "ec2_imds_provider_client"; - ctx->aws_ec2_filter_client->has_auth = FLB_FALSE; - ctx->aws_ec2_filter_client->provider = NULL; - ctx->aws_ec2_filter_client->region = NULL; - ctx->aws_ec2_filter_client->service = NULL; - ctx->aws_ec2_filter_client->port = FLB_AWS_IMDS_PORT; - ctx->aws_ec2_filter_client->flags = 0; - ctx->aws_ec2_filter_client->proxy = NULL; - - struct flb_upstream *upstream; - upstream = flb_upstream_create(config, FLB_AWS_IMDS_HOST, FLB_AWS_IMDS_PORT, - FLB_IO_TCP, NULL); - if (!upstream) { - flb_plg_debug(ctx->ins, "unable to connect to EC2 IMDS"); - return -1; - } - - /* IMDSv2 token request will timeout if hops = 1 and running within container */ - upstream->base.net.connect_timeout = FLB_AWS_IMDS_TIMEOUT; - upstream->base.net.io_timeout = FLB_AWS_IMDS_TIMEOUT; - upstream->base.net.keepalive = FLB_FALSE; /* On timeout, the connection is broken */ - ctx->aws_ec2_filter_client->upstream = upstream; - flb_stream_disable_async_mode(&ctx->aws_ec2_filter_client->upstream->base); - - ctx->client_imds = flb_aws_imds_create(&flb_aws_imds_config_default, - ctx->aws_ec2_filter_client); - if (!ctx->client_imds) { - flb_plg_error(ctx->ins, "failed to create aws client"); - flb_free(ctx); - return -1; - } - ctx->client_imds->imds_version = imds_version; - - /* Populate context with config map defaults and incoming properties */ - ret = flb_filter_config_map_set(f_ins, (void *) ctx); - if (ret == -1) { - flb_plg_error(f_ins, "configuration error"); - flb_free(ctx); - return -1; - } - - ctx->metadata_retrieved = FLB_FALSE; - - /* Retrieve metadata */ - ret = get_ec2_metadata(ctx); - if (ret < 0) { - /* If the metadata fetch fails, the plugin continues to work. */ - /* Every flush will attempt to fetch ec2 metadata, if needed. */ - /* In the error is unrecoverable (-3), it exits and does not retry. */ - if (ret == -3) { - flb_free(ctx); - return -1; - } - } - else { - expose_aws_meta(ctx); - } - - flb_filter_set_context(f_ins, ctx); - return 0; -} - - -/* Get VPC ID from the metadata server. - * Initializes ctx->vpc_id and ctx->vpc_id_len. - */ -static int get_vpc_id(struct flb_filter_aws *ctx) -{ - ctx->vpc_id = flb_aws_imds_get_vpc_id(ctx->client_imds); - if (ctx->vpc_id == NULL) { - return -1; - } - ctx->vpc_id_len = flb_sds_len(ctx->vpc_id); - return 0; -} - -void flb_filter_aws_tags_destroy(struct flb_filter_aws *ctx) -{ - int i; - if (!ctx) { - return; - } - if (ctx->tag_keys) { - for (i = 0; i < ctx->tags_count; i++) { - if (ctx->tag_keys[i]) { - flb_sds_destroy(ctx->tag_keys[i]); - } - } - flb_free(ctx->tag_keys); - ctx->tag_keys = NULL; - } - if (ctx->tag_values) { - for (i = 0; i < ctx->tags_count; i++) { - if (ctx->tag_values[i]) { - flb_sds_destroy(ctx->tag_values[i]); - } - } - flb_free(ctx->tag_values); - ctx->tag_values = NULL; - } - if (ctx->tag_keys_len) { - flb_free(ctx->tag_keys_len); - } - ctx->tag_keys_len = NULL; - if (ctx->tag_values_len) { - flb_free(ctx->tag_values_len); - } - ctx->tag_values_len = NULL; - if (ctx->tag_is_enabled) { - flb_free(ctx->tag_is_enabled); - } - ctx->tag_is_enabled = NULL; - ctx->tags_count = 0; -} - -/* Get EC2 instance tag keys from /latest/meta-data/tags/instance. - * Initializes ctx->tags_count, ctx->tag_keys and ctx->tag_keys_len. - * - * In case EC2 metadata server doesn't return tags, either due to the fact that tags are - * disabled in the metadata server or EC2 has no tags, function returns -2. - */ -static int get_ec2_tag_keys(struct flb_filter_aws *ctx) -{ - int ret; - flb_sds_t tags_list = NULL; - size_t len = 0; - size_t tag_index = 0; - size_t tag_start = 0; - size_t tag_end = 0; - flb_sds_t tag_key; - flb_sds_t tmp; - size_t tag_key_len; - int i; - - /* get a list of tag keys from the meta data server */ - ret = flb_aws_imds_request(ctx->client_imds, FLB_AWS_IMDS_INSTANCE_TAG, &tags_list, - &len); - if (ret < 0) { - ctx->tags_count = 0; - if (ret == -2) { /* if there are no tags, response status code is 404 */ - flb_plg_warn(ctx->ins, "EC2 instance metadata tag request returned 404. " - "This likely indicates your instance has no tags " - "or the EC2 tagging metadata API is not enabled"); - return -2; - } - flb_sds_destroy(tags_list); - return -1; - } - - /* if endpoint returned 200, normally at least 1 tag should be present */ - /* for the sake of correctness, let's check the edge case when response is empty */ - if (len == 0) { - ctx->tags_count = 0; - flb_sds_destroy(tags_list); - return -1; - } - - /* count number of tag keys and allocate memory for pointers and lengths */ - /* since get_metadata returned 0, we assume there is at least 1 tag */ - /* \n is separator, therefore number of items = number of \n + 1 */ - ctx->tags_count = 1; - for (i = 0; i < len; i++) { - if (tags_list[i] == '\n') { - ctx->tags_count++; - } - } - ctx->tag_keys = flb_calloc(ctx->tags_count, sizeof(flb_sds_t)); - if (!ctx->tag_keys) { - flb_errno(); - flb_sds_destroy(tags_list); - return -1; - } - ctx->tag_keys_len = flb_calloc(ctx->tags_count, sizeof(size_t*)); - if (!ctx->tag_keys_len) { - flb_errno(); - flb_sds_destroy(tags_list); - return -1; - } - - /* go over the response and initialize tag_keys values */ - /* code below finds two indices which define tag key and copies them to ctx */ - while (tag_end <= len) { - /* replace \n with \0 to 'clearly' separate tag key strings */ - if (tags_list[tag_end] == '\n') { - tags_list[tag_end] = '\0'; - } - if ((tags_list[tag_end] == '\0' || tag_end == len) && (tag_start < tag_end)) { - /* length of tag key characters is the difference between start and end */ - /* for instance, if tag name is 'Name\0...', the corresponding values are */ - /* tag_start = 0, points to 'N' */ - /* tag_end = 4, points to '\0' just after 'e' */ - /* f.e.: 4 - 0 = 4, which is equal to len("Name") */ - tag_key_len = tag_end - tag_start; - ctx->tag_keys_len[tag_index] = tag_key_len; - - /* allocate new memory for the tag key value */ - /* + 1, because we need one more character for \0 */ - tmp = flb_sds_create_size(tag_key_len + 1); - if (!tmp) { - flb_errno(); - flb_sds_destroy(tags_list); - return -2; - } - tmp[tag_key_len] = '\0'; - ctx->tag_keys[tag_index] = tmp; - - /* tag_key points to the first character of tag key as char* */ - tag_key = tags_list + tag_start; - memcpy(ctx->tag_keys[tag_index], tag_key, tag_key_len); - - tag_index++; - tag_start = tag_end + 1; - } - tag_end++; - } - - flb_sds_destroy(tags_list); - - return ret; -} - -/* Get EC2 instance tag values from /latest/meta-data/tags/instance/{tag_key}. - * Initializes ctx->tag_values and ctx->tag_values_len. - */ -static int get_ec2_tag_values(struct flb_filter_aws *ctx) -{ - int ret; - size_t i; - flb_sds_t tag_value = NULL; - size_t tag_value_len = 0; - size_t tag_value_path_len; - flb_sds_t tag_value_path; - flb_sds_t tmp; - - /* initialize array for the tag values */ - ctx->tag_values = flb_calloc(ctx->tags_count, sizeof(flb_sds_t)); - if (!ctx->tag_values) { - flb_errno(); - return -1; - } - ctx->tag_values_len = flb_calloc(ctx->tags_count, sizeof(size_t)); - if (!ctx->tag_values_len) { - flb_errno(); - return -1; - } - - for (i = 0; i < ctx->tags_count; i++) { - /* fetch tag value using path: /latest/meta-data/tags/instance/{tag_name} */ - tag_value_path_len = ctx->tag_keys_len[i] + 1 + - strlen(FLB_AWS_IMDS_INSTANCE_TAG); - tag_value_path = flb_sds_create_size(tag_value_path_len + 1); - if (!tag_value_path) { - flb_errno(); - return -1; - } - tmp = flb_sds_printf(&tag_value_path, "%s/%s", - FLB_AWS_IMDS_INSTANCE_TAG, - ctx->tag_keys[i]); - if (!tmp) { - flb_errno(); - flb_sds_destroy(tag_value_path); - return -1; - } - tag_value_path = tmp; - - ret = flb_aws_imds_request(ctx->client_imds, tag_value_path, &tag_value, - &tag_value_len); - if (ret < 0) { - flb_sds_destroy(tag_value_path); - if (ret == -2) { - flb_plg_error(ctx->ins, "no value for tag %s", ctx->tag_keys[i]); - } else { - flb_plg_error(ctx->ins, "could not fetch value for tag %s", - ctx->tag_keys[i]); - } - return ret; - } - - ctx->tag_values[i] = tag_value; - ctx->tag_values_len[i] = tag_value_len; - - flb_sds_destroy(tag_value_path); - } - - return 0; -} - -static int tag_is_present_in_list(struct flb_filter_aws *ctx, flb_sds_t tag, - flb_sds_t *tags, int tags_n) -{ - int i; - for (i = 0; i < tags_n; i++) { - if (strcmp(tag, tags[i]) == 0) { - return FLB_TRUE; - } - } - return FLB_FALSE; -} - -static int tags_split(char *tags, flb_sds_t **tags_list, int *tags_list_n) { - flb_sds_t token; - int i; - int n; - n = 1; - for (i = 0; i < strlen(tags); i++) { - if (tags[i] == ',') { - n++; - } - } - - *tags_list = flb_calloc(sizeof(flb_sds_t), n); - if (*tags_list == NULL) { - return -2; - } - - token = strtok(tags, ","); - i = 0; - while (token != NULL) { - (*tags_list)[i] = token; - i++; - token = strtok(NULL, ","); - } - - *tags_list_n = n; - - return 0; -} - -static int get_ec2_tag_enabled(struct flb_filter_aws *ctx) -{ - const char *tags_include; - const char *tags_exclude; - char *tags_copy; - flb_sds_t *tags; - int tags_n; - int i; - int tag_present; - int result; - - /* if there are no tags, there is no need to evaluate which tag is enabled */ - if (ctx->tags_count == 0) { - return 0; - } - - - /* allocate memory for 'tag_is_enabled' for all tags */ - ctx->tag_is_enabled = flb_calloc(ctx->tags_count, sizeof(int)); - if (!ctx->tag_is_enabled) { - flb_plg_error(ctx->ins, "Failed to allocate memory for tag_is_enabled"); - return -1; - } - - /* if tags_include and tags_exclude are not defined, set all tags as enabled */ - for (i = 0; i < ctx->tags_count; i++) { - ctx->tag_is_enabled[i] = FLB_TRUE; - } - - /* apply tags_included configuration */ - tags_include = flb_filter_get_property("tags_include", ctx->ins); - if (tags_include) { - /* copy const string in order to use strtok which modifes the string */ - tags_copy = flb_strdup(tags_include); - if (!tags_copy) { - return -1; - } - result = tags_split(tags_copy, &tags, &tags_n); - if (result < 0) { - free(tags_copy); - return -1; - } - for (i = 0; i < ctx->tags_count; i++) { - tag_present = tag_is_present_in_list(ctx, ctx->tag_keys[i], tags, tags_n); - /* tag is enabled if present in included list */ - ctx->tag_is_enabled[i] = tag_present; - } - free(tags_copy); - free(tags); - } - - /* apply tags_excluded configuration, only if tags_included is not defined */ - tags_exclude = flb_filter_get_property("tags_exclude", ctx->ins); - if (tags_include && tags_exclude) { - flb_plg_error(ctx->ins, "configuration is invalid, both tags_include" - " and tags_exclude are specified at the same time"); - return -3; - } - if (!tags_include && tags_exclude) { - /* copy const string in order to use strtok which modifes the string */ - tags_copy = flb_strdup(tags_exclude); - if (!tags_copy) { - return -1; - } - result = tags_split(tags_copy, &tags, &tags_n); - if (result < 0) { - free(tags_copy); - return -1; - } - for (i = 0; i < ctx->tags_count; i++) { - tag_present = tag_is_present_in_list(ctx, ctx->tag_keys[i], tags, tags_n); - if (tag_present == FLB_TRUE) { - /* tag is excluded, so should be disabled */ - ctx->tag_is_enabled[i] = FLB_FALSE; - } else { - /* tag is not excluded, therefore should be enabled */ - ctx->tag_is_enabled[i] = FLB_TRUE; - } - } - free(tags_copy); - free(tags); - } - - return 0; -} - -static int get_ec2_tags(struct flb_filter_aws *ctx) -{ - int i; - int ret; - - ctx->tags_fetched = FLB_FALSE; - - /* get_ec2_tags function might be called multiple times, so we need to always */ - /* free memory for tags in case of previous allocations */ - flb_filter_aws_tags_destroy(ctx); - - ret = get_ec2_tag_keys(ctx); - if (ret < 0) { - flb_filter_aws_tags_destroy(ctx); - if (ret == -2) { - /* -2 means there are no tags, */ - /* to avoid requesting ec2 tags repeatedly for each flush */ - /* it marks fetching tags as done */ - ctx->tags_fetched = FLB_TRUE; - return 0; - } - return ret; - } - ret = get_ec2_tag_values(ctx); - if (ret < 0) { - flb_filter_aws_tags_destroy(ctx); - return ret; - } - - ret = get_ec2_tag_enabled(ctx); - if (ret < 0) { - flb_filter_aws_tags_destroy(ctx); - return ret; - } - - /* log tags debug information */ - for (i = 0; i < ctx->tags_count; i++) { - flb_plg_debug(ctx->ins, "found tag %s which is included=%d", - ctx->tag_keys[i], ctx->tag_is_enabled[i]); - } - - ctx->tags_fetched = FLB_TRUE; - return 0; -} - -/* - * Makes a call to IMDS to set get the values of all metadata fields. - * It can be called repeatedly if some metadata calls initially do not succeed. - */ -static int get_ec2_metadata(struct flb_filter_aws *ctx) -{ - int ret; - int i; - - if (ctx->instance_id_include && !ctx->instance_id) { - ret = flb_aws_imds_request(ctx->client_imds, FLB_AWS_IMDS_INSTANCE_ID_PATH, - &ctx->instance_id, - &ctx->instance_id_len); - if (ret < 0) { - flb_plg_error(ctx->ins, "Failed to get instance ID"); - return -1; - } - ctx->new_keys++; - } - - if (ctx->availability_zone_include && !ctx->availability_zone) { - ret = flb_aws_imds_request(ctx->client_imds, FLB_AWS_IMDS_AZ_PATH, - &ctx->availability_zone, - &ctx->availability_zone_len); - - if (ret < 0) { - flb_plg_error(ctx->ins, "Failed to get instance AZ"); - return -1; - } - ctx->new_keys++; - } - - if (ctx->instance_type_include && !ctx->instance_type) { - ret = flb_aws_imds_request(ctx->client_imds, FLB_AWS_IMDS_INSTANCE_TYPE_PATH, - &ctx->instance_type, &ctx->instance_type_len); - - if (ret < 0) { - flb_plg_error(ctx->ins, "Failed to get instance type"); - return -1; - } - ctx->new_keys++; - } - - if (ctx->private_ip_include && !ctx->private_ip) { - ret = flb_aws_imds_request(ctx->client_imds, FLB_AWS_IMDS_PRIVATE_IP_PATH, - &ctx->private_ip, &ctx->private_ip_len); - - if (ret < 0) { - flb_plg_error(ctx->ins, "Failed to get instance private IP"); - return -1; - } - ctx->new_keys++; - } - - if (ctx->vpc_id_include && !ctx->vpc_id) { - ret = get_vpc_id(ctx); - - if (ret < 0) { - flb_plg_error(ctx->ins, "Failed to get instance VPC ID"); - return -1; - } - ctx->new_keys++; - } - - if (ctx->ami_id_include && !ctx->ami_id) { - ret = flb_aws_imds_request(ctx->client_imds, FLB_AWS_IMDS_AMI_ID_PATH, - &ctx->ami_id, &ctx->ami_id_len); - - if (ret < 0) { - flb_plg_error(ctx->ins, "Failed to get AMI ID"); - return -1; - } - ctx->new_keys++; - } - - if (ctx->account_id_include && !ctx->account_id) { - ret = flb_aws_imds_request_by_key(ctx->client_imds, FLB_AWS_IMDS_ACCOUNT_ID_PATH, - &ctx->account_id, &ctx->account_id_len, - "accountId"); - - if (ret < 0) { - flb_plg_error(ctx->ins, "Failed to get Account ID"); - return -1; - } - ctx->new_keys++; - } - - if (ctx->hostname_include && !ctx->hostname) { - ret = flb_aws_imds_request(ctx->client_imds, FLB_AWS_IMDS_HOSTNAME_PATH, - &ctx->hostname, &ctx->hostname_len); - - if (ret < 0) { - flb_plg_error(ctx->ins, "Failed to get Hostname"); - return -1; - } - ctx->new_keys++; - } - - if (ctx->tags_enabled && !ctx->tags_fetched) { - ret = get_ec2_tags(ctx); - if (ret < 0) { - flb_plg_error(ctx->ins, "Failed to get instance EC2 Tags"); - return ret; - } - for (i = 0; i < ctx->tags_count; i++) { - if (ctx->tag_is_enabled[i] == FLB_TRUE) { - ctx->new_keys++; - } - } - } - - ctx->metadata_retrieved = FLB_TRUE; - return 0; -} - -static int cb_aws_filter(const void *data, size_t bytes, - const char *tag, int tag_len, - void **out_buf, size_t *out_size, - struct flb_filter_instance *f_ins, - struct flb_input_instance *i_ins, - void *context, - struct flb_config *config) -{ - struct flb_filter_aws *ctx = context; - int i = 0; - int ret; - msgpack_object *obj; - msgpack_object_kv *kv; - struct flb_log_event_encoder log_encoder; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - (void) f_ins; - (void) i_ins; - (void) config; - - /* First check that the metadata has been retrieved */ - if (!ctx->metadata_retrieved) { - ret = get_ec2_metadata(ctx); - if (ret < 0) { - return FLB_FILTER_NOTOUCH; - } - expose_aws_meta(ctx); - } - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return FLB_FILTER_NOTOUCH; - } - - ret = flb_log_event_encoder_init(&log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event encoder initialization error : %d", ret); - - flb_log_event_decoder_destroy(&log_decoder); - - return FLB_FILTER_NOTOUCH; - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - obj = log_event.body; - - ret = flb_log_event_encoder_begin_record(&log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_timestamp( - &log_encoder, - &log_event.timestamp); - } - - /* iterate through the old record map and add it to the new buffer */ - kv = obj->via.map.ptr; - - for(i=0; - i < obj->via.map.size && - ret == FLB_EVENT_ENCODER_SUCCESS; - i++) { - ret = flb_log_event_encoder_append_body_values( - &log_encoder, - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&kv[i].key), - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&kv[i].val)); - } - - /* append new keys */ - if (ctx->availability_zone_include && - ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - &log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE(FLB_FILTER_AWS_AVAILABILITY_ZONE_KEY), - FLB_LOG_EVENT_STRING_VALUE(ctx->availability_zone, - ctx->availability_zone_len)); - } - - if (ctx->instance_id_include && - ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - &log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE(FLB_FILTER_AWS_INSTANCE_ID_KEY), - FLB_LOG_EVENT_STRING_VALUE(ctx->instance_id, - ctx->instance_id_len)); - } - - if (ctx->instance_type_include && - ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - &log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE(FLB_FILTER_AWS_INSTANCE_TYPE_KEY), - FLB_LOG_EVENT_STRING_VALUE(ctx->instance_type, - ctx->instance_type_len)); - } - - if (ctx->private_ip_include && - ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - &log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE(FLB_FILTER_AWS_PRIVATE_IP_KEY), - FLB_LOG_EVENT_STRING_VALUE(ctx->private_ip, - ctx->private_ip_len)); - } - - if (ctx->vpc_id_include && - ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - &log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE(FLB_FILTER_AWS_VPC_ID_KEY), - FLB_LOG_EVENT_STRING_VALUE(ctx->vpc_id, - ctx->vpc_id_len)); - } - - if (ctx->ami_id_include && - ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - &log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE(FLB_FILTER_AWS_AMI_ID_KEY), - FLB_LOG_EVENT_STRING_VALUE(ctx->ami_id, - ctx->ami_id_len)); - } - - if (ctx->account_id_include && - ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - &log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE(FLB_FILTER_AWS_ACCOUNT_ID_KEY), - FLB_LOG_EVENT_STRING_VALUE(ctx->account_id, - ctx->account_id_len)); - } - - if (ctx->hostname_include && - ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - &log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE(FLB_FILTER_AWS_HOSTNAME_KEY), - FLB_LOG_EVENT_STRING_VALUE(ctx->hostname, - ctx->hostname_len)); - } - - if (ctx->tags_enabled && ctx->tags_fetched) { - for (i = 0; - i < ctx->tags_count && - ret == FLB_EVENT_ENCODER_SUCCESS; - i++) { - if (ctx->tag_is_enabled[i] == FLB_TRUE) { - ret = flb_log_event_encoder_append_body_values( - &log_encoder, - FLB_LOG_EVENT_STRING_VALUE(ctx->tag_keys[i], - ctx->tag_keys_len[i]), - FLB_LOG_EVENT_STRING_VALUE(ctx->tag_values[i], - ctx->tag_values_len[i])); - } - } - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(&log_encoder); - } - } - - if (ret == FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA && - log_decoder.offset == bytes) { - ret = FLB_EVENT_ENCODER_SUCCESS; - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - *out_buf = log_encoder.output_buffer; - *out_size = log_encoder.output_length; - - ret = FLB_FILTER_MODIFIED; - - flb_log_event_encoder_claim_internal_buffer_ownership(&log_encoder); - } - else { - flb_plg_error(ctx->ins, - "Log event encoder error : %d", ret); - - ret = FLB_FILTER_NOTOUCH; - } - - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - return ret; -} - -static void flb_filter_aws_destroy(struct flb_filter_aws *ctx) -{ - if (ctx->options == NULL) { - /* non null options are only provided by unit tests and since */ - /* aws client mock must clean up the memory with some special behaviour */ - /* if options are NOT null (which means we are running unit tests), */ - /* we rely on unit tests to perform memory cleanup */ - if (ctx->aws_ec2_filter_client) { - flb_aws_client_destroy(ctx->aws_ec2_filter_client); - } - } - if (ctx->client_imds) { - flb_aws_imds_destroy(ctx->client_imds); - } - - if (ctx->availability_zone) { - flb_sds_destroy(ctx->availability_zone); - } - - if (ctx->instance_id) { - flb_sds_destroy(ctx->instance_id); - } - - if (ctx->instance_type) { - flb_sds_destroy(ctx->instance_type); - } - - if (ctx->private_ip) { - flb_sds_destroy(ctx->private_ip); - } - - if (ctx->vpc_id) { - flb_sds_destroy(ctx->vpc_id); - } - - if (ctx->ami_id) { - flb_sds_destroy(ctx->ami_id); - } - - if (ctx->account_id) { - flb_sds_destroy(ctx->account_id); - } - - if (ctx->hostname) { - flb_sds_destroy(ctx->hostname); - } - - flb_filter_aws_tags_destroy(ctx); - - flb_free(ctx); -} - -static int cb_aws_exit(void *data, struct flb_config *config) -{ - struct flb_filter_aws *ctx = data; - - if (ctx != NULL) { - flb_filter_aws_destroy(ctx); - } - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "imds_version", "v2", - 0, FLB_FALSE, 0, - "Specifies which version of the EC2 instance metadata service" - " will be used: 'v1' or 'v2'. 'v2' may not work" - " if you run Fluent Bit in a container." - }, - { - FLB_CONFIG_MAP_BOOL, "az", "true", - 0, FLB_TRUE, offsetof(struct flb_filter_aws, availability_zone_include), - "Enable EC2 instance availability zone" - }, - { - FLB_CONFIG_MAP_BOOL, "ec2_instance_id", "true", - 0, FLB_TRUE, offsetof(struct flb_filter_aws, instance_id_include), - "Enable EC2 instance ID" - }, - { - FLB_CONFIG_MAP_BOOL, "ec2_instance_type", "false", - 0, FLB_TRUE, offsetof(struct flb_filter_aws, instance_type_include), - "Enable EC2 instance type" - }, - { - FLB_CONFIG_MAP_BOOL, "private_ip", "false", - 0, FLB_TRUE, offsetof(struct flb_filter_aws, private_ip_include), - "Enable EC2 instance private IP" - }, - { - FLB_CONFIG_MAP_BOOL, "vpc_id", "false", - 0, FLB_TRUE, offsetof(struct flb_filter_aws, vpc_id_include), - "Enable EC2 instance VPC ID" - }, - { - FLB_CONFIG_MAP_BOOL, "ami_id", "false", - 0, FLB_TRUE, offsetof(struct flb_filter_aws, ami_id_include), - "Enable EC2 instance Image ID" - }, - { - FLB_CONFIG_MAP_BOOL, "account_id", "false", - 0, FLB_TRUE, offsetof(struct flb_filter_aws, account_id_include), - "Enable EC2 instance Account ID" - }, - { - FLB_CONFIG_MAP_BOOL, "hostname", "false", - 0, FLB_TRUE, offsetof(struct flb_filter_aws, hostname_include), - "Enable EC2 instance hostname" - }, - { - FLB_CONFIG_MAP_BOOL, "tags_enabled", "false", - 0, FLB_TRUE, offsetof(struct flb_filter_aws, tags_enabled), - "Enable EC2 instance tags, " - "injects all tags if tags_include and tags_exclude are empty" - }, - { - FLB_CONFIG_MAP_STR, "tags_include", "", - 0, FLB_FALSE, 0, - "Defines list of specific EC2 tag keys to inject into the logs; " - "tag keys must be separated by \",\" character; " - "tags which are not present in this list will be ignored; " - "e.g.: \"Name,tag1,tag2\"" - }, - { - FLB_CONFIG_MAP_STR, "tags_exclude", "", - 0, FLB_FALSE, 0, - "Defines list of specific EC2 tag keys not to inject into the logs; " - "tag keys must be separated by \",\" character; " - "if both tags_include and tags_exclude are specified, configuration is invalid" - " and plugin fails" - }, - {0} -}; - -struct flb_filter_plugin filter_aws_plugin = { - .name = "aws", - .description = "Add AWS Metadata", - .cb_init = cb_aws_init, - .cb_filter = cb_aws_filter, - .cb_exit = cb_aws_exit, - .config_map = config_map, - .flags = 0 -}; diff --git a/fluent-bit/plugins/filter_aws/aws.h b/fluent-bit/plugins/filter_aws/aws.h deleted file mode 100644 index d165de5a2..000000000 --- a/fluent-bit/plugins/filter_aws/aws.h +++ /dev/null @@ -1,131 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_FILTER_AWS_H -#define FLB_FILTER_AWS_H - -#include -#include - -#define FLB_FILTER_AWS_AVAILABILITY_ZONE_KEY "az" -#define FLB_FILTER_AWS_AVAILABILITY_ZONE_KEY_LEN 2 -#define FLB_FILTER_AWS_INSTANCE_ID_KEY "ec2_instance_id" -#define FLB_FILTER_AWS_INSTANCE_ID_KEY_LEN 15 -#define FLB_FILTER_AWS_INSTANCE_TYPE_KEY "ec2_instance_type" -#define FLB_FILTER_AWS_INSTANCE_TYPE_KEY_LEN 17 -#define FLB_FILTER_AWS_PRIVATE_IP_KEY "private_ip" -#define FLB_FILTER_AWS_PRIVATE_IP_KEY_LEN 10 -#define FLB_FILTER_AWS_VPC_ID_KEY "vpc_id" -#define FLB_FILTER_AWS_VPC_ID_KEY_LEN 6 -#define FLB_FILTER_AWS_AMI_ID_KEY "ami_id" -#define FLB_FILTER_AWS_AMI_ID_KEY_LEN 6 -#define FLB_FILTER_AWS_ACCOUNT_ID_KEY "account_id" -#define FLB_FILTER_AWS_ACCOUNT_ID_KEY_LEN 10 -#define FLB_FILTER_AWS_HOSTNAME_KEY "hostname" -#define FLB_FILTER_AWS_HOSTNAME_KEY_LEN 8 - -struct flb_filter_aws { - struct flb_filter_aws_init_options *options; - - /* upstream connection to ec2 IMDS */ - struct flb_aws_client *aws_ec2_filter_client; - struct flb_aws_imds *client_imds; - - /* - * IMDSv2 requires a token which must be present in metadata requests - * This plugin does not refresh the token - */ - flb_sds_t imds_v2_token; - size_t imds_v2_token_len; - - /* Metadata fields - * These are queried only once; ec2 metadata is assumed to be immutable - */ - flb_sds_t availability_zone; - size_t availability_zone_len; - int availability_zone_include; - - flb_sds_t instance_id; - size_t instance_id_len; - int instance_id_include; - - flb_sds_t instance_type; - size_t instance_type_len; - int instance_type_include; - - flb_sds_t private_ip; - size_t private_ip_len; - int private_ip_include; - - flb_sds_t vpc_id; - size_t vpc_id_len; - int vpc_id_include; - - flb_sds_t ami_id; - size_t ami_id_len; - int ami_id_include; - - flb_sds_t account_id; - size_t account_id_len; - int account_id_include; - - flb_sds_t hostname; - size_t hostname_len; - int hostname_include; - - /* tags_* fields are related to exposing EC2 tags in log labels - * tags_enabled defines if EC2 tags functionality is enabled */ - int tags_enabled; - - /* tags_fetched defines if tag keys and values were fetched successfully - * and might be used to inject into msgpack */ - int tags_fetched; - /* tags_count defines how many tags are available to use - * it could be 0 if there are no tags defined or if metadata server has - * disabled exposing tags functionality */ - size_t tags_count; - /* tag_keys is an array of tag key strings */ - flb_sds_t *tag_keys; - /* tag_keys_len is an array of lengths corresponding to tag_keys items */ - size_t *tag_keys_len; - /* tag_values is an array of tag values strings */ - flb_sds_t *tag_values; - /* tag_values_len is an array of lengths related to tag_values items */ - size_t *tag_values_len; - /* tag_is_enabled is an array of bools which define if corresponding tag should be injected */ - /* e.g.: if tag_is_enabled[0] = FALSE, then filter aws should not inject first tag */ - int *tag_is_enabled; - - /* number of new keys added by this plugin */ - int new_keys; - - int metadata_retrieved; - - /* Plugin can use EC2 metadata v1 or v2; default is v2 */ - int use_v2; - - /* Filter plugin instance reference */ - struct flb_filter_instance *ins; -}; - -struct flb_filter_aws_init_options { - struct flb_aws_client_generator *client_generator; -}; - -#endif diff --git a/fluent-bit/plugins/filter_checklist/CMakeLists.txt b/fluent-bit/plugins/filter_checklist/CMakeLists.txt deleted file mode 100644 index eb8a13529..000000000 --- a/fluent-bit/plugins/filter_checklist/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - checklist.c) - -FLB_PLUGIN(filter_checklist "${src}" "") diff --git a/fluent-bit/plugins/filter_checklist/checklist.c b/fluent-bit/plugins/filter_checklist/checklist.c deleted file mode 100644 index 0664777c6..000000000 --- a/fluent-bit/plugins/filter_checklist/checklist.c +++ /dev/null @@ -1,656 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "checklist.h" -#include - -static int db_init(struct checklist *ctx) -{ - int ret; - - /* initialize databse */ - ctx->db = flb_sqldb_open(":memory:", "filter_check", ctx->config); - if (!ctx->db) { - flb_plg_error(ctx->ins, "could not create in-memory database"); - return -1; - } - - /* create table */ - ret = flb_sqldb_query(ctx->db, SQL_CREATE_TABLE, NULL, NULL); - if (ret != FLB_OK) { - flb_plg_error(ctx->ins, "db: could not create table"); - return -1; - } - - /* create table */ - ret = flb_sqldb_query(ctx->db, SQL_CASE_SENSITIVE, NULL, NULL); - if (ret != FLB_OK) { - flb_plg_error(ctx->ins, "db: could not set CASE SENSITIVE"); - return -1; - } - - /* - * Prepare SQL statements - * ----------------------- - */ - - /* SQL_INSERT */ - ret = sqlite3_prepare_v2(ctx->db->handler, - SQL_INSERT, - -1, - &ctx->stmt_insert, - 0); - if (ret != SQLITE_OK) { - flb_plg_error(ctx->ins, "error preparing database SQL statement: insert"); - return -1; - } - - /* SQL_CHECK */ - ret = sqlite3_prepare_v2(ctx->db->handler, - SQL_CHECK, - -1, - &ctx->stmt_check, - 0); - if (ret != SQLITE_OK) { - flb_plg_error(ctx->ins, "error preparing database SQL statement: check"); - return -1; - } - - return 0; -} - -static int db_insert(struct checklist *ctx, char *buf, int len) -{ - int ret; - - /* Bind parameter */ - sqlite3_bind_text(ctx->stmt_insert, 1, buf, len, 0); - - /* Run the insert */ - ret = sqlite3_step(ctx->stmt_insert); - if (ret != SQLITE_DONE) { - sqlite3_clear_bindings(ctx->stmt_insert); - sqlite3_reset(ctx->stmt_insert); - flb_plg_warn(ctx->ins, "cannot execute insert for value: %s", buf); - return -1; - } - - sqlite3_clear_bindings(ctx->stmt_insert); - sqlite3_reset(ctx->stmt_insert); - - return flb_sqldb_last_id(ctx->db); -} - -static int db_check(struct checklist *ctx, char *buf, size_t size) -{ - int ret; - int match = FLB_FALSE; - - /* Bind parameter */ - sqlite3_bind_text(ctx->stmt_check, 1, buf, size, 0); - - /* Run the check */ - ret = sqlite3_step(ctx->stmt_check); - if (ret == SQLITE_ROW) { - match = FLB_TRUE; - } - - sqlite3_clear_bindings(ctx->stmt_check); - sqlite3_reset(ctx->stmt_check); - - return match; -} - -static int load_file_patterns(struct checklist *ctx) -{ - int i; - int ret; - int len; - int line = 0; - int size = LINE_SIZE; - char buf[LINE_SIZE]; - FILE *f; - - /* open file */ - f = fopen(ctx->file, "r"); - if (!f) { - flb_errno(); - flb_plg_error(ctx->ins, "could not open file: %s", ctx->file); - return -1; - } - - /* read and process rules on lines */ - while (fgets(buf, size - 1, f)) { - len = strlen(buf); - if (buf[len - 1] == '\n') { - buf[--len] = 0; - if (len && buf[len - 1] == '\r') { - buf[--len] = 0; - } - } - else if (!feof(f)) { - flb_plg_error(ctx->ins, "length of content has exceeded limit"); - fclose(f); - return -1; - } - - /* skip empty and commented lines */ - if (!buf[0] || buf[0] == '#') { - line++; - continue; - } - - /* convert to lowercase if needed */ - if (ctx->ignore_case) { - for (i = 0; i < len; i++) { - buf[i] = tolower(buf[i]); - } - } - - /* add the entry as a hash table key, no value reference is needed */ - if (ctx->mode == CHECK_EXACT_MATCH) { - ret = flb_hash_table_add(ctx->ht, buf, len, "", 0); - } - else if (ctx->mode == CHECK_PARTIAL_MATCH) { - ret = db_insert(ctx, buf, len); - } - - if (ret >= 0) { - flb_plg_debug(ctx->ins, "file list: line=%i adds value='%s'", line, buf); - } - line++; - } - - fclose(f); - return 0; -} - -static int init_config(struct checklist *ctx) -{ - int ret; - char *tmp; - struct flb_time t0; - struct flb_time t1; - struct flb_time t_diff; - - /* check if we have 'records' to add */ - if (mk_list_size(ctx->records) == 0) { - flb_plg_warn(ctx->ins, "no 'record' options has been specified"); - } - - /* lookup mode */ - ctx->mode = CHECK_EXACT_MATCH; - tmp = (char *) flb_filter_get_property("mode", ctx->ins); - if (tmp) { - if (strcasecmp(tmp, "exact") == 0) { - ctx->mode = CHECK_EXACT_MATCH; - } - else if (strcasecmp(tmp, "partial") == 0) { - ctx->mode = CHECK_PARTIAL_MATCH; - } - } - - if (ctx->mode == CHECK_EXACT_MATCH) { - /* create hash table */ - ctx->ht = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, - CHECK_HASH_TABLE_SIZE, -1); - if (!ctx->ht) { - flb_plg_error(ctx->ins, "could not create hash table"); - return -1; - } - } - else if (ctx->mode == CHECK_PARTIAL_MATCH) { - ret = db_init(ctx); - if (ret < 0) { - return -1; - } - } - - /* record accessor pattern / key name */ - ctx->ra_lookup_key = flb_ra_create(ctx->lookup_key, FLB_TRUE); - if (!ctx->ra_lookup_key) { - flb_plg_error(ctx->ins, "invalid lookup_key pattern: %s", - ctx->lookup_key); - return -1; - } - - /* validate file */ - if (!ctx->file) { - flb_plg_error(ctx->ins, "option 'file' is not set"); - return -1; - } - - - /* load file content */ - flb_time_get(&t0); - ret = load_file_patterns(ctx); - flb_time_get(&t1); - - /* load time */ - flb_time_diff(&t1, &t0, &t_diff); - flb_plg_info(ctx->ins, "load file elapsed time (sec.ns): %lu.%lu", - t_diff.tm.tv_sec, t_diff.tm.tv_nsec); - - return ret; -} - -static int cb_checklist_init(struct flb_filter_instance *ins, - struct flb_config *config, - void *data) -{ - int ret; - struct checklist *ctx; - - ctx = flb_calloc(1, sizeof(struct checklist)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->ins = ins; - ctx->config = config; - - /* set context */ - flb_filter_set_context(ins, ctx); - - /* Set config_map properties in our local context */ - ret = flb_filter_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return -1; - } - - ret = init_config(ctx); - - return 0; -} - -static int set_record(struct checklist *ctx, - struct flb_log_event_encoder *log_encoder, - struct flb_log_event *log_event) -{ - int i; - int len; - int ret; - int skip; - msgpack_object k; - msgpack_object v; - msgpack_object *map; - struct mk_list *head; - struct flb_slist_entry *r_key; - struct flb_slist_entry *r_val; - struct flb_config_map_val *mv; - - ret = flb_log_event_encoder_begin_record(log_encoder); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - return -1; - } - - ret = flb_log_event_encoder_set_timestamp(log_encoder, &log_event->timestamp); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - return -2; - } - - ret = flb_log_event_encoder_set_metadata_from_msgpack_object(log_encoder, - log_event->metadata); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - return -3; - } - - map = log_event->body; - - for (i = 0; i < map->via.map.size; i++) { - k = map->via.map.ptr[i].key; - v = map->via.map.ptr[i].val; - - if (k.type != MSGPACK_OBJECT_STR) { - continue; - } - - /* iterate 'records' list, check if this key is a duplicated */ - skip = FLB_FALSE; - flb_config_map_foreach(head, mv, ctx->records) { - r_key = mk_list_entry_first(mv->val.list, struct flb_slist_entry, _head); - r_val = mk_list_entry_last(mv->val.list, struct flb_slist_entry, _head); - - len = flb_sds_len(r_key->str); - if (k.via.str.size != len) { - continue; - } - - if (strncmp(k.via.str.ptr, r_key->str, len) == 0) { - skip = FLB_TRUE; - break; - } - } - - /* - * skip is true if the current key will be overrided by some entry of - * the 'records' list. - */ - if (skip) { - continue; - } - - ret = flb_log_event_encoder_append_body_values( - log_encoder, - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&k), - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&v)); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - return -4; - } - } - - /* Pack custom records */ - flb_config_map_foreach(head, mv, ctx->records) { - r_key = mk_list_entry_first(mv->val.list, struct flb_slist_entry, _head); - r_val = mk_list_entry_last(mv->val.list, struct flb_slist_entry, _head); - - ret = flb_log_event_encoder_append_body_string( - log_encoder, r_key->str, flb_sds_len(r_key->str)); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - return -5; - } - - if (strcasecmp(r_val->str, "true") == 0) { - ret = flb_log_event_encoder_append_body_boolean( - log_encoder, FLB_TRUE); - } - else if (strcasecmp(r_val->str, "false") == 0) { - ret = flb_log_event_encoder_append_body_boolean( - log_encoder, FLB_FALSE); - } - else if (strcasecmp(r_val->str, "null") == 0) { - ret = flb_log_event_encoder_append_body_null( - log_encoder); - } - else { - ret = flb_log_event_encoder_append_body_string( - log_encoder, r_val->str, flb_sds_len(r_val->str)); - } - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - return -3; - } - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(log_encoder); - } - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - return -4; - } - - return 0; -} - -static int cb_checklist_filter(const void *data, size_t bytes, - const char *tag, int tag_len, - void **out_buf, size_t *out_bytes, - struct flb_filter_instance *ins, - struct flb_input_instance *i_ins, - void *filter_context, - struct flb_config *config) -{ - int i; - int id; - int found; - int matches = 0; - size_t pre = 0; - size_t off = 0; - size_t cmp_size; - char *cmp_buf; - char *tmp_buf; - size_t tmp_size; - struct checklist *ctx = filter_context; - struct flb_ra_value *rval; - struct flb_time t0; - struct flb_time t1; - struct flb_time t_diff; - struct flb_log_event_encoder log_encoder; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - int ret; - - (void) ins; - (void) i_ins; - (void) config; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return FLB_FILTER_NOTOUCH; - } - - ret = flb_log_event_encoder_init(&log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ins, "Log event encoder initialization error : %d", ret); - - flb_log_event_decoder_destroy(&log_decoder); - - return FLB_FILTER_NOTOUCH; - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - off = log_decoder.offset; - found = FLB_FALSE; - - rval = flb_ra_get_value_object(ctx->ra_lookup_key, *log_event.body); - if (rval) { - if (ctx->print_query_time) { - flb_time_get(&t0); - } - - cmp_buf = NULL; - if (rval->type == FLB_RA_STRING) { - /* convert to lowercase */ - if (ctx->ignore_case) { - cmp_buf = flb_calloc(1, rval->o.via.str.size + 1); - if (!cmp_buf) { - flb_errno(); - flb_ra_key_value_destroy(rval); - continue; - } - memcpy(cmp_buf, rval->o.via.str.ptr, rval->o.via.str.size); - for (i = 0; i < rval->o.via.str.size; i++) { - cmp_buf[i] = tolower(cmp_buf[i]); - } - } - else { - cmp_buf = (char *) rval->o.via.str.ptr; - } - cmp_size = rval->o.via.str.size; - - if (ctx->mode == CHECK_EXACT_MATCH) { - id = flb_hash_table_get(ctx->ht, cmp_buf, cmp_size, - (void *) &tmp_buf, &tmp_size); - if (id >= 0) { - found = FLB_TRUE; - } - } - else if (ctx->mode == CHECK_PARTIAL_MATCH) { - found = db_check(ctx, cmp_buf, cmp_size); - } - - if (cmp_buf && cmp_buf != (char *) rval->o.via.str.ptr) { - flb_free(cmp_buf); - } - } - - /* print elapsed time */ - if (ctx->print_query_time && found) { - flb_time_get(&t1); - flb_time_diff(&t1, &t0, &t_diff); - - flb_plg_info(ctx->ins, - "query time (sec.ns): %lu.%lu : '%.*s'", - t_diff.tm.tv_sec, - t_diff.tm.tv_nsec, - (int) rval->o.via.str.size, - rval->o.via.str.ptr); - } - - flb_ra_key_value_destroy(rval); - } - - if (found) { - /* add any previous content that not matched */ - if (log_encoder.output_length == 0 && pre > 0) { - ret = flb_log_event_encoder_emit_raw_record( - &log_encoder, - data, - pre); - } - - ret = set_record(ctx, &log_encoder, &log_event); - - if (ret < -1) { - flb_log_event_encoder_rollback_record(&log_encoder); - } - - matches++; - } - else { - if (log_encoder.output_length > 0) { - /* append current record to new buffer */ - ret = flb_log_event_encoder_emit_raw_record( - &log_encoder, - &((char *) data)[pre], - off - pre); - } - } - pre = off; - } - - if (log_encoder.output_length > 0 && matches > 0) { - *out_buf = log_encoder.output_buffer; - *out_bytes = log_encoder.output_length; - - flb_log_event_encoder_claim_internal_buffer_ownership(&log_encoder); - - ret = FLB_FILTER_MODIFIED; - } - else { - ret = FLB_FILTER_NOTOUCH; - } - - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - return ret; -} - -static int cb_exit(void *data, struct flb_config *config) -{ - struct checklist *ctx = data; - - if (!ctx) { - return 0; - } - - if (ctx->ra_lookup_key) { - flb_ra_destroy(ctx->ra_lookup_key); - } - - if (ctx->ht) { - flb_hash_table_destroy(ctx->ht); - } - - if (ctx->db) { - sqlite3_finalize(ctx->stmt_insert); - sqlite3_finalize(ctx->stmt_check); - flb_sqldb_close(ctx->db); - } - - flb_free(ctx); - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "file", NULL, - 0, FLB_TRUE, offsetof(struct checklist, file), - "Specify the file that contains the patterns to lookup." - }, - - { - FLB_CONFIG_MAP_STR, "mode", "exact", - 0, FLB_FALSE, 0, - "Set the check mode: 'exact' or 'partial'." - }, - - { - FLB_CONFIG_MAP_BOOL, "print_query_time", "false", - 0, FLB_TRUE, offsetof(struct checklist, print_query_time), - "Print to stdout the elapseed query time for every matched record" - }, - - { - FLB_CONFIG_MAP_BOOL, "ignore_case", "false", - 0, FLB_TRUE, offsetof(struct checklist, ignore_case), - "Compare strings by ignoring case." - }, - - { - FLB_CONFIG_MAP_STR, "lookup_key", "log", - 0, FLB_TRUE, offsetof(struct checklist, lookup_key), - "Name of the key to lookup." - }, - - { - FLB_CONFIG_MAP_SLIST_2, "record", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct checklist, records), - "Name of record key to add and its value, it accept two values,e.g " - "'record mykey my val'. You can add many 'record' entries as needed." - }, - - /* EOF */ - {0} -}; - -struct flb_filter_plugin filter_checklist_plugin = { - .name = "checklist", - .description = "Check records and flag them", - .cb_init = cb_checklist_init, - .cb_filter = cb_checklist_filter, - .cb_exit = cb_exit, - .config_map = config_map, - .flags = 0 -}; diff --git a/fluent-bit/plugins/filter_checklist/checklist.h b/fluent-bit/plugins/filter_checklist/checklist.h deleted file mode 100644 index 8cd39516d..000000000 --- a/fluent-bit/plugins/filter_checklist/checklist.h +++ /dev/null @@ -1,69 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_FILTER_CHECK_H -#define FLB_FILTER_CHECK_H - -#include -#include -#include -#include - -#define LINE_SIZE 2048 -#define CHECK_HASH_TABLE_SIZE 100000 -#define CHECK_EXACT_MATCH 0 /* exact string match */ -#define CHECK_PARTIAL_MATCH 1 /* partial match */ - -/* plugin context */ -struct checklist { - /* config options */ - int mode; - int ignore_case; - int print_query_time; - flb_sds_t file; - flb_sds_t lookup_key; - struct mk_list *records; - - /* internal */ - struct flb_sqldb *db; - sqlite3_stmt *stmt_insert; - sqlite3_stmt *stmt_check; - struct flb_hash_table *ht; - struct flb_record_accessor *ra_lookup_key; - struct flb_filter_instance *ins; - struct flb_config *config; -}; - -/* create table */ -#define SQL_CREATE_TABLE \ - "CREATE TABLE IF NOT EXISTS list (" \ - " pattern text " \ - ");" - -#define SQL_CASE_SENSITIVE \ - "PRAGMA case_sensitive_like = true;" - -/* insert pattern into list table */ -#define SQL_INSERT "INSERT INTO list (pattern) VALUES (@val);" - -/* validate incoming value against list */ -#define SQL_CHECK \ - "SELECT pattern FROM list WHERE @val LIKE (pattern || '%');" - -#endif diff --git a/fluent-bit/plugins/filter_ecs/CMakeLists.txt b/fluent-bit/plugins/filter_ecs/CMakeLists.txt deleted file mode 100644 index 335a870f7..000000000 --- a/fluent-bit/plugins/filter_ecs/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -set(src - ecs.c - ) - -FLB_PLUGIN(filter_ecs "${src}" "") diff --git a/fluent-bit/plugins/filter_ecs/ecs.c b/fluent-bit/plugins/filter_ecs/ecs.c deleted file mode 100644 index 82339e60e..000000000 --- a/fluent-bit/plugins/filter_ecs/ecs.c +++ /dev/null @@ -1,1760 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "ecs.h" - -static int get_ecs_cluster_metadata(struct flb_filter_ecs *ctx); -static void flb_filter_ecs_destroy(struct flb_filter_ecs *ctx); - -/* cluster meta is static so we can expose it on global ctx for other plugins to use */ -static void expose_ecs_cluster_meta(struct flb_filter_ecs *ctx) -{ - struct flb_env *env; - struct flb_config *config = ctx->ins->config; - - env = config->env; - - flb_env_set(env, "ecs", "enabled"); - - if (ctx->cluster_metadata.cluster_name) { - flb_env_set(env, - "aws.ecs.cluster_name", - ctx->cluster_metadata.cluster_name); - } - - if (ctx->cluster_metadata.container_instance_arn) { - flb_env_set(env, - "aws.ecs.container_instance_arn", - ctx->cluster_metadata.container_instance_arn); - } - - if (ctx->cluster_metadata.container_instance_id) { - flb_env_set(env, - "aws.ecs.container_instance_id", - ctx->cluster_metadata.container_instance_id); - } - - if (ctx->cluster_metadata.ecs_agent_version) { - flb_env_set(env, - "aws.ecs.ecs_agent_version", - ctx->cluster_metadata.container_instance_id); - } -} - -static int cb_ecs_init(struct flb_filter_instance *f_ins, - struct flb_config *config, - void *data) -{ - int ret; - struct flb_filter_ecs *ctx = NULL; - struct mk_list *head; - struct mk_list *split; - struct flb_kv *kv; - struct flb_split_entry *sentry; - int list_size; - struct flb_ecs_metadata_key *ecs_meta = NULL; - (void) data; - - /* Create context */ - ctx = flb_calloc(1, sizeof(struct flb_filter_ecs)); - if (!ctx) { - flb_errno(); - return -1; - } - - ctx->ins = f_ins; - - /* Populate context with config map defaults and incoming properties */ - ret = flb_filter_config_map_set(f_ins, (void *) ctx); - if (ret == -1) { - flb_plg_error(f_ins, "configuration error"); - flb_free(ctx); - return -1; - } - - mk_list_init(&ctx->metadata_keys); - ctx->metadata_keys_len = 0; - mk_list_init(&ctx->metadata_buffers); - - mk_list_foreach(head, &f_ins->properties) { - kv = mk_list_entry(head, struct flb_kv, _head); - - if (strcasecmp(kv->key, "add") == 0) { - split = flb_utils_split(kv->val, ' ', 2); - list_size = mk_list_size(split); - - if (list_size == 0 || list_size > 2) { - flb_plg_error(ctx->ins, "Invalid config for %s", kv->key); - flb_utils_split_free(split); - goto error; - } - - sentry = mk_list_entry_first(split, struct flb_split_entry, _head); - ecs_meta = flb_calloc(1, sizeof(struct flb_ecs_metadata_key)); - if (!ecs_meta) { - flb_errno(); - flb_utils_split_free(split); - goto error; - } - - ecs_meta->key = flb_sds_create_len(sentry->value, sentry->len); - if (!ecs_meta->key) { - flb_errno(); - flb_utils_split_free(split); - goto error; - } - - sentry = mk_list_entry_last(split, struct flb_split_entry, _head); - - ecs_meta->template = flb_sds_create_len(sentry->value, sentry->len); - if (!ecs_meta->template) { - flb_errno(); - flb_utils_split_free(split); - goto error; - } - - ecs_meta->ra = flb_ra_create(ecs_meta->template, FLB_FALSE); - if (ecs_meta->ra == NULL) { - flb_plg_error(ctx->ins, "Could not parse template for `%s`", ecs_meta->key); - flb_utils_split_free(split); - goto error; - } - - mk_list_add(&ecs_meta->_head, &ctx->metadata_keys); - ctx->metadata_keys_len++; - flb_utils_split_free(split); - } - } - - ctx->ecs_upstream = flb_upstream_create(config, - ctx->ecs_host, - ctx->ecs_port, - FLB_IO_TCP, - NULL); - - if (!ctx->ecs_upstream) { - flb_errno(); - flb_plg_error(ctx->ins, "Could not create upstream connection to ECS Agent"); - goto error; - } - - flb_stream_disable_async_mode(&ctx->ecs_upstream->base); - ctx->has_cluster_metadata = FLB_FALSE; - - /* entries are only evicted when TTL is reached and a get is issued */ - ctx->container_hash_table = flb_hash_table_create_with_ttl(ctx->ecs_meta_cache_ttl, - FLB_HASH_TABLE_EVICT_OLDER, - FLB_ECS_FILTER_HASH_TABLE_SIZE, - FLB_ECS_FILTER_HASH_TABLE_SIZE); - if (!ctx->container_hash_table) { - flb_plg_error(f_ins, "failed to create container_hash_table"); - goto error; - } - - ctx->failed_metadata_request_tags = flb_hash_table_create_with_ttl(ctx->ecs_meta_cache_ttl, - FLB_HASH_TABLE_EVICT_OLDER, - FLB_ECS_FILTER_HASH_TABLE_SIZE, - FLB_ECS_FILTER_HASH_TABLE_SIZE); - if (!ctx->failed_metadata_request_tags) { - flb_plg_error(f_ins, "failed to create failed_metadata_request_tags table"); - goto error; - } - - ctx->ecs_tag_prefix_len = strlen(ctx->ecs_tag_prefix); - - /* attempt to get metadata in init, can retry in cb_filter */ - ret = get_ecs_cluster_metadata(ctx); - - flb_filter_set_context(f_ins, ctx); - return 0; - -error: - flb_plg_error(ctx->ins, "Initialization failed."); - flb_filter_ecs_destroy(ctx); - return -1; -} - -static int plugin_under_test() -{ - if (getenv("FLB_ECS_PLUGIN_UNDER_TEST") != NULL) { - return FLB_TRUE; - } - - return FLB_FALSE; -} - -static char *mock_error_response(char *error_env_var) -{ - char *err_val = NULL; - char *error = NULL; - int len = 0; - - err_val = getenv(error_env_var); - if (err_val != NULL && strlen(err_val) > 0) { - error = flb_malloc(strlen(err_val) + sizeof(char)); - if (error == NULL) { - flb_errno(); - return NULL; - } - - len = strlen(err_val); - memcpy(error, err_val, len); - error[len] = '\0'; - return error; - } - - return NULL; -} - -static struct flb_http_client *mock_http_call(char *error_env_var, char *api) -{ - /* create an http client so that we can set the response */ - struct flb_http_client *c = NULL; - char *error = mock_error_response(error_env_var); - - c = flb_calloc(1, sizeof(struct flb_http_client)); - if (!c) { - flb_errno(); - flb_free(error); - return NULL; - } - mk_list_init(&c->headers); - - if (error != NULL) { - c->resp.status = 400; - /* resp.data is freed on destroy, payload is supposed to reference it */ - c->resp.data = error; - c->resp.payload = c->resp.data; - c->resp.payload_size = strlen(error); - } - else { - c->resp.status = 200; - if (strcmp(api, "Cluster") == 0) { - /* mocked success response */ - c->resp.payload = "{\"Cluster\": \"cluster_name\",\"ContainerInstanceArn\": \"arn:aws:ecs:region:aws_account_id:container-instance/cluster_name/container_instance_id\",\"Version\": \"Amazon ECS Agent - v1.30.0 (02ff320c)\"}"; - c->resp.payload_size = strlen(c->resp.payload); - } - else { - c->resp.payload = "{\"Arn\": \"arn:aws:ecs:us-west-2:012345678910:task/default/e01d58a8-151b-40e8-bc01-22647b9ecfec\",\"Containers\": [{\"DockerId\": \"79c796ed2a7f864f485c76f83f3165488097279d296a7c05bd5201a1c69b2920\",\"DockerName\": \"ecs-nginx-efs-2-nginx-9ac0808dd0afa495f001\",\"Name\": \"nginx\"}],\"DesiredStatus\": \"RUNNING\",\"Family\": \"nginx-efs\",\"KnownStatus\": \"RUNNING\",\"Version\": \"2\"}"; - c->resp.payload_size = strlen(c->resp.payload); - } - } - - return c; -} - -/* - * Both container instance and task ARNs have the ID at the end after last '/' - */ -static flb_sds_t parse_id_from_arn(const char *arn, int len) -{ - int i; - flb_sds_t ID = NULL; - int last_slash = 0; - int id_start = 0; - - for (i = 0; i < len; i++) { - if (arn[i] == '/') { - last_slash = i; - } - } - - if (last_slash == 0 || last_slash >= len - 2) { - return NULL; - } - id_start = last_slash + 1; - - ID = flb_sds_create_len(arn + id_start, len - id_start); - if (ID == NULL) { - flb_errno(); - return NULL; - } - - return ID; -} - -/* - * This deserializes the msgpack metadata buf to msgpack_object - * which can be used with flb_ra_translate in the main filter callback - */ -static int flb_ecs_metadata_buffer_init(struct flb_filter_ecs *ctx, - struct flb_ecs_metadata_buffer *meta) -{ - msgpack_unpacked result; - msgpack_object root; - size_t off = 0; - int ret; - - msgpack_unpacked_init(&result); - ret = msgpack_unpack_next(&result, meta->buf, meta->size, &off); - if (ret != MSGPACK_UNPACK_SUCCESS) { - flb_plg_error(ctx->ins, "Cannot unpack flb_ecs_metadata_buffer"); - msgpack_unpacked_destroy(&result); - return -1; - } - - root = result.data; - if (root.type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "Cannot unpack flb_ecs_metadata_buffer, msgpack_type=%i", - root.type); - msgpack_unpacked_destroy(&result); - return -1; - } - - meta->unpacked = result; - meta->obj = root; - meta->last_used_time = time(NULL); - meta->free_packer = FLB_TRUE; - - return 0; -} - -static void flb_ecs_metadata_buffer_destroy(struct flb_ecs_metadata_buffer *meta) -{ - if (meta) { - flb_free(meta->buf); - if (meta->free_packer == FLB_TRUE) { - msgpack_unpacked_destroy(&meta->unpacked); - } - if (meta->id) { - flb_sds_destroy(meta->id); - } - flb_free(meta); - } -} - -/* - * Get cluster and container instance info, which are static and never change - */ -static int get_ecs_cluster_metadata(struct flb_filter_ecs *ctx) -{ - struct flb_http_client *c; - struct flb_connection *u_conn; - int ret; - int root_type; - int found_cluster = FLB_FALSE; - int found_version = FLB_FALSE; - int found_instance = FLB_FALSE; - int free_conn = FLB_FALSE; - int i; - int len; - char *buffer; - size_t size; - size_t b_sent; - size_t off = 0; - msgpack_unpacked result; - msgpack_object root; - msgpack_object key; - msgpack_object val; - msgpack_sbuffer tmp_sbuf; - msgpack_packer tmp_pck; - flb_sds_t container_instance_id = NULL; - flb_sds_t tmp = NULL; - - /* Compose HTTP Client request*/ - if (plugin_under_test() == FLB_TRUE) { - c = mock_http_call("TEST_CLUSTER_ERROR", "Cluster"); - ret = 0; - } - else { - u_conn = flb_upstream_conn_get(ctx->ecs_upstream); - - if (!u_conn) { - flb_plg_error(ctx->ins, "ECS agent introspection endpoint connection error"); - return -1; - } - free_conn = FLB_TRUE; - c = flb_http_client(u_conn, FLB_HTTP_GET, - FLB_ECS_FILTER_CLUSTER_PATH, - NULL, 0, - ctx->ecs_host, ctx->ecs_port, - NULL, 0); - flb_http_buffer_size(c, 0); /* 0 means unlimited */ - - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - - ret = flb_http_do(c, &b_sent); - flb_plg_debug(ctx->ins, "http_do=%i, " - "HTTP Status: %i", - ret, c->resp.status); - } - - if (ret != 0 || c->resp.status != 200) { - if (c->resp.payload_size > 0) { - flb_plg_warn(ctx->ins, "Failed to get metadata from %s, will retry", - FLB_ECS_FILTER_CLUSTER_PATH); - flb_plg_debug(ctx->ins, "HTTP response\n%s", - c->resp.payload); - } else { - flb_plg_warn(ctx->ins, "%s response status was %d with no payload, will retry", - FLB_ECS_FILTER_CLUSTER_PATH, - c->resp.status); - } - flb_http_client_destroy(c); - if (free_conn == FLB_TRUE) { - flb_upstream_conn_release(u_conn); - } - return -1; - } - - if (free_conn == FLB_TRUE) { - flb_upstream_conn_release(u_conn); - } - - ret = flb_pack_json(c->resp.payload, c->resp.payload_size, - &buffer, &size, &root_type, NULL); - - if (ret < 0) { - flb_plg_warn(ctx->ins, "Could not parse response from %s; response=\n%s", - FLB_ECS_FILTER_CLUSTER_PATH, c->resp.payload); - flb_http_client_destroy(c); - return -1; - } - - /* parse metadata response */ - msgpack_unpacked_init(&result); - ret = msgpack_unpack_next(&result, buffer, size, &off); - if (ret != MSGPACK_UNPACK_SUCCESS) { - flb_plg_error(ctx->ins, "Cannot unpack %s response to find metadata\n%s", - FLB_ECS_FILTER_CLUSTER_PATH, c->resp.payload); - flb_free(buffer); - msgpack_unpacked_destroy(&result); - flb_http_client_destroy(c); - return -1; - } - - flb_http_client_destroy(c); - - root = result.data; - if (root.type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "%s response parsing failed, msgpack_type=%i", - FLB_ECS_FILTER_CLUSTER_PATH, - root.type); - flb_free(buffer); - msgpack_unpacked_destroy(&result); - return -1; - } - - /* -Metadata Response: -{ - "Cluster": "cluster_name", - "ContainerInstanceArn": "arn:aws:ecs:region:aws_account_id:container-instance/cluster_name/container_instance_id", - "Version": "Amazon ECS Agent - v1.30.0 (02ff320c)" -} -But our metadata keys names are: -{ - "ClusterName": "cluster_name", - "ContainerInstanceArn": "arn:aws:ecs:region:aws_account_id:container-instance/cluster_name/container_instance_id", - "ContainerInstanceID": "container_instance_id" - "ECSAgentVersion": "Amazon ECS Agent - v1.30.0 (02ff320c)" -} - */ - - for (i = 0; i < root.via.map.size; i++) { - key = root.via.map.ptr[i].key; - if (key.type != MSGPACK_OBJECT_STR) { - flb_plg_error(ctx->ins, "%s response parsing failed, msgpack key type=%i", - FLB_ECS_FILTER_CLUSTER_PATH, - key.type); - continue; - } - - if (key.via.str.size == 7 && strncmp(key.via.str.ptr, "Cluster", 7) == 0) { - val = root.via.map.ptr[i].val; - if (val.type != MSGPACK_OBJECT_STR) { - flb_plg_error(ctx->ins, "metadata parsing: unexpected 'Cluster' value type=%i", - val.type); - flb_free(buffer); - msgpack_unpacked_destroy(&result); - return -1; - } - - found_cluster = FLB_TRUE; - if (ctx->cluster_metadata.cluster_name == NULL) { - tmp = flb_sds_create_len(val.via.str.ptr, (int) val.via.str.size); - if (!tmp) { - flb_errno(); - flb_free(buffer); - msgpack_unpacked_destroy(&result); - return -1; - } - ctx->cluster_metadata.cluster_name = tmp; - } - - } - else if (key.via.str.size == 20 && strncmp(key.via.str.ptr, "ContainerInstanceArn", 20) == 0) { - val = root.via.map.ptr[i].val; - if (val.type != MSGPACK_OBJECT_STR) { - flb_plg_error(ctx->ins, "metadata parsing: unexpected 'ContainerInstanceArn' value type=%i", - val.type); - flb_free(buffer); - msgpack_unpacked_destroy(&result); - return -1; - } - - /* first the ARN */ - found_instance = FLB_TRUE; - if (ctx->cluster_metadata.container_instance_arn == NULL) { - tmp = flb_sds_create_len(val.via.str.ptr, (int) val.via.str.size); - if (!tmp) { - flb_errno(); - flb_free(buffer); - msgpack_unpacked_destroy(&result); - return -1; - } - ctx->cluster_metadata.container_instance_arn = tmp; - } - - /* then the ID */ - if (ctx->cluster_metadata.container_instance_id == NULL) { - container_instance_id = parse_id_from_arn(val.via.str.ptr, (int) val.via.str.size); - if (container_instance_id == NULL) { - flb_plg_error(ctx->ins, "metadata parsing: failed to get ID from %.*s", - (int) val.via.str.size, val.via.str.ptr); - flb_free(buffer); - msgpack_unpacked_destroy(&result); - return -1; - } - ctx->cluster_metadata.container_instance_id = container_instance_id; - } - - } else if (key.via.str.size == 7 && strncmp(key.via.str.ptr, "Version", 7) == 0) { - val = root.via.map.ptr[i].val; - if (val.type != MSGPACK_OBJECT_STR) { - flb_plg_error(ctx->ins, "metadata parsing: unexpected 'Version' value type=%i", - val.type); - flb_free(buffer); - msgpack_unpacked_destroy(&result); - return -1; - } - - found_version = FLB_TRUE; - if (ctx->cluster_metadata.ecs_agent_version == NULL) { - tmp = flb_sds_create_len(val.via.str.ptr, (int) val.via.str.size); - if (!tmp) { - flb_errno(); - flb_free(buffer); - msgpack_unpacked_destroy(&result); - return -1; - } - ctx->cluster_metadata.ecs_agent_version = tmp; - } - } - - } - - flb_free(buffer); - msgpack_unpacked_destroy(&result); - - if (found_cluster == FLB_FALSE) { - flb_plg_error(ctx->ins, "Could not parse 'Cluster' from %s response", - FLB_ECS_FILTER_CLUSTER_PATH); - return -1; - } - if (found_instance == FLB_FALSE) { - flb_plg_error(ctx->ins, "Could not parse 'ContainerInstanceArn' from %s response", - FLB_ECS_FILTER_CLUSTER_PATH); - return -1; - } - if (found_version == FLB_FALSE) { - flb_plg_error(ctx->ins, "Could not parse 'Version' from %s response", - FLB_ECS_FILTER_CLUSTER_PATH); - return -1; - } - - /* - * We also create a standalone cluster metadata msgpack object - * This is used as a fallback for logs when we can't find the - * task metadata for a log. It is valid to attach cluster meta - * to eg. Docker daemon logs which are not an AWS ECS Task via - * the `cluster_metadata_only` setting. - */ - msgpack_sbuffer_init(&tmp_sbuf); - msgpack_packer_init(&tmp_pck, &tmp_sbuf, msgpack_sbuffer_write); - msgpack_pack_map(&tmp_pck, 4); - - msgpack_pack_str(&tmp_pck, 11); - msgpack_pack_str_body(&tmp_pck, - "ClusterName", - 11); - len = flb_sds_len(ctx->cluster_metadata.cluster_name); - msgpack_pack_str(&tmp_pck, len); - msgpack_pack_str_body(&tmp_pck, - ctx->cluster_metadata.cluster_name, - len); - - msgpack_pack_str(&tmp_pck, 20); - msgpack_pack_str_body(&tmp_pck, - "ContainerInstanceArn", - 20); - len = flb_sds_len(ctx->cluster_metadata.container_instance_arn); - msgpack_pack_str(&tmp_pck, len); - msgpack_pack_str_body(&tmp_pck, - ctx->cluster_metadata.container_instance_arn, - len); - - msgpack_pack_str(&tmp_pck, 19); - msgpack_pack_str_body(&tmp_pck, - "ContainerInstanceID", - 19); - len = flb_sds_len(ctx->cluster_metadata.container_instance_id); - msgpack_pack_str(&tmp_pck, len); - msgpack_pack_str_body(&tmp_pck, - ctx->cluster_metadata.container_instance_id, - len); - - msgpack_pack_str(&tmp_pck, 15); - msgpack_pack_str_body(&tmp_pck, - "ECSAgentVersion", - 15); - len = flb_sds_len(ctx->cluster_metadata.ecs_agent_version); - msgpack_pack_str(&tmp_pck, len); - msgpack_pack_str_body(&tmp_pck, - ctx->cluster_metadata.ecs_agent_version, - len); - - ctx->cluster_meta_buf.buf = tmp_sbuf.data; - ctx->cluster_meta_buf.size = tmp_sbuf.size; - - ret = flb_ecs_metadata_buffer_init(ctx, &ctx->cluster_meta_buf); - if (ret < 0) { - flb_plg_error(ctx->ins, "Could not init metadata buffer from %s response", - FLB_ECS_FILTER_CLUSTER_PATH); - msgpack_sbuffer_destroy(&tmp_sbuf); - ctx->cluster_meta_buf.buf = NULL; - ctx->cluster_meta_buf.size = 0; - return -1; - } - - ctx->has_cluster_metadata = FLB_TRUE; - expose_ecs_cluster_meta(ctx); - return 0; -} - -/* - * This is the helper function used by get_task_metadata() - * that actually creates the final metadata msgpack buffer - * with our final key names. - * It collects cluster, task, and container metadata into one -The new metadata msgpack is flat and looks like: -{ - "ContainerID": "79c796ed2a7f864f485c76f83f3165488097279d296a7c05bd5201a1c69b2920", - "DockerContainerName": "ecs-nginx-efs-2-nginx-9ac0808dd0afa495f001", - "ECSContainerName": "nginx", - - "ClusterName": "cluster_name", - "ContainerInstanceArn": "arn:aws:ecs:region:aws_account_id:container-instance/cluster_name/container_instance_id", - "ContainerInstanceID": "container_instance_id" - "ECSAgentVersion": "Amazon ECS Agent - v1.30.0 (02ff320c)" - - "TaskARN": "arn:aws:ecs:us-west-2:012345678910:task/default/example5-58ff-46c9-ae05-543f8example", - "TaskID: "example5-58ff-46c9-ae05-543f8example", - "TaskDefinitionFamily": "hello_world", - "TaskDefinitionVersion": "8", -} - */ -static int process_container_response(struct flb_filter_ecs *ctx, - msgpack_object container, - struct flb_ecs_task_metadata task_meta) -{ - int ret; - int found_id = FLB_FALSE; - int found_ecs_name = FLB_FALSE; - int found_docker_name = FLB_FALSE; - int i; - int len; - struct flb_ecs_metadata_buffer *cont_meta_buf; - msgpack_object key; - msgpack_object val; - msgpack_sbuffer tmp_sbuf; - msgpack_packer tmp_pck; - flb_sds_t short_id = NULL; - - /* - * We copy the metadata response to a new buffer - * So we can define the metadata key names - */ - msgpack_sbuffer_init(&tmp_sbuf); - msgpack_packer_init(&tmp_pck, &tmp_sbuf, msgpack_sbuffer_write); - - /* 3 container metadata keys, 4 for instance/cluster, 4 for the task */ - msgpack_pack_map(&tmp_pck, 11); - - /* 1st- process/pack the raw container metadata response */ - for (i = 0; i < container.via.map.size; i++) { - key = container.via.map.ptr[i].key; - if (key.type != MSGPACK_OBJECT_STR) { - flb_plg_error(ctx->ins, "Container metadata parsing failed, msgpack key type=%i", - key.type); - continue; - } - - if (key.via.str.size == 8 && strncmp(key.via.str.ptr, "DockerId", 8) == 0) { - val = container.via.map.ptr[i].val; - if (val.type != MSGPACK_OBJECT_STR) { - flb_plg_error(ctx->ins, "metadata parsing: unexpected 'DockerId' value type=%i", - val.type); - msgpack_sbuffer_destroy(&tmp_sbuf); - if (short_id != NULL) { - flb_sds_destroy(short_id); - } - return -1; - } - - /* save the short ID for hash table key */ - short_id = flb_sds_create_len(val.via.str.ptr, 12); - if (!short_id) { - flb_errno(); - msgpack_sbuffer_destroy(&tmp_sbuf); - return -1; - } - - found_id = FLB_TRUE; - msgpack_pack_str(&tmp_pck, 11); - msgpack_pack_str_body(&tmp_pck, - "ContainerID", - 11); - msgpack_pack_str(&tmp_pck, (int) val.via.str.size); - msgpack_pack_str_body(&tmp_pck, - val.via.str.ptr, - (int) val.via.str.size); - } - else if (key.via.str.size == 10 && strncmp(key.via.str.ptr, "DockerName", 10) == 0) { - val = container.via.map.ptr[i].val; - if (val.type != MSGPACK_OBJECT_STR) { - flb_plg_error(ctx->ins, "metadata parsing: unexpected 'DockerName' value type=%i", - val.type); - msgpack_sbuffer_destroy(&tmp_sbuf); - if (short_id != NULL) { - flb_sds_destroy(short_id); - } - return -1; - } - - /* first pack the ARN */ - found_docker_name = FLB_TRUE; - msgpack_pack_str(&tmp_pck, 19); - msgpack_pack_str_body(&tmp_pck, - "DockerContainerName", - 19); - msgpack_pack_str(&tmp_pck, (int) val.via.str.size); - msgpack_pack_str_body(&tmp_pck, - val.via.str.ptr, - (int) val.via.str.size); - } else if (key.via.str.size == 4 && strncmp(key.via.str.ptr, "Name", 4) == 0) { - val = container.via.map.ptr[i].val; - if (val.type != MSGPACK_OBJECT_STR) { - flb_plg_error(ctx->ins, "metadata parsing: unexpected 'Name' value type=%i", - val.type); - msgpack_sbuffer_destroy(&tmp_sbuf); - if (short_id != NULL) { - flb_sds_destroy(short_id); - } - return -1; - } - - found_ecs_name = FLB_TRUE; - msgpack_pack_str(&tmp_pck, 16); - msgpack_pack_str_body(&tmp_pck, - "ECSContainerName", - 16); - msgpack_pack_str(&tmp_pck, (int) val.via.str.size); - msgpack_pack_str_body(&tmp_pck, - val.via.str.ptr, - (int) val.via.str.size); - } - } - - if (found_id == FLB_FALSE) { - flb_plg_error(ctx->ins, "Could not parse Task 'DockerId' from container response"); - msgpack_sbuffer_destroy(&tmp_sbuf); - return -1; - } - if (found_docker_name == FLB_FALSE) { - flb_plg_error(ctx->ins, "Could not parse 'DockerName' from container response"); - msgpack_sbuffer_destroy(&tmp_sbuf); - if (short_id != NULL) { - flb_sds_destroy(short_id); - } - return -1; - } - if (found_ecs_name == FLB_FALSE) { - flb_plg_error(ctx->ins, "Could not parse 'Name' from container response"); - msgpack_sbuffer_destroy(&tmp_sbuf); - if (short_id != NULL) { - flb_sds_destroy(short_id); - } - return -1; - } - - /* 2nd - Add the task fields from the task_meta temp buf we were given */ - msgpack_pack_str(&tmp_pck, 20); - msgpack_pack_str_body(&tmp_pck, - "TaskDefinitionFamily", - 20); - msgpack_pack_str(&tmp_pck, task_meta.task_def_family_len); - msgpack_pack_str_body(&tmp_pck, - task_meta.task_def_family, - task_meta.task_def_family_len); - - msgpack_pack_str(&tmp_pck, 7); - msgpack_pack_str_body(&tmp_pck, - "TaskARN", - 7); - msgpack_pack_str(&tmp_pck, task_meta.task_arn_len); - msgpack_pack_str_body(&tmp_pck, - task_meta.task_arn, - task_meta.task_arn_len); - msgpack_pack_str(&tmp_pck, 6); - msgpack_pack_str_body(&tmp_pck, - "TaskID", - 6); - msgpack_pack_str(&tmp_pck, task_meta.task_id_len); - msgpack_pack_str_body(&tmp_pck, - task_meta.task_id, - task_meta.task_id_len); - - msgpack_pack_str(&tmp_pck, 21); - msgpack_pack_str_body(&tmp_pck, - "TaskDefinitionVersion", - 21); - msgpack_pack_str(&tmp_pck, task_meta.task_def_version_len); - msgpack_pack_str_body(&tmp_pck, - task_meta.task_def_version, - task_meta.task_def_version_len); - - /* 3rd - Add the static cluster fields from the plugin context */ - msgpack_pack_str(&tmp_pck, 11); - msgpack_pack_str_body(&tmp_pck, - "ClusterName", - 11); - len = flb_sds_len(ctx->cluster_metadata.cluster_name); - msgpack_pack_str(&tmp_pck, len); - msgpack_pack_str_body(&tmp_pck, - ctx->cluster_metadata.cluster_name, - len); - - msgpack_pack_str(&tmp_pck, 20); - msgpack_pack_str_body(&tmp_pck, - "ContainerInstanceArn", - 20); - len = flb_sds_len(ctx->cluster_metadata.container_instance_arn); - msgpack_pack_str(&tmp_pck, len); - msgpack_pack_str_body(&tmp_pck, - ctx->cluster_metadata.container_instance_arn, - len); - - msgpack_pack_str(&tmp_pck, 19); - msgpack_pack_str_body(&tmp_pck, - "ContainerInstanceID", - 19); - len = flb_sds_len(ctx->cluster_metadata.container_instance_id); - msgpack_pack_str(&tmp_pck, len); - msgpack_pack_str_body(&tmp_pck, - ctx->cluster_metadata.container_instance_id, - len); - - msgpack_pack_str(&tmp_pck, 15); - msgpack_pack_str_body(&tmp_pck, - "ECSAgentVersion", - 15); - len = flb_sds_len(ctx->cluster_metadata.ecs_agent_version); - msgpack_pack_str(&tmp_pck, len); - msgpack_pack_str_body(&tmp_pck, - ctx->cluster_metadata.ecs_agent_version, - len); - - cont_meta_buf = flb_calloc(1, sizeof(struct flb_ecs_metadata_buffer)); - if (!cont_meta_buf) { - flb_errno(); - msgpack_sbuffer_destroy(&tmp_sbuf); - flb_sds_destroy(short_id); - return -1; - } - - cont_meta_buf->buf = tmp_sbuf.data; - cont_meta_buf->size = tmp_sbuf.size; - - ret = flb_ecs_metadata_buffer_init(ctx, cont_meta_buf); - if (ret < 0) { - flb_plg_error(ctx->ins, "Could not init metadata buffer from container response"); - msgpack_sbuffer_destroy(&tmp_sbuf); - flb_free(cont_meta_buf); - flb_sds_destroy(short_id); - return -1; - } - cont_meta_buf->id = short_id; - mk_list_add(&cont_meta_buf->_head, &ctx->metadata_buffers); - - /* - * Size is set to 0 so the table just stores our pointer - * Otherwise it will try to copy the memory to a new buffer - */ - ret = flb_hash_table_add(ctx->container_hash_table, - short_id, strlen(short_id), - cont_meta_buf, 0); - - if (ret == -1) { - flb_plg_error(ctx->ins, "Could not add container ID %s to metadata hash table", - short_id); - flb_ecs_metadata_buffer_destroy(cont_meta_buf); - } else { - ret = 0; - flb_plg_debug(ctx->ins, "Added `%s` to container metadata hash table", - short_id); - } - return ret; -} - -/* - * Gets the container and task metadata for a task via a container's - * 12 char short ID. This can be used with the ECS Agent - * Introspection API: http://localhost:51678/v1/tasks?dockerid={short_id} - * Entries in the hash table will be added for all containers in the task - */ -static int get_task_metadata(struct flb_filter_ecs *ctx, char* short_id) -{ - struct flb_http_client *c; - struct flb_connection *u_conn; - int ret; - int root_type; - int found_task = FLB_FALSE; - int found_version = FLB_FALSE; - int found_family = FLB_FALSE; - int found_containers = FLB_FALSE; - int free_conn = FLB_FALSE; - int i; - int k; - char *buffer; - size_t size; - size_t b_sent; - size_t off = 0; - msgpack_unpacked result; - msgpack_object root; - msgpack_object key; - msgpack_object val; - msgpack_object container; - flb_sds_t tmp; - flb_sds_t http_path; - flb_sds_t task_id = NULL; - struct flb_ecs_task_metadata task_meta; - - tmp = flb_sds_create_size(64); - if (!tmp) { - return -1; - } - http_path = flb_sds_printf(&tmp, FLB_ECS_FILTER_TASK_PATH_FORMAT, short_id); - if (!http_path) { - flb_sds_destroy(tmp); - return -1; - } - - /* Compose HTTP Client request*/ - if (plugin_under_test() == FLB_TRUE) { - c = mock_http_call("TEST_TASK_ERROR", "Task"); - ret = 0; - } - else { - u_conn = flb_upstream_conn_get(ctx->ecs_upstream); - - if (!u_conn) { - flb_plg_error(ctx->ins, "ECS agent introspection endpoint connection error"); - flb_sds_destroy(http_path); - return -1; - } - free_conn = FLB_TRUE; - c = flb_http_client(u_conn, FLB_HTTP_GET, - http_path, - NULL, 0, - ctx->ecs_host, ctx->ecs_port, - NULL, 0); - flb_http_buffer_size(c, 0); /* 0 means unlimited */ - - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - - ret = flb_http_do(c, &b_sent); - flb_plg_debug(ctx->ins, "http_do=%i, " - "HTTP Status: %i", - ret, c->resp.status); - } - - if (ret != 0 || c->resp.status != 200) { - if (c->resp.payload_size > 0) { - flb_plg_warn(ctx->ins, "Failed to get metadata from %s, will retry", - http_path); - flb_plg_debug(ctx->ins, "HTTP response\n%s", - c->resp.payload); - } else { - flb_plg_warn(ctx->ins, "%s response status was %d with no payload, will retry", - http_path, - c->resp.status); - } - flb_http_client_destroy(c); - if (free_conn == FLB_TRUE) { - flb_upstream_conn_release(u_conn); - } - flb_sds_destroy(http_path); - return -1; - } - - if (free_conn == FLB_TRUE) { - flb_upstream_conn_release(u_conn); - } - - ret = flb_pack_json(c->resp.payload, c->resp.payload_size, - &buffer, &size, &root_type, NULL); - - if (ret < 0) { - flb_plg_warn(ctx->ins, "Could not parse response from %s; response=\n%s", - http_path, c->resp.payload); - flb_sds_destroy(http_path); - flb_http_client_destroy(c); - return -1; - } - - /* parse metadata response */ - msgpack_unpacked_init(&result); - ret = msgpack_unpack_next(&result, buffer, size, &off); - if (ret != MSGPACK_UNPACK_SUCCESS) { - flb_plg_error(ctx->ins, "Cannot unpack %s response to find metadata\n%s", - http_path, c->resp.payload); - flb_free(buffer); - msgpack_unpacked_destroy(&result); - flb_sds_destroy(http_path); - flb_http_client_destroy(c); - return -1; - } - - flb_http_client_destroy(c); - - root = result.data; - if (root.type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "%s response parsing failed, msgpack_type=%i", - http_path, - root.type); - flb_free(buffer); - msgpack_unpacked_destroy(&result); - flb_sds_destroy(http_path); - return -1; - } - - /* -Metadata Response: -{ - "Arn": "arn:aws:ecs:us-west-2:012345678910:task/default/e01d58a8-151b-40e8-bc01-22647b9ecfec", - "Containers": [ - { - "DockerId": "79c796ed2a7f864f485c76f83f3165488097279d296a7c05bd5201a1c69b2920", - "DockerName": "ecs-nginx-efs-2-nginx-9ac0808dd0afa495f001", - "Name": "nginx" - } - ], - "DesiredStatus": "RUNNING", - "Family": "nginx-efs", - "KnownStatus": "RUNNING", - "Version": "2" -} - */ - - for (i = 0; i < root.via.map.size; i++) { - key = root.via.map.ptr[i].key; - if (key.type != MSGPACK_OBJECT_STR) { - flb_plg_error(ctx->ins, "%s response parsing failed, msgpack key type=%i", - http_path, - key.type); - continue; - } - - if (key.via.str.size == 6 && strncmp(key.via.str.ptr, "Family", 6) == 0) { - val = root.via.map.ptr[i].val; - if (val.type != MSGPACK_OBJECT_STR) { - flb_plg_error(ctx->ins, "metadata parsing: unexpected 'Family' value type=%i", - val.type); - flb_free(buffer); - msgpack_unpacked_destroy(&result); - flb_sds_destroy(http_path); - if (task_id) { - flb_sds_destroy(task_id); - } - return -1; - } - found_family = FLB_TRUE; - task_meta.task_def_family = val.via.str.ptr; - task_meta.task_def_family_len = (int) val.via.str.size; - } - else if (key.via.str.size == 3 && strncmp(key.via.str.ptr, "Arn", 3) == 0) { - val = root.via.map.ptr[i].val; - if (val.type != MSGPACK_OBJECT_STR) { - flb_plg_error(ctx->ins, "metadata parsing: unexpected 'Arn' value type=%i", - val.type); - flb_free(buffer); - msgpack_unpacked_destroy(&result); - flb_sds_destroy(http_path); - if (task_id) { - flb_sds_destroy(task_id); - } - return -1; - } - /* first get the ARN */ - found_task = FLB_TRUE; - task_meta.task_arn = val.via.str.ptr; - task_meta.task_arn_len = (int) val.via.str.size; - - /* then get the ID */ - task_id = parse_id_from_arn(val.via.str.ptr, (int) val.via.str.size); - if (task_id == NULL) { - flb_plg_error(ctx->ins, "metadata parsing: failed to get ID from %.*s", - (int) val.via.str.size, val.via.str.ptr); - flb_free(buffer); - msgpack_unpacked_destroy(&result); - flb_sds_destroy(http_path); - if (task_id) { - flb_sds_destroy(task_id); - } - return -1; - } - - task_meta.task_id = task_id; - task_meta.task_id_len = flb_sds_len(task_id); - } else if (key.via.str.size == 7 && strncmp(key.via.str.ptr, "Version", 7) == 0) { - val = root.via.map.ptr[i].val; - if (val.type != MSGPACK_OBJECT_STR) { - flb_plg_error(ctx->ins, "metadata parsing: unexpected 'Version' value type=%i", - val.type); - flb_free(buffer); - msgpack_unpacked_destroy(&result); - flb_sds_destroy(http_path); - if (task_id) { - flb_sds_destroy(task_id); - } - return -1; - } - found_version = FLB_TRUE; - task_meta.task_def_version = val.via.str.ptr; - task_meta.task_def_version_len = (int) val.via.str.size; - } else if (key.via.str.size == 10 && strncmp(key.via.str.ptr, "Containers", 10) == 0) { - val = root.via.map.ptr[i].val; - if (val.type != MSGPACK_OBJECT_ARRAY ) { - flb_plg_error(ctx->ins, "metadata parsing: unexpected 'Containers' value type=%i", - val.type); - flb_free(buffer); - msgpack_unpacked_destroy(&result); - flb_sds_destroy(http_path); - if (task_id) { - flb_sds_destroy(task_id); - } - return -1; - } - found_containers = FLB_TRUE; - } - } - - if (found_task == FLB_FALSE) { - flb_plg_error(ctx->ins, "Could not parse Task 'Arn' from %s response", - http_path); - flb_sds_destroy(http_path); - flb_free(buffer); - msgpack_unpacked_destroy(&result); - return -1; - } - if (found_family == FLB_FALSE) { - flb_plg_error(ctx->ins, "Could not parse 'Family' from %s response", - http_path); - flb_sds_destroy(http_path); - flb_free(buffer); - msgpack_unpacked_destroy(&result); - if (task_id) { - flb_sds_destroy(task_id); - } - return -1; - } - if (found_version == FLB_FALSE) { - flb_plg_error(ctx->ins, "Could not parse 'Version' from %s response", - http_path); - flb_sds_destroy(http_path); - flb_free(buffer); - msgpack_unpacked_destroy(&result); - if (task_id) { - flb_sds_destroy(task_id); - } - return -1; - } - if (found_containers == FLB_FALSE) { - flb_plg_error(ctx->ins, "Could not parse 'Containers' from %s response", - http_path); - flb_sds_destroy(http_path); - flb_free(buffer); - msgpack_unpacked_destroy(&result); - if (task_id) { - flb_sds_destroy(task_id); - } - return -1; - } - - /* - * Process metadata response a 2nd time to get the Containers list - * This is because we need one complete metadata buf per container - * with all task metadata. So we collect task before we process containers. - */ - for (i = 0; i < root.via.map.size; i++) { - key = root.via.map.ptr[i].key; - if (key.type != MSGPACK_OBJECT_STR) { - flb_plg_error(ctx->ins, "%s response parsing failed, msgpack key type=%i", - http_path, - key.type); - continue; - } - - if (key.via.str.size == 10 && strncmp(key.via.str.ptr, "Containers", 10) == 0) { - val = root.via.map.ptr[i].val; - if (val.type != MSGPACK_OBJECT_ARRAY ) { - flb_plg_error(ctx->ins, "metadata parsing: unexpected 'Containers' value type=%i", - val.type); - flb_free(buffer); - msgpack_unpacked_destroy(&result); - flb_sds_destroy(http_path); - flb_sds_destroy(task_id); - return -1; - } - - /* iterate through list of containers and process them*/ - for (k = 0; k < val.via.array.size; k++) { - container = val.via.array.ptr[k]; - if (container.type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "metadata parsing: unexpected 'Containers[%d]' inner value type=%i", - k, - container.type); - flb_free(buffer); - msgpack_unpacked_destroy(&result); - flb_sds_destroy(http_path); - flb_sds_destroy(task_id); - return -1; - } - ret = process_container_response(ctx, container, task_meta); - if (ret < 0) { - flb_plg_error(ctx->ins, "metadata parsing: failed to parse 'Containers[%d]'", - k); - flb_free(buffer); - msgpack_unpacked_destroy(&result); - flb_sds_destroy(http_path); - flb_sds_destroy(task_id); - return -1; - } - } - } - } - - flb_free(buffer); - msgpack_unpacked_destroy(&result); - flb_sds_destroy(task_id); - flb_sds_destroy(http_path); - return 0; -} - -static int get_metadata_by_id(struct flb_filter_ecs *ctx, - const char *tag, int tag_len, - struct flb_ecs_metadata_buffer **metadata_buffer) -{ - flb_sds_t container_short_id = NULL; - const char *tmp; - int ret; - size_t size; - - if (ctx->ecs_tag_prefix_len + 12 > tag_len) { - flb_plg_warn(ctx->ins, "Tag '%s' length check failed: tag is expected " - "to be or be prefixed with '{ecs_tag_prefix}{12 character container short ID}'", - tag); - return -1; - } - - ret = strncmp(ctx->ecs_tag_prefix, tag, ctx->ecs_tag_prefix_len); - if (ret != 0) { - flb_plg_warn(ctx->ins, "Tag '%s' is not prefixed with ecs_tag_prefix '%s'", - tag, ctx->ecs_tag_prefix); - return -1; - } - - tmp = tag + ctx->ecs_tag_prefix_len; - container_short_id = flb_sds_create_len(tmp, 12); - if (!container_short_id) { - flb_errno(); - return -1; - } - - /* get metadata for this container */ - ret = flb_hash_table_get(ctx->container_hash_table, - container_short_id, flb_sds_len(container_short_id), - (void **) metadata_buffer, &size); - - if (ret == -1) { - /* try fetch metadata */ - ret = get_task_metadata(ctx, container_short_id); - if (ret < 0) { - flb_plg_info(ctx->ins, "Requesting metadata from ECS Agent introspection endpoint failed for tag %s", - tag); - flb_sds_destroy(container_short_id); - return -1; - } - /* get from hash table */ - ret = flb_hash_table_get(ctx->container_hash_table, - container_short_id, flb_sds_len(container_short_id), - (void **) metadata_buffer, &size); - } - - flb_sds_destroy(container_short_id); - return ret; -} - -static void clean_old_metadata_buffers(struct flb_filter_ecs *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_ecs_metadata_buffer *buf; - time_t now = time(NULL); - - mk_list_foreach_safe(head, tmp, &ctx->metadata_buffers) { - buf = mk_list_entry(head, struct flb_ecs_metadata_buffer, _head); - if (now > (buf->last_used_time + ctx->ecs_meta_cache_ttl)) { - flb_plg_debug(ctx->ins, "cleaning buffer: now=%ld, ttl=%d, last_used_time=%ld", - (long)now, ctx->ecs_meta_cache_ttl, (long)buf->last_used_time); - mk_list_del(&buf->_head); - flb_hash_table_del(ctx->container_hash_table, buf->id); - flb_ecs_metadata_buffer_destroy(buf); - } - } -} - -static int is_tag_marked_failed(struct flb_filter_ecs *ctx, - const char *tag, int tag_len) -{ - int ret; - int *val = NULL; - size_t val_size; - - ret = flb_hash_table_get(ctx->failed_metadata_request_tags, - tag, tag_len, - (void **) &val, &val_size); - if (ret != -1) { - if (*val >= ctx->agent_endpoint_retries) { - return FLB_TRUE; - } - } - - return FLB_FALSE; -} - -static void mark_tag_failed(struct flb_filter_ecs *ctx, - const char *tag, int tag_len) -{ - int ret; - int *val = NULL; - int *new_val = NULL; - size_t val_size; - - ret = flb_hash_table_get(ctx->failed_metadata_request_tags, - tag, tag_len, - (void **) &val, &val_size); - - if (ret == -1) { - /* hash table copies memory to new heap block */ - val = flb_malloc(sizeof(int)); - if (!val) { - flb_errno(); - return; - } - *val = 1; - flb_hash_table_add(ctx->failed_metadata_request_tags, - tag, tag_len, - val, sizeof(int)); - /* hash table will contain a copy */ - flb_free(val); - } else { - /* - * val is memory returned from hash table - * if we simply update the value here and call flb_hash_add - * it first frees the old memory (which is what we passed it) - * then tries to copy over the memory we passed in to a new location - * flb_hash stores all entries as if they were strings, so we also - * can't simply increment the value returned by flb_hash_get - */ - new_val = flb_malloc(sizeof(int)); - if (!new_val) { - flb_errno(); - return; - } - /* increment number of failed metadata requests for this tag */ - *new_val = *val + 1; - flb_hash_table_add(ctx->failed_metadata_request_tags, - tag, tag_len, - new_val, sizeof(int)); - flb_plg_info(ctx->ins, "Failed to get ECS Metadata for tag %s %d times. " - "This might be because the logs for this tag do not come from an ECS Task Container. " - "This plugin will retry metadata requests at most %d times total for this tag.", - tag, *new_val, ctx->agent_endpoint_retries); - flb_free(new_val); - } -} - -static int cb_ecs_filter(const void *data, size_t bytes, - const char *tag, int tag_len, - void **out_buf, size_t *out_size, - struct flb_filter_instance *f_ins, - struct flb_input_instance *i_ins, - void *context, - struct flb_config *config) -{ - struct flb_filter_ecs *ctx = context; - int i = 0; - int ret; - int check = FLB_FALSE; - msgpack_object *obj; - msgpack_object_kv *kv; - struct mk_list *tmp; - struct mk_list *head; - struct flb_ecs_metadata_key *metadata_key; - struct flb_ecs_metadata_buffer *metadata_buffer; - flb_sds_t val; - struct flb_log_event_encoder log_encoder; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - (void) f_ins; - (void) i_ins; - (void) config; - - /* First check that the static cluster metadata has been retrieved */ - if (ctx->has_cluster_metadata == FLB_FALSE) { - ret = get_ecs_cluster_metadata(ctx); - if (ret < 0) { - flb_plg_warn(ctx->ins, "Could not retrieve cluster metadata " - "from ECS Agent"); - return FLB_FILTER_NOTOUCH; - } - } - - /* check if the current tag is marked as failed */ - check = is_tag_marked_failed(ctx, tag, tag_len); - if (check == FLB_TRUE) { - flb_plg_debug(ctx->ins, "Failed to get ECS Metadata for tag %s %d times. " - "Will not attempt to retry the metadata request. Will attach cluster metadata only.", - tag, ctx->agent_endpoint_retries); - } - - if (check == FLB_FALSE && ctx->cluster_metadata_only == FLB_FALSE) { - ret = get_metadata_by_id(ctx, tag, tag_len, &metadata_buffer); - if (ret == -1) { - flb_plg_info(ctx->ins, "Failed to get ECS Task metadata for %s, " - "falling back to process cluster metadata only. If " - "this is intentional, set `Cluster_Metadata_Only On`", - tag); - mark_tag_failed(ctx, tag, tag_len); - metadata_buffer = &ctx->cluster_meta_buf; - } - } else { - metadata_buffer = &ctx->cluster_meta_buf; - } - - metadata_buffer->last_used_time = time(NULL); - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return FLB_FILTER_NOTOUCH; - } - - ret = flb_log_event_encoder_init(&log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event encoder initialization error : %d", ret); - - flb_log_event_decoder_destroy(&log_decoder); - - return FLB_FILTER_NOTOUCH; - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - obj = log_event.body; - - ret = flb_log_event_encoder_begin_record(&log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_timestamp( - &log_encoder, &log_event.timestamp); - } - - /* iterate through the old record map and add it to the new buffer */ - kv = obj->via.map.ptr; - for(i=0; - i < obj->via.map.size && - ret == FLB_EVENT_ENCODER_SUCCESS; - i++) { - ret = flb_log_event_encoder_append_body_values( - &log_encoder, - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&kv[i].key), - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&kv[i].val)); - } - - /* append new keys */ - mk_list_foreach_safe(head, tmp, &ctx->metadata_keys) { - metadata_key = mk_list_entry(head, struct flb_ecs_metadata_key, _head); - val = flb_ra_translate(metadata_key->ra, NULL, 0, - metadata_buffer->obj, NULL); - if (!val) { - flb_plg_info(ctx->ins, "Translation failed for %s : %s", - metadata_key->key, metadata_key->template); - - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - return FLB_FILTER_NOTOUCH; - } - - ret = flb_log_event_encoder_append_body_values( - &log_encoder, - FLB_LOG_EVENT_STRING_VALUE(metadata_key->key, - flb_sds_len(metadata_key->key)), - FLB_LOG_EVENT_STRING_VALUE(val, flb_sds_len(val))); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_info(ctx->ins, - "Metadata appendage failed for %.*s", - (int) flb_sds_len(metadata_key->key), - metadata_key->key); - - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - return FLB_FILTER_NOTOUCH; - } - - flb_sds_destroy(val); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_log_event_encoder_commit_record(&log_encoder); - } - } - - if (ctx->cluster_metadata_only == FLB_FALSE) { - clean_old_metadata_buffers(ctx); - } - - if (ret == FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA && - log_decoder.offset == bytes) { - ret = FLB_EVENT_ENCODER_SUCCESS; - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - *out_buf = log_encoder.output_buffer; - *out_size = log_encoder.output_length; - - ret = FLB_FILTER_MODIFIED; - - flb_log_event_encoder_claim_internal_buffer_ownership(&log_encoder); - } - else { - flb_plg_error(ctx->ins, - "Log event encoder error : %d", ret); - - ret = FLB_FILTER_NOTOUCH; - } - - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - return ret; -} - -static void flb_ecs_metadata_key_destroy(struct flb_ecs_metadata_key *metadata_key) -{ - if (metadata_key) { - if (metadata_key->key) { - flb_sds_destroy(metadata_key->key); - } - if (metadata_key->template) { - flb_sds_destroy(metadata_key->template); - } - if (metadata_key->ra) { - flb_ra_destroy(metadata_key->ra); - } - flb_free(metadata_key); - } -} - -static void flb_filter_ecs_destroy(struct flb_filter_ecs *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_ecs_metadata_key *metadata_key; - struct flb_ecs_metadata_buffer *buf; - - if (ctx) { - if (ctx->ecs_upstream) { - flb_upstream_destroy(ctx->ecs_upstream); - } - if (ctx->cluster_metadata.cluster_name) { - flb_sds_destroy(ctx->cluster_metadata.cluster_name); - } - if (ctx->cluster_metadata.container_instance_arn) { - flb_sds_destroy(ctx->cluster_metadata.container_instance_arn); - } - if (ctx->cluster_metadata.container_instance_id) { - flb_sds_destroy(ctx->cluster_metadata.container_instance_id); - } - if (ctx->cluster_metadata.ecs_agent_version) { - flb_sds_destroy(ctx->cluster_metadata.ecs_agent_version); - } - if (ctx->cluster_meta_buf.buf) { - flb_free(ctx->cluster_meta_buf.buf); - msgpack_unpacked_destroy(&ctx->cluster_meta_buf.unpacked); - } - mk_list_foreach_safe(head, tmp, &ctx->metadata_keys) { - metadata_key = mk_list_entry(head, struct flb_ecs_metadata_key, _head); - mk_list_del(&metadata_key->_head); - flb_ecs_metadata_key_destroy(metadata_key); - } - mk_list_foreach_safe(head, tmp, &ctx->metadata_buffers) { - buf = mk_list_entry(head, struct flb_ecs_metadata_buffer, _head); - mk_list_del(&buf->_head); - flb_hash_table_del(ctx->container_hash_table, buf->id); - flb_ecs_metadata_buffer_destroy(buf); - } - if (ctx->container_hash_table) { - flb_hash_table_destroy(ctx->container_hash_table); - } - if (ctx->failed_metadata_request_tags) { - flb_hash_table_destroy(ctx->failed_metadata_request_tags); - } - flb_free(ctx); - } -} - -static int cb_ecs_exit(void *data, struct flb_config *config) -{ - struct flb_filter_ecs *ctx = data; - - flb_filter_ecs_destroy(ctx); - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - - { - FLB_CONFIG_MAP_STR, "add", NULL, - FLB_CONFIG_MAP_MULT, FLB_FALSE, 0, - "Add a metadata key/value pair with the given key and given value from the given template. " - "Format is `Add KEY TEMPLATE`." - }, - - { - FLB_CONFIG_MAP_STR, "ecs_tag_prefix", "", - 0, FLB_TRUE, offsetof(struct flb_filter_ecs, ecs_tag_prefix), - "This filter must obtain the 12 character container short ID to query " - "for ECS Task metadata. The filter removes the prefx from the tag and then assumes " - "the next 12 characters are the short container ID. If the container short ID, " - "is not found in the tag, the filter can/must fallback to only attaching cluster metadata " - "(cluster name, container instance ID/ARN, and ECS Agent version)." - }, - - { - FLB_CONFIG_MAP_BOOL, "cluster_metadata_only", "false", - 0, FLB_TRUE, offsetof(struct flb_filter_ecs, cluster_metadata_only), - "Only attempt to attach the cluster related metadata to logs " - "(cluster name, container instance ID/ARN, and ECS Agent version). " - "With this option off, if this filter can not obtain the task metadata for a log, it will " - "output errors. Use this option if you have logs that are not part of an " - "ECS task (ex: Docker Daemon logs)." - }, - - { - FLB_CONFIG_MAP_TIME, "ecs_meta_cache_ttl", "3600", - 0, FLB_TRUE, offsetof(struct flb_filter_ecs, ecs_meta_cache_ttl), - "Configurable TTL for cached ECS Task Metadata. Default 3600s (1 hour)" - "For example, set this value to 600 or 600s or 10m and cache entries " - "which have been created more than 10 minutes will be evicted." - "Cache eviction is needed to purge task metadata for tasks that " - "have been stopped." - }, - - { - FLB_CONFIG_MAP_STR, "ecs_meta_host", FLB_ECS_FILTER_HOST, - 0, FLB_TRUE, offsetof(struct flb_filter_ecs, ecs_host), - "The host name at which the ECS Agent Introspection endpoint is reachable. " - "Defaults to 127.0.0.1" - }, - - { - FLB_CONFIG_MAP_INT, "ecs_meta_port", FLB_ECS_FILTER_PORT, - 0, FLB_TRUE, offsetof(struct flb_filter_ecs, ecs_port), - "The port at which the ECS Agent Introspection endpoint is reachable. " - "Defaults to 51678" - }, - - { - FLB_CONFIG_MAP_INT, "agent_endpoint_retries", FLB_ECS_FILTER_METADATA_RETRIES, - 0, FLB_TRUE, offsetof(struct flb_filter_ecs, agent_endpoint_retries), - "Number of retries for failed metadata requests to ECS Agent Introspection " - "endpoint. The most common cause of failed metadata requests is that the " - "container the metadata request was made for is not part of an ECS Task. " - "Check if you have non-task containers and docker dual logging enabled." - }, - - {0} -}; - -struct flb_filter_plugin filter_ecs_plugin = { - .name = "ecs", - .description = "Add AWS ECS Metadata", - .cb_init = cb_ecs_init, - .cb_filter = cb_ecs_filter, - .cb_exit = cb_ecs_exit, - .config_map = config_map, - .flags = 0 -}; diff --git a/fluent-bit/plugins/filter_ecs/ecs.h b/fluent-bit/plugins/filter_ecs/ecs.h deleted file mode 100644 index 71d0248fa..000000000 --- a/fluent-bit/plugins/filter_ecs/ecs.h +++ /dev/null @@ -1,152 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_FILTER_ECS_H -#define FLB_FILTER_ECS_H - -#include -#include -#include -#include -#include - -#define FLB_ECS_FILTER_HOST "127.0.0.1" -#define FLB_ECS_FILTER_PORT "51678" -#define FLB_ECS_FILTER_CLUSTER_PATH "/v1/metadata" -#define FLB_ECS_FILTER_TASK_PATH_FORMAT "/v1/tasks?dockerid=%s" -#define FLB_ECS_FILTER_METADATA_RETRIES "2" - -/* - * Kubernetes recommends not running more than 110 pods per node - * In ECS, the number of tasks per instance will vary considerably - * But this should be a very safe starting size for the table - * Since we use the TTL hash table there is no max size. - */ -#define FLB_ECS_FILTER_HASH_TABLE_SIZE 100 - - -struct flb_ecs_metadata_key { - flb_sds_t key; - flb_sds_t template; - struct flb_record_accessor *ra; - - struct mk_list _head; -}; - -struct flb_ecs_metadata_buffer { - /* msgpack_sbuffer */ - char *buf; - size_t size; - - /* unpacked object to use with flb_ra_translate */ - msgpack_unpacked unpacked; - msgpack_object obj; - int free_packer; - - /* the hash table only stores a pointer- we need the list to track and free these */ - struct mk_list _head; - /* we clean up the memory for these once ecs_meta_cache_ttl has expired */ - time_t last_used_time; - - /* - * To remove from the hash table on TTL expiration, we need the ID - * While we use a TTL hash, it won't clean up the memory, so we have a separate routine for that - * and it needs to ensure that the list and hash table has the same contents - */ - flb_sds_t id; -}; - -struct flb_ecs_cluster_metadata { - flb_sds_t cluster_name; - flb_sds_t container_instance_arn; - flb_sds_t container_instance_id; - flb_sds_t ecs_agent_version; -}; - -/* - * The ECS Agent task response gives us both task & container at the same time - * We need a temporary structure to organize the task metadata - * Before we create the final flb_ecs_metadata_buffer objects with all metadata - * So this struct just stores tmp pointers to the deserialized msgpack - */ -struct flb_ecs_task_metadata { - const char* task_arn; - int task_arn_len; - const char *task_id; - int task_id_len; - const char *task_def_family; - int task_def_family_len; - const char *task_def_version; - int task_def_version_len; -}; - -struct flb_filter_ecs { - /* upstream connection to ECS Agent */ - struct flb_upstream *ecs_upstream; - - /* Filter plugin instance reference */ - struct flb_filter_instance *ins; - - struct mk_list metadata_keys; - int metadata_keys_len; - - flb_sds_t ecs_host; - int ecs_port; - - int agent_endpoint_retries; - - /* - * This field is used when we build new container metadata objects - */ - struct flb_ecs_cluster_metadata cluster_metadata; - int has_cluster_metadata; - /* - * If looking up the container fails, we should still always be able to - * attach cluster metadata. So we have a fallback metadata buffer for that. - * For example, users may want to attach cluster name to Docker Daemon logs, - * even though Docker is not an AWS ECS Task/container. - */ - struct flb_ecs_metadata_buffer cluster_meta_buf; - - /* - * Maps 12 char container short ID to metadata buffer - */ - struct flb_hash_table *container_hash_table; - - /* - * The hash table only stores pointers, so we keep a list of meta objects - * that need to be freed - */ - struct mk_list metadata_buffers; - - /* - * Fluent Bit may pick up logs for containers that were not scheduled by ECS - * These will lead to continuous error messages. Therefore, we store - * a hash table of tags for which we could not get metadata so we can stop - * retrying on them. - */ - struct flb_hash_table *failed_metadata_request_tags; - - int ecs_meta_cache_ttl; - char *ecs_tag_prefix; - int ecs_tag_prefix_len; - int cluster_metadata_only; -}; - -#endif diff --git a/fluent-bit/plugins/filter_expect/CMakeLists.txt b/fluent-bit/plugins/filter_expect/CMakeLists.txt deleted file mode 100644 index cc6c03d1a..000000000 --- a/fluent-bit/plugins/filter_expect/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - expect.c) - -FLB_PLUGIN(filter_expect "${src}" "") diff --git a/fluent-bit/plugins/filter_expect/expect.c b/fluent-bit/plugins/filter_expect/expect.c deleted file mode 100644 index 102085c02..000000000 --- a/fluent-bit/plugins/filter_expect/expect.c +++ /dev/null @@ -1,614 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "expect.h" -#include - -static int key_to_type(char *key) -{ - if (strcasecmp(key, "key_exists") == 0) { - return FLB_EXP_KEY_EXISTS; - } - else if (strcasecmp(key, "key_not_exists") == 0) { - return FLB_EXP_KEY_NOT_EXISTS; - } - else if (strcasecmp(key, "key_val_is_null") == 0) { - return FLB_EXP_KEY_VAL_NULL; - } - else if (strcasecmp(key, "key_val_is_not_null") == 0) { - return FLB_EXP_KEY_VAL_NOT_NULL; - } - else if (strcasecmp(key, "key_val_eq") == 0) { - return FLB_EXP_KEY_VAL_EQ; - } - - return -1; -} - -/* Create a rule */ -static struct flb_expect_rule *rule_create(struct flb_expect *ctx, - int type, char *value) -{ - int ret; - struct mk_list *list; - struct flb_slist_entry *key; - struct flb_slist_entry *val; - struct flb_expect_rule *rule; - - rule = flb_calloc(1, sizeof(struct flb_expect_rule)); - if (!rule) { - flb_errno(); - return NULL; - } - rule->type = type; - rule->value = value; - rule->expect = NULL; - - /* Only the rule 'key_val_eq' expects two values from the configuration */ - if (type == FLB_EXP_KEY_VAL_EQ) { - list = flb_malloc(sizeof(struct mk_list)); - if (!list) { - flb_errno(); - flb_free(rule); - return NULL; - } - mk_list_init(list); - ret = flb_slist_split_string(list, value, ' ', 1); - if (ret == -1) { - flb_plg_error(ctx->ins, "error reading list of options '%s'", - value); - flb_free(rule); - return NULL; - } - - /* Get the 'key' and the expected value */ - key = mk_list_entry_first(list, struct flb_slist_entry, _head); - val = mk_list_entry_last(list, struct flb_slist_entry, _head); - - rule->ra = flb_ra_create(key->str, FLB_TRUE); - if (!rule->ra) { - flb_plg_error(ctx->ins, "error processing accessor key '%s'", - key->str); - flb_slist_destroy(list); - flb_free(list); - flb_free(rule); - return NULL; - } - rule->expect = flb_sds_create(val->str); - flb_slist_destroy(list); - flb_free(list); - } - else { - rule->ra = flb_ra_create(value, FLB_TRUE); - if (!rule->ra) { - flb_plg_error(ctx->ins, "error processing accessor key '%s'", - value); - flb_free(rule); - return NULL; - } - } - - return rule; -} - -static void rule_destroy(struct flb_expect_rule *rule) -{ - if (rule->expect) { - flb_sds_destroy(rule->expect); - } - if (rule->ra) { - flb_ra_destroy(rule->ra); - } - - flb_free(rule); -} - -static void context_destroy(struct flb_expect *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_expect_rule *rule; - - mk_list_foreach_safe(head, tmp, &ctx->rules) { - rule = mk_list_entry(head, struct flb_expect_rule, _head); - mk_list_del(&rule->_head); - rule_destroy(rule); - } - flb_free(ctx); -} - -static struct flb_expect *context_create(struct flb_filter_instance *ins, - struct flb_config *config) -{ - int i = 0; - int type; - int ret; - flb_sds_t tmp; - struct flb_kv *kv; - struct mk_list *head; - struct flb_expect *ctx; - struct flb_expect_rule *rule; - - ctx = flb_calloc(1, sizeof(struct flb_expect)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - ctx->action = FLB_EXP_WARN; - mk_list_init(&ctx->rules); - - /* Get the action property */ - tmp = (char *) flb_filter_get_property("action", ins); - if (tmp) { - if (strcasecmp(tmp, "warn") == 0) { - ctx->action = FLB_EXP_WARN; - } - else if (strcasecmp(tmp, "exit") == 0) { - ctx->action = FLB_EXP_EXIT; - } - else if (strcasecmp(tmp, "result_key") == 0) { - ctx->action = FLB_EXP_RESULT_KEY; - } - else { - flb_plg_error(ctx->ins, "unexpected 'action' value '%s'", tmp); - flb_free(ctx); - return NULL; - } - } - - /* Load config map */ - ret = flb_filter_config_map_set(ins, ctx); - if (ret == -1) { - flb_free(ctx); - return NULL; - } - - /* Read the configuration properties */ - mk_list_foreach(head, &ins->properties) { - kv = mk_list_entry(head, struct flb_kv, _head); - - /* Validate the type of the rule */ - type = key_to_type(kv->key); - if (strcasecmp(kv->key, "result_key") == 0) { - /* skip */ - continue; - } - - if (type == -1 && strcasecmp(kv->key, "action") != 0) { - flb_plg_error(ctx->ins, "unknown configuration rule '%s'", kv->key); - context_destroy(ctx); - return NULL; - } - - rule = rule_create(ctx, type, kv->val); - if (!rule) { - context_destroy(ctx); - return NULL; - } - mk_list_add(&rule->_head, &ctx->rules); - - /* Debug message */ - if (rule->type == -1) { - flb_plg_debug(ctx->ins, "action : '%s'", kv->val); - } - else { - flb_plg_debug(ctx->ins, "rule #%i: '%s', expects: '%s'", - i, kv->key, kv->val); - } - i++; - } - - return ctx; - -} - -static int cb_expect_init(struct flb_filter_instance *ins, - struct flb_config *config, - void *data) -{ - struct flb_expect *ctx; - - /* Create the plugin context */ - ctx = context_create(ins, config); - if (!ctx) { - return -1; - } - - /* Set filter context */ - flb_filter_set_context(ins, ctx); - - if (mk_list_size(&ctx->rules) == 0) { - flb_plg_warn(ctx->ins, "no rules has been defined"); - } - - return 0; -} - -static char *ra_value_type_to_str(struct flb_ra_value *val) -{ - if (val->type == FLB_RA_BOOL) { - return "boolean"; - } - else if (val->type == FLB_RA_INT) { - return "integer"; - } - else if (val->type == FLB_RA_FLOAT) { - return "float / double"; - } - else if (val->type == FLB_RA_STRING) { - return "string"; - } - else if (val->type == FLB_RA_NULL) { - return "null"; - } - - return "UNKNOWN"; -} - -static int rule_apply(struct flb_expect *ctx, msgpack_object map) -{ - int n = 0; - char *json; - size_t size = 1024; - struct mk_list *head; - struct flb_expect_rule *rule; - struct flb_ra_value *val; - - mk_list_foreach(head, &ctx->rules) { - rule = mk_list_entry(head, struct flb_expect_rule, _head); - - val = flb_ra_get_value_object(rule->ra, map); - if (rule->type == FLB_EXP_KEY_EXISTS) { - if (val) { - flb_ra_key_value_destroy(val); - n++; - continue; - } - - json = flb_msgpack_to_json_str(size, &map); - flb_plg_error(ctx->ins, - "exception on rule #%i 'key_exists', key '%s' " - "not found. Record content:\n%s", - n, rule->value, json); - flb_free(json); - return FLB_FALSE; - } - else if (rule->type == FLB_EXP_KEY_NOT_EXISTS) { - if (!val) { - n++; - continue; - } - json = flb_msgpack_to_json_str(size, &map); - flb_plg_error(ctx->ins, - "exception on rule #%i 'key_not_exists', key '%s' " - "exists. Record content:\n%s", - n, rule->value, json); - flb_free(json); - flb_ra_key_value_destroy(val); - return FLB_FALSE; - } - else if (rule->type == FLB_EXP_KEY_VAL_NULL) { - if (!val) { - json = flb_msgpack_to_json_str(size, &map); - flb_plg_error(ctx->ins, - "exception on rule #%i 'key_val_is_null', " - "key '%s' not found. Record content:\n%s", - n, rule->value, json); - flb_free(json); - return FLB_FALSE; - } - if (val->type != FLB_RA_NULL) { - json = flb_msgpack_to_json_str(size, &map); - flb_plg_error(ctx->ins, - "exception on rule #%i 'key_val_is_null', " - "key '%s' contains a value type '%s'. " - "Record content:\n%s", - n, rule->value, - ra_value_type_to_str(val), json); - flb_free(json); - flb_ra_key_value_destroy(val); - return FLB_FALSE; - } - flb_ra_key_value_destroy(val); - } - else if (rule->type == FLB_EXP_KEY_VAL_NOT_NULL) { - if (!val) { - json = flb_msgpack_to_json_str(size, &map); - flb_plg_error(ctx->ins, - "exception on rule #%i 'key_val_is_not_null', " - "key '%s' not found. Record content:\n%s", - n, rule->value, json); - flb_free(json); - return FLB_FALSE; - } - if (val->type == FLB_RA_NULL) { - json = flb_msgpack_to_json_str(size, &map); - flb_plg_error(ctx->ins, - "exception on rule #%i 'key_val_is_not_null', " - "key '%s' contains a value type '%s'. " - "Record content:\n%s", - n, rule->value, - ra_value_type_to_str(val), json); - flb_free(json); - flb_ra_key_value_destroy(val); - return FLB_FALSE; - } - flb_ra_key_value_destroy(val); - } - else if (rule->type == FLB_EXP_KEY_VAL_EQ) { - if (!val) { - json = flb_msgpack_to_json_str(size, &map); - flb_plg_error(ctx->ins, - "exception on rule #%i 'key_val_is_null', " - "key '%s' not found. Record content:\n%s", - n, rule->value, json); - flb_free(json); - return FLB_FALSE; - } - - if (val->type == FLB_RA_STRING) { - if (flb_sds_cmp(val->val.string, rule->expect, - flb_sds_len(rule->expect)) != 0) { - json = flb_msgpack_to_json_str(size, &map); - flb_plg_error(ctx->ins, - "exception on rule #%i 'key_val_eq', " - "key value '%s' is different than " - "expected: '%s'. Record content:\n%s", - n, val->val.string, rule->expect, json); - flb_free(json); - flb_ra_key_value_destroy(val); - return FLB_FALSE; - } - } - flb_ra_key_value_destroy(val); - } - n++; - } - - return FLB_TRUE; -} - -static int cb_expect_filter(const void *data, size_t bytes, - const char *tag, int tag_len, - void **out_buf, size_t *out_bytes, - struct flb_filter_instance *f_ins, - struct flb_input_instance *i_ins, - void *filter_context, - struct flb_config *config) -{ - int ret; - int i; - int rule_matched = FLB_TRUE; - msgpack_object_kv *kv; - struct flb_expect *ctx = filter_context; - struct flb_log_event_encoder log_encoder; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - (void) out_buf; - (void) out_bytes; - (void) f_ins; - (void) i_ins; - (void) config; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return FLB_FILTER_NOTOUCH; - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - ret = rule_apply(ctx, *log_event.body); - if (ret == FLB_TRUE) { - /* rule matches, we are good */ - continue; - } - else { - if (ctx->action == FLB_EXP_WARN) { - flb_plg_warn(ctx->ins, "expect check failed"); - } - else if (ctx->action == FLB_EXP_EXIT) { - flb_engine_exit_status(config, 255); - } - else if (ctx->action == FLB_EXP_RESULT_KEY) { - rule_matched = FLB_FALSE; - } - break; - } - } - - ret = 0; - /* Append result key when action is "result_key"*/ - if (ctx->action == FLB_EXP_RESULT_KEY) { - flb_log_event_decoder_reset(&log_decoder, (char *) data, bytes); - - ret = flb_log_event_encoder_init(&log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event encoder initialization error : %d", ret); - - flb_log_event_decoder_destroy(&log_decoder); - - return FLB_FILTER_NOTOUCH; - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - ret = flb_log_event_encoder_begin_record(&log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_timestamp( - &log_encoder, &log_event.timestamp); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_metadata_from_msgpack_object(&log_encoder, - log_event.metadata); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - &log_encoder, - FLB_LOG_EVENT_STRING_VALUE(ctx->result_key, flb_sds_len(ctx->result_key)), - FLB_LOG_EVENT_BOOLEAN_VALUE(rule_matched)); - } - - kv = log_event.body->via.map.ptr; - for (i=0 ; - i < log_event.body->via.map.size && - ret == FLB_EVENT_ENCODER_SUCCESS ; - i++) { - ret = flb_log_event_encoder_append_body_values( - &log_encoder, - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&kv[i].key), - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&kv[i].val)); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(&log_encoder); - } - } - - if (ret == FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA && - log_decoder.offset == bytes) { - ret = FLB_EVENT_ENCODER_SUCCESS; - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - *out_buf = log_encoder.output_buffer; - *out_bytes = log_encoder.output_length; - - ret = FLB_FILTER_MODIFIED; - - flb_log_event_encoder_claim_internal_buffer_ownership(&log_encoder); - } - else { - flb_plg_error(ctx->ins, - "Log event encoder error : %d", ret); - - ret = FLB_FILTER_NOTOUCH; - } - - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - return ret; - } - - flb_log_event_decoder_destroy(&log_decoder); - - return FLB_FILTER_NOTOUCH; -} - -static int cb_expect_exit(void *data, struct flb_config *config) -{ - struct flb_expect *ctx = data; - (void) config; - - if (!ctx) { - return 0; - } - - context_destroy(ctx); - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = -{ - /* rule: the key exists in the record */ - { - FLB_CONFIG_MAP_STR, "key_exists", NULL, - FLB_CONFIG_MAP_MULT, FLB_FALSE, 0, - "check that the given key name exists in the record" - }, - - /* rule: the key not exists in the record */ - { - FLB_CONFIG_MAP_STR, "key_not_exists", NULL, - FLB_CONFIG_MAP_MULT, FLB_FALSE, 0, - "check that the given key name do not exists in the record" - }, - - /* rule: the value of the key is NULL */ - { - FLB_CONFIG_MAP_STR, "key_val_is_null", NULL, - FLB_CONFIG_MAP_MULT, FLB_FALSE, 0, - "check that the value of the key is NULL" - }, - - /* rule: the value of the key is NOT NULL */ - { - FLB_CONFIG_MAP_STR, "key_val_is_not_null", NULL, - FLB_CONFIG_MAP_MULT, FLB_FALSE, 0, - "check that the value of the key is NOT NULL" - }, - - /* rule: the value of the key is equal a given value */ - { - FLB_CONFIG_MAP_SLIST_1, "key_val_eq", NULL, - FLB_CONFIG_MAP_MULT, FLB_FALSE, 0, - "check that the value of the key equals the given value" - }, - - /* rule action: the value of the key is equal a given value */ - { - FLB_CONFIG_MAP_STR, "action", "warn", - 0, FLB_FALSE, 0, - "action to take when a rule does not match: 'warn', 'exit' or 'result_key'." - }, - { - FLB_CONFIG_MAP_STR, "result_key", "matched", - 0, FLB_TRUE, offsetof(struct flb_expect, result_key), - "specify the key name to append a boolean that indicates rule is matched or not. " - "This key is to be used only when 'action' is 'result_key'." - }, - - /* EOF */ - {0} -}; - -struct flb_filter_plugin filter_expect_plugin = { - .name = "expect", - .description = "Validate expected keys and values", - .cb_init = cb_expect_init, - .cb_filter = cb_expect_filter, - .cb_exit = cb_expect_exit, - .config_map = config_map, - .flags = 0 -}; diff --git a/fluent-bit/plugins/filter_expect/expect.h b/fluent-bit/plugins/filter_expect/expect.h deleted file mode 100644 index bc7939d3d..000000000 --- a/fluent-bit/plugins/filter_expect/expect.h +++ /dev/null @@ -1,53 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_FILTER_EXPECT_H -#define FLB_FILTER_EXPECT_H - -#include -#include -#include - -#define FLB_EXP_WARN 0 -#define FLB_EXP_EXIT 1 -#define FLB_EXP_RESULT_KEY 2 - -/* Rule types */ -#define FLB_EXP_KEY_EXISTS 0 /* key exists */ -#define FLB_EXP_KEY_NOT_EXISTS 1 /* key not exists */ -#define FLB_EXP_KEY_VAL_NULL 2 /* key value has a NULL value */ -#define FLB_EXP_KEY_VAL_NOT_NULL 3 /* key value has a NULL value */ -#define FLB_EXP_KEY_VAL_EQ 4 /* key value is equal some given value */ - -struct flb_expect_rule { - int type; - flb_sds_t value; /* original value given in the config */ - flb_sds_t expect; /* specific value match (FLB_EXP_KEY_VAL_EQ */ - struct flb_record_accessor *ra; - struct mk_list _head; -}; - -struct flb_expect { - int action; - flb_sds_t result_key; - struct mk_list rules; - struct flb_filter_instance *ins; -}; - -#endif diff --git a/fluent-bit/plugins/filter_geoip2/.gitignore b/fluent-bit/plugins/filter_geoip2/.gitignore deleted file mode 100644 index 54b22adfa..000000000 --- a/fluent-bit/plugins/filter_geoip2/.gitignore +++ /dev/null @@ -1 +0,0 @@ -libmaxminddb/include/maxminddb_config.h diff --git a/fluent-bit/plugins/filter_geoip2/CMakeLists.txt b/fluent-bit/plugins/filter_geoip2/CMakeLists.txt deleted file mode 100644 index b33799e79..000000000 --- a/fluent-bit/plugins/filter_geoip2/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -# libmaxminddb - -# The subdirectory 'libmaxminddb' was imported from the following repo. -# (The minimum set of files required to compile geoip2.c were imported) -# -# https://github.com/maxmind/libmaxminddb -# -# We keep the exact version in the file 'libmaxminddb/VERSION', -# Please update the content when you upgrade libmaxminddb. -# -option(BUILD_TESTING "" OFF) -set(CMAKE_C_FLAGS "-std=gnu99 ${CMAKE_C_FLAGS}") -add_subdirectory(libmaxminddb EXCLUDE_FROM_ALL) -include_directories(libmaxminddb/include/) - -set(src - geoip2.c) - -FLB_PLUGIN(filter_geoip2 "${src}" "maxminddb") diff --git a/fluent-bit/plugins/filter_geoip2/geoip2.c b/fluent-bit/plugins/filter_geoip2/geoip2.c deleted file mode 100644 index 28559dfef..000000000 --- a/fluent-bit/plugins/filter_geoip2/geoip2.c +++ /dev/null @@ -1,519 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "geoip2.h" - -static int configure(struct geoip2_ctx *ctx, - struct flb_filter_instance *f_ins) -{ - struct flb_kv *kv = NULL; - struct mk_list *head = NULL; - struct mk_list *split; - int status; - struct geoip2_record *record; - struct flb_split_entry *sentry; - struct flb_config_map_val *record_key; - int ret; - - ctx->mmdb = flb_malloc(sizeof(MMDB_s)); - ctx->lookup_keys_num = 0; - ctx->records_num = 0; - - ret = flb_filter_config_map_set(f_ins, (void *)ctx); - if (ret == -1) { - flb_plg_error(f_ins, "unable to load configuration"); - flb_free(ctx->mmdb); - return -1; - } - - if (ctx->database) { - status = MMDB_open(ctx->database, MMDB_MODE_MMAP, ctx->mmdb); - if (status != MMDB_SUCCESS) { - flb_plg_error(f_ins, "Cannot open geoip2 database: %s: %s", - ctx->database, MMDB_strerror(status)); - flb_free(ctx->mmdb); - return -1; - } - } else { - flb_plg_error(f_ins, "no geoip2 database has been loaded"); - flb_free(ctx->mmdb); - return -1; - } - - mk_list_foreach(head, ctx->lookup_keys) { - ctx->lookup_keys_num++; - } - - flb_config_map_foreach(head, record_key, ctx->record_keys) { - record = flb_malloc(sizeof(struct geoip2_record)); - if (!record) { - flb_errno(); - continue; - } - split = flb_utils_split(record_key->val.str, ' ', 2); - if (mk_list_size(split) != 3) { - flb_plg_error(f_ins, "invalid record parameter: '%s'", kv->val); - flb_plg_error(f_ins, "expects 'KEY LOOKUP_KEY VALUE'"); - flb_free(record); - flb_utils_split_free(split); - continue; - } - - /* Get first value (field) */ - sentry = mk_list_entry_first(split, struct flb_split_entry, _head); - record->key = flb_strndup(sentry->value, sentry->len); - record->key_len = sentry->len; - - sentry = mk_list_entry_next(&sentry->_head, struct flb_split_entry, - _head, split); - record->lookup_key = flb_strndup(sentry->value, sentry->len); - record->lookup_key_len = sentry->len; - - sentry = mk_list_entry_last(split, struct flb_split_entry, _head); - record->val = flb_strndup(sentry->value, sentry->len); - record->val_len = sentry->len; - - flb_utils_split_free(split); - mk_list_add(&record->_head, &ctx->records); - ctx->records_num++; - } - - if (ctx->lookup_keys_num <= 0) { - flb_plg_error(f_ins, "at least one lookup_key is required"); - return -1; - } - if (ctx->records_num <= 0) { - flb_plg_error(f_ins, "at least one record is required"); - return -1; - } - return 0; -} - -static int delete_list(struct geoip2_ctx *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - struct geoip2_record *record; - - mk_list_foreach_safe(head, tmp, &ctx->records) { - record = mk_list_entry(head, struct geoip2_record, _head); - flb_free(record->lookup_key); - flb_free(record->key); - flb_free(record->val); - mk_list_del(&record->_head); - flb_free(record); - } - return 0; -} - -static struct flb_hash_table *prepare_lookup_keys(msgpack_object *map, - struct geoip2_ctx *ctx) -{ - msgpack_object_kv *kv; - msgpack_object *key; - msgpack_object *val; - struct mk_list *head; - struct flb_config_map_val *lookup_key; - struct flb_hash_table *ht; - - ht = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, ctx->lookup_keys_num, -1); - if (!ht) { - return NULL; - } - - kv = map->via.map.ptr; - for (int i = 0; i < map->via.map.size; i++) { - key = &(kv + i)->key; - val = &(kv + i)->val; - if (key->type != MSGPACK_OBJECT_STR) { - continue; - } - if (val->type != MSGPACK_OBJECT_STR) { - continue; - } - - flb_config_map_foreach(head, lookup_key, ctx->lookup_keys) { - if (strncasecmp(key->via.str.ptr, lookup_key->val.str, - flb_sds_len(lookup_key->val.str)) == 0) { - flb_hash_table_add(ht, lookup_key->val.str, flb_sds_len(lookup_key->val.str), - (void *) val->via.str.ptr, val->via.str.size); - } - } - } - - return ht; -} - -static MMDB_lookup_result_s mmdb_lookup(struct geoip2_ctx *ctx, const char *ip) -{ - int gai_error; - int mmdb_error; - MMDB_lookup_result_s result; - - result = MMDB_lookup_string(ctx->mmdb, ip, &gai_error, &mmdb_error); - if (gai_error != 0) { - flb_plg_error(ctx->ins, "getaddrinfo failed: %s", gai_strerror(gai_error)); - } - if (mmdb_error != MMDB_SUCCESS) { - flb_plg_error(ctx->ins, "lookup failed : %s", MMDB_strerror(mmdb_error)); - } - - return result; -} - -static void add_geoip_fields(msgpack_object *map, - struct flb_hash_table *lookup_keys, - struct geoip2_ctx *ctx, - struct flb_log_event_encoder *encoder) -{ - int ret; - struct mk_list *head; - struct mk_list *tmp; - struct geoip2_record *record; - const char *ip; - size_t ip_size; - MMDB_lookup_result_s result; - MMDB_entry_s entry; - MMDB_entry_data_s entry_data; - char **path; - int status; - char *pos; - char key[64]; - struct mk_list *split; - int split_size; - struct mk_list *path_head; - struct mk_list *path_tmp; - struct flb_split_entry *sentry; - int i = 0; - - mk_list_foreach_safe(head, tmp, &ctx->records) { - record = mk_list_entry(head, struct geoip2_record, _head); - - flb_log_event_encoder_append_body_string( - encoder, record->key, record->key_len); - - ret = flb_hash_table_get(lookup_keys, record->lookup_key, record->lookup_key_len, - (void *) &ip, &ip_size); - if (ret == -1) { - flb_log_event_encoder_append_body_null(encoder); - continue; - } - - result = mmdb_lookup(ctx, ip); - if (!result.found_entry) { - flb_log_event_encoder_append_body_null(encoder); - continue; - } - entry = result.entry; - pos = strstr(record->val, "}"); - memset(key, '\0', sizeof(key)); - strncpy(key, record->val + 2, pos - (record->val + 2)); - split = flb_utils_split(key, '.', 2); - split_size = mk_list_size(split); - path = flb_malloc(sizeof(char *) * (split_size + 1)); - i = 0; - mk_list_foreach_safe(path_head, path_tmp, split) { - sentry = mk_list_entry(path_head, struct flb_split_entry, _head); - path[i] = flb_strndup(sentry->value, sentry->len); - i++; - } - path[split_size] = NULL; - status = MMDB_aget_value(&entry, &entry_data, (const char *const *const)path); - flb_utils_split_free(split); - for (int j = 0; j < split_size; j++) { - flb_free(path[j]); - } - flb_free(path); - if (status != MMDB_SUCCESS) { - flb_plg_warn(ctx->ins, "cannot get value: %s", MMDB_strerror(status)); - flb_log_event_encoder_append_body_null(encoder); - continue; - } - if (!entry_data.has_data) { - flb_plg_warn(ctx->ins, "found entry does not have data"); - flb_log_event_encoder_append_body_null(encoder); - continue; - } - if (entry_data.type == MMDB_DATA_TYPE_MAP || - entry_data.type == MMDB_DATA_TYPE_ARRAY) { - flb_plg_warn(ctx->ins, "Not supported MAP and ARRAY"); - flb_log_event_encoder_append_body_null(encoder); - continue; - } - - switch (entry_data.type) { - case MMDB_DATA_TYPE_EXTENDED: - /* TODO: not implemented */ - flb_log_event_encoder_append_body_null(encoder); - break; - case MMDB_DATA_TYPE_POINTER: - /* TODO: not implemented */ - flb_log_event_encoder_append_body_null(encoder); - break; - case MMDB_DATA_TYPE_UTF8_STRING: - flb_log_event_encoder_append_body_string( - encoder, - (char *) entry_data.utf8_string, - entry_data.data_size); - break; - case MMDB_DATA_TYPE_DOUBLE: - flb_log_event_encoder_append_body_double( - encoder, entry_data.double_value); - break; - case MMDB_DATA_TYPE_BYTES: - flb_log_event_encoder_append_body_string( - encoder, - (char *) entry_data.bytes, - entry_data.data_size); - break; - case MMDB_DATA_TYPE_UINT16: - flb_log_event_encoder_append_body_uint16( - encoder, entry_data.uint16); - break; - case MMDB_DATA_TYPE_UINT32: - flb_log_event_encoder_append_body_uint32( - encoder, entry_data.uint32); - break; - case MMDB_DATA_TYPE_MAP: - /* TODO: not implemented */ - flb_log_event_encoder_append_body_null(encoder); - break; - case MMDB_DATA_TYPE_INT32: - flb_log_event_encoder_append_body_int32( - encoder, entry_data.int32); - break; - case MMDB_DATA_TYPE_UINT64: - flb_log_event_encoder_append_body_uint64( - encoder, entry_data.uint64); - break; - case MMDB_DATA_TYPE_UINT128: -#if !(MMDB_UINT128_IS_BYTE_ARRAY) - /* entry_data.uint128; */ - flb_warn("Not supported uint128"); -#else - flb_warn("Not implemented when MMDB_UINT128_IS_BYTE_ARRAY"); -#endif - flb_log_event_encoder_append_body_null(encoder); - break; - case MMDB_DATA_TYPE_ARRAY: - /* TODO: not implemented */ - flb_log_event_encoder_append_body_null(encoder); - break; - case MMDB_DATA_TYPE_CONTAINER: - /* TODO: not implemented */ - flb_log_event_encoder_append_body_null(encoder); - break; - case MMDB_DATA_TYPE_END_MARKER: - break; - case MMDB_DATA_TYPE_BOOLEAN: - flb_log_event_encoder_append_body_boolean( - encoder, (int) entry_data.boolean); - break; - case MMDB_DATA_TYPE_FLOAT: - flb_log_event_encoder_append_body_double( - encoder, entry_data.float_value); - break; - default: - flb_error("Unknown type: %d", entry_data.type); - break; - } - } -} - -static int cb_geoip2_init(struct flb_filter_instance *f_ins, - struct flb_config *config, - void *data) -{ - struct geoip2_ctx *ctx = NULL; - /* Create context */ - ctx = flb_calloc(1, sizeof(struct geoip2_ctx)); - if (!ctx) { - flb_errno(); - return -1; - } - mk_list_init(&ctx->records); - - - if (configure(ctx, f_ins) < 0) { - delete_list(ctx); - return -1; - } - - ctx->ins = f_ins; - flb_filter_set_context(f_ins, ctx); - - return 0; -} - -static int cb_geoip2_filter(const void *data, size_t bytes, - const char *tag, int tag_len, - void **out_buf, size_t *out_size, - struct flb_filter_instance *f_ins, - struct flb_input_instance *i_ins, - void *context, - struct flb_config *config) -{ - struct geoip2_ctx *ctx = context; - msgpack_object_kv *kv; - struct flb_hash_table *lookup_keys_hash; - struct flb_log_event_encoder log_encoder; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - int ret; - int i; - - (void) i_ins; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return FLB_FILTER_NOTOUCH; - } - - ret = flb_log_event_encoder_init(&log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event encoder initialization error : %d", ret); - - flb_log_event_decoder_destroy(&log_decoder); - - return FLB_FILTER_NOTOUCH; - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - - ret = flb_log_event_encoder_begin_record(&log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_timestamp( - &log_encoder, &log_event.timestamp); - } - - kv = log_event.body->via.map.ptr; - for (i = 0; - i < log_event.body->via.map.size && - ret == FLB_EVENT_ENCODER_SUCCESS ; - i++) { - ret = flb_log_event_encoder_append_body_values( - &log_encoder, - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&kv[i].key), - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&kv[i].val)); - } - - lookup_keys_hash = prepare_lookup_keys(log_event.body, ctx); - add_geoip_fields(log_event.body, lookup_keys_hash, ctx, &log_encoder); - flb_hash_table_destroy(lookup_keys_hash); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(&log_encoder); - } - } - - if (ret == FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA && - log_decoder.offset == bytes) { - ret = FLB_EVENT_ENCODER_SUCCESS; - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - *out_buf = log_encoder.output_buffer; - *out_size = log_encoder.output_length; - - ret = FLB_FILTER_MODIFIED; - - flb_log_event_encoder_claim_internal_buffer_ownership(&log_encoder); - } - else { - flb_plg_error(ctx->ins, - "Log event encoder error : %d", ret); - - ret = FLB_FILTER_NOTOUCH; - } - - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - return ret; -} - -static int cb_geoip2_exit(void *data, struct flb_config *config) -{ - struct geoip2_ctx *ctx = data; - - if (ctx != NULL) { - delete_list(ctx); - MMDB_close(ctx->mmdb); - flb_free(ctx->mmdb); - flb_free(ctx); - } - - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "database", (char *)NULL, - 0, FLB_TRUE, offsetof(struct geoip2_ctx, database), - "Set the geoip2 database path" - }, - { - FLB_CONFIG_MAP_STR, "lookup_key", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct geoip2_ctx, lookup_keys), - "Add a lookup_key" - }, - { - FLB_CONFIG_MAP_STR, "record", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct geoip2_ctx, record_keys), - "Add a record to the output base on geoip2" - }, - /* EOF */ - {0} -}; - -struct flb_filter_plugin filter_geoip2_plugin = { - .name = "geoip2", - .description = "add geoip information to records", - .cb_init = cb_geoip2_init, - .cb_filter = cb_geoip2_filter, - .cb_exit = cb_geoip2_exit, - .config_map = config_map, - .flags = 0, -}; diff --git a/fluent-bit/plugins/filter_geoip2/geoip2.h b/fluent-bit/plugins/filter_geoip2/geoip2.h deleted file mode 100644 index 5c23d60f6..000000000 --- a/fluent-bit/plugins/filter_geoip2/geoip2.h +++ /dev/null @@ -1,46 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#ifndef FLB_FILTER_GEOIP2_H -#define FLB_FILTER_GEOIP2_H - -struct geoip2_record { - char *lookup_key; - char *key; - char *val; - int lookup_key_len; - int key_len; - int val_len; - struct mk_list _head; -}; - -struct geoip2_ctx { - flb_sds_t database; - MMDB_s *mmdb; - int lookup_keys_num; - int records_num; - struct mk_list *lookup_keys; - struct mk_list *record_keys; - struct mk_list records; - struct flb_filter_instance *ins; -}; - -#endif diff --git a/fluent-bit/plugins/filter_geoip2/libmaxminddb/CMakeLists.txt b/fluent-bit/plugins/filter_geoip2/libmaxminddb/CMakeLists.txt deleted file mode 100644 index 2ed068af2..000000000 --- a/fluent-bit/plugins/filter_geoip2/libmaxminddb/CMakeLists.txt +++ /dev/null @@ -1,101 +0,0 @@ -project(maxminddb - LANGUAGES C -) -set(MAXMINDDB_SOVERSION 0.0.7) - -option(BUILD_SHARED_LIBS "Build shared libraries (.dll/.so) instead of static ones (.lib/.a)" OFF) -option(BUILD_TESTING "Build test programs" ON) - -include(CheckTypeSize) -check_type_size("unsigned __int128" UINT128) -check_type_size("unsigned int __attribute__((mode(TI)))" UINT128_USING_MODE) -if(HAVE_UINT128) - set(MMDB_UINT128_USING_MODE 0) - set(MMDB_UINT128_IS_BYTE_ARRAY 0) -elseif(HAVE_UINT128_USING_MODE) - set(MMDB_UINT128_USING_MODE 1) - set(MMDB_UINT128_IS_BYTE_ARRAY 0) -else() - set(MMDB_UINT128_USING_MODE 0) - set(MMDB_UINT128_IS_BYTE_ARRAY 1) -endif() - -include (TestBigEndian) -TEST_BIG_ENDIAN(IS_BIG_ENDIAN) - -if (${CMAKE_SYSTEM_NAME} MATCHES "Linux") - set(CMAKE_POSITION_INDEPENDENT_CODE ON) -endif() - -configure_file(${PROJECT_SOURCE_DIR}/include/maxminddb_config.h.cmake.in - ${PROJECT_SOURCE_DIR}/include/maxminddb_config.h) - -add_library(maxminddb STATIC - src/maxminddb.c - src/data-pool.c -) -add_library(maxminddb::maxminddb ALIAS maxminddb) - -set_target_properties(maxminddb PROPERTIES VERSION ${MAXMINDDB_SOVERSION}) - -target_compile_definitions(maxminddb PUBLIC PACKAGE_VERSION="${PROJECT_VERSION}") - -if(NOT IS_BIG_ENDIAN) - target_compile_definitions(maxminddb PRIVATE MMDB_LITTLE_ENDIAN=1) -endif() - -if(MSVC) - target_compile_definitions(maxminddb PRIVATE _CRT_SECURE_NO_WARNINGS) -endif() - -if(WIN32) - target_link_libraries(maxminddb ws2_32) -endif() - -set(CMAKE_SHARED_LIBRARY_PREFIX lib) -set(CMAKE_STATIC_LIBRARY_PREFIX lib) - -set(MAXMINDB_INCLUDE_DIR - . - include -) - - - -# -# NOTE: This function call was modified for Fluent Bit. -# The original first argument was the following: -# -# $ -# -target_include_directories(maxminddb PUBLIC - $ - $ -) - -set(MAXMINDB_HEADERS - include/maxminddb.h - include/maxminddb_config.h -) -set_target_properties(maxminddb PROPERTIES PUBLIC_HEADER "${MAXMINDB_HEADERS}") - -#install(TARGETS maxminddb -# EXPORT maxminddb -# ARCHIVE DESTINATION lib -# PUBLIC_HEADER DESTINATION include/ -#) -# -## This is required to work with FetchContent -#install(EXPORT maxminddb -# FILE maxminddb-config.cmake -# NAMESPACE maxminddb:: -# DESTINATION lib/cmake/maxminddb) - -# We always want to build mmdblookup -add_subdirectory(bin) - -if (BUILD_TESTING) - enable_testing() - add_subdirectory(t) -endif() - diff --git a/fluent-bit/plugins/filter_geoip2/libmaxminddb/LICENSE b/fluent-bit/plugins/filter_geoip2/libmaxminddb/LICENSE deleted file mode 100644 index d64569567..000000000 --- a/fluent-bit/plugins/filter_geoip2/libmaxminddb/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/fluent-bit/plugins/filter_geoip2/libmaxminddb/NOTICE b/fluent-bit/plugins/filter_geoip2/libmaxminddb/NOTICE deleted file mode 100644 index 6b8694752..000000000 --- a/fluent-bit/plugins/filter_geoip2/libmaxminddb/NOTICE +++ /dev/null @@ -1,13 +0,0 @@ -Copyright 2013-2014 MaxMind, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. \ No newline at end of file diff --git a/fluent-bit/plugins/filter_geoip2/libmaxminddb/VERSION b/fluent-bit/plugins/filter_geoip2/libmaxminddb/VERSION deleted file mode 100644 index dbdbf984c..000000000 --- a/fluent-bit/plugins/filter_geoip2/libmaxminddb/VERSION +++ /dev/null @@ -1 +0,0 @@ -ad35e6af diff --git a/fluent-bit/plugins/filter_geoip2/libmaxminddb/bin/CMakeLists.txt b/fluent-bit/plugins/filter_geoip2/libmaxminddb/bin/CMakeLists.txt deleted file mode 100644 index 9026be83b..000000000 --- a/fluent-bit/plugins/filter_geoip2/libmaxminddb/bin/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -# getopt is required by mmdblookup which is not available by default on Windows -if(NOT WIN32) - add_executable(mmdblookup - mmdblookup.c - ) - - target_link_libraries(mmdblookup maxminddb pthread) - - install( - TARGETS mmdblookup - RUNTIME DESTINATION bin - ) -endif() diff --git a/fluent-bit/plugins/filter_geoip2/libmaxminddb/bin/Makefile.am b/fluent-bit/plugins/filter_geoip2/libmaxminddb/bin/Makefile.am deleted file mode 100644 index c00ba95b5..000000000 --- a/fluent-bit/plugins/filter_geoip2/libmaxminddb/bin/Makefile.am +++ /dev/null @@ -1,10 +0,0 @@ -include $(top_srcdir)/common.mk - -AM_LDFLAGS = $(top_builddir)/src/libmaxminddb.la - -bin_PROGRAMS = mmdblookup - -if !WINDOWS -AM_CPPFLAGS += -pthread -AM_LDFLAGS += -pthread -endif diff --git a/fluent-bit/plugins/filter_geoip2/libmaxminddb/bin/mmdblookup.c b/fluent-bit/plugins/filter_geoip2/libmaxminddb/bin/mmdblookup.c deleted file mode 100644 index d7ec3fff2..000000000 --- a/fluent-bit/plugins/filter_geoip2/libmaxminddb/bin/mmdblookup.c +++ /dev/null @@ -1,762 +0,0 @@ -#ifdef HAVE_CONFIG_H -#include -#endif -#include "maxminddb.h" -#include -#include -#ifndef _WIN32 -#include -#endif -#include -#include -#include -#include -#include - -#ifdef _WIN32 -#ifndef UNICODE -#define UNICODE -#endif -#include -#else -#include -#include -#endif - -#define LOCAL static - -LOCAL void usage(char *program, int exit_code, const char *error); -LOCAL const char **get_options( - int argc, - char **argv, - char **mmdb_file, - char **ip_address, - int *verbose, - int *iterations, - int *lookup_path_length, - int *const thread_count, - char **const ip_file); -LOCAL MMDB_s open_or_die(const char *fname); -LOCAL void dump_meta(MMDB_s *mmdb); -LOCAL bool lookup_from_file(MMDB_s *const mmdb, - char const *const ip_file, - bool const dump); -LOCAL int lookup_and_print(MMDB_s *mmdb, const char *ip_address, - const char **lookup_path, - int lookup_path_length, - bool verbose); -LOCAL int benchmark(MMDB_s *mmdb, int iterations); -LOCAL MMDB_lookup_result_s lookup_or_die(MMDB_s *mmdb, const char *ipstr); -LOCAL void random_ipv4(char *ip); - -#ifndef _WIN32 -// These aren't with the automatically generated prototypes as we'd lose the -// enclosing macros. -static bool start_threaded_benchmark( - MMDB_s *const mmdb, - int const thread_count, - int const iterations); -static long double get_time(void); -static void *thread(void *arg); -#endif - -#ifdef _WIN32 -int wmain(int argc, wchar_t **wargv) -{ - // Convert our argument list from UTF-16 to UTF-8. - char **argv = (char **)calloc(argc, sizeof(char *)); - if (!argv) { - fprintf(stderr, "calloc(): %s\n", strerror(errno)); - exit(1); - } - for (int i = 0; i < argc; i++) { - int utf8_width; - char *utf8_string; - utf8_width = WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, NULL, 0, - NULL, NULL); - if (utf8_width < 1) { - fprintf(stderr, "WideCharToMultiByte() failed: %d\n", - GetLastError()); - exit(1); - } - utf8_string = calloc(utf8_width, sizeof(char)); - if (!utf8_string) { - fprintf(stderr, "calloc(): %s\n", strerror(errno)); - exit(1); - } - if (WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, utf8_string, - utf8_width, NULL, NULL) < 1) { - fprintf(stderr, "WideCharToMultiByte() failed: %d\n", - GetLastError()); - exit(1); - } - argv[i] = utf8_string; - } -#else // _WIN32 -int main(int argc, char **argv) -{ -#endif // _WIN32 - char *mmdb_file = NULL; - char *ip_address = NULL; - int verbose = 0; - int iterations = 0; - int lookup_path_length = 0; - int thread_count = 0; - char *ip_file = NULL; - - const char **lookup_path = - get_options(argc, argv, &mmdb_file, &ip_address, &verbose, &iterations, - &lookup_path_length, &thread_count, &ip_file); - - MMDB_s mmdb = open_or_die(mmdb_file); - - if (verbose) { - dump_meta(&mmdb); - } - - // The benchmarking and lookup from file modes are hidden features mainly - // intended for development right now. This means there are several flags - // that exist but are intentionally not mentioned in the usage or man page. - - // The lookup from file mode may be useful to expose publicly in the usage, - // but we should have it respect the lookup_path functionality if we do so. - if (ip_file) { - free((void *)lookup_path); - if (!lookup_from_file(&mmdb, ip_file, verbose == 1)) { - MMDB_close(&mmdb); - return 1; - } - MMDB_close(&mmdb); - return 0; - } - - if (0 == iterations) { - exit(lookup_and_print(&mmdb, ip_address, lookup_path, - lookup_path_length, verbose)); - } - - free((void *)lookup_path); - - srand( (int)time(NULL) ); - -#ifndef _WIN32 - if (thread_count > 0) { - if (!start_threaded_benchmark(&mmdb, thread_count, iterations)) { - MMDB_close(&mmdb); - exit(1); - } - MMDB_close(&mmdb); - exit(0); - } -#endif - - exit(benchmark(&mmdb, iterations)); -} - -LOCAL void usage(char *program, int exit_code, const char *error) -{ - if (NULL != error) { - fprintf(stderr, "\n *ERROR: %s\n", error); - } - - char *usage = "\n" - " %s --file /path/to/file.mmdb --ip 1.2.3.4 [path to lookup]\n" - "\n" - " This application accepts the following options:\n" - "\n" - " --file (-f) The path to the MMDB file. Required.\n" - "\n" - " --ip (-i) The IP address to look up. Required.\n" - "\n" - " --verbose (-v) Turns on verbose output. Specifically, this causes this\n" - " application to output the database metadata.\n" - "\n" - " --version Print the program's version number and exit.\n" - "\n" - " --help (-h -?) Show usage information.\n" - "\n" - " If an IP's data entry resolves to a map or array, you can provide\n" - " a lookup path to only show part of that data.\n" - "\n" - " For example, given a JSON structure like this:\n" - "\n" - " {\n" - " \"names\": {\n" - " \"en\": \"Germany\",\n" - " \"de\": \"Deutschland\"\n" - " },\n" - " \"cities\": [ \"Berlin\", \"Frankfurt\" ]\n" - " }\n" - "\n" - " You could look up just the English name by calling mmdblookup with a lookup path of:\n" - "\n" - " mmdblookup --file ... --ip ... names en\n" - "\n" - " Or you could look up the second city in the list with:\n" - "\n" - " mmdblookup --file ... --ip ... cities 1\n" - "\n" - " Array numbering begins with zero (0).\n" - "\n" - " If you do not provide a path to lookup, all of the information for a given IP\n" - " will be shown.\n" - "\n"; - - fprintf(stdout, usage, program); - exit(exit_code); -} - -LOCAL const char **get_options( - int argc, - char **argv, - char **mmdb_file, - char **ip_address, - int *verbose, - int *iterations, - int *lookup_path_length, - int *const thread_count, - char **const ip_file) -{ - static int help = 0; - static int version = 0; - - while (1) { - static struct option options[] = { - { "file", required_argument, 0, 'f' }, - { "ip", required_argument, 0, 'i' }, - { "verbose", no_argument, 0, 'v' }, - { "version", no_argument, 0, 'n' }, - { "benchmark", required_argument, 0, 'b' }, -#ifndef _WIN32 - { "threads", required_argument, 0, 't' }, -#endif - { "ip-file", required_argument, 0, 'I' }, - { "help", no_argument, 0, 'h' }, - { "?", no_argument, 0, 1 }, - { 0, 0, 0, 0 } - }; - - int opt_index; -#ifdef _WIN32 - char const * const optstring = "f:i:b:I:vnh?"; -#else - char const * const optstring = "f:i:b:t:I:vnh?"; -#endif - int opt_char = getopt_long(argc, argv, optstring, options, - &opt_index); - - if (-1 == opt_char) { - break; - } - - if ('f' == opt_char) { - *mmdb_file = optarg; - } else if ('i' == opt_char) { - *ip_address = optarg; - } else if ('v' == opt_char) { - *verbose = 1; - } else if ('n' == opt_char) { - version = 1; - } else if ('b' == opt_char) { - *iterations = strtol(optarg, NULL, 10); - } else if ('h' == opt_char || '?' == opt_char) { - help = 1; - } else if (opt_char == 't') { - *thread_count = strtol(optarg, NULL, 10); - } else if (opt_char == 'I') { - *ip_file = optarg; - } - } - -#ifdef _WIN32 - char *program = alloca(strlen(argv[0])); - _splitpath(argv[0], NULL, NULL, program, NULL); - _splitpath(argv[0], NULL, NULL, NULL, program + strlen(program)); -#else - char *program = basename(argv[0]); -#endif - - if (help) { - usage(program, 0, NULL); - } - - if (version) { - fprintf(stdout, "\n %s version %s\n\n", program, PACKAGE_VERSION); - exit(0); - } - - if (NULL == *mmdb_file) { - usage(program, 1, "You must provide a filename with --file"); - } - - if (*ip_address == NULL && *iterations == 0 && !*ip_file) { - usage(program, 1, "You must provide an IP address with --ip"); - } - - const char **lookup_path = - calloc((argc - optind) + 1, sizeof(const char *)); - int i; - for (i = 0; i < argc - optind; i++) { - lookup_path[i] = argv[i + optind]; - (*lookup_path_length)++; - } - lookup_path[i] = NULL; - - return lookup_path; -} - -LOCAL MMDB_s open_or_die(const char *fname) -{ - MMDB_s mmdb; - int status = MMDB_open(fname, MMDB_MODE_MMAP, &mmdb); - - if (MMDB_SUCCESS != status) { - fprintf(stderr, "\n Can't open %s - %s\n", fname, - MMDB_strerror(status)); - - if (MMDB_IO_ERROR == status) { - fprintf(stderr, " IO error: %s\n", strerror(errno)); - } - - fprintf(stderr, "\n"); - - exit(2); - } - - return mmdb; -} - -LOCAL void dump_meta(MMDB_s *mmdb) -{ - const char *meta_dump = "\n" - " Database metadata\n" - " Node count: %i\n" - " Record size: %i bits\n" - " IP version: IPv%i\n" - " Binary format: %i.%i\n" - " Build epoch: %llu (%s)\n" - " Type: %s\n" - " Languages: "; - - char date[40]; - const time_t epoch = (const time_t)mmdb->metadata.build_epoch; - strftime(date, 40, "%F %T UTC", gmtime(&epoch)); - - fprintf(stdout, meta_dump, - mmdb->metadata.node_count, - mmdb->metadata.record_size, - mmdb->metadata.ip_version, - mmdb->metadata.binary_format_major_version, - mmdb->metadata.binary_format_minor_version, - mmdb->metadata.build_epoch, - date, - mmdb->metadata.database_type); - - for (size_t i = 0; i < mmdb->metadata.languages.count; i++) { - fprintf(stdout, "%s", mmdb->metadata.languages.names[i]); - if (i < mmdb->metadata.languages.count - 1) { - fprintf(stdout, " "); - } - } - fprintf(stdout, "\n"); - - fprintf(stdout, " Description:\n"); - for (size_t i = 0; i < mmdb->metadata.description.count; i++) { - fprintf(stdout, " %s: %s\n", - mmdb->metadata.description.descriptions[i]->language, - mmdb->metadata.description.descriptions[i]->description); - } - fprintf(stdout, "\n"); -} - -// The input file should have one IP per line. -// -// We look up each IP. -// -// If dump is true, we dump the data for each IP to stderr. This is useful for -// comparison in that you can dump out the data for the IPs before and after -// making changes. It goes to stderr rather than stdout so that the report does -// not get included in what you will compare (since it will almost always be -// different). -// -// In addition to being useful for comparisons, this function provides a way to -// have a more deterministic set of lookups for benchmarking. -LOCAL bool lookup_from_file(MMDB_s *const mmdb, - char const *const ip_file, - bool const dump) -{ - FILE *const fh = fopen(ip_file, "r"); - if (!fh) { - fprintf(stderr, "fopen(): %s: %s\n", ip_file, strerror(errno)); - return false; - } - - clock_t const clock_start = clock(); - char buf[1024] = { 0 }; - // I'd normally use uint64_t, but support for it is optional in C99. - unsigned long long i = 0; - while (1) { - if (fgets(buf, sizeof(buf), fh) == NULL) { - if (!feof(fh)) { - fprintf(stderr, "fgets(): %s\n", strerror(errno)); - fclose(fh); - return false; - } - if (fclose(fh) != 0) { - fprintf(stderr, "fclose(): %s\n", strerror(errno)); - return false; - } - break; - } - - char *ptr = buf; - while (*ptr != '\0') { - if (*ptr == '\n') { - *ptr = '\0'; - break; - } - ptr++; - } - if (strlen(buf) == 0) { - continue; - } - - i++; - - MMDB_lookup_result_s result = lookup_or_die(mmdb, buf); - if (!result.found_entry) { - continue; - } - - MMDB_entry_data_list_s *entry_data_list = NULL; - int const status = MMDB_get_entry_data_list(&result.entry, - &entry_data_list); - if (status != MMDB_SUCCESS) { - fprintf(stderr, "MMDB_get_entry_data_list(): %s\n", - MMDB_strerror(status)); - fclose(fh); - MMDB_free_entry_data_list(entry_data_list); - return false; - } - - if (!entry_data_list) { - fprintf(stderr, "entry_data_list is NULL\n"); - fclose(fh); - return false; - } - - if (dump) { - fprintf(stdout, "%s:\n", buf); - int const status = MMDB_dump_entry_data_list(stderr, - entry_data_list, 0); - if (status != MMDB_SUCCESS) { - fprintf(stderr, "MMDB_dump_entry_data_list(): %s\n", - MMDB_strerror(status)); - fclose(fh); - MMDB_free_entry_data_list(entry_data_list); - return false; - } - } - - MMDB_free_entry_data_list(entry_data_list); - } - - clock_t const clock_diff = clock() - clock_start; - double const seconds = (double)clock_diff / CLOCKS_PER_SEC; - - fprintf( - stdout, - "Looked up %llu addresses in %.2f seconds. %.2f lookups per second.\n", - i, seconds, i / seconds); - - return true; -} - -LOCAL int lookup_and_print(MMDB_s *mmdb, const char *ip_address, - const char **lookup_path, - int lookup_path_length, - bool verbose) -{ - - MMDB_lookup_result_s result = lookup_or_die(mmdb, ip_address); - MMDB_entry_data_list_s *entry_data_list = NULL; - - int exit_code = 0; - - if (verbose) { - fprintf( - stdout, - "\n Record prefix length: %d\n", - result.netmask - ); - } - - if (result.found_entry) { - int status; - if (lookup_path_length) { - MMDB_entry_data_s entry_data; - status = MMDB_aget_value(&result.entry, &entry_data, - lookup_path); - if (MMDB_SUCCESS == status) { - if (entry_data.offset) { - MMDB_entry_s entry = - { .mmdb = mmdb, .offset = entry_data.offset }; - status = MMDB_get_entry_data_list(&entry, - &entry_data_list); - } else { - fprintf( - stdout, - "\n No data was found at the lookup path you provided\n\n"); - } - } - } else { - status = MMDB_get_entry_data_list(&result.entry, - &entry_data_list); - } - - if (MMDB_SUCCESS != status) { - fprintf(stderr, "Got an error looking up the entry data - %s\n", - MMDB_strerror(status)); - exit_code = 5; - goto end; - } - - if (NULL != entry_data_list) { - fprintf(stdout, "\n"); - MMDB_dump_entry_data_list(stdout, entry_data_list, 2); - fprintf(stdout, "\n"); - } - } else { - fprintf(stderr, - "\n Could not find an entry for this IP address (%s)\n\n", - ip_address); - exit_code = 6; - } - - end: - MMDB_free_entry_data_list(entry_data_list); - MMDB_close(mmdb); - free((void *)lookup_path); - - return exit_code; -} - -LOCAL int benchmark(MMDB_s *mmdb, int iterations) -{ - char ip_address[16]; - int exit_code = 0; - - clock_t time = clock(); - - for (int i = 0; i < iterations; i++) { - random_ipv4(ip_address); - - MMDB_lookup_result_s result = lookup_or_die(mmdb, ip_address); - MMDB_entry_data_list_s *entry_data_list = NULL; - - if (result.found_entry) { - - int status = MMDB_get_entry_data_list(&result.entry, - &entry_data_list); - - if (MMDB_SUCCESS != status) { - fprintf(stderr, "Got an error looking up the entry data - %s\n", - MMDB_strerror(status)); - exit_code = 5; - MMDB_free_entry_data_list(entry_data_list); - goto end; - } - } - - MMDB_free_entry_data_list(entry_data_list); - } - - time = clock() - time; - double seconds = ((double)time / CLOCKS_PER_SEC); - fprintf( - stdout, - "\n Looked up %i addresses in %.2f seconds. %.2f lookups per second.\n\n", - iterations, seconds, iterations / seconds); - - end: - MMDB_close(mmdb); - - return exit_code; -} - -LOCAL MMDB_lookup_result_s lookup_or_die(MMDB_s *mmdb, const char *ipstr) -{ - int gai_error, mmdb_error; - MMDB_lookup_result_s result = - MMDB_lookup_string(mmdb, ipstr, &gai_error, &mmdb_error); - - if (0 != gai_error) { - fprintf(stderr, - "\n Error from call to getaddrinfo for %s - %s\n\n", - ipstr, -#ifdef _WIN32 - gai_strerrorA(gai_error) -#else - gai_strerror(gai_error) -#endif - ); - exit(3); - } - - if (MMDB_SUCCESS != mmdb_error) { - fprintf(stderr, "\n Got an error from the maxminddb library: %s\n\n", - MMDB_strerror(mmdb_error)); - exit(4); - } - - return result; -} - -LOCAL void random_ipv4(char *ip) -{ - // rand() is perfectly fine for this use case - // coverity[dont_call] - int ip_int = rand(); - uint8_t *bytes = (uint8_t *)&ip_int; - - snprintf(ip, 16, "%u.%u.%u.%u", - *bytes, *(bytes + 1), *(bytes + 2), *(bytes + 3)); -} - -#ifndef _WIN32 -struct thread_info { - pthread_t id; - int num; - MMDB_s *mmdb; - int iterations; -}; - -static bool start_threaded_benchmark( - MMDB_s *const mmdb, - int const thread_count, - int const iterations) -{ - struct thread_info *const tinfo = calloc(thread_count, - sizeof(struct thread_info)); - if (!tinfo) { - fprintf(stderr, "calloc(): %s\n", strerror(errno)); - return false; - } - - // Using clock() isn't appropriate for multiple threads. It's CPU time, not - // wall time. - long double const start_time = get_time(); - if (start_time == -1) { - free(tinfo); - return false; - } - - for (int i = 0; i < thread_count; i++) { - tinfo[i].num = i; - tinfo[i].mmdb = mmdb; - tinfo[i].iterations = iterations; - - if (pthread_create(&tinfo[i].id, NULL, &thread, &tinfo[i]) != 0) { - fprintf(stderr, "pthread_create() failed\n"); - free(tinfo); - return false; - } - } - - for (int i = 0; i < thread_count; i++) { - if (pthread_join(tinfo[i].id, NULL) != 0) { - fprintf(stderr, "pthread_join() failed\n"); - free(tinfo); - return false; - } - } - - free(tinfo); - - long double const end_time = get_time(); - if (end_time == -1) { - return false; - } - - long double const elapsed = end_time - start_time; - unsigned long long const total_ips = iterations * thread_count; - long double rate = total_ips; - if (elapsed != 0) { - rate = total_ips / elapsed; - } - - fprintf( - stdout, - "Looked up %llu addresses using %d threads in %.2Lf seconds. %.2Lf lookups per second.\n", - total_ips, thread_count, elapsed, rate); - - return true; -} - -static long double get_time(void) -{ - // clock_gettime() is not present on OSX until 10.12. -#ifdef HAVE_CLOCK_GETTIME - struct timespec tp = { - .tv_sec = 0, - .tv_nsec = 0, - }; - clockid_t clk_id = CLOCK_REALTIME; -#ifdef _POSIX_MONOTONIC_CLOCK - clk_id = CLOCK_MONOTONIC; -#endif - if (clock_gettime(clk_id, &tp) != 0) { - fprintf(stderr, "clock_gettime(): %s\n", strerror(errno)); - return -1; - } - return tp.tv_sec + ((float)tp.tv_nsec / 1e9); -#else - time_t t = time(NULL); - if (t == (time_t)-1) { - fprintf(stderr, "time(): %s\n", strerror(errno)); - return -1; - } - return (long double)t; -#endif -} - -static void *thread(void *arg) -{ - const struct thread_info *const tinfo = arg; - if (!tinfo) { - fprintf(stderr, "thread(): %s\n", strerror(EINVAL)); - return NULL; - } - - char ip_address[16] = { 0 }; - - for (int i = 0; i < tinfo->iterations; i++) { - memset(ip_address, 0, 16); - random_ipv4(ip_address); - - MMDB_lookup_result_s result = lookup_or_die(tinfo->mmdb, ip_address); - if (!result.found_entry) { - continue; - } - - MMDB_entry_data_list_s *entry_data_list = NULL; - int const status = MMDB_get_entry_data_list(&result.entry, - &entry_data_list); - if (status != MMDB_SUCCESS) { - fprintf(stderr, "MMDB_get_entry_data_list(): %s\n", - MMDB_strerror(status)); - MMDB_free_entry_data_list(entry_data_list); - return NULL; - } - - if (!entry_data_list) { - fprintf(stderr, "entry_data_list is NULL\n"); - return NULL; - } - - MMDB_free_entry_data_list(entry_data_list); - } - - return NULL; -} -#endif diff --git a/fluent-bit/plugins/filter_geoip2/libmaxminddb/include/maxminddb.h b/fluent-bit/plugins/filter_geoip2/libmaxminddb/include/maxminddb.h deleted file mode 100644 index 13b276f14..000000000 --- a/fluent-bit/plugins/filter_geoip2/libmaxminddb/include/maxminddb.h +++ /dev/null @@ -1,255 +0,0 @@ -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef MAXMINDDB_H -#define MAXMINDDB_H - -/* Request POSIX.1-2008. However, we want to remain compatible with - * POSIX.1-2001 (since we have been historically and see no reason to drop - * compatibility). By requesting POSIX.1-2008, we can conditionally use - * features provided by that standard if the implementation provides it. We can - * check for what the implementation provides by checking the _POSIX_VERSION - * macro after including unistd.h. If a feature is in POSIX.1-2008 but not - * POSIX.1-2001, check that macro before using the feature (or check for the - * feature directly if possible). */ -#ifndef _POSIX_C_SOURCE -#define _POSIX_C_SOURCE 200809L -#endif - -#include "maxminddb_config.h" -#include -#include -#include -#include -#include - -#ifdef _WIN32 -#include -#include -/* libmaxminddb package version from configure */ -#define PACKAGE_VERSION "1.4.3" - -typedef ADDRESS_FAMILY sa_family_t; - -#if defined(_MSC_VER) -/* MSVC doesn't define signed size_t, copy it from configure */ -#define ssize_t SSIZE_T - -/* MSVC doesn't support restricted pointers */ -#define restrict -#endif -#else -#include -#include -#include -#endif - -#define MMDB_DATA_TYPE_EXTENDED (0) -#define MMDB_DATA_TYPE_POINTER (1) -#define MMDB_DATA_TYPE_UTF8_STRING (2) -#define MMDB_DATA_TYPE_DOUBLE (3) -#define MMDB_DATA_TYPE_BYTES (4) -#define MMDB_DATA_TYPE_UINT16 (5) -#define MMDB_DATA_TYPE_UINT32 (6) -#define MMDB_DATA_TYPE_MAP (7) -#define MMDB_DATA_TYPE_INT32 (8) -#define MMDB_DATA_TYPE_UINT64 (9) -#define MMDB_DATA_TYPE_UINT128 (10) -#define MMDB_DATA_TYPE_ARRAY (11) -#define MMDB_DATA_TYPE_CONTAINER (12) -#define MMDB_DATA_TYPE_END_MARKER (13) -#define MMDB_DATA_TYPE_BOOLEAN (14) -#define MMDB_DATA_TYPE_FLOAT (15) - -#define MMDB_RECORD_TYPE_SEARCH_NODE (0) -#define MMDB_RECORD_TYPE_EMPTY (1) -#define MMDB_RECORD_TYPE_DATA (2) -#define MMDB_RECORD_TYPE_INVALID (3) - -/* flags for open */ -#define MMDB_MODE_MMAP (1) -#define MMDB_MODE_MASK (7) - -/* error codes */ -#define MMDB_SUCCESS (0) -#define MMDB_FILE_OPEN_ERROR (1) -#define MMDB_CORRUPT_SEARCH_TREE_ERROR (2) -#define MMDB_INVALID_METADATA_ERROR (3) -#define MMDB_IO_ERROR (4) -#define MMDB_OUT_OF_MEMORY_ERROR (5) -#define MMDB_UNKNOWN_DATABASE_FORMAT_ERROR (6) -#define MMDB_INVALID_DATA_ERROR (7) -#define MMDB_INVALID_LOOKUP_PATH_ERROR (8) -#define MMDB_LOOKUP_PATH_DOES_NOT_MATCH_DATA_ERROR (9) -#define MMDB_INVALID_NODE_NUMBER_ERROR (10) -#define MMDB_IPV6_LOOKUP_IN_IPV4_DATABASE_ERROR (11) - -#if !(MMDB_UINT128_IS_BYTE_ARRAY) -#if MMDB_UINT128_USING_MODE -typedef unsigned int mmdb_uint128_t __attribute__ ((__mode__(TI))); -#else -typedef unsigned __int128 mmdb_uint128_t; -#endif -#endif - -/* This is a pointer into the data section for a given IP address lookup */ -typedef struct MMDB_entry_s { - const struct MMDB_s *mmdb; - uint32_t offset; -} MMDB_entry_s; - -typedef struct MMDB_lookup_result_s { - bool found_entry; - MMDB_entry_s entry; - uint16_t netmask; -} MMDB_lookup_result_s; - -typedef struct MMDB_entry_data_s { - bool has_data; - union { - uint32_t pointer; - const char *utf8_string; - double double_value; - const uint8_t *bytes; - uint16_t uint16; - uint32_t uint32; - int32_t int32; - uint64_t uint64; -#if MMDB_UINT128_IS_BYTE_ARRAY - uint8_t uint128[16]; -#else - mmdb_uint128_t uint128; -#endif - bool boolean; - float float_value; - }; - /* This is a 0 if a given entry cannot be found. This can only happen - * when a call to MMDB_(v)get_value() asks for hash keys or array - * indices that don't exist. */ - uint32_t offset; - /* This is the next entry in the data section, but it's really only - * relevant for entries that part of a larger map or array - * struct. There's no good reason for an end user to look at this - * directly. */ - uint32_t offset_to_next; - /* This is only valid for strings, utf8_strings or binary data */ - uint32_t data_size; - /* This is an MMDB_DATA_TYPE_* constant */ - uint32_t type; -} MMDB_entry_data_s; - -/* This is the return type when someone asks for all the entry data in a map or array */ -typedef struct MMDB_entry_data_list_s { - MMDB_entry_data_s entry_data; - struct MMDB_entry_data_list_s *next; - void *pool; -} MMDB_entry_data_list_s; - -typedef struct MMDB_description_s { - const char *language; - const char *description; -} MMDB_description_s; - -/* WARNING: do not add new fields to this struct without bumping the SONAME. - * The struct is allocated by the users of this library and increasing the - * size will cause existing users to allocate too little space when the shared - * library is upgraded */ -typedef struct MMDB_metadata_s { - uint32_t node_count; - uint16_t record_size; - uint16_t ip_version; - const char *database_type; - struct { - size_t count; - const char **names; - } languages; - uint16_t binary_format_major_version; - uint16_t binary_format_minor_version; - uint64_t build_epoch; - struct { - size_t count; - MMDB_description_s **descriptions; - } description; - /* See above warning before adding fields */ -} MMDB_metadata_s; - -/* WARNING: do not add new fields to this struct without bumping the SONAME. - * The struct is allocated by the users of this library and increasing the - * size will cause existing users to allocate too little space when the shared - * library is upgraded */ -typedef struct MMDB_ipv4_start_node_s { - uint16_t netmask; - uint32_t node_value; - /* See above warning before adding fields */ -} MMDB_ipv4_start_node_s; - -/* WARNING: do not add new fields to this struct without bumping the SONAME. - * The struct is allocated by the users of this library and increasing the - * size will cause existing users to allocate too little space when the shared - * library is upgraded */ -typedef struct MMDB_s { - uint32_t flags; - const char *filename; - ssize_t file_size; - const uint8_t *file_content; - const uint8_t *data_section; - uint32_t data_section_size; - const uint8_t *metadata_section; - uint32_t metadata_section_size; - uint16_t full_record_byte_size; - uint16_t depth; - MMDB_ipv4_start_node_s ipv4_start_node; - MMDB_metadata_s metadata; - /* See above warning before adding fields */ -} MMDB_s; - -typedef struct MMDB_search_node_s { - uint64_t left_record; - uint64_t right_record; - uint8_t left_record_type; - uint8_t right_record_type; - MMDB_entry_s left_record_entry; - MMDB_entry_s right_record_entry; -} MMDB_search_node_s; - -extern int MMDB_open(const char *const filename, uint32_t flags, - MMDB_s *const mmdb); -extern MMDB_lookup_result_s MMDB_lookup_string(const MMDB_s *const mmdb, - const char *const ipstr, - int *const gai_error, - int *const mmdb_error); -extern MMDB_lookup_result_s MMDB_lookup_sockaddr( - const MMDB_s *const mmdb, - const struct sockaddr *const sockaddr, - int *const mmdb_error); -extern int MMDB_read_node(const MMDB_s *const mmdb, - uint32_t node_number, - MMDB_search_node_s *const node); -extern int MMDB_get_value(MMDB_entry_s *const start, - MMDB_entry_data_s *const entry_data, - ...); -extern int MMDB_vget_value(MMDB_entry_s *const start, - MMDB_entry_data_s *const entry_data, - va_list va_path); -extern int MMDB_aget_value(MMDB_entry_s *const start, - MMDB_entry_data_s *const entry_data, - const char *const *const path); -extern int MMDB_get_metadata_as_entry_data_list( - const MMDB_s *const mmdb, MMDB_entry_data_list_s **const entry_data_list); -extern int MMDB_get_entry_data_list( - MMDB_entry_s *start, MMDB_entry_data_list_s **const entry_data_list); -extern void MMDB_free_entry_data_list( - MMDB_entry_data_list_s *const entry_data_list); -extern void MMDB_close(MMDB_s *const mmdb); -extern const char *MMDB_lib_version(void); -extern int MMDB_dump_entry_data_list(FILE *const stream, - MMDB_entry_data_list_s *const entry_data_list, - int indent); -extern const char *MMDB_strerror(int error_code); - -#endif /* MAXMINDDB_H */ - -#ifdef __cplusplus -} -#endif diff --git a/fluent-bit/plugins/filter_geoip2/libmaxminddb/include/maxminddb_config.h.cmake.in b/fluent-bit/plugins/filter_geoip2/libmaxminddb/include/maxminddb_config.h.cmake.in deleted file mode 100644 index 8b1977f86..000000000 --- a/fluent-bit/plugins/filter_geoip2/libmaxminddb/include/maxminddb_config.h.cmake.in +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef MAXMINDDB_CONFIG_H -#define MAXMINDDB_CONFIG_H - -#ifndef MMDB_UINT128_USING_MODE -/* Define as 1 if we use unsigned int __atribute__ ((__mode__(TI))) for uint128 values */ -#cmakedefine MMDB_UINT128_USING_MODE @MMDB_UINT128_USING_MODE@ -#endif - -#ifndef MMDB_UINT128_IS_BYTE_ARRAY -/* Define as 1 if we don't have an unsigned __int128 type */ -#cmakedefine MMDB_UINT128_IS_BYTE_ARRAY @MMDB_UINT128_IS_BYTE_ARRAY@ -#endif - -#endif /* MAXMINDDB_CONFIG_H */ diff --git a/fluent-bit/plugins/filter_geoip2/libmaxminddb/include/maxminddb_config.h.in b/fluent-bit/plugins/filter_geoip2/libmaxminddb/include/maxminddb_config.h.in deleted file mode 100644 index 314d559d3..000000000 --- a/fluent-bit/plugins/filter_geoip2/libmaxminddb/include/maxminddb_config.h.in +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef MAXMINDDB_CONFIG_H -#define MAXMINDDB_CONFIG_H - -#ifndef MMDB_UINT128_USING_MODE -/* Define as 1 if we use unsigned int __atribute__ ((__mode__(TI))) for uint128 values */ -#define MMDB_UINT128_USING_MODE 0 -#endif - -#ifndef MMDB_UINT128_IS_BYTE_ARRAY -/* Define as 1 if we don't have an unsigned __int128 type */ -#undef MMDB_UINT128_IS_BYTE_ARRAY -#endif - -#endif /* MAXMINDDB_CONFIG_H */ diff --git a/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/Makefile.am b/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/Makefile.am deleted file mode 100644 index 6d57acaae..000000000 --- a/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/Makefile.am +++ /dev/null @@ -1,25 +0,0 @@ -include $(top_srcdir)/common.mk - -lib_LTLIBRARIES = libmaxminddb.la - -libmaxminddb_la_SOURCES = maxminddb.c maxminddb-compat-util.h \ - data-pool.c data-pool.h -libmaxminddb_la_LDFLAGS = -version-info 0:7:0 -export-symbols-regex '^MMDB_.*' -include_HEADERS = $(top_srcdir)/include/maxminddb.h - -pkgconfig_DATA = libmaxminddb.pc - -TESTS = test-data-pool - -check_PROGRAMS = test-data-pool - -test_data_pool_SOURCES = data-pool.c data-pool.h -test_data_pool_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/t -DTEST_DATA_POOL -test_data_pool_LDADD = $(top_srcdir)/t/libmmdbtest.la \ - $(top_srcdir)/t/libtap/libtap.a - -$(top_srcdir)/t/libmmdbtest.la: - $(MAKE) -C $(top_srcdir)/t libmmdbtest.la - -$(top_srcdir)/t/libtap/libtap.a: - $(MAKE) -C $(top_srcdir)/t/libtap libtap.a diff --git a/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/data-pool.c b/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/data-pool.c deleted file mode 100644 index 48521b64d..000000000 --- a/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/data-pool.c +++ /dev/null @@ -1,180 +0,0 @@ -#include "data-pool.h" -#include "maxminddb.h" - -#include -#include -#include - -static bool can_multiply(size_t const, size_t const, size_t const); - -// Allocate an MMDB_data_pool_s. It initially has space for size -// MMDB_entry_data_list_s structs. -MMDB_data_pool_s *data_pool_new(size_t const size) -{ - MMDB_data_pool_s *const pool = calloc(1, sizeof(MMDB_data_pool_s)); - if (!pool) { - return NULL; - } - - if (size == 0 || - !can_multiply(SIZE_MAX, size, sizeof(MMDB_entry_data_list_s))) { - data_pool_destroy(pool); - return NULL; - } - pool->size = size; - pool->blocks[0] = calloc(pool->size, sizeof(MMDB_entry_data_list_s)); - if (!pool->blocks[0]) { - data_pool_destroy(pool); - return NULL; - } - pool->blocks[0]->pool = pool; - - pool->sizes[0] = size; - - pool->block = pool->blocks[0]; - - return pool; -} - -// Determine if we can multiply m*n. We can do this if the result will be below -// the given max. max will typically be SIZE_MAX. -// -// We want to know if we'll wrap around. -static bool can_multiply(size_t const max, size_t const m, size_t const n) -{ - if (m == 0) { - return false; - } - - return n <= max / m; -} - -// Clean up the data pool. -void data_pool_destroy(MMDB_data_pool_s *const pool) -{ - if (!pool) { - return; - } - - for (size_t i = 0; i <= pool->index; i++) { - free(pool->blocks[i]); - } - - free(pool); -} - -// Claim a new struct from the pool. Doing this may cause the pool's size to -// grow. -MMDB_entry_data_list_s *data_pool_alloc(MMDB_data_pool_s *const pool) -{ - if (!pool) { - return NULL; - } - - if (pool->used < pool->size) { - MMDB_entry_data_list_s *const element = pool->block + pool->used; - pool->used++; - return element; - } - - // Take it from a new block of memory. - - size_t const new_index = pool->index + 1; - if (new_index == DATA_POOL_NUM_BLOCKS) { - // See the comment about not growing this on DATA_POOL_NUM_BLOCKS. - return NULL; - } - - if (!can_multiply(SIZE_MAX, pool->size, 2)) { - return NULL; - } - size_t const new_size = pool->size * 2; - - if (!can_multiply(SIZE_MAX, new_size, sizeof(MMDB_entry_data_list_s))) { - return NULL; - } - pool->blocks[new_index] = calloc(new_size, sizeof(MMDB_entry_data_list_s)); - if (!pool->blocks[new_index]) { - return NULL; - } - - // We don't need to set this, but it's useful for introspection in tests. - pool->blocks[new_index]->pool = pool; - - pool->index = new_index; - pool->block = pool->blocks[pool->index]; - - pool->size = new_size; - pool->sizes[pool->index] = pool->size; - - MMDB_entry_data_list_s *const element = pool->block; - pool->used = 1; - return element; -} - -// Turn the structs in the array-like pool into a linked list. -// -// Before calling this function, the list isn't linked up. -MMDB_entry_data_list_s *data_pool_to_list(MMDB_data_pool_s *const pool) -{ - if (!pool) { - return NULL; - } - - if (pool->index == 0 && pool->used == 0) { - return NULL; - } - - for (size_t i = 0; i <= pool->index; i++) { - MMDB_entry_data_list_s *const block = pool->blocks[i]; - - size_t size = pool->sizes[i]; - if (i == pool->index) { - size = pool->used; - } - - for (size_t j = 0; j < size - 1; j++) { - MMDB_entry_data_list_s *const cur = block + j; - cur->next = block + j + 1; - } - - if (i < pool->index) { - MMDB_entry_data_list_s *const last = block + size - 1; - last->next = pool->blocks[i + 1]; - } - } - - return pool->blocks[0]; -} - -#ifdef TEST_DATA_POOL - -#include -#include - -static void test_can_multiply(void); - -int main(void) -{ - plan(NO_PLAN); - test_can_multiply(); - done_testing(); -} - -static void test_can_multiply(void) -{ - { - ok(can_multiply(SIZE_MAX, 1, SIZE_MAX), "1*SIZE_MAX is ok"); - } - - { - ok(!can_multiply(SIZE_MAX, 2, SIZE_MAX), "2*SIZE_MAX is not ok"); - } - - { - ok(can_multiply(SIZE_MAX, 10240, sizeof(MMDB_entry_data_list_s)), - "1024 entry_data_list_s's are okay"); - } -} - -#endif diff --git a/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/data-pool.h b/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/data-pool.h deleted file mode 100644 index 25d09923e..000000000 --- a/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/data-pool.h +++ /dev/null @@ -1,52 +0,0 @@ -#ifndef DATA_POOL_H -#define DATA_POOL_H - -#include "maxminddb.h" - -#include -#include - -// This should be large enough that we never need to grow the array of pointers -// to blocks. 32 is enough. Even starting out of with size 1 (1 struct), the -// 32nd element alone will provide 2**32 structs as we exponentially increase -// the number in each block. Being confident that we do not have to grow the -// array lets us avoid writing code to do that. That code would be risky as it -// would rarely be hit and likely not be well tested. -#define DATA_POOL_NUM_BLOCKS 32 - -// A pool of memory for MMDB_entry_data_list_s structs. This is so we can -// allocate multiple up front rather than one at a time for performance -// reasons. -// -// The order you add elements to it (by calling data_pool_alloc()) ends up as -// the order of the list. -// -// The memory only grows. There is no support for releasing an element you take -// back to the pool. -typedef struct MMDB_data_pool_s { - // Index of the current block we're allocating out of. - size_t index; - - // The size of the current block, counting by structs. - size_t size; - - // How many used in the current block, counting by structs. - size_t used; - - // The current block we're allocating out of. - MMDB_entry_data_list_s *block; - - // The size of each block. - size_t sizes[DATA_POOL_NUM_BLOCKS]; - - // An array of pointers to blocks of memory holding space for list - // elements. - MMDB_entry_data_list_s *blocks[DATA_POOL_NUM_BLOCKS]; -} MMDB_data_pool_s; - -MMDB_data_pool_s *data_pool_new(size_t const); -void data_pool_destroy(MMDB_data_pool_s *const); -MMDB_entry_data_list_s *data_pool_alloc(MMDB_data_pool_s *const); -MMDB_entry_data_list_s *data_pool_to_list(MMDB_data_pool_s *const); - -#endif diff --git a/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/libmaxminddb.pc.in b/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/libmaxminddb.pc.in deleted file mode 100644 index 00ced3ba9..000000000 --- a/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/libmaxminddb.pc.in +++ /dev/null @@ -1,11 +0,0 @@ -prefix=@prefix@ -exec_prefix=@prefix@ -libdir=@libdir@ -includedir=@includedir@ - -Name: libmaxminddb -Description: C library for the MaxMind DB file format -URL: http://maxmind.github.io/libmaxminddb/ -Version: @PACKAGE_VERSION@ -Libs: -L${libdir} -lmaxminddb -Cflags: -I${includedir} diff --git a/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/maxminddb-compat-util.h b/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/maxminddb-compat-util.h deleted file mode 100644 index e3f0320f2..000000000 --- a/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/maxminddb-compat-util.h +++ /dev/null @@ -1,167 +0,0 @@ -#include -#include - -/* *INDENT-OFF* */ - -/* The memmem, strdup, and strndup functions were all copied from the - * FreeBSD source, along with the relevant copyright notice. - * - * It'd be nicer to simply use the functions available on the system if they - * exist, but there doesn't seem to be a good way to detect them without also - * defining things like _GNU_SOURCE, which we want to avoid, because then we - * end up _accidentally_ using GNU features without noticing, which then - * breaks on systems like OSX. - * - * C is fun! */ - -/* Applies to memmem implementation */ -/*- - * Copyright (c) 2005 Pascal Gloor - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. The name of the author may not be used to endorse or promote - * products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ -static void * -mmdb_memmem(const void *l, size_t l_len, const void *s, size_t s_len) -{ - register char *cur, *last; - const char *cl = (const char *)l; - const char *cs = (const char *)s; - - /* we need something to compare */ - if (l_len == 0 || s_len == 0) - return NULL; - - /* "s" must be smaller or equal to "l" */ - if (l_len < s_len) - return NULL; - - /* special case where s_len == 1 */ - if (s_len == 1) - return memchr(l, (int)*cs, l_len); - - /* the last position where its possible to find "s" in "l" */ - last = (char *)cl + l_len - s_len; - - for (cur = (char *)cl; cur <= last; cur++) - if (cur[0] == cs[0] && memcmp(cur, cs, s_len) == 0) - return cur; - - return NULL; -} - -/* Applies to strnlen implementation */ -/*- - * Copyright (c) 2009 David Schultz - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ -static size_t -mmdb_strnlen(const char *s, size_t maxlen) -{ - size_t len; - - for (len = 0; len < maxlen; len++, s++) { - if (!*s) - break; - } - return (len); -} - -/* Applies to strdup and strndup implementation */ -/* - * Copyright (c) 1988, 1993 - * The Regents of the University of California. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the University nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ -static char * -mmdb_strdup(const char *str) -{ - size_t len; - char *copy; - - len = strlen(str) + 1; - if ((copy = malloc(len)) == NULL) - return (NULL); - memcpy(copy, str, len); - return (copy); -} - -static char * -mmdb_strndup(const char *str, size_t n) -{ - size_t len; - char *copy; - - len = mmdb_strnlen(str, n); - if ((copy = malloc(len + 1)) == NULL) - return (NULL); - memcpy(copy, str, len); - copy[len] = '\0'; - return (copy); -} -/* *INDENT-ON* */ diff --git a/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/maxminddb.c b/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/maxminddb.c deleted file mode 100644 index 427f48afd..000000000 --- a/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/maxminddb.c +++ /dev/null @@ -1,2157 +0,0 @@ -#if HAVE_CONFIG_H -#include -#endif -#include "data-pool.h" -#include "maxminddb.h" -#include "maxminddb-compat-util.h" -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef _WIN32 -#ifndef UNICODE -#define UNICODE -#endif -#include -#include -#else -#include -#include -#include -#endif - -#define MMDB_DATA_SECTION_SEPARATOR (16) -#define MAXIMUM_DATA_STRUCTURE_DEPTH (512) - -#ifdef MMDB_DEBUG -#define LOCAL -#define DEBUG_MSG(msg) fprintf(stderr, msg "\n") -#define DEBUG_MSGF(fmt, ...) fprintf(stderr, fmt "\n", __VA_ARGS__) -#define DEBUG_BINARY(fmt, byte) \ - do { \ - char *binary = byte_to_binary(byte); \ - if (NULL == binary) { \ - fprintf(stderr, "Calloc failed in DEBUG_BINARY\n"); \ - abort(); \ - } \ - fprintf(stderr, fmt "\n", binary); \ - free(binary); \ - } while (0) -#define DEBUG_NL fprintf(stderr, "\n") -#else -#define LOCAL static -#define DEBUG_MSG(...) -#define DEBUG_MSGF(...) -#define DEBUG_BINARY(...) -#define DEBUG_NL -#endif - -#ifdef MMDB_DEBUG -char *byte_to_binary(uint8_t byte) -{ - char *bits = calloc(9, sizeof(char)); - if (NULL == bits) { - return bits; - } - - for (uint8_t i = 0; i < 8; i++) { - bits[i] = byte & (128 >> i) ? '1' : '0'; - } - bits[8] = '\0'; - - return bits; -} - -char *type_num_to_name(uint8_t num) -{ - switch (num) { - case 0: - return "extended"; - case 1: - return "pointer"; - case 2: - return "utf8_string"; - case 3: - return "double"; - case 4: - return "bytes"; - case 5: - return "uint16"; - case 6: - return "uint32"; - case 7: - return "map"; - case 8: - return "int32"; - case 9: - return "uint64"; - case 10: - return "uint128"; - case 11: - return "array"; - case 12: - return "container"; - case 13: - return "end_marker"; - case 14: - return "boolean"; - case 15: - return "float"; - default: - return "unknown type"; - } -} -#endif - -/* None of the values we check on the lhs are bigger than uint32_t, so on - * platforms where SIZE_MAX is a 64-bit integer, this would be a no-op, and it - * makes the compiler complain if we do the check anyway. */ -#if SIZE_MAX == UINT32_MAX -#define MAYBE_CHECK_SIZE_OVERFLOW(lhs, rhs, error) \ - if ((lhs) > (rhs)) { \ - return error; \ - } -#else -#define MAYBE_CHECK_SIZE_OVERFLOW(...) -#endif - -typedef struct record_info_s { - uint16_t record_length; - uint32_t (*left_record_getter)(const uint8_t *); - uint32_t (*right_record_getter)(const uint8_t *); - uint8_t right_record_offset; -} record_info_s; - -#define METADATA_MARKER "\xab\xcd\xefMaxMind.com" -/* This is 128kb */ -#define METADATA_BLOCK_MAX_SIZE 131072 - -// 64 leads us to allocating 4 KiB on a 64bit system. -#define MMDB_POOL_INIT_SIZE 64 - -LOCAL int map_file(MMDB_s *const mmdb); -LOCAL const uint8_t *find_metadata(const uint8_t *file_content, - ssize_t file_size, uint32_t *metadata_size); -LOCAL int read_metadata(MMDB_s *mmdb); -LOCAL MMDB_s make_fake_metadata_db(const MMDB_s *const mmdb); -LOCAL int value_for_key_as_uint16(MMDB_entry_s *start, char *key, - uint16_t *value); -LOCAL int value_for_key_as_uint32(MMDB_entry_s *start, char *key, - uint32_t *value); -LOCAL int value_for_key_as_uint64(MMDB_entry_s *start, char *key, - uint64_t *value); -LOCAL int value_for_key_as_string(MMDB_entry_s *start, char *key, - char const **value); -LOCAL int populate_languages_metadata(MMDB_s *mmdb, MMDB_s *metadata_db, - MMDB_entry_s *metadata_start); -LOCAL int populate_description_metadata(MMDB_s *mmdb, MMDB_s *metadata_db, - MMDB_entry_s *metadata_start); -LOCAL int resolve_any_address(const char *ipstr, struct addrinfo **addresses); -LOCAL int find_address_in_search_tree(const MMDB_s *const mmdb, - uint8_t *address, - sa_family_t address_family, - MMDB_lookup_result_s *result); -LOCAL record_info_s record_info_for_database(const MMDB_s *const mmdb); -LOCAL int find_ipv4_start_node(MMDB_s *const mmdb); -LOCAL uint8_t record_type(const MMDB_s *const mmdb, uint64_t record); -LOCAL uint32_t get_left_28_bit_record(const uint8_t *record); -LOCAL uint32_t get_right_28_bit_record(const uint8_t *record); -LOCAL uint32_t data_section_offset_for_record(const MMDB_s *const mmdb, - uint64_t record); -LOCAL int path_length(va_list va_path); -LOCAL int lookup_path_in_array(const char *path_elem, const MMDB_s *const mmdb, - MMDB_entry_data_s *entry_data); -LOCAL int lookup_path_in_map(const char *path_elem, const MMDB_s *const mmdb, - MMDB_entry_data_s *entry_data); -LOCAL int skip_map_or_array(const MMDB_s *const mmdb, - MMDB_entry_data_s *entry_data); -LOCAL int decode_one_follow(const MMDB_s *const mmdb, uint32_t offset, - MMDB_entry_data_s *entry_data); -LOCAL int decode_one(const MMDB_s *const mmdb, uint32_t offset, - MMDB_entry_data_s *entry_data); -LOCAL int get_ext_type(int raw_ext_type); -LOCAL uint32_t get_ptr_from(uint8_t ctrl, uint8_t const *const ptr, - int ptr_size); -LOCAL int get_entry_data_list(const MMDB_s *const mmdb, - uint32_t offset, - MMDB_entry_data_list_s *const entry_data_list, - MMDB_data_pool_s *const pool, - int depth); -LOCAL float get_ieee754_float(const uint8_t *restrict p); -LOCAL double get_ieee754_double(const uint8_t *restrict p); -LOCAL uint32_t get_uint32(const uint8_t *p); -LOCAL uint32_t get_uint24(const uint8_t *p); -LOCAL uint32_t get_uint16(const uint8_t *p); -LOCAL uint64_t get_uintX(const uint8_t *p, int length); -LOCAL int32_t get_sintX(const uint8_t *p, int length); -LOCAL void free_mmdb_struct(MMDB_s *const mmdb); -LOCAL void free_languages_metadata(MMDB_s *mmdb); -LOCAL void free_descriptions_metadata(MMDB_s *mmdb); -LOCAL MMDB_entry_data_list_s *dump_entry_data_list( - FILE *stream, MMDB_entry_data_list_s *entry_data_list, int indent, - int *status); -LOCAL void print_indentation(FILE *stream, int i); -LOCAL char *bytes_to_hex(uint8_t *bytes, uint32_t size); - -#define CHECKED_DECODE_ONE(mmdb, offset, entry_data) \ - do { \ - int status = decode_one(mmdb, offset, entry_data); \ - if (MMDB_SUCCESS != status) { \ - DEBUG_MSGF("CHECKED_DECODE_ONE failed." \ - " status = %d (%s)", status, MMDB_strerror(status)); \ - return status; \ - } \ - } while (0) - -#define CHECKED_DECODE_ONE_FOLLOW(mmdb, offset, entry_data) \ - do { \ - int status = decode_one_follow(mmdb, offset, entry_data); \ - if (MMDB_SUCCESS != status) { \ - DEBUG_MSGF("CHECKED_DECODE_ONE_FOLLOW failed." \ - " status = %d (%s)", status, MMDB_strerror(status)); \ - return status; \ - } \ - } while (0) - -#define FREE_AND_SET_NULL(p) { free((void *)(p)); (p) = NULL; } - -int MMDB_open(const char *const filename, uint32_t flags, MMDB_s *const mmdb) -{ - int status = MMDB_SUCCESS; - - mmdb->file_content = NULL; - mmdb->data_section = NULL; - mmdb->metadata.database_type = NULL; - mmdb->metadata.languages.count = 0; - mmdb->metadata.languages.names = NULL; - mmdb->metadata.description.count = 0; - - mmdb->filename = mmdb_strdup(filename); - if (NULL == mmdb->filename) { - status = MMDB_OUT_OF_MEMORY_ERROR; - goto cleanup; - } - - if ((flags & MMDB_MODE_MASK) == 0) { - flags |= MMDB_MODE_MMAP; - } - mmdb->flags = flags; - - if (MMDB_SUCCESS != (status = map_file(mmdb))) { - goto cleanup; - } - -#ifdef _WIN32 - WSADATA wsa; - WSAStartup(MAKEWORD(2, 2), &wsa); -#endif - - uint32_t metadata_size = 0; - const uint8_t *metadata = find_metadata(mmdb->file_content, mmdb->file_size, - &metadata_size); - if (NULL == metadata) { - status = MMDB_INVALID_METADATA_ERROR; - goto cleanup; - } - - mmdb->metadata_section = metadata; - mmdb->metadata_section_size = metadata_size; - - status = read_metadata(mmdb); - if (MMDB_SUCCESS != status) { - goto cleanup; - } - - if (mmdb->metadata.binary_format_major_version != 2) { - status = MMDB_UNKNOWN_DATABASE_FORMAT_ERROR; - goto cleanup; - } - - uint32_t search_tree_size = mmdb->metadata.node_count * - mmdb->full_record_byte_size; - - mmdb->data_section = mmdb->file_content + search_tree_size - + MMDB_DATA_SECTION_SEPARATOR; - if (search_tree_size + MMDB_DATA_SECTION_SEPARATOR > - (uint32_t)mmdb->file_size) { - status = MMDB_INVALID_METADATA_ERROR; - goto cleanup; - } - mmdb->data_section_size = (uint32_t)mmdb->file_size - search_tree_size - - MMDB_DATA_SECTION_SEPARATOR; - - // Although it is likely not possible to construct a database with valid - // valid metadata, as parsed above, and a data_section_size less than 3, - // we do this check as later we assume it is at least three when doing - // bound checks. - if (mmdb->data_section_size < 3) { - status = MMDB_INVALID_DATA_ERROR; - goto cleanup; - } - - mmdb->metadata_section = metadata; - mmdb->ipv4_start_node.node_value = 0; - mmdb->ipv4_start_node.netmask = 0; - - // We do this immediately as otherwise there is a race to set - // ipv4_start_node.node_value and ipv4_start_node.netmask. - if (mmdb->metadata.ip_version == 6) { - status = find_ipv4_start_node(mmdb); - if (status != MMDB_SUCCESS) { - goto cleanup; - } - } - - cleanup: - if (MMDB_SUCCESS != status) { - int saved_errno = errno; - free_mmdb_struct(mmdb); - errno = saved_errno; - } - return status; -} - -#ifdef _WIN32 - -LOCAL LPWSTR utf8_to_utf16(const char *utf8_str) -{ - int wide_chars = MultiByteToWideChar(CP_UTF8, 0, utf8_str, -1, NULL, 0); - wchar_t *utf16_str = (wchar_t *)calloc(wide_chars, sizeof(wchar_t)); - - if (MultiByteToWideChar(CP_UTF8, 0, utf8_str, -1, utf16_str, - wide_chars) < 1) { - free(utf16_str); - return NULL; - } - - return utf16_str; -} - -LOCAL int map_file(MMDB_s *const mmdb) -{ - DWORD size; - int status = MMDB_SUCCESS; - HANDLE mmh = NULL; - HANDLE fd = INVALID_HANDLE_VALUE; - LPWSTR utf16_filename = utf8_to_utf16(mmdb->filename); - if (!utf16_filename) { - status = MMDB_FILE_OPEN_ERROR; - goto cleanup; - } - fd = CreateFileW(utf16_filename, GENERIC_READ, FILE_SHARE_READ, NULL, - OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); - if (fd == INVALID_HANDLE_VALUE) { - status = MMDB_FILE_OPEN_ERROR; - goto cleanup; - } - size = GetFileSize(fd, NULL); - if (size == INVALID_FILE_SIZE) { - status = MMDB_FILE_OPEN_ERROR; - goto cleanup; - } - mmh = CreateFileMapping(fd, NULL, PAGE_READONLY, 0, size, NULL); - /* Microsoft documentation for CreateFileMapping indicates this returns - NULL not INVALID_HANDLE_VALUE on error */ - if (NULL == mmh) { - status = MMDB_IO_ERROR; - goto cleanup; - } - uint8_t *file_content = - (uint8_t *)MapViewOfFile(mmh, FILE_MAP_READ, 0, 0, 0); - if (file_content == NULL) { - status = MMDB_IO_ERROR; - goto cleanup; - } - - mmdb->file_size = size; - mmdb->file_content = file_content; - - cleanup:; - int saved_errno = errno; - if (INVALID_HANDLE_VALUE != fd) { - CloseHandle(fd); - } - if (NULL != mmh) { - CloseHandle(mmh); - } - errno = saved_errno; - free(utf16_filename); - - return status; -} - -#else // _WIN32 - -LOCAL int map_file(MMDB_s *const mmdb) -{ - ssize_t size; - int status = MMDB_SUCCESS; - - int flags = O_RDONLY; -#ifdef O_CLOEXEC - flags |= O_CLOEXEC; -#endif - int fd = open(mmdb->filename, flags); - struct stat s; - if (fd < 0 || fstat(fd, &s)) { - status = MMDB_FILE_OPEN_ERROR; - goto cleanup; - } - - size = s.st_size; - if (size < 0 || size != s.st_size) { - status = MMDB_OUT_OF_MEMORY_ERROR; - goto cleanup; - } - - uint8_t *file_content = - (uint8_t *)mmap(NULL, size, PROT_READ, MAP_SHARED, fd, 0); - if (MAP_FAILED == file_content) { - if (ENOMEM == errno) { - status = MMDB_OUT_OF_MEMORY_ERROR; - } else { - status = MMDB_IO_ERROR; - } - goto cleanup; - } - - mmdb->file_size = size; - mmdb->file_content = file_content; - - cleanup:; - int saved_errno = errno; - if (fd >= 0) { - close(fd); - } - errno = saved_errno; - - return status; -} - -#endif // _WIN32 - -LOCAL const uint8_t *find_metadata(const uint8_t *file_content, - ssize_t file_size, uint32_t *metadata_size) -{ - const ssize_t marker_len = sizeof(METADATA_MARKER) - 1; - ssize_t max_size = file_size > - METADATA_BLOCK_MAX_SIZE ? METADATA_BLOCK_MAX_SIZE : - file_size; - - uint8_t *search_area = (uint8_t *)(file_content + (file_size - max_size)); - uint8_t *start = search_area; - uint8_t *tmp; - do { - tmp = mmdb_memmem(search_area, max_size, - METADATA_MARKER, marker_len); - - if (NULL != tmp) { - max_size -= tmp - search_area; - search_area = tmp; - - /* Continue searching just after the marker we just read, in case - * there are multiple markers in the same file. This would be odd - * but is certainly not impossible. */ - max_size -= marker_len; - search_area += marker_len; - } - } while (NULL != tmp); - - if (search_area == start) { - return NULL; - } - - *metadata_size = (uint32_t)max_size; - - return search_area; -} - -LOCAL int read_metadata(MMDB_s *mmdb) -{ - /* We need to create a fake MMDB_s struct in order to decode values from - the metadata. The metadata is basically just like the data section, so we - want to use the same functions we use for the data section to get metadata - values. */ - MMDB_s metadata_db = make_fake_metadata_db(mmdb); - - MMDB_entry_s metadata_start = { - .mmdb = &metadata_db, - .offset = 0 - }; - - int status = - value_for_key_as_uint32(&metadata_start, "node_count", - &mmdb->metadata.node_count); - if (MMDB_SUCCESS != status) { - return status; - } - if (!mmdb->metadata.node_count) { - DEBUG_MSG("could not find node_count value in metadata"); - return MMDB_INVALID_METADATA_ERROR; - } - - status = value_for_key_as_uint16(&metadata_start, "record_size", - &mmdb->metadata.record_size); - if (MMDB_SUCCESS != status) { - return status; - } - if (!mmdb->metadata.record_size) { - DEBUG_MSG("could not find record_size value in metadata"); - return MMDB_INVALID_METADATA_ERROR; - } - - if (mmdb->metadata.record_size != 24 && mmdb->metadata.record_size != 28 - && mmdb->metadata.record_size != 32) { - DEBUG_MSGF("bad record size in metadata: %i", - mmdb->metadata.record_size); - return MMDB_UNKNOWN_DATABASE_FORMAT_ERROR; - } - - status = value_for_key_as_uint16(&metadata_start, "ip_version", - &mmdb->metadata.ip_version); - if (MMDB_SUCCESS != status) { - return status; - } - if (!mmdb->metadata.ip_version) { - DEBUG_MSG("could not find ip_version value in metadata"); - return MMDB_INVALID_METADATA_ERROR; - } - if (!(mmdb->metadata.ip_version == 4 || mmdb->metadata.ip_version == 6)) { - DEBUG_MSGF("ip_version value in metadata is not 4 or 6 - it was %i", - mmdb->metadata.ip_version); - return MMDB_INVALID_METADATA_ERROR; - } - - status = value_for_key_as_string(&metadata_start, "database_type", - &mmdb->metadata.database_type); - if (MMDB_SUCCESS != status) { - DEBUG_MSG("error finding database_type value in metadata"); - return status; - } - - status = - populate_languages_metadata(mmdb, &metadata_db, &metadata_start); - if (MMDB_SUCCESS != status) { - DEBUG_MSG("could not populate languages from metadata"); - return status; - } - - status = value_for_key_as_uint16( - &metadata_start, "binary_format_major_version", - &mmdb->metadata.binary_format_major_version); - if (MMDB_SUCCESS != status) { - return status; - } - if (!mmdb->metadata.binary_format_major_version) { - DEBUG_MSG( - "could not find binary_format_major_version value in metadata"); - return MMDB_INVALID_METADATA_ERROR; - } - - status = value_for_key_as_uint16( - &metadata_start, "binary_format_minor_version", - &mmdb->metadata.binary_format_minor_version); - if (MMDB_SUCCESS != status) { - return status; - } - - status = value_for_key_as_uint64(&metadata_start, "build_epoch", - &mmdb->metadata.build_epoch); - if (MMDB_SUCCESS != status) { - return status; - } - if (!mmdb->metadata.build_epoch) { - DEBUG_MSG("could not find build_epoch value in metadata"); - return MMDB_INVALID_METADATA_ERROR; - } - - status = populate_description_metadata(mmdb, &metadata_db, &metadata_start); - if (MMDB_SUCCESS != status) { - DEBUG_MSG("could not populate description from metadata"); - return status; - } - - mmdb->full_record_byte_size = mmdb->metadata.record_size * 2 / 8U; - - mmdb->depth = mmdb->metadata.ip_version == 4 ? 32 : 128; - - return MMDB_SUCCESS; -} - -LOCAL MMDB_s make_fake_metadata_db(const MMDB_s *const mmdb) -{ - MMDB_s fake_metadata_db = { - .data_section = mmdb->metadata_section, - .data_section_size = mmdb->metadata_section_size - }; - - return fake_metadata_db; -} - -LOCAL int value_for_key_as_uint16(MMDB_entry_s *start, char *key, - uint16_t *value) -{ - MMDB_entry_data_s entry_data; - const char *path[] = { key, NULL }; - int status = MMDB_aget_value(start, &entry_data, path); - if (MMDB_SUCCESS != status) { - return status; - } - if (MMDB_DATA_TYPE_UINT16 != entry_data.type) { - DEBUG_MSGF("expect uint16 for %s but received %s", key, - type_num_to_name( - entry_data.type)); - return MMDB_INVALID_METADATA_ERROR; - } - *value = entry_data.uint16; - return MMDB_SUCCESS; -} - -LOCAL int value_for_key_as_uint32(MMDB_entry_s *start, char *key, - uint32_t *value) -{ - MMDB_entry_data_s entry_data; - const char *path[] = { key, NULL }; - int status = MMDB_aget_value(start, &entry_data, path); - if (MMDB_SUCCESS != status) { - return status; - } - if (MMDB_DATA_TYPE_UINT32 != entry_data.type) { - DEBUG_MSGF("expect uint32 for %s but received %s", key, - type_num_to_name( - entry_data.type)); - return MMDB_INVALID_METADATA_ERROR; - } - *value = entry_data.uint32; - return MMDB_SUCCESS; -} - -LOCAL int value_for_key_as_uint64(MMDB_entry_s *start, char *key, - uint64_t *value) -{ - MMDB_entry_data_s entry_data; - const char *path[] = { key, NULL }; - int status = MMDB_aget_value(start, &entry_data, path); - if (MMDB_SUCCESS != status) { - return status; - } - if (MMDB_DATA_TYPE_UINT64 != entry_data.type) { - DEBUG_MSGF("expect uint64 for %s but received %s", key, - type_num_to_name( - entry_data.type)); - return MMDB_INVALID_METADATA_ERROR; - } - *value = entry_data.uint64; - return MMDB_SUCCESS; -} - -LOCAL int value_for_key_as_string(MMDB_entry_s *start, char *key, - char const **value) -{ - MMDB_entry_data_s entry_data; - const char *path[] = { key, NULL }; - int status = MMDB_aget_value(start, &entry_data, path); - if (MMDB_SUCCESS != status) { - return status; - } - if (MMDB_DATA_TYPE_UTF8_STRING != entry_data.type) { - DEBUG_MSGF("expect string for %s but received %s", key, - type_num_to_name( - entry_data.type)); - return MMDB_INVALID_METADATA_ERROR; - } - *value = mmdb_strndup((char *)entry_data.utf8_string, entry_data.data_size); - if (NULL == *value) { - return MMDB_OUT_OF_MEMORY_ERROR; - } - return MMDB_SUCCESS; -} - -LOCAL int populate_languages_metadata(MMDB_s *mmdb, MMDB_s *metadata_db, - MMDB_entry_s *metadata_start) -{ - MMDB_entry_data_s entry_data; - - const char *path[] = { "languages", NULL }; - int status = MMDB_aget_value(metadata_start, &entry_data, path); - if (MMDB_SUCCESS != status) { - return status; - } - if (MMDB_DATA_TYPE_ARRAY != entry_data.type) { - return MMDB_INVALID_METADATA_ERROR; - } - - MMDB_entry_s array_start = { - .mmdb = metadata_db, - .offset = entry_data.offset - }; - - MMDB_entry_data_list_s *member; - status = MMDB_get_entry_data_list(&array_start, &member); - if (MMDB_SUCCESS != status) { - return status; - } - - MMDB_entry_data_list_s *first_member = member; - - uint32_t array_size = member->entry_data.data_size; - MAYBE_CHECK_SIZE_OVERFLOW(array_size, SIZE_MAX / sizeof(char *), - MMDB_INVALID_METADATA_ERROR); - - mmdb->metadata.languages.count = 0; - mmdb->metadata.languages.names = calloc(array_size, sizeof(char *)); - if (NULL == mmdb->metadata.languages.names) { - return MMDB_OUT_OF_MEMORY_ERROR; - } - - for (uint32_t i = 0; i < array_size; i++) { - member = member->next; - if (MMDB_DATA_TYPE_UTF8_STRING != member->entry_data.type) { - return MMDB_INVALID_METADATA_ERROR; - } - - mmdb->metadata.languages.names[i] = - mmdb_strndup((char *)member->entry_data.utf8_string, - member->entry_data.data_size); - - if (NULL == mmdb->metadata.languages.names[i]) { - return MMDB_OUT_OF_MEMORY_ERROR; - } - // We assign this as we go so that if we fail a calloc and need to - // free it, the count is right. - mmdb->metadata.languages.count = i + 1; - } - - MMDB_free_entry_data_list(first_member); - - return MMDB_SUCCESS; -} - -LOCAL int populate_description_metadata(MMDB_s *mmdb, MMDB_s *metadata_db, - MMDB_entry_s *metadata_start) -{ - MMDB_entry_data_s entry_data; - - const char *path[] = { "description", NULL }; - int status = MMDB_aget_value(metadata_start, &entry_data, path); - if (MMDB_SUCCESS != status) { - return status; - } - - if (MMDB_DATA_TYPE_MAP != entry_data.type) { - DEBUG_MSGF("Unexpected entry_data type: %d", entry_data.type); - return MMDB_INVALID_METADATA_ERROR; - } - - MMDB_entry_s map_start = { - .mmdb = metadata_db, - .offset = entry_data.offset - }; - - MMDB_entry_data_list_s *member; - status = MMDB_get_entry_data_list(&map_start, &member); - if (MMDB_SUCCESS != status) { - DEBUG_MSGF( - "MMDB_get_entry_data_list failed while populating description." - " status = %d (%s)", status, MMDB_strerror(status)); - return status; - } - - MMDB_entry_data_list_s *first_member = member; - - uint32_t map_size = member->entry_data.data_size; - mmdb->metadata.description.count = 0; - if (0 == map_size) { - mmdb->metadata.description.descriptions = NULL; - goto cleanup; - } - MAYBE_CHECK_SIZE_OVERFLOW(map_size, SIZE_MAX / sizeof(MMDB_description_s *), - MMDB_INVALID_METADATA_ERROR); - - mmdb->metadata.description.descriptions = - calloc(map_size, sizeof(MMDB_description_s *)); - if (NULL == mmdb->metadata.description.descriptions) { - status = MMDB_OUT_OF_MEMORY_ERROR; - goto cleanup; - } - - for (uint32_t i = 0; i < map_size; i++) { - mmdb->metadata.description.descriptions[i] = - calloc(1, sizeof(MMDB_description_s)); - if (NULL == mmdb->metadata.description.descriptions[i]) { - status = MMDB_OUT_OF_MEMORY_ERROR; - goto cleanup; - } - - mmdb->metadata.description.count = i + 1; - mmdb->metadata.description.descriptions[i]->language = NULL; - mmdb->metadata.description.descriptions[i]->description = NULL; - - member = member->next; - - if (MMDB_DATA_TYPE_UTF8_STRING != member->entry_data.type) { - status = MMDB_INVALID_METADATA_ERROR; - goto cleanup; - } - - mmdb->metadata.description.descriptions[i]->language = - mmdb_strndup((char *)member->entry_data.utf8_string, - member->entry_data.data_size); - - if (NULL == mmdb->metadata.description.descriptions[i]->language) { - status = MMDB_OUT_OF_MEMORY_ERROR; - goto cleanup; - } - - member = member->next; - - if (MMDB_DATA_TYPE_UTF8_STRING != member->entry_data.type) { - status = MMDB_INVALID_METADATA_ERROR; - goto cleanup; - } - - mmdb->metadata.description.descriptions[i]->description = - mmdb_strndup((char *)member->entry_data.utf8_string, - member->entry_data.data_size); - - if (NULL == mmdb->metadata.description.descriptions[i]->description) { - status = MMDB_OUT_OF_MEMORY_ERROR; - goto cleanup; - } - } - - cleanup: - MMDB_free_entry_data_list(first_member); - - return status; -} - -MMDB_lookup_result_s MMDB_lookup_string(const MMDB_s *const mmdb, - const char *const ipstr, - int *const gai_error, - int *const mmdb_error) -{ - MMDB_lookup_result_s result = { - .found_entry = false, - .netmask = 0, - .entry = { - .mmdb = mmdb, - .offset = 0 - } - }; - - struct addrinfo *addresses = NULL; - *gai_error = resolve_any_address(ipstr, &addresses); - - if (!*gai_error) { - result = MMDB_lookup_sockaddr(mmdb, addresses->ai_addr, mmdb_error); - } - - if (NULL != addresses) { - freeaddrinfo(addresses); - } - - return result; -} - -LOCAL int resolve_any_address(const char *ipstr, struct addrinfo **addresses) -{ - struct addrinfo hints = { - .ai_family = AF_UNSPEC, - .ai_flags = AI_NUMERICHOST, - // We set ai_socktype so that we only get one result back - .ai_socktype = SOCK_STREAM - }; - - int gai_status = getaddrinfo(ipstr, NULL, &hints, addresses); - if (gai_status) { - return gai_status; - } - - return 0; -} - -MMDB_lookup_result_s MMDB_lookup_sockaddr( - const MMDB_s *const mmdb, - const struct sockaddr *const sockaddr, - int *const mmdb_error) -{ - MMDB_lookup_result_s result = { - .found_entry = false, - .netmask = 0, - .entry = { - .mmdb = mmdb, - .offset = 0 - } - }; - - uint8_t mapped_address[16], *address; - if (mmdb->metadata.ip_version == 4) { - if (sockaddr->sa_family == AF_INET6) { - *mmdb_error = MMDB_IPV6_LOOKUP_IN_IPV4_DATABASE_ERROR; - return result; - } - address = (uint8_t *)&((struct sockaddr_in *)sockaddr)->sin_addr.s_addr; - } else { - if (sockaddr->sa_family == AF_INET6) { - address = - (uint8_t *)&((struct sockaddr_in6 *)sockaddr)->sin6_addr. - s6_addr; - } else { - address = mapped_address; - memset(address, 0, 12); - memcpy(address + 12, - &((struct sockaddr_in *)sockaddr)->sin_addr.s_addr, 4); - } - } - - *mmdb_error = - find_address_in_search_tree(mmdb, address, sockaddr->sa_family, - &result); - - return result; -} - -LOCAL int find_address_in_search_tree(const MMDB_s *const mmdb, - uint8_t *address, - sa_family_t address_family, - MMDB_lookup_result_s *result) -{ - record_info_s record_info = record_info_for_database(mmdb); - if (0 == record_info.right_record_offset) { - return MMDB_UNKNOWN_DATABASE_FORMAT_ERROR; - } - - uint32_t value = 0; - uint16_t current_bit = 0; - if (mmdb->metadata.ip_version == 6 && address_family == AF_INET) { - value = mmdb->ipv4_start_node.node_value; - current_bit = mmdb->ipv4_start_node.netmask; - } - - uint32_t node_count = mmdb->metadata.node_count; - const uint8_t *search_tree = mmdb->file_content; - const uint8_t *record_pointer; - for (; current_bit < mmdb->depth && value < node_count; current_bit++) { - uint8_t bit = 1U & - (address[current_bit >> 3] >> (7 - (current_bit % 8))); - - record_pointer = &search_tree[value * record_info.record_length]; - if (record_pointer + record_info.record_length > mmdb->data_section) { - return MMDB_CORRUPT_SEARCH_TREE_ERROR; - } - if (bit) { - record_pointer += record_info.right_record_offset; - value = record_info.right_record_getter(record_pointer); - } else { - value = record_info.left_record_getter(record_pointer); - } - } - - result->netmask = current_bit; - - if (value >= node_count + mmdb->data_section_size) { - // The pointer points off the end of the database. - return MMDB_CORRUPT_SEARCH_TREE_ERROR; - } - - if (value == node_count) { - // record is empty - result->found_entry = false; - return MMDB_SUCCESS; - } - result->found_entry = true; - result->entry.offset = data_section_offset_for_record(mmdb, value); - - return MMDB_SUCCESS; -} - -LOCAL record_info_s record_info_for_database(const MMDB_s *const mmdb) -{ - record_info_s record_info = { - .record_length = mmdb->full_record_byte_size, - .right_record_offset = 0 - }; - - if (record_info.record_length == 6) { - record_info.left_record_getter = &get_uint24; - record_info.right_record_getter = &get_uint24; - record_info.right_record_offset = 3; - } else if (record_info.record_length == 7) { - record_info.left_record_getter = &get_left_28_bit_record; - record_info.right_record_getter = &get_right_28_bit_record; - record_info.right_record_offset = 3; - } else if (record_info.record_length == 8) { - record_info.left_record_getter = &get_uint32; - record_info.right_record_getter = &get_uint32; - record_info.right_record_offset = 4; - } else { - assert(false); - } - - return record_info; -} - -LOCAL int find_ipv4_start_node(MMDB_s *const mmdb) -{ - /* In a pathological case of a database with a single node search tree, - * this check will be true even after we've found the IPv4 start node, but - * that doesn't seem worth trying to fix. */ - if (mmdb->ipv4_start_node.node_value != 0) { - return MMDB_SUCCESS; - } - - record_info_s record_info = record_info_for_database(mmdb); - - const uint8_t *search_tree = mmdb->file_content; - uint32_t node_value = 0; - const uint8_t *record_pointer; - uint16_t netmask; - uint32_t node_count = mmdb->metadata.node_count; - - for (netmask = 0; netmask < 96 && node_value < node_count; netmask++) { - record_pointer = &search_tree[node_value * record_info.record_length]; - if (record_pointer + record_info.record_length > mmdb->data_section) { - return MMDB_CORRUPT_SEARCH_TREE_ERROR; - } - node_value = record_info.left_record_getter(record_pointer); - } - - mmdb->ipv4_start_node.node_value = node_value; - mmdb->ipv4_start_node.netmask = netmask; - - return MMDB_SUCCESS; -} - -LOCAL uint8_t record_type(const MMDB_s *const mmdb, uint64_t record) -{ - uint32_t node_count = mmdb->metadata.node_count; - - /* Ideally we'd check to make sure that a record never points to a - * previously seen value, but that's more complicated. For now, we can - * at least check that we don't end up at the top of the tree again. */ - if (record == 0) { - DEBUG_MSG("record has a value of 0"); - return MMDB_RECORD_TYPE_INVALID; - } - - if (record < node_count) { - return MMDB_RECORD_TYPE_SEARCH_NODE; - } - - if (record == node_count) { - return MMDB_RECORD_TYPE_EMPTY; - } - - if (record - node_count < mmdb->data_section_size) { - return MMDB_RECORD_TYPE_DATA; - } - - DEBUG_MSG("record has a value that points outside of the database"); - return MMDB_RECORD_TYPE_INVALID; -} - -LOCAL uint32_t get_left_28_bit_record(const uint8_t *record) -{ - return record[0] * 65536 + record[1] * 256 + record[2] + - ((record[3] & 0xf0) << 20); -} - -LOCAL uint32_t get_right_28_bit_record(const uint8_t *record) -{ - uint32_t value = get_uint32(record); - return value & 0xfffffff; -} - -int MMDB_read_node(const MMDB_s *const mmdb, uint32_t node_number, - MMDB_search_node_s *const node) -{ - record_info_s record_info = record_info_for_database(mmdb); - if (0 == record_info.right_record_offset) { - return MMDB_UNKNOWN_DATABASE_FORMAT_ERROR; - } - - if (node_number > mmdb->metadata.node_count) { - return MMDB_INVALID_NODE_NUMBER_ERROR; - } - - const uint8_t *search_tree = mmdb->file_content; - const uint8_t *record_pointer = - &search_tree[node_number * record_info.record_length]; - node->left_record = record_info.left_record_getter(record_pointer); - record_pointer += record_info.right_record_offset; - node->right_record = record_info.right_record_getter(record_pointer); - - node->left_record_type = record_type(mmdb, node->left_record); - node->right_record_type = record_type(mmdb, node->right_record); - - // Note that offset will be invalid if the record type is not - // MMDB_RECORD_TYPE_DATA, but that's ok. Any use of the record entry - // for other data types is a programming error. - node->left_record_entry = (struct MMDB_entry_s) { - .mmdb = mmdb, - .offset = data_section_offset_for_record(mmdb, node->left_record), - }; - node->right_record_entry = (struct MMDB_entry_s) { - .mmdb = mmdb, - .offset = data_section_offset_for_record(mmdb, node->right_record), - }; - - return MMDB_SUCCESS; -} - -LOCAL uint32_t data_section_offset_for_record(const MMDB_s *const mmdb, - uint64_t record) -{ - return (uint32_t)record - mmdb->metadata.node_count - - MMDB_DATA_SECTION_SEPARATOR; -} - -int MMDB_get_value(MMDB_entry_s *const start, - MMDB_entry_data_s *const entry_data, - ...) -{ - va_list path; - va_start(path, entry_data); - int status = MMDB_vget_value(start, entry_data, path); - va_end(path); - return status; -} - -int MMDB_vget_value(MMDB_entry_s *const start, - MMDB_entry_data_s *const entry_data, - va_list va_path) -{ - int length = path_length(va_path); - const char *path_elem; - int i = 0; - - MAYBE_CHECK_SIZE_OVERFLOW(length, SIZE_MAX / sizeof(const char *) - 1, - MMDB_INVALID_METADATA_ERROR); - - const char **path = calloc(length + 1, sizeof(const char *)); - if (NULL == path) { - return MMDB_OUT_OF_MEMORY_ERROR; - } - - while (NULL != (path_elem = va_arg(va_path, char *))) { - path[i] = path_elem; - i++; - } - path[i] = NULL; - - int status = MMDB_aget_value(start, entry_data, path); - - free((char **)path); - - return status; -} - -LOCAL int path_length(va_list va_path) -{ - int i = 0; - const char *ignore; - va_list path_copy; - va_copy(path_copy, va_path); - - while (NULL != (ignore = va_arg(path_copy, char *))) { - i++; - } - - va_end(path_copy); - - return i; -} - -int MMDB_aget_value(MMDB_entry_s *const start, - MMDB_entry_data_s *const entry_data, - const char *const *const path) -{ - const MMDB_s *const mmdb = start->mmdb; - uint32_t offset = start->offset; - - memset(entry_data, 0, sizeof(MMDB_entry_data_s)); - DEBUG_NL; - DEBUG_MSG("looking up value by path"); - - CHECKED_DECODE_ONE_FOLLOW(mmdb, offset, entry_data); - - DEBUG_NL; - DEBUG_MSGF("top level element is a %s", type_num_to_name(entry_data->type)); - - /* Can this happen? It'd probably represent a pathological case under - * normal use, but there's nothing preventing someone from passing an - * invalid MMDB_entry_s struct to this function */ - if (!entry_data->has_data) { - return MMDB_INVALID_LOOKUP_PATH_ERROR; - } - - const char *path_elem; - int i = 0; - while (NULL != (path_elem = path[i++])) { - DEBUG_NL; - DEBUG_MSGF("path elem = %s", path_elem); - - /* XXX - it'd be good to find a quicker way to skip through these - entries that doesn't involve decoding them - completely. Basically we need to just use the size from the - control byte to advance our pointer rather than calling - decode_one(). */ - if (entry_data->type == MMDB_DATA_TYPE_ARRAY) { - int status = lookup_path_in_array(path_elem, mmdb, entry_data); - if (MMDB_SUCCESS != status) { - memset(entry_data, 0, sizeof(MMDB_entry_data_s)); - return status; - } - } else if (entry_data->type == MMDB_DATA_TYPE_MAP) { - int status = lookup_path_in_map(path_elem, mmdb, entry_data); - if (MMDB_SUCCESS != status) { - memset(entry_data, 0, sizeof(MMDB_entry_data_s)); - return status; - } - } else { - /* Once we make the code traverse maps & arrays without calling - * decode_one() we can get rid of this. */ - memset(entry_data, 0, sizeof(MMDB_entry_data_s)); - return MMDB_LOOKUP_PATH_DOES_NOT_MATCH_DATA_ERROR; - } - } - - return MMDB_SUCCESS; -} - -LOCAL int lookup_path_in_array(const char *path_elem, - const MMDB_s *const mmdb, - MMDB_entry_data_s *entry_data) -{ - uint32_t size = entry_data->data_size; - char *first_invalid; - - int saved_errno = errno; - errno = 0; - int array_index = strtol(path_elem, &first_invalid, 10); - if (ERANGE == errno) { - errno = saved_errno; - return MMDB_INVALID_LOOKUP_PATH_ERROR; - } - errno = saved_errno; - - if (array_index < 0) { - array_index += size; - - if (array_index < 0) { - return MMDB_LOOKUP_PATH_DOES_NOT_MATCH_DATA_ERROR; - } - } - - if (*first_invalid || (uint32_t)array_index >= size) { - return MMDB_LOOKUP_PATH_DOES_NOT_MATCH_DATA_ERROR; - } - - for (int i = 0; i < array_index; i++) { - /* We don't want to follow a pointer here. If the next element is a - * pointer we simply skip it and keep going */ - CHECKED_DECODE_ONE(mmdb, entry_data->offset_to_next, entry_data); - int status = skip_map_or_array(mmdb, entry_data); - if (MMDB_SUCCESS != status) { - return status; - } - } - - MMDB_entry_data_s value; - CHECKED_DECODE_ONE_FOLLOW(mmdb, entry_data->offset_to_next, &value); - memcpy(entry_data, &value, sizeof(MMDB_entry_data_s)); - - return MMDB_SUCCESS; -} - -LOCAL int lookup_path_in_map(const char *path_elem, - const MMDB_s *const mmdb, - MMDB_entry_data_s *entry_data) -{ - uint32_t size = entry_data->data_size; - uint32_t offset = entry_data->offset_to_next; - size_t path_elem_len = strlen(path_elem); - - while (size-- > 0) { - MMDB_entry_data_s key, value; - CHECKED_DECODE_ONE_FOLLOW(mmdb, offset, &key); - - uint32_t offset_to_value = key.offset_to_next; - - if (MMDB_DATA_TYPE_UTF8_STRING != key.type) { - return MMDB_INVALID_DATA_ERROR; - } - - if (key.data_size == path_elem_len && - !memcmp(path_elem, key.utf8_string, path_elem_len)) { - - DEBUG_MSG("found key matching path elem"); - - CHECKED_DECODE_ONE_FOLLOW(mmdb, offset_to_value, &value); - memcpy(entry_data, &value, sizeof(MMDB_entry_data_s)); - return MMDB_SUCCESS; - } else { - /* We don't want to follow a pointer here. If the next element is - * a pointer we simply skip it and keep going */ - CHECKED_DECODE_ONE(mmdb, offset_to_value, &value); - int status = skip_map_or_array(mmdb, &value); - if (MMDB_SUCCESS != status) { - return status; - } - offset = value.offset_to_next; - } - } - - memset(entry_data, 0, sizeof(MMDB_entry_data_s)); - return MMDB_LOOKUP_PATH_DOES_NOT_MATCH_DATA_ERROR; -} - -LOCAL int skip_map_or_array(const MMDB_s *const mmdb, - MMDB_entry_data_s *entry_data) -{ - if (entry_data->type == MMDB_DATA_TYPE_MAP) { - uint32_t size = entry_data->data_size; - while (size-- > 0) { - CHECKED_DECODE_ONE(mmdb, entry_data->offset_to_next, entry_data); // key - CHECKED_DECODE_ONE(mmdb, entry_data->offset_to_next, entry_data); // value - int status = skip_map_or_array(mmdb, entry_data); - if (MMDB_SUCCESS != status) { - return status; - } - } - } else if (entry_data->type == MMDB_DATA_TYPE_ARRAY) { - uint32_t size = entry_data->data_size; - while (size-- > 0) { - CHECKED_DECODE_ONE(mmdb, entry_data->offset_to_next, entry_data); // value - int status = skip_map_or_array(mmdb, entry_data); - if (MMDB_SUCCESS != status) { - return status; - } - } - } - - return MMDB_SUCCESS; -} - -LOCAL int decode_one_follow(const MMDB_s *const mmdb, uint32_t offset, - MMDB_entry_data_s *entry_data) -{ - CHECKED_DECODE_ONE(mmdb, offset, entry_data); - if (entry_data->type == MMDB_DATA_TYPE_POINTER) { - uint32_t next = entry_data->offset_to_next; - CHECKED_DECODE_ONE(mmdb, entry_data->pointer, entry_data); - /* Pointers to pointers are illegal under the spec */ - if (entry_data->type == MMDB_DATA_TYPE_POINTER) { - DEBUG_MSG("pointer points to another pointer"); - return MMDB_INVALID_DATA_ERROR; - } - - /* The pointer could point to any part of the data section but the - * next entry for this particular offset may be the one after the - * pointer, not the one after whatever the pointer points to. This - * depends on whether the pointer points to something that is a simple - * value or a compound value. For a compound value, the next one is - * the one after the pointer result, not the one after the pointer. */ - if (entry_data->type != MMDB_DATA_TYPE_MAP - && entry_data->type != MMDB_DATA_TYPE_ARRAY) { - - entry_data->offset_to_next = next; - } - } - - return MMDB_SUCCESS; -} - -#if !MMDB_UINT128_IS_BYTE_ARRAY -LOCAL mmdb_uint128_t get_uint128(const uint8_t *p, int length) -{ - mmdb_uint128_t value = 0; - while (length-- > 0) { - value <<= 8; - value += *p++; - } - return value; -} -#endif - -LOCAL int decode_one(const MMDB_s *const mmdb, uint32_t offset, - MMDB_entry_data_s *entry_data) -{ - const uint8_t *mem = mmdb->data_section; - - // We subtract rather than add as it possible that offset + 1 - // could overflow for a corrupt database while an underflow - // from data_section_size - 1 should not be possible. - if (offset > mmdb->data_section_size - 1) { - DEBUG_MSGF("Offset (%d) past data section (%d)", offset, - mmdb->data_section_size); - return MMDB_INVALID_DATA_ERROR; - } - - entry_data->offset = offset; - entry_data->has_data = true; - - DEBUG_NL; - DEBUG_MSGF("Offset: %i", offset); - - uint8_t ctrl = mem[offset++]; - DEBUG_BINARY("Control byte: %s", ctrl); - - int type = (ctrl >> 5) & 7; - DEBUG_MSGF("Type: %i (%s)", type, type_num_to_name(type)); - - if (type == MMDB_DATA_TYPE_EXTENDED) { - // Subtracting 1 to avoid possible overflow on offset + 1 - if (offset > mmdb->data_section_size - 1) { - DEBUG_MSGF("Extended type offset (%d) past data section (%d)", - offset, - mmdb->data_section_size); - return MMDB_INVALID_DATA_ERROR; - } - type = get_ext_type(mem[offset++]); - DEBUG_MSGF("Extended type: %i (%s)", type, type_num_to_name(type)); - } - - entry_data->type = type; - - if (type == MMDB_DATA_TYPE_POINTER) { - uint8_t psize = ((ctrl >> 3) & 3) + 1; - DEBUG_MSGF("Pointer size: %i", psize); - - // We check that the offset does not extend past the end of the - // database and that the subtraction of psize did not underflow. - if (offset > mmdb->data_section_size - psize || - mmdb->data_section_size < psize) { - DEBUG_MSGF("Pointer offset (%d) past data section (%d)", offset + - psize, - mmdb->data_section_size); - return MMDB_INVALID_DATA_ERROR; - } - entry_data->pointer = get_ptr_from(ctrl, &mem[offset], psize); - DEBUG_MSGF("Pointer to: %i", entry_data->pointer); - - entry_data->data_size = psize; - entry_data->offset_to_next = offset + psize; - return MMDB_SUCCESS; - } - - uint32_t size = ctrl & 31; - switch (size) { - case 29: - // We subtract when checking offset to avoid possible overflow - if (offset > mmdb->data_section_size - 1) { - DEBUG_MSGF("String end (%d, case 29) past data section (%d)", - offset, - mmdb->data_section_size); - return MMDB_INVALID_DATA_ERROR; - } - size = 29 + mem[offset++]; - break; - case 30: - // We subtract when checking offset to avoid possible overflow - if (offset > mmdb->data_section_size - 2) { - DEBUG_MSGF("String end (%d, case 30) past data section (%d)", - offset, - mmdb->data_section_size); - return MMDB_INVALID_DATA_ERROR; - } - size = 285 + get_uint16(&mem[offset]); - offset += 2; - break; - case 31: - // We subtract when checking offset to avoid possible overflow - if (offset > mmdb->data_section_size - 3) { - DEBUG_MSGF("String end (%d, case 31) past data section (%d)", - offset, - mmdb->data_section_size); - return MMDB_INVALID_DATA_ERROR; - } - size = 65821 + get_uint24(&mem[offset]); - offset += 3; - default: - break; - } - - DEBUG_MSGF("Size: %i", size); - - if (type == MMDB_DATA_TYPE_MAP || type == MMDB_DATA_TYPE_ARRAY) { - entry_data->data_size = size; - entry_data->offset_to_next = offset; - return MMDB_SUCCESS; - } - - if (type == MMDB_DATA_TYPE_BOOLEAN) { - entry_data->boolean = size ? true : false; - entry_data->data_size = 0; - entry_data->offset_to_next = offset; - DEBUG_MSGF("boolean value: %s", entry_data->boolean ? "true" : "false"); - return MMDB_SUCCESS; - } - - // Check that the data doesn't extend past the end of the memory - // buffer and that the calculation in doing this did not underflow. - if (offset > mmdb->data_section_size - size || - mmdb->data_section_size < size) { - DEBUG_MSGF("Data end (%d) past data section (%d)", offset + size, - mmdb->data_section_size); - return MMDB_INVALID_DATA_ERROR; - } - - if (type == MMDB_DATA_TYPE_UINT16) { - if (size > 2) { - DEBUG_MSGF("uint16 of size %d", size); - return MMDB_INVALID_DATA_ERROR; - } - entry_data->uint16 = (uint16_t)get_uintX(&mem[offset], size); - DEBUG_MSGF("uint16 value: %u", entry_data->uint16); - } else if (type == MMDB_DATA_TYPE_UINT32) { - if (size > 4) { - DEBUG_MSGF("uint32 of size %d", size); - return MMDB_INVALID_DATA_ERROR; - } - entry_data->uint32 = (uint32_t)get_uintX(&mem[offset], size); - DEBUG_MSGF("uint32 value: %u", entry_data->uint32); - } else if (type == MMDB_DATA_TYPE_INT32) { - if (size > 4) { - DEBUG_MSGF("int32 of size %d", size); - return MMDB_INVALID_DATA_ERROR; - } - entry_data->int32 = get_sintX(&mem[offset], size); - DEBUG_MSGF("int32 value: %i", entry_data->int32); - } else if (type == MMDB_DATA_TYPE_UINT64) { - if (size > 8) { - DEBUG_MSGF("uint64 of size %d", size); - return MMDB_INVALID_DATA_ERROR; - } - entry_data->uint64 = get_uintX(&mem[offset], size); - DEBUG_MSGF("uint64 value: %" PRIu64, entry_data->uint64); - } else if (type == MMDB_DATA_TYPE_UINT128) { - if (size > 16) { - DEBUG_MSGF("uint128 of size %d", size); - return MMDB_INVALID_DATA_ERROR; - } -#if MMDB_UINT128_IS_BYTE_ARRAY - memset(entry_data->uint128, 0, 16); - if (size > 0) { - memcpy(entry_data->uint128 + 16 - size, &mem[offset], size); - } -#else - entry_data->uint128 = get_uint128(&mem[offset], size); -#endif - } else if (type == MMDB_DATA_TYPE_FLOAT) { - if (size != 4) { - DEBUG_MSGF("float of size %d", size); - return MMDB_INVALID_DATA_ERROR; - } - size = 4; - entry_data->float_value = get_ieee754_float(&mem[offset]); - DEBUG_MSGF("float value: %f", entry_data->float_value); - } else if (type == MMDB_DATA_TYPE_DOUBLE) { - if (size != 8) { - DEBUG_MSGF("double of size %d", size); - return MMDB_INVALID_DATA_ERROR; - } - size = 8; - entry_data->double_value = get_ieee754_double(&mem[offset]); - DEBUG_MSGF("double value: %f", entry_data->double_value); - } else if (type == MMDB_DATA_TYPE_UTF8_STRING) { - entry_data->utf8_string = size == 0 ? "" : (char *)&mem[offset]; - entry_data->data_size = size; -#ifdef MMDB_DEBUG - char *string = mmdb_strndup(entry_data->utf8_string, - size > 50 ? 50 : size); - if (NULL == string) { - abort(); - } - DEBUG_MSGF("string value: %s", string); - free(string); -#endif - } else if (type == MMDB_DATA_TYPE_BYTES) { - entry_data->bytes = &mem[offset]; - entry_data->data_size = size; - } - - entry_data->offset_to_next = offset + size; - - return MMDB_SUCCESS; -} - -LOCAL int get_ext_type(int raw_ext_type) -{ - return 7 + raw_ext_type; -} - -LOCAL uint32_t get_ptr_from(uint8_t ctrl, uint8_t const *const ptr, - int ptr_size) -{ - uint32_t new_offset; - switch (ptr_size) { - case 1: - new_offset = ( (ctrl & 7) << 8) + ptr[0]; - break; - case 2: - new_offset = 2048 + ( (ctrl & 7) << 16 ) + ( ptr[0] << 8) + ptr[1]; - break; - case 3: - new_offset = 2048 + 524288 + ( (ctrl & 7) << 24 ) + get_uint24(ptr); - break; - case 4: - default: - new_offset = get_uint32(ptr); - break; - } - return new_offset; -} - -int MMDB_get_metadata_as_entry_data_list( - const MMDB_s *const mmdb, MMDB_entry_data_list_s **const entry_data_list) -{ - MMDB_s metadata_db = make_fake_metadata_db(mmdb); - - MMDB_entry_s metadata_start = { - .mmdb = &metadata_db, - .offset = 0 - }; - - return MMDB_get_entry_data_list(&metadata_start, entry_data_list); -} - -int MMDB_get_entry_data_list( - MMDB_entry_s *start, MMDB_entry_data_list_s **const entry_data_list) -{ - MMDB_data_pool_s *const pool = data_pool_new(MMDB_POOL_INIT_SIZE); - if (!pool) { - return MMDB_OUT_OF_MEMORY_ERROR; - } - - MMDB_entry_data_list_s *const list = data_pool_alloc(pool); - if (!list) { - data_pool_destroy(pool); - return MMDB_OUT_OF_MEMORY_ERROR; - } - - int const status = get_entry_data_list(start->mmdb, start->offset, list, - pool, 0); - - *entry_data_list = data_pool_to_list(pool); - if (!*entry_data_list) { - data_pool_destroy(pool); - return MMDB_OUT_OF_MEMORY_ERROR; - } - - return status; -} - -LOCAL int get_entry_data_list(const MMDB_s *const mmdb, - uint32_t offset, - MMDB_entry_data_list_s *const entry_data_list, - MMDB_data_pool_s *const pool, - int depth) -{ - if (depth >= MAXIMUM_DATA_STRUCTURE_DEPTH) { - DEBUG_MSG("reached the maximum data structure depth"); - return MMDB_INVALID_DATA_ERROR; - } - depth++; - CHECKED_DECODE_ONE(mmdb, offset, &entry_data_list->entry_data); - - switch (entry_data_list->entry_data.type) { - case MMDB_DATA_TYPE_POINTER: - { - uint32_t next_offset = entry_data_list->entry_data.offset_to_next; - uint32_t last_offset; - CHECKED_DECODE_ONE(mmdb, last_offset = - entry_data_list->entry_data.pointer, - &entry_data_list->entry_data); - - /* Pointers to pointers are illegal under the spec */ - if (entry_data_list->entry_data.type == MMDB_DATA_TYPE_POINTER) { - DEBUG_MSG("pointer points to another pointer"); - return MMDB_INVALID_DATA_ERROR; - } - - if (entry_data_list->entry_data.type == MMDB_DATA_TYPE_ARRAY - || entry_data_list->entry_data.type == MMDB_DATA_TYPE_MAP) { - - int status = - get_entry_data_list(mmdb, last_offset, entry_data_list, - pool, depth); - if (MMDB_SUCCESS != status) { - DEBUG_MSG("get_entry_data_list on pointer failed."); - return status; - } - } - entry_data_list->entry_data.offset_to_next = next_offset; - } - break; - case MMDB_DATA_TYPE_ARRAY: - { - uint32_t array_size = entry_data_list->entry_data.data_size; - uint32_t array_offset = entry_data_list->entry_data.offset_to_next; - while (array_size-- > 0) { - MMDB_entry_data_list_s *entry_data_list_to = - data_pool_alloc(pool); - if (!entry_data_list_to) { - return MMDB_OUT_OF_MEMORY_ERROR; - } - - int status = - get_entry_data_list(mmdb, array_offset, entry_data_list_to, - pool, depth); - if (MMDB_SUCCESS != status) { - DEBUG_MSG("get_entry_data_list on array element failed."); - return status; - } - - array_offset = entry_data_list_to->entry_data.offset_to_next; - } - entry_data_list->entry_data.offset_to_next = array_offset; - - } - break; - case MMDB_DATA_TYPE_MAP: - { - uint32_t size = entry_data_list->entry_data.data_size; - - offset = entry_data_list->entry_data.offset_to_next; - while (size-- > 0) { - MMDB_entry_data_list_s *list_key = data_pool_alloc(pool); - if (!list_key) { - return MMDB_OUT_OF_MEMORY_ERROR; - } - - int status = - get_entry_data_list(mmdb, offset, list_key, pool, depth); - if (MMDB_SUCCESS != status) { - DEBUG_MSG("get_entry_data_list on map key failed."); - return status; - } - - offset = list_key->entry_data.offset_to_next; - - MMDB_entry_data_list_s *list_value = data_pool_alloc(pool); - if (!list_value) { - return MMDB_OUT_OF_MEMORY_ERROR; - } - - status = get_entry_data_list(mmdb, offset, list_value, pool, - depth); - if (MMDB_SUCCESS != status) { - DEBUG_MSG("get_entry_data_list on map element failed."); - return status; - } - offset = list_value->entry_data.offset_to_next; - } - entry_data_list->entry_data.offset_to_next = offset; - } - break; - default: - break; - } - - return MMDB_SUCCESS; -} - -LOCAL float get_ieee754_float(const uint8_t *restrict p) -{ - volatile float f; - uint8_t *q = (void *)&f; -/* Windows builds don't use autoconf but we can assume they're all - * little-endian. */ -#if MMDB_LITTLE_ENDIAN || _WIN32 - q[3] = p[0]; - q[2] = p[1]; - q[1] = p[2]; - q[0] = p[3]; -#else - memcpy(q, p, 4); -#endif - return f; -} - -LOCAL double get_ieee754_double(const uint8_t *restrict p) -{ - volatile double d; - uint8_t *q = (void *)&d; -#if MMDB_LITTLE_ENDIAN || _WIN32 - q[7] = p[0]; - q[6] = p[1]; - q[5] = p[2]; - q[4] = p[3]; - q[3] = p[4]; - q[2] = p[5]; - q[1] = p[6]; - q[0] = p[7]; -#else - memcpy(q, p, 8); -#endif - - return d; -} - -LOCAL uint32_t get_uint32(const uint8_t *p) -{ - return p[0] * 16777216U + p[1] * 65536 + p[2] * 256 + p[3]; -} - -LOCAL uint32_t get_uint24(const uint8_t *p) -{ - return p[0] * 65536U + p[1] * 256 + p[2]; -} - -LOCAL uint32_t get_uint16(const uint8_t *p) -{ - return p[0] * 256U + p[1]; -} - -LOCAL uint64_t get_uintX(const uint8_t *p, int length) -{ - uint64_t value = 0; - while (length-- > 0) { - value <<= 8; - value += *p++; - } - return value; -} - -LOCAL int32_t get_sintX(const uint8_t *p, int length) -{ - return (int32_t)get_uintX(p, length); -} - -void MMDB_free_entry_data_list(MMDB_entry_data_list_s *const entry_data_list) -{ - if (entry_data_list == NULL) { - return; - } - data_pool_destroy(entry_data_list->pool); -} - -void MMDB_close(MMDB_s *const mmdb) -{ - free_mmdb_struct(mmdb); -} - -LOCAL void free_mmdb_struct(MMDB_s *const mmdb) -{ - if (!mmdb) { - return; - } - - if (NULL != mmdb->filename) { - FREE_AND_SET_NULL(mmdb->filename); - } - if (NULL != mmdb->file_content) { -#ifdef _WIN32 - UnmapViewOfFile(mmdb->file_content); - /* Winsock is only initialized if open was successful so we only have - * to cleanup then. */ - WSACleanup(); -#else - munmap((void *)mmdb->file_content, mmdb->file_size); -#endif - } - - if (NULL != mmdb->metadata.database_type) { - FREE_AND_SET_NULL(mmdb->metadata.database_type); - } - - free_languages_metadata(mmdb); - free_descriptions_metadata(mmdb); -} - -LOCAL void free_languages_metadata(MMDB_s *mmdb) -{ - if (!mmdb->metadata.languages.names) { - return; - } - - for (size_t i = 0; i < mmdb->metadata.languages.count; i++) { - FREE_AND_SET_NULL(mmdb->metadata.languages.names[i]); - } - FREE_AND_SET_NULL(mmdb->metadata.languages.names); -} - -LOCAL void free_descriptions_metadata(MMDB_s *mmdb) -{ - if (!mmdb->metadata.description.count) { - return; - } - - for (size_t i = 0; i < mmdb->metadata.description.count; i++) { - if (NULL != mmdb->metadata.description.descriptions[i]) { - if (NULL != - mmdb->metadata.description.descriptions[i]->language) { - FREE_AND_SET_NULL( - mmdb->metadata.description.descriptions[i]->language); - } - - if (NULL != - mmdb->metadata.description.descriptions[i]->description) { - FREE_AND_SET_NULL( - mmdb->metadata.description.descriptions[i]->description); - } - FREE_AND_SET_NULL(mmdb->metadata.description.descriptions[i]); - } - } - - FREE_AND_SET_NULL(mmdb->metadata.description.descriptions); -} - -const char *MMDB_lib_version(void) -{ - return PACKAGE_VERSION; -} - -int MMDB_dump_entry_data_list(FILE *const stream, - MMDB_entry_data_list_s *const entry_data_list, - int indent) -{ - int status; - dump_entry_data_list(stream, entry_data_list, indent, &status); - return status; -} - -LOCAL MMDB_entry_data_list_s *dump_entry_data_list( - FILE *stream, MMDB_entry_data_list_s *entry_data_list, int indent, - int *status) -{ - switch (entry_data_list->entry_data.type) { - case MMDB_DATA_TYPE_MAP: - { - uint32_t size = entry_data_list->entry_data.data_size; - - print_indentation(stream, indent); - fprintf(stream, "{\n"); - indent += 2; - - for (entry_data_list = entry_data_list->next; - size && entry_data_list; size--) { - - if (MMDB_DATA_TYPE_UTF8_STRING != - entry_data_list->entry_data.type) { - *status = MMDB_INVALID_DATA_ERROR; - return NULL; - } - char *key = - mmdb_strndup( - (char *)entry_data_list->entry_data.utf8_string, - entry_data_list->entry_data.data_size); - if (NULL == key) { - *status = MMDB_OUT_OF_MEMORY_ERROR; - return NULL; - } - - print_indentation(stream, indent); - fprintf(stream, "\"%s\": \n", key); - free(key); - - entry_data_list = entry_data_list->next; - entry_data_list = - dump_entry_data_list(stream, entry_data_list, indent + 2, - status); - - if (MMDB_SUCCESS != *status) { - return NULL; - } - } - - indent -= 2; - print_indentation(stream, indent); - fprintf(stream, "}\n"); - } - break; - case MMDB_DATA_TYPE_ARRAY: - { - uint32_t size = entry_data_list->entry_data.data_size; - - print_indentation(stream, indent); - fprintf(stream, "[\n"); - indent += 2; - - for (entry_data_list = entry_data_list->next; - size && entry_data_list; size--) { - entry_data_list = - dump_entry_data_list(stream, entry_data_list, indent, - status); - if (MMDB_SUCCESS != *status) { - return NULL; - } - } - - indent -= 2; - print_indentation(stream, indent); - fprintf(stream, "]\n"); - } - break; - case MMDB_DATA_TYPE_UTF8_STRING: - { - char *string = - mmdb_strndup((char *)entry_data_list->entry_data.utf8_string, - entry_data_list->entry_data.data_size); - if (NULL == string) { - *status = MMDB_OUT_OF_MEMORY_ERROR; - return NULL; - } - print_indentation(stream, indent); - fprintf(stream, "\"%s\" \n", string); - free(string); - entry_data_list = entry_data_list->next; - } - break; - case MMDB_DATA_TYPE_BYTES: - { - char *hex_string = - bytes_to_hex((uint8_t *)entry_data_list->entry_data.bytes, - entry_data_list->entry_data.data_size); - - if (NULL == hex_string) { - *status = MMDB_OUT_OF_MEMORY_ERROR; - return NULL; - } - - print_indentation(stream, indent); - fprintf(stream, "%s \n", hex_string); - free(hex_string); - - entry_data_list = entry_data_list->next; - } - break; - case MMDB_DATA_TYPE_DOUBLE: - print_indentation(stream, indent); - fprintf(stream, "%f \n", - entry_data_list->entry_data.double_value); - entry_data_list = entry_data_list->next; - break; - case MMDB_DATA_TYPE_FLOAT: - print_indentation(stream, indent); - fprintf(stream, "%f \n", - entry_data_list->entry_data.float_value); - entry_data_list = entry_data_list->next; - break; - case MMDB_DATA_TYPE_UINT16: - print_indentation(stream, indent); - fprintf(stream, "%u \n", entry_data_list->entry_data.uint16); - entry_data_list = entry_data_list->next; - break; - case MMDB_DATA_TYPE_UINT32: - print_indentation(stream, indent); - fprintf(stream, "%u \n", entry_data_list->entry_data.uint32); - entry_data_list = entry_data_list->next; - break; - case MMDB_DATA_TYPE_BOOLEAN: - print_indentation(stream, indent); - fprintf(stream, "%s \n", - entry_data_list->entry_data.boolean ? "true" : "false"); - entry_data_list = entry_data_list->next; - break; - case MMDB_DATA_TYPE_UINT64: - print_indentation(stream, indent); - fprintf(stream, "%" PRIu64 " \n", - entry_data_list->entry_data.uint64); - entry_data_list = entry_data_list->next; - break; - case MMDB_DATA_TYPE_UINT128: - print_indentation(stream, indent); -#if MMDB_UINT128_IS_BYTE_ARRAY - char *hex_string = - bytes_to_hex((uint8_t *)entry_data_list->entry_data.uint128, 16); - if (NULL == hex_string) { - *status = MMDB_OUT_OF_MEMORY_ERROR; - return NULL; - } - fprintf(stream, "0x%s \n", hex_string); - free(hex_string); -#else - uint64_t high = entry_data_list->entry_data.uint128 >> 64; - uint64_t low = (uint64_t)entry_data_list->entry_data.uint128; - fprintf(stream, "0x%016" PRIX64 "%016" PRIX64 " \n", high, - low); -#endif - entry_data_list = entry_data_list->next; - break; - case MMDB_DATA_TYPE_INT32: - print_indentation(stream, indent); - fprintf(stream, "%d \n", entry_data_list->entry_data.int32); - entry_data_list = entry_data_list->next; - break; - default: - *status = MMDB_INVALID_DATA_ERROR; - return NULL; - } - - *status = MMDB_SUCCESS; - return entry_data_list; -} - -LOCAL void print_indentation(FILE *stream, int i) -{ - char buffer[1024]; - int size = i >= 1024 ? 1023 : i; - memset(buffer, 32, size); - buffer[size] = '\0'; - fputs(buffer, stream); -} - -LOCAL char *bytes_to_hex(uint8_t *bytes, uint32_t size) -{ - char *hex_string; - MAYBE_CHECK_SIZE_OVERFLOW(size, SIZE_MAX / 2 - 1, NULL); - - hex_string = calloc((size * 2) + 1, sizeof(char)); - if (NULL == hex_string) { - return NULL; - } - - for (uint32_t i = 0; i < size; i++) { - sprintf(hex_string + (2 * i), "%02X", bytes[i]); - } - - - - return hex_string; -} - -const char *MMDB_strerror(int error_code) -{ - switch (error_code) { - case MMDB_SUCCESS: - return "Success (not an error)"; - case MMDB_FILE_OPEN_ERROR: - return "Error opening the specified MaxMind DB file"; - case MMDB_CORRUPT_SEARCH_TREE_ERROR: - return "The MaxMind DB file's search tree is corrupt"; - case MMDB_INVALID_METADATA_ERROR: - return "The MaxMind DB file contains invalid metadata"; - case MMDB_IO_ERROR: - return "An attempt to read data from the MaxMind DB file failed"; - case MMDB_OUT_OF_MEMORY_ERROR: - return "A memory allocation call failed"; - case MMDB_UNKNOWN_DATABASE_FORMAT_ERROR: - return - "The MaxMind DB file is in a format this library can't handle (unknown record size or binary format version)"; - case MMDB_INVALID_DATA_ERROR: - return - "The MaxMind DB file's data section contains bad data (unknown data type or corrupt data)"; - case MMDB_INVALID_LOOKUP_PATH_ERROR: - return - "The lookup path contained an invalid value (like a negative integer for an array index)"; - case MMDB_LOOKUP_PATH_DOES_NOT_MATCH_DATA_ERROR: - return - "The lookup path does not match the data (key that doesn't exist, array index bigger than the array, expected array or map where none exists)"; - case MMDB_INVALID_NODE_NUMBER_ERROR: - return - "The MMDB_read_node function was called with a node number that does not exist in the search tree"; - case MMDB_IPV6_LOOKUP_IN_IPV4_DATABASE_ERROR: - return - "You attempted to look up an IPv6 address in an IPv4-only database"; - default: - return "Unknown error code"; - } -} diff --git a/fluent-bit/plugins/filter_grep/CMakeLists.txt b/fluent-bit/plugins/filter_grep/CMakeLists.txt deleted file mode 100644 index 056b94092..000000000 --- a/fluent-bit/plugins/filter_grep/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - grep.c) - -FLB_PLUGIN(filter_grep "${src}" "") diff --git a/fluent-bit/plugins/filter_grep/grep.c b/fluent-bit/plugins/filter_grep/grep.c deleted file mode 100644 index 7a4657093..000000000 --- a/fluent-bit/plugins/filter_grep/grep.c +++ /dev/null @@ -1,434 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "grep.h" - -static void delete_rules(struct grep_ctx *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - struct grep_rule *rule; - - mk_list_foreach_safe(head, tmp, &ctx->rules) { - rule = mk_list_entry(head, struct grep_rule, _head); - flb_sds_destroy(rule->field); - flb_free(rule->regex_pattern); - flb_ra_destroy(rule->ra); - flb_regex_destroy(rule->regex); - mk_list_del(&rule->_head); - flb_free(rule); - } -} - -static int set_rules(struct grep_ctx *ctx, struct flb_filter_instance *f_ins) -{ - int first_rule = GREP_NO_RULE; - flb_sds_t tmp; - struct mk_list *head; - struct mk_list *split; - struct flb_split_entry *sentry; - struct flb_kv *kv; - struct grep_rule *rule; - - /* Iterate all filter properties */ - mk_list_foreach(head, &f_ins->properties) { - kv = mk_list_entry(head, struct flb_kv, _head); - - /* Create a new rule */ - rule = flb_malloc(sizeof(struct grep_rule)); - if (!rule) { - flb_errno(); - return -1; - } - - /* Get the type */ - if (strcasecmp(kv->key, "regex") == 0) { - rule->type = GREP_REGEX; - } - else if (strcasecmp(kv->key, "exclude") == 0) { - rule->type = GREP_EXCLUDE; - } - else { - /* Other property. Skip */ - flb_free(rule); - continue; - } - - if (ctx->logical_op != GREP_LOGICAL_OP_LEGACY && first_rule != GREP_NO_RULE) { - /* 'AND'/'OR' case */ - if (first_rule != rule->type) { - flb_plg_error(ctx->ins, "Both 'regex' and 'exclude' are set."); - delete_rules(ctx); - flb_free(rule); - return -1; - } - } - first_rule = rule->type; - - /* As a value we expect a pair of field name and a regular expression */ - split = flb_utils_split(kv->val, ' ', 1); - if (mk_list_size(split) != 2) { - flb_plg_error(ctx->ins, - "invalid regex, expected field and regular expression"); - delete_rules(ctx); - flb_free(rule); - flb_utils_split_free(split); - return -1; - } - - /* Get first value (field) */ - sentry = mk_list_entry_first(split, struct flb_split_entry, _head); - if (*sentry->value == '$') { - rule->field = flb_sds_create_len(sentry->value, sentry->len); - } - else { - rule->field = flb_sds_create_size(sentry->len + 2); - tmp = flb_sds_cat(rule->field, "$", 1); - rule->field = tmp; - - tmp = flb_sds_cat(rule->field, sentry->value, sentry->len); - rule->field = tmp; - } - - /* Get remaining content (regular expression) */ - sentry = mk_list_entry_last(split, struct flb_split_entry, _head); - rule->regex_pattern = flb_strndup(sentry->value, sentry->len); - if (rule->regex_pattern == NULL) { - flb_errno(); - delete_rules(ctx); - flb_free(rule); - flb_utils_split_free(split); - return -1; - } - - /* Release split */ - flb_utils_split_free(split); - - /* Create a record accessor context for this rule */ - rule->ra = flb_ra_create(rule->field, FLB_FALSE); - if (!rule->ra) { - flb_plg_error(ctx->ins, "invalid record accessor? '%s'", rule->field); - delete_rules(ctx); - flb_free(rule); - return -1; - } - - /* Convert string to regex pattern */ - rule->regex = flb_regex_create(rule->regex_pattern); - if (!rule->regex) { - flb_plg_error(ctx->ins, "could not compile regex pattern '%s'", - rule->regex_pattern); - delete_rules(ctx); - flb_free(rule); - return -1; - } - - /* Link to parent list */ - mk_list_add(&rule->_head, &ctx->rules); - } - - return 0; -} - -/* Given a msgpack record, do some filter action based on the defined rules */ -static inline int grep_filter_data(msgpack_object map, struct grep_ctx *ctx) -{ - ssize_t ret; - struct mk_list *head; - struct grep_rule *rule; - - /* For each rule, validate against map fields */ - mk_list_foreach(head, &ctx->rules) { - rule = mk_list_entry(head, struct grep_rule, _head); - - ret = flb_ra_regex_match(rule->ra, map, rule->regex, NULL); - if (ret <= 0) { /* no match */ - if (rule->type == GREP_REGEX) { - return GREP_RET_EXCLUDE; - } - } - else { - if (rule->type == GREP_EXCLUDE) { - return GREP_RET_EXCLUDE; - } - else { - return GREP_RET_KEEP; - } - } - } - - return GREP_RET_KEEP; -} - -static int cb_grep_init(struct flb_filter_instance *f_ins, - struct flb_config *config, - void *data) -{ - int ret; - size_t len; - const char* val; - struct grep_ctx *ctx; - - /* Create context */ - ctx = flb_malloc(sizeof(struct grep_ctx)); - if (!ctx) { - flb_errno(); - return -1; - } - if (flb_filter_config_map_set(f_ins, ctx) < 0) { - flb_errno(); - flb_plg_error(f_ins, "configuration error"); - flb_free(ctx); - return -1; - } - mk_list_init(&ctx->rules); - ctx->ins = f_ins; - - ctx->logical_op = GREP_LOGICAL_OP_LEGACY; - val = flb_filter_get_property("logical_op", f_ins); - if (val != NULL) { - len = strlen(val); - if (len == 3 && strncasecmp("AND", val, len) == 0) { - flb_plg_info(ctx->ins, "AND mode"); - ctx->logical_op = GREP_LOGICAL_OP_AND; - } - else if (len == 2 && strncasecmp("OR", val, len) == 0) { - flb_plg_info(ctx->ins, "OR mode"); - ctx->logical_op = GREP_LOGICAL_OP_OR; - } - else if (len == 6 && strncasecmp("legacy", val, len) == 0) { - flb_plg_info(ctx->ins, "legacy mode"); - ctx->logical_op = GREP_LOGICAL_OP_LEGACY; - } - } - - /* Load rules */ - ret = set_rules(ctx, f_ins); - if (ret == -1) { - flb_free(ctx); - return -1; - } - - /* Set our context */ - flb_filter_set_context(f_ins, ctx); - return 0; -} - -static inline int grep_filter_data_and_or(msgpack_object map, struct grep_ctx *ctx) -{ - ssize_t ra_ret; - int found = FLB_FALSE; - struct mk_list *head; - struct grep_rule *rule; - - /* For each rule, validate against map fields */ - mk_list_foreach(head, &ctx->rules) { - found = FLB_FALSE; - rule = mk_list_entry(head, struct grep_rule, _head); - - ra_ret = flb_ra_regex_match(rule->ra, map, rule->regex, NULL); - if (ra_ret > 0) { - found = FLB_TRUE; - } - - if (ctx->logical_op == GREP_LOGICAL_OP_OR && found == FLB_TRUE) { - /* OR case: One rule is matched. */ - goto grep_filter_data_and_or_end; - } - else if (ctx->logical_op == GREP_LOGICAL_OP_AND && found == FLB_FALSE) { - /* AND case: One rule is not matched */ - goto grep_filter_data_and_or_end; - } - } - - grep_filter_data_and_or_end: - if (rule->type == GREP_REGEX) { - return found ? GREP_RET_KEEP : GREP_RET_EXCLUDE; - } - - /* rule is exclude */ - return found ? GREP_RET_EXCLUDE : GREP_RET_KEEP; -} - -static int cb_grep_filter(const void *data, size_t bytes, - const char *tag, int tag_len, - void **out_buf, size_t *out_size, - struct flb_filter_instance *f_ins, - struct flb_input_instance *i_ins, - void *context, - struct flb_config *config) -{ - int ret; - int old_size = 0; - int new_size = 0; - msgpack_object map; - struct flb_log_event_encoder log_encoder; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - struct grep_ctx *ctx; - - (void) f_ins; - (void) i_ins; - (void) config; - - ctx = (struct grep_ctx *) context; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return FLB_FILTER_NOTOUCH; - } - - ret = flb_log_event_encoder_init(&log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event encoder initialization error : %d", ret); - - flb_log_event_decoder_destroy(&log_decoder); - - return FLB_FILTER_NOTOUCH; - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - old_size++; - - /* get time and map */ - map = *log_event.body; - - if (ctx->logical_op == GREP_LOGICAL_OP_LEGACY) { - ret = grep_filter_data(map, ctx); - } - else { - ret = grep_filter_data_and_or(map, ctx); - } - - if (ret == GREP_RET_KEEP) { - ret = flb_log_event_encoder_emit_raw_record( - &log_encoder, - log_decoder.record_base, - log_decoder.record_length); - - new_size++; - } - else if (ret == GREP_RET_EXCLUDE) { - /* Do nothing */ - } - } - - if (ret == FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA && - log_decoder.offset == bytes) { - ret = FLB_EVENT_ENCODER_SUCCESS; - } - - flb_log_event_decoder_destroy(&log_decoder); - - /* we keep everything ? */ - if (old_size == new_size) { - flb_log_event_encoder_destroy(&log_encoder); - - /* Destroy the buffer to avoid more overhead */ - return FLB_FILTER_NOTOUCH; - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - *out_buf = log_encoder.output_buffer; - *out_size = log_encoder.output_length; - - ret = FLB_FILTER_MODIFIED; - - flb_log_event_encoder_claim_internal_buffer_ownership(&log_encoder); - } - else { - flb_plg_error(ctx->ins, - "Log event encoder error : %d", ret); - - ret = FLB_FILTER_NOTOUCH; - } - - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - return ret; -} - -static int cb_grep_exit(void *data, struct flb_config *config) -{ - struct grep_ctx *ctx = data; - - if (!ctx) { - return 0; - } - - delete_rules(ctx); - flb_free(ctx); - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "regex", NULL, - FLB_CONFIG_MAP_MULT, FLB_FALSE, 0, - "Keep records in which the content of KEY matches the regular expression." - }, - { - FLB_CONFIG_MAP_STR, "exclude", NULL, - FLB_CONFIG_MAP_MULT, FLB_FALSE, 0, - "Exclude records in which the content of KEY matches the regular expression." - }, - { - FLB_CONFIG_MAP_STR, "logical_op", "legacy", - 0, FLB_FALSE, 0, - "Specify whether to use logical conjuciton or disjunction. legacy, AND and OR are allowed." - }, - {0} -}; - -struct flb_filter_plugin filter_grep_plugin = { - .name = "grep", - .description = "grep events by specified field values", - .cb_init = cb_grep_init, - .cb_filter = cb_grep_filter, - .cb_exit = cb_grep_exit, - .config_map = config_map, - .flags = 0 -}; diff --git a/fluent-bit/plugins/filter_grep/grep.h b/fluent-bit/plugins/filter_grep/grep.h deleted file mode 100644 index dc48c8f61..000000000 --- a/fluent-bit/plugins/filter_grep/grep.h +++ /dev/null @@ -1,58 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_FILTER_GREP_H -#define FLB_FILTER_GREP_H - -#include -#include -#include -#include - -/* rule types */ -#define GREP_NO_RULE 0 -#define GREP_REGEX 1 -#define GREP_EXCLUDE 2 - -/* actions */ -#define GREP_RET_KEEP 0 -#define GREP_RET_EXCLUDE 1 - -enum _logical_op{ - GREP_LOGICAL_OP_LEGACY, - GREP_LOGICAL_OP_OR, - GREP_LOGICAL_OP_AND -} logical_op; - -struct grep_ctx { - struct mk_list rules; - int logical_op; - struct flb_filter_instance *ins; -}; - -struct grep_rule { - int type; - flb_sds_t field; - char *regex_pattern; - struct flb_regex *regex; - struct flb_record_accessor *ra; - struct mk_list _head; -}; - -#endif diff --git a/fluent-bit/plugins/filter_kubernetes/CMakeLists.txt b/fluent-bit/plugins/filter_kubernetes/CMakeLists.txt deleted file mode 100644 index d3ebac7ef..000000000 --- a/fluent-bit/plugins/filter_kubernetes/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -set(src - kube_conf.c - kube_meta.c - kube_regex.c - kube_property.c - kubernetes.c - ) - -FLB_PLUGIN(filter_kubernetes "${src}" "") - -# K8s token command is currently Linux-only. -if (${CMAKE_SYSTEM_NAME} MATCHES "Linux") - FLB_DEFINITION(FLB_HAVE_KUBE_TOKEN_COMMAND) -endif() \ No newline at end of file diff --git a/fluent-bit/plugins/filter_kubernetes/kube_conf.c b/fluent-bit/plugins/filter_kubernetes/kube_conf.c deleted file mode 100644 index 9dc360b48..000000000 --- a/fluent-bit/plugins/filter_kubernetes/kube_conf.c +++ /dev/null @@ -1,232 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifndef FLB_HAVE_TLS -#error "Fluent Bit was built without TLS support" -#endif - -#include "kube_meta.h" -#include "kube_conf.h" - -struct flb_kube *flb_kube_conf_create(struct flb_filter_instance *ins, - struct flb_config *config) -{ - int off; - int ret; - const char *url; - const char *tmp; - const char *p; - const char *cmd; - struct flb_kube *ctx; - - ctx = flb_calloc(1, sizeof(struct flb_kube)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->config = config; - ctx->ins = ins; - - /* Set config_map properties in our local context */ - ret = flb_filter_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return NULL; - } - - /* K8s Token Command */ - cmd = flb_filter_get_property("kube_token_command", ins); - if (cmd) { - ctx->kube_token_command = cmd; - } - else { - ctx->kube_token_command = NULL; - } - ctx->kube_token_create = 0; - - /* Merge Parser */ - tmp = flb_filter_get_property("merge_parser", ins); - if (tmp) { - ctx->merge_parser = flb_parser_get(tmp, config); - if (!ctx->merge_parser) { - flb_plg_error(ctx->ins, "parser '%s' is not registered", tmp); - } - } - else { - ctx->merge_parser = NULL; - } - - /* Get Kubernetes API server */ - url = flb_filter_get_property("kube_url", ins); - - if (ctx->use_tag_for_meta) { - ctx->api_https = FLB_FALSE; - } - else if (ctx->use_kubelet) { - ctx->api_host = flb_strdup(ctx->kubelet_host); - ctx->api_port = ctx->kubelet_port; - ctx->api_https = FLB_TRUE; - - /* This is for unit test diagnostic purposes */ - if (ctx->meta_preload_cache_dir) { - ctx->api_https = FLB_FALSE; - } - - } - else if (!url) { - ctx->api_host = flb_strdup(FLB_API_HOST); - ctx->api_port = FLB_API_PORT; - ctx->api_https = FLB_API_TLS; - } - else { - tmp = url; - - /* Check the protocol */ - if (strncmp(tmp, "http://", 7) == 0) { - off = 7; - ctx->api_https = FLB_FALSE; - } - else if (strncmp(tmp, "https://", 8) == 0) { - off = 8; - ctx->api_https = FLB_TRUE; - } - else { - flb_kube_conf_destroy(ctx); - return NULL; - } - - /* Get hostname and TCP port */ - p = url + off; - tmp = strchr(p, ':'); - if (tmp) { - ctx->api_host = flb_strndup(p, tmp - p); - tmp++; - ctx->api_port = atoi(tmp); - } - else { - ctx->api_host = flb_strdup(p); - ctx->api_port = FLB_API_PORT; - } - } - - snprintf(ctx->kube_url, sizeof(ctx->kube_url) - 1, - "%s://%s:%i", - ctx->api_https ? "https" : "http", - ctx->api_host, ctx->api_port); - - if (ctx->kube_meta_cache_ttl > 0) { - ctx->hash_table = flb_hash_table_create_with_ttl(ctx->kube_meta_cache_ttl, - FLB_HASH_TABLE_EVICT_OLDER, - FLB_HASH_TABLE_SIZE, - FLB_HASH_TABLE_SIZE); - } - else { - ctx->hash_table = flb_hash_table_create(FLB_HASH_TABLE_EVICT_RANDOM, - FLB_HASH_TABLE_SIZE, - FLB_HASH_TABLE_SIZE); - } - - if (!ctx->hash_table) { - flb_kube_conf_destroy(ctx); - return NULL; - } - - /* Merge log buffer */ - if (ctx->merge_log == FLB_TRUE) { - ctx->unesc_buf = flb_malloc(FLB_MERGE_BUF_SIZE); - ctx->unesc_buf_size = FLB_MERGE_BUF_SIZE; - } - - /* Custom Regex */ - tmp = flb_filter_get_property("regex_parser", ins); - if (tmp) { - /* Get custom parser */ - ctx->parser = flb_parser_get(tmp, config); - if (!ctx->parser) { - flb_plg_error(ctx->ins, "invalid parser '%s'", tmp); - flb_kube_conf_destroy(ctx); - return NULL; - } - - /* Force to regex parser */ - if (ctx->parser->type != FLB_PARSER_REGEX) { - flb_plg_error(ctx->ins, "invalid parser type '%s'", tmp); - flb_kube_conf_destroy(ctx); - return NULL; - } - else { - ctx->regex = ctx->parser->regex; - } - } - - if (!ctx->use_tag_for_meta) { - flb_plg_info(ctx->ins, "https=%i host=%s port=%i", - ctx->api_https, ctx->api_host, ctx->api_port); - } - return ctx; -} - -void flb_kube_conf_destroy(struct flb_kube *ctx) -{ - if (ctx == NULL) { - return; - } - - if (ctx->hash_table) { - flb_hash_table_destroy(ctx->hash_table); - } - - if (ctx->merge_log == FLB_TRUE) { - flb_free(ctx->unesc_buf); - } - - /* Destroy regex content only if a parser was not defined */ - if (ctx->parser == NULL && ctx->regex) { - flb_regex_destroy(ctx->regex); - } - - flb_free(ctx->api_host); - flb_free(ctx->token); - flb_free(ctx->namespace); - flb_free(ctx->podname); - flb_free(ctx->auth); - - if (ctx->upstream) { - flb_upstream_destroy(ctx->upstream); - } - -#ifdef FLB_HAVE_TLS - if (ctx->tls) { - flb_tls_destroy(ctx->tls); - } -#endif - - flb_free(ctx); -} diff --git a/fluent-bit/plugins/filter_kubernetes/kube_conf.h b/fluent-bit/plugins/filter_kubernetes/kube_conf.h deleted file mode 100644 index f12b6dc27..000000000 --- a/fluent-bit/plugins/filter_kubernetes/kube_conf.h +++ /dev/null @@ -1,174 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_FILTER_KUBE_CONF_H -#define FLB_FILTER_KUBE_CONF_H - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* - * Since this filter might get a high number of request per second, - * we need to keep some cached data to perform filtering, e.g: - * - * tag -> regex: pod name, container ID, container name, etc - * - * By default, we define a hash table for 256 entries. - */ -#define FLB_HASH_TABLE_SIZE 256 - -/* - * When merging nested JSON strings from Docker logs, we need a temporary - * buffer to perform the convertion. To optimize the process, we pre-allocate - * a buffer for that purpose. The FLB_MERGE_BUF_SIZE defines the buffer size. - * - * Note: this is only the initial buffer size, it can grow depending on needs - * for every incoming json-string. - */ -#define FLB_MERGE_BUF_SIZE 2048 /* 2KB */ - -/* Kubernetes API server info */ -#define FLB_API_HOST "kubernetes.default.svc" -#define FLB_API_PORT 443 -#define FLB_API_TLS FLB_TRUE - -/* - * Default expected Kubernetes tag prefix, this is used mostly when source - * data comes from in_tail with custom tags like: kube.service.* - */ -#ifdef FLB_SYSTEM_WINDOWS -#define FLB_KUBE_TAG_PREFIX "kube.c.var.log.containers." -#else -#define FLB_KUBE_TAG_PREFIX "kube.var.log.containers." -#endif - -struct kube_meta; - -/* Filter context */ -struct flb_kube { - /* Configuration parameters */ - char *api_host; - int api_port; - int api_https; - int use_journal; - int cache_use_docker_id; - int labels; - int annotations; - int dummy_meta; - int tls_debug; - int tls_verify; - int kube_token_ttl; - flb_sds_t meta_preload_cache_dir; - - /* Configuration proposed through Annotations (boolean) */ - int k8s_logging_parser; /* allow to process a suggested parser ? */ - int k8s_logging_exclude; /* allowed to suggest to exclude logs ? */ - - /* HTTP Client Setup */ - size_t buffer_size; - - /* Merge Log feature */ - int merge_log; /* old merge_json_log */ - - struct flb_parser *merge_parser; - - /* Temporal buffer to unescape strings */ - size_t unesc_buf_size; - size_t unesc_buf_len; - char *unesc_buf; - - /* - * Merge Log Trim: if merge_log is enabled, this flag allows to trim - * the value and remove any trailing \n or \r. - */ - int merge_log_trim; - - /* Log key, old merge_json_key (default 'log') */ - flb_sds_t merge_log_key; - - /* Keep original log key after successful parsing */ - int keep_log; - - /* API Server end point */ - char kube_url[1024]; - - /* Kubernetes tag prefix */ - flb_sds_t kube_tag_prefix; - - /* Regex context to parse records */ - struct flb_regex *regex; - struct flb_parser *parser; - - /* TLS CA certificate file */ - char *tls_ca_path; - char *tls_ca_file; - - /* TLS virtual host (optional), set by configmap */ - flb_sds_t tls_vhost; - - /* Kubernetes Namespace */ - char *namespace; - size_t namespace_len; - - /* POD Name where Fluent Bit is running */ - char *podname; - size_t podname_len; - - /* Kubernetes Token from FLB_KUBE_TOKEN file */ - char *token_file; - char *token; - size_t token_len; - /* Command to get Kubernetes Authorization Token */ - const char *kube_token_command; - int kube_token_create; - - /* Pre-formatted HTTP Authorization header value */ - char *auth; - size_t auth_len; - - int dns_retries; - int dns_wait_time; - - int use_tag_for_meta; - int use_kubelet; - char *kubelet_host; - int kubelet_port; - - int kube_meta_cache_ttl; - - struct flb_tls *tls; - - struct flb_config *config; - struct flb_hash_table *hash_table; - struct flb_upstream *upstream; - struct flb_filter_instance *ins; -}; - -struct flb_kube *flb_kube_conf_create(struct flb_filter_instance *i, - struct flb_config *config); -void flb_kube_conf_destroy(struct flb_kube *ctx); - -#endif diff --git a/fluent-bit/plugins/filter_kubernetes/kube_meta.c b/fluent-bit/plugins/filter_kubernetes/kube_meta.c deleted file mode 100644 index fbad2bb02..000000000 --- a/fluent-bit/plugins/filter_kubernetes/kube_meta.c +++ /dev/null @@ -1,1650 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "kube_conf.h" -#include "kube_meta.h" -#include "kube_property.h" - -#define FLB_KUBE_META_CONTAINER_STATUSES_KEY "containerStatuses" -#define FLB_KUBE_META_CONTAINER_STATUSES_KEY_LEN \ - (sizeof(FLB_KUBE_META_CONTAINER_STATUSES_KEY) - 1) -#define FLB_KUBE_META_INIT_CONTAINER_STATUSES_KEY "initContainerStatuses" -#define FLB_KUBE_META_INIT_CONTAINER_STATUSES_KEY_LEN \ - (sizeof(FLB_KUBE_META_INIT_CONTAINER_STATUSES_KEY) - 1) -#define FLB_KUBE_TOKEN_BUF_SIZE 8192 /* 8KB */ - -static int file_to_buffer(const char *path, - char **out_buf, size_t *out_size) -{ - int ret; - char *buf; - ssize_t bytes; - FILE *fp; - struct stat st; - - if (!(fp = fopen(path, "r"))) { - return -1; - } - - ret = stat(path, &st); - if (ret == -1) { - flb_errno(); - fclose(fp); - return -1; - } - - buf = flb_calloc(1, (st.st_size + 1)); - if (!buf) { - flb_errno(); - fclose(fp); - return -1; - } - - bytes = fread(buf, st.st_size, 1, fp); - if (bytes < 1) { - flb_free(buf); - fclose(fp); - return -1; - } - - fclose(fp); - - *out_buf = buf; - *out_size = st.st_size; - - return 0; -} - -#ifdef FLB_HAVE_KUBE_TOKEN_COMMAND -/* Run command to get Kubernetes authorization token */ -static int get_token_with_command(const char *command, - char **out_buf, size_t *out_size) -{ - FILE *fp; - char buf[FLB_KUBE_TOKEN_BUF_SIZE]; - char *temp; - char *res; - size_t size = 0; - size_t len = 0; - - fp = popen(command, "r"); - if (fp == NULL) { - return -1; - } - - res = flb_calloc(1, FLB_KUBE_TOKEN_BUF_SIZE); - if (!res) { - flb_errno(); - pclose(fp); - return -1; - } - - while (fgets(buf, sizeof(buf), fp) != NULL) { - len = strlen(buf); - if (len >= FLB_KUBE_TOKEN_BUF_SIZE - 1) { - temp = flb_realloc(res, (FLB_KUBE_TOKEN_BUF_SIZE + size) * 2); - if (temp == NULL) { - flb_errno(); - flb_free(res); - pclose(fp); - return -1; - } - res = temp; - } - strcpy(res + size, buf); - size += len; - } - - if (strlen(res) < 1) { - flb_free(res); - pclose(fp); - return -1; - } - - pclose(fp); - - *out_buf = res; - *out_size = strlen(res); - - return 0; -} -#endif - -/* Set K8s Authorization Token and get HTTP Auth Header */ -static int get_http_auth_header(struct flb_kube *ctx) -{ - int ret; - char *temp; - char *tk = NULL; - size_t tk_size = 0; - - if (ctx->kube_token_command != NULL) { -#ifdef FLB_HAVE_KUBE_TOKEN_COMMAND - ret = get_token_with_command(ctx->kube_token_command, &tk, &tk_size); -#else - ret = -1; -#endif - if (ret == -1) { - flb_plg_warn(ctx->ins, "failed to run command %s", ctx->kube_token_command); - } - } - else { - ret = file_to_buffer(ctx->token_file, &tk, &tk_size); - if (ret == -1) { - flb_plg_warn(ctx->ins, "cannot open %s", FLB_KUBE_TOKEN); - } - flb_plg_info(ctx->ins, " token updated"); - } - ctx->kube_token_create = time(NULL); - - /* Token */ - if (ctx->token != NULL) { - flb_free(ctx->token); - } - ctx->token = tk; - ctx->token_len = tk_size; - - /* HTTP Auth Header */ - if (ctx->auth == NULL) { - ctx->auth = flb_malloc(tk_size + 32); - } - else if (ctx->auth_len < tk_size + 32) { - temp = flb_realloc(ctx->auth, tk_size + 32); - if (temp == NULL) { - flb_free(ctx->auth); - ctx->auth = NULL; - return -1; - } - ctx->auth = temp; - } - - if (!ctx->auth) { - return -1; - } - ctx->auth_len = snprintf(ctx->auth, tk_size + 32, - "Bearer %s", - tk); - - return 0; -} - -/* Refresh HTTP Auth Header if K8s Authorization Token is expired */ -static int refresh_token_if_needed(struct flb_kube *ctx) -{ - int expired = 0; - int ret; - - if (ctx->kube_token_create > 0) { - if (time(NULL) > ctx->kube_token_create + ctx->kube_token_ttl) { - expired = FLB_TRUE; - } - } - - if (expired || ctx->kube_token_create == 0) { - ret = get_http_auth_header(ctx); - if (ret == -1) { - flb_plg_warn(ctx->ins, "failed to set http auth header"); - return -1; - } - } - - return 0; -} - -static void expose_k8s_meta(struct flb_kube *ctx) -{ - char *tmp; - struct flb_env *env; - - env = ctx->config->env; - - flb_env_set(env, "k8s", "enabled"); - flb_env_set(env, "k8s.namespace", ctx->namespace); - flb_env_set(env, "k8s.pod_name", ctx->podname); - - tmp = (char *) flb_env_get(env, "NODE_NAME"); - if (tmp) { - flb_env_set(env, "k8s.node_name", tmp); - } -} - -/* Load local information from a POD context */ -static int get_local_pod_info(struct flb_kube *ctx) -{ - int ret; - char *ns; - size_t ns_size; - char *hostname; - - /* Get the namespace name */ - ret = file_to_buffer(FLB_KUBE_NAMESPACE, &ns, &ns_size); - if (ret == -1) { - /* - * If it fails, it's just informational, as likely the caller - * wanted to connect using the Proxy instead from inside a POD. - */ - flb_plg_warn(ctx->ins, "cannot open %s", FLB_KUBE_NAMESPACE); - return FLB_FALSE; - } - - /* Namespace */ - ctx->namespace = ns; - ctx->namespace_len = ns_size; - - /* POD Name */ - hostname = getenv("HOSTNAME"); - if (hostname) { - ctx->podname = flb_strdup(hostname); - ctx->podname_len = strlen(ctx->podname); - } - else { - char tmp[256]; - gethostname(tmp, 256); - ctx->podname = flb_strdup(tmp); - ctx->podname_len = strlen(ctx->podname); - } - - /* If a namespace was recognized, a token is mandatory */ - /* Use the token to get HTTP Auth Header*/ - ret = get_http_auth_header(ctx); - if (ret == -1) { - flb_plg_warn(ctx->ins, "failed to set http auth header"); - return FLB_FALSE; - } - - expose_k8s_meta(ctx); - return FLB_TRUE; -} - -/* - * If a file exists called namespace_podname.meta, load it and use it. - * If not, fall back to API. This is primarily for diagnostic purposes, - * e.g. debugging new parsers. - */ -static int get_meta_file_info(struct flb_kube *ctx, const char *namespace, - const char *podname, char **buffer, size_t *size, - int *root_type) { - - int fd = -1; - char *payload = NULL; - size_t payload_size = 0; - struct stat sb; - int packed = -1; - int ret; - char uri[1024]; - - if (ctx->meta_preload_cache_dir && namespace && podname) { - - ret = snprintf(uri, sizeof(uri) - 1, "%s/%s_%s.meta", - ctx->meta_preload_cache_dir, namespace, podname); - if (ret > 0) { - fd = open(uri, O_RDONLY, 0); - if (fd != -1) { - if (fstat(fd, &sb) == 0) { - payload = flb_malloc(sb.st_size); - if (!payload) { - flb_errno(); - } - else { - ret = read(fd, payload, sb.st_size); - if (ret == sb.st_size) { - payload_size = ret; - } - } - } - close(fd); - } - } - - if (payload_size) { - packed = flb_pack_json(payload, payload_size, - buffer, size, root_type, - NULL); - } - - if (payload) { - flb_free(payload); - } - } - - return packed; -} - -/* Gather metadata from HTTP Request, - * this could send out HTTP Request either to KUBE Server API or Kubelet - */ -static int get_meta_info_from_request(struct flb_kube *ctx, - const char *namespace, - const char *podname, - char **buffer, size_t *size, - int *root_type, - char* uri) -{ - struct flb_http_client *c; - struct flb_connection *u_conn; - int ret; - size_t b_sent; - int packed; - - if (!ctx->upstream) { - return -1; - } - - u_conn = flb_upstream_conn_get(ctx->upstream); - - if (!u_conn) { - flb_plg_error(ctx->ins, "kubelet upstream connection error"); - return -1; - } - - ret = refresh_token_if_needed(ctx); - if (ret == -1) { - flb_plg_error(ctx->ins, "failed to refresh token"); - flb_upstream_conn_release(u_conn); - return -1; - } - - /* Compose HTTP Client request*/ - c = flb_http_client(u_conn, FLB_HTTP_GET, - uri, - NULL, 0, NULL, 0, NULL, 0); - flb_http_buffer_size(c, ctx->buffer_size); - - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - flb_http_add_header(c, "Connection", 10, "close", 5); - if (ctx->auth_len > 0) { - flb_http_add_header(c, "Authorization", 13, ctx->auth, ctx->auth_len); - } - - ret = flb_http_do(c, &b_sent); - flb_plg_debug(ctx->ins, "Request (ns=%s, pod=%s) http_do=%i, " - "HTTP Status: %i", - namespace, podname, ret, c->resp.status); - - if (ret != 0 || c->resp.status != 200) { - if (c->resp.payload_size > 0) { - flb_plg_debug(ctx->ins, "HTTP response\n%s", - c->resp.payload); - } - flb_http_client_destroy(c); - flb_upstream_conn_release(u_conn); - return -1; - } - - packed = flb_pack_json(c->resp.payload, c->resp.payload_size, - buffer, size, root_type, NULL); - - /* release resources */ - flb_http_client_destroy(c); - flb_upstream_conn_release(u_conn); - - return packed; - -} - -/* Gather pods list information from Kubelet */ -static int get_pods_from_kubelet(struct flb_kube *ctx, - const char *namespace, const char *podname, - char **out_buf, size_t *out_size) -{ - int ret; - int packed = -1; - int root_type; - char uri[1024]; - char *buf; - size_t size; - - *out_buf = NULL; - *out_size = 0; - - /* used for unit test purposes*/ - packed = get_meta_file_info(ctx, namespace, podname, &buf, &size, - &root_type); - - if (packed == -1) { - - ret = snprintf(uri, sizeof(uri) - 1, FLB_KUBELET_PODS); - if (ret == -1) { - return -1; - } - flb_plg_debug(ctx->ins, - "Send out request to Kubelet for pods information."); - packed = get_meta_info_from_request(ctx, namespace, podname, - &buf, &size, &root_type, uri); - } - - /* validate pack */ - if (packed == -1) { - return -1; - } - - *out_buf = buf; - *out_size = size; - - return 0; -} - -/* Gather metadata from API Server */ -static int get_api_server_info(struct flb_kube *ctx, - const char *namespace, const char *podname, - char **out_buf, size_t *out_size) -{ - int ret; - int packed = -1; - int root_type; - char uri[1024]; - char *buf; - size_t size; - - *out_buf = NULL; - *out_size = 0; - - /* used for unit test purposes*/ - packed = get_meta_file_info(ctx, namespace, podname, - &buf, &size, &root_type); - - if (packed == -1) { - - ret = snprintf(uri, sizeof(uri) - 1, FLB_KUBE_API_FMT, namespace, - podname); - - if (ret == -1) { - return -1; - } - flb_plg_debug(ctx->ins, - "Send out request to API Server for pods information"); - packed = get_meta_info_from_request(ctx, namespace, podname, - &buf, &size, &root_type, uri); - } - - /* validate pack */ - if (packed == -1) { - return -1; - } - - *out_buf = buf; - *out_size = size; - - return 0; -} - -static void cb_results(const char *name, const char *value, - size_t vlen, void *data) -{ - struct flb_kube_meta *meta = data; - - if (vlen == 0) { - return; - } - - if (meta->podname == NULL && strcmp(name, "pod_name") == 0) { - meta->podname = flb_strndup(value, vlen); - meta->podname_len = vlen; - meta->fields++; - } - else if (meta->namespace == NULL && - strcmp(name, "namespace_name") == 0) { - meta->namespace = flb_strndup(value, vlen); - meta->namespace_len = vlen; - meta->fields++; - } - else if (meta->container_name == NULL && - strcmp(name, "container_name") == 0) { - meta->container_name = flb_strndup(value, vlen); - meta->container_name_len = vlen; - meta->fields++; - } - else if (meta->docker_id == NULL && - strcmp(name, "docker_id") == 0) { - meta->docker_id = flb_strndup(value, vlen); - meta->docker_id_len = vlen; - meta->fields++; - } - else if (meta->container_hash == NULL && - strcmp(name, "container_hash") == 0) { - meta->container_hash = flb_strndup(value, vlen); - meta->container_hash_len = vlen; - meta->fields++; - } - - return; -} - -static int extract_hash(const char * im, int sz, const char ** out, int * outsz) -{ - char * colon = NULL; - char * slash = NULL; - - *out = NULL; - *outsz = 0; - - if (sz <= 1) { - return -1; - } - - colon = memchr(im, ':', sz); - - if (colon == NULL) { - return -1; - } else { - slash = colon; - while ((im + sz - slash + 1) > 0 && *(slash + 1) == '/') { - slash++; - } - if (slash == colon) { - slash = NULL; - } - } - - if (slash == NULL && (im + sz - colon) > 0) { - *out = im; - } - - if (slash != NULL && (colon - slash) < 0 && (im + sz - slash) > 0) { - *out = slash + 1; - } - - if (*out) { - *outsz = im + sz - *out; - return 0; - } - return -1; -} - -/* - * As per Kubernetes Pod spec, - * https://kubernetes.io/docs/concepts/workloads/pods/pod/, we look - * for status.{initContainerStatuses, containerStatuses}.{containerID, imageID, image} - * where status.{initContainerStatuses, containerStatus}.name == our container - * name - * status: - * ... - * containerStatuses: - * - containerID: XXX - * image: YYY - * imageID: ZZZ - * ... - * name: nginx-ingress-microk8s -*/ -static void extract_container_hash(struct flb_kube_meta *meta, - msgpack_object status) -{ - int i; - msgpack_object k, v; - int docker_id_len = 0; - int container_hash_len = 0; - int container_image_len = 0; - const char *container_hash; - const char *docker_id; - const char *container_image; - const char *tmp; - int tmp_len = 0; - int name_found = FLB_FALSE; - /* Process status/containerStatus map for docker_id, container_hash, container_image */ - for (i = 0; - (meta->docker_id_len == 0 || meta->container_hash_len == 0 || - meta->container_image_len == 0) && - i < status.via.map.size; i++) { - k = status.via.map.ptr[i].key; - if ((k.via.str.size == FLB_KUBE_META_CONTAINER_STATUSES_KEY_LEN && - strncmp(k.via.str.ptr, - FLB_KUBE_META_CONTAINER_STATUSES_KEY, - FLB_KUBE_META_CONTAINER_STATUSES_KEY_LEN) == 0) || - (k.via.str.size == FLB_KUBE_META_INIT_CONTAINER_STATUSES_KEY_LEN && - strncmp(k.via.str.ptr, - FLB_KUBE_META_INIT_CONTAINER_STATUSES_KEY, - FLB_KUBE_META_INIT_CONTAINER_STATUSES_KEY_LEN) == 0)) { - int j; - v = status.via.map.ptr[i].val; - for (j = 0; - (meta->docker_id_len == 0 || - meta->container_hash_len == 0 || - meta->container_image_len == 0) && j < v.via.array.size; - j++) { - int l; - msgpack_object k1, k2; - msgpack_object_str v2; - k1 = v.via.array.ptr[j]; - for (l = 0; - (meta->docker_id_len == 0 || - meta->container_hash_len == 0 || - meta->container_image_len == 0) && - l < k1.via.map.size; l++) { - k2 = k1.via.map.ptr[l].key; - v2 = k1.via.map.ptr[l].val.via.str; - if (k2.via.str.size == sizeof("name") - 1 && - !strncmp(k2.via.str.ptr, "name", k2.via.str.size)) { - if (v2.size == meta->container_name_len && - !strncmp(v2.ptr, - meta->container_name, - meta->container_name_len)) { - name_found = FLB_TRUE; - } - else { - break; - } - } - else if (k2.via.str.size == sizeof("containerID") - 1 && - !strncmp(k2.via.str.ptr, - "containerID", - k2.via.str.size)) { - if (extract_hash(v2.ptr, v2.size, &tmp, &tmp_len) == 0) { - docker_id = tmp; - docker_id_len = tmp_len; - } - } - else if (k2.via.str.size == sizeof("imageID") - 1 && - !strncmp(k2.via.str.ptr, - "imageID", - k2.via.str.size)) { - if (extract_hash(v2.ptr, v2.size, &tmp, &tmp_len) == 0) { - container_hash = tmp; - container_hash_len = tmp_len; - } - } - else if (k2.via.str.size == sizeof("image") - 1 && - !strncmp(k2.via.str.ptr, - "image", - k2.via.str.size)) { - container_image = v2.ptr; - container_image_len = v2.size; - } - } - if (name_found) { - if (container_hash_len && !meta->container_hash_len) { - meta->container_hash_len = container_hash_len; - meta->container_hash = flb_strndup(container_hash, - container_hash_len); - meta->fields++; - } - if (docker_id_len && !meta->docker_id_len) { - meta->docker_id_len = docker_id_len; - meta->docker_id = flb_strndup(docker_id, docker_id_len); - meta->fields++; - } - if (container_image_len && !meta->container_image_len) { - meta->container_image_len = container_image_len; - meta->container_image = flb_strndup(container_image, container_image_len); - meta->fields++; - } - return; - } - } - } - } -} - -static int search_podname_and_namespace(struct flb_kube_meta *meta, - struct flb_kube *ctx, - msgpack_object map) -{ - int i; - int podname_found = FLB_FALSE; - int namespace_found = FLB_FALSE; - int target_podname_found = FLB_FALSE; - int target_namespace_found = FLB_FALSE; - - msgpack_object k; - msgpack_object v; - - for (i = 0; (!podname_found || !namespace_found) && - i < map.via.map.size; i++) { - - k = map.via.map.ptr[i].key; - v = map.via.map.ptr[i].val; - if (k.via.str.size == 4 && !strncmp(k.via.str.ptr, "name", 4)) { - - podname_found = FLB_TRUE; - if (!strncmp(v.via.str.ptr, meta->podname, meta->podname_len)) { - target_podname_found = FLB_TRUE; - } - - } - else if (k.via.str.size == 9 && !strncmp(k.via.str.ptr, - "namespace", 9)) { - - namespace_found = FLB_TRUE; - if (!strncmp((char *)v.via.str.ptr, - meta->namespace, - meta->namespace_len)) { - target_namespace_found = FLB_TRUE; - } - } - } - - if (!target_podname_found || !target_namespace_found) { - return -1; - } - - return 0; -} - -static int search_metadata_in_items(struct flb_kube_meta *meta, - struct flb_kube *ctx, - msgpack_object items_array, - msgpack_object *target_item_map) -{ - int i, j; - - int target_found = FLB_FALSE; - msgpack_object item_info_map; - msgpack_object k; - msgpack_object v; - - for (i = 0; !target_found && i < items_array.via.array.size; i++) { - - item_info_map = items_array.via.array.ptr[i]; - if (item_info_map.type != MSGPACK_OBJECT_MAP) { - continue; - } - - for (j = 0; j < item_info_map.via.map.size; j++) { - - k = item_info_map.via.map.ptr[j].key; - if (k.via.str.size == 8 && - !strncmp(k.via.str.ptr, "metadata", 8)) { - - v = item_info_map.via.map.ptr[j].val; - if (search_podname_and_namespace(meta, ctx, v) == 0) { - target_found = FLB_TRUE; - *target_item_map = item_info_map; - flb_plg_debug(ctx->ins, - "kubelet find pod: %s and ns: %s match", - meta->podname, meta->namespace); - } - break; - } - } - } - - if (!target_found) { - flb_plg_debug(ctx->ins, - "kubelet didn't find pod: %s, ns: %s match", - meta->podname, meta->namespace); - return -1; - } - return 0; -} - -/* At this point map points to the ROOT map, eg: - * - * { - * "kind": "PodList", - * "apiVersion": "v1", - * "metadata": {}, - * "items": [{ - * "metadata": { - * "name": "fluent-bit-rz47v", - * "generateName": "fluent-bit-", - * "namespace": "kube-system", - * "selfLink": "/api/v1/namespaces/kube-system/pods/fluent-bit-rz47v", - * .... - * } - * }] - * - */ -static int search_item_in_items(struct flb_kube_meta *meta, - struct flb_kube *ctx, - msgpack_object api_map, - msgpack_object *target_item_map) -{ - - int i; - int items_array_found = FLB_FALSE; - - msgpack_object k; - msgpack_object v; - msgpack_object items_array; - - for (i = 0; !items_array_found && i < api_map.via.map.size; i++) { - - k = api_map.via.map.ptr[i].key; - if (k.via.str.size == 5 && !strncmp(k.via.str.ptr, "items", 5)) { - - v = api_map.via.map.ptr[i].val; - if (v.type == MSGPACK_OBJECT_ARRAY) { - items_array = v; - items_array_found = FLB_TRUE; - } - } - } - - int ret = search_metadata_in_items(meta, ctx, items_array, - target_item_map); - - return ret; -} - - -static int merge_meta_from_tag(struct flb_kube *ctx, struct flb_kube_meta *meta, - char **out_buf, size_t *out_size) -{ - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - struct flb_mp_map_header mh; - - /* Initialize output msgpack buffer */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - flb_mp_map_header_init(&mh, &mp_pck); - - if (meta->podname != NULL) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 8); - msgpack_pack_str_body(&mp_pck, "pod_name", 8); - msgpack_pack_str(&mp_pck, meta->podname_len); - msgpack_pack_str_body(&mp_pck, meta->podname, meta->podname_len); - } - - if (meta->namespace != NULL) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 14); - msgpack_pack_str_body(&mp_pck, "namespace_name", 14); - msgpack_pack_str(&mp_pck, meta->namespace_len); - msgpack_pack_str_body(&mp_pck, meta->namespace, meta->namespace_len); - } - - if (meta->container_name != NULL) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 14); - msgpack_pack_str_body(&mp_pck, "container_name", 14); - msgpack_pack_str(&mp_pck, meta->container_name_len); - msgpack_pack_str_body(&mp_pck, meta->container_name, - meta->container_name_len); - } - if (meta->docker_id != NULL) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 9); - msgpack_pack_str_body(&mp_pck, "docker_id", 9); - msgpack_pack_str(&mp_pck, meta->docker_id_len); - msgpack_pack_str_body(&mp_pck, meta->docker_id, - meta->docker_id_len); - } - - flb_mp_map_header_end(&mh); - - /* Set outgoing msgpack buffer */ - *out_buf = mp_sbuf.data; - *out_size = mp_sbuf.size; - - return 0; -} - -static int merge_meta(struct flb_kube_meta *meta, struct flb_kube *ctx, - const char *api_buf, size_t api_size, - char **out_buf, size_t *out_size) -{ - int i; - int ret; - int map_size = 0; - int meta_found = FLB_FALSE; - int spec_found = FLB_FALSE; - int status_found = FLB_FALSE; - int target_found = FLB_TRUE; - int have_uid = -1; - int have_labels = -1; - int have_annotations = -1; - int have_nodename = -1; - size_t off = 0; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - - msgpack_unpacked api_result; - msgpack_unpacked meta_result; - msgpack_object item_result; - msgpack_object k; - msgpack_object v; - msgpack_object meta_val; - msgpack_object spec_val; - msgpack_object status_val; - msgpack_object api_map; - msgpack_object ann_map; - struct flb_kube_props props = {0}; - - /* - * - reg_buf: is a msgpack Map containing meta captured using Regex - * - * - api_buf: metadata associated to namespace and POD Name coming from - * the API server. - * - * When merging data we aim to add the following keys from the API server: - * - * - pod_id - * - labels - * - annotations - */ - - /* Initialize output msgpack buffer */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - /* Iterate API server msgpack and lookup specific fields */ - if (api_buf != NULL) { - msgpack_unpacked_init(&api_result); - ret = msgpack_unpack_next(&api_result, api_buf, api_size, &off); - if (ret == MSGPACK_UNPACK_SUCCESS) { - - if (ctx->use_kubelet) { - ret = search_item_in_items(meta, ctx, api_result.data, &item_result); - if (ret == -1) { - target_found = FLB_FALSE; - } - api_map = target_found ? item_result : api_result.data; - } else { - api_map = api_result.data; - } - - /* At this point map points to the ROOT map, eg: - * - * { - * "kind": "Pod", - * "apiVersion": "v1", - * "metadata": { - * "name": "fluent-bit-rz47v", - * "generateName": "fluent-bit-", - * "namespace": "kube-system", - * "selfLink": "/api/v1/namespaces/kube-system/pods/fluent-bit-rz47v", - * .... - * } - * - * We are interested into the 'metadata' map value. - * We are also interested in the spec.nodeName. - * We are also interested in the status.containerStatuses. - */ - for (i = 0; target_found && !(meta_found && spec_found && status_found) && - i < api_map.via.map.size; i++) { - k = api_map.via.map.ptr[i].key; - if (k.via.str.size == 8 && !strncmp(k.via.str.ptr, "metadata", 8)) { - meta_val = api_map.via.map.ptr[i].val; - if (meta_val.type == MSGPACK_OBJECT_MAP) { - meta_found = FLB_TRUE; - } - } - else if (k.via.str.size == 4 && !strncmp(k.via.str.ptr, "spec", 4)) { - spec_val = api_map.via.map.ptr[i].val; - spec_found = FLB_TRUE; - } - else if (k.via.str.size == 6 && !strncmp(k.via.str.ptr, "status", 6)) { - status_val = api_map.via.map.ptr[i].val; - status_found = FLB_TRUE; - } - } - - if (meta_found == FLB_TRUE) { - /* Process metadata map value */ - msgpack_unpacked_init(&meta_result); - for (i = 0; i < meta_val.via.map.size; i++) { - k = meta_val.via.map.ptr[i].key; - - char *ptr = (char *) k.via.str.ptr; - size_t size = k.via.str.size; - - if (size == 3 && strncmp(ptr, "uid", 3) == 0) { - have_uid = i; - map_size++; - } - else if (size == 6 && strncmp(ptr, "labels", 6) == 0) { - have_labels = i; - if (ctx->labels == FLB_TRUE) { - map_size++; - } - } - - else if (size == 11 && strncmp(ptr, "annotations", 11) == 0) { - have_annotations = i; - if (ctx->annotations == FLB_TRUE) { - map_size++; - } - } - - if (have_uid >= 0 && have_labels >= 0 && have_annotations >= 0) { - break; - } - } - } - - /* Process spec map value for nodeName */ - if (spec_found == FLB_TRUE) { - for (i = 0; i < spec_val.via.map.size; i++) { - k = spec_val.via.map.ptr[i].key; - if (k.via.str.size == 8 && - strncmp(k.via.str.ptr, "nodeName", 8) == 0) { - have_nodename = i; - map_size++; - break; - } - } - } - - if ((!meta->container_hash || !meta->docker_id || !meta->container_image) && status_found) { - extract_container_hash(meta, status_val); - } - } - } - - /* Set map size: current + pod_id, labels and annotations */ - map_size += meta->fields; - - /* Append Regex fields */ - msgpack_pack_map(&mp_pck, map_size); - if (meta->podname != NULL) { - msgpack_pack_str(&mp_pck, 8); - msgpack_pack_str_body(&mp_pck, "pod_name", 8); - msgpack_pack_str(&mp_pck, meta->podname_len); - msgpack_pack_str_body(&mp_pck, meta->podname, meta->podname_len); - } - if (meta->namespace != NULL) { - msgpack_pack_str(&mp_pck, 14); - msgpack_pack_str_body(&mp_pck, "namespace_name", 14); - msgpack_pack_str(&mp_pck, meta->namespace_len); - msgpack_pack_str_body(&mp_pck, meta->namespace, meta->namespace_len); - } - - /* Append API Server content */ - if (have_uid >= 0) { - v = meta_val.via.map.ptr[have_uid].val; - - msgpack_pack_str(&mp_pck, 6); - msgpack_pack_str_body(&mp_pck, "pod_id", 6); - msgpack_pack_object(&mp_pck, v); - } - - if (have_labels >= 0 && ctx->labels == FLB_TRUE) { - k = meta_val.via.map.ptr[have_labels].key; - v = meta_val.via.map.ptr[have_labels].val; - - msgpack_pack_object(&mp_pck, k); - msgpack_pack_object(&mp_pck, v); - } - - if (have_annotations >= 0 && ctx->annotations == FLB_TRUE) { - k = meta_val.via.map.ptr[have_annotations].key; - v = meta_val.via.map.ptr[have_annotations].val; - - msgpack_pack_object(&mp_pck, k); - msgpack_pack_object(&mp_pck, v); - } - - if (have_nodename >= 0) { - v = spec_val.via.map.ptr[have_nodename].val; - - msgpack_pack_str(&mp_pck, 4); - msgpack_pack_str_body(&mp_pck, "host", 4); - msgpack_pack_object(&mp_pck, v); - } - - if (meta->container_name != NULL) { - msgpack_pack_str(&mp_pck, 14); - msgpack_pack_str_body(&mp_pck, "container_name", 14); - msgpack_pack_str(&mp_pck, meta->container_name_len); - msgpack_pack_str_body(&mp_pck, meta->container_name, - meta->container_name_len); - } - if (meta->docker_id != NULL) { - msgpack_pack_str(&mp_pck, 9); - msgpack_pack_str_body(&mp_pck, "docker_id", 9); - msgpack_pack_str(&mp_pck, meta->docker_id_len); - msgpack_pack_str_body(&mp_pck, meta->docker_id, - meta->docker_id_len); - } - if (meta->container_hash != NULL) { - msgpack_pack_str(&mp_pck, 14); - msgpack_pack_str_body(&mp_pck, "container_hash", 14); - msgpack_pack_str(&mp_pck, meta->container_hash_len); - msgpack_pack_str_body(&mp_pck, meta->container_hash, - meta->container_hash_len); - } - if (meta->container_image != NULL) { - msgpack_pack_str(&mp_pck, 15); - msgpack_pack_str_body(&mp_pck, "container_image", 15); - msgpack_pack_str(&mp_pck, meta->container_image_len); - msgpack_pack_str_body(&mp_pck, meta->container_image, - meta->container_image_len); - } - - /* Process configuration suggested through Annotations */ - if (have_annotations >= 0) { - ann_map = meta_val.via.map.ptr[have_annotations].val; - - /* Iterate annotations keys and look for 'logging' key */ - if (ann_map.type == MSGPACK_OBJECT_MAP) { - for (i = 0; i < ann_map.via.map.size; i++) { - k = ann_map.via.map.ptr[i].key; - v = ann_map.via.map.ptr[i].val; - - if (k.via.str.size > 13 && /* >= 'fluentbit.io/' */ - strncmp(k.via.str.ptr, "fluentbit.io/", 13) == 0) { - - /* Validate and set the property */ - flb_kube_prop_set(ctx, meta, - k.via.str.ptr + 13, - k.via.str.size - 13, - v.via.str.ptr, - v.via.str.size, - &props); - } - } - } - - /* Pack Annotation properties */ - void *prop_buf; - size_t prop_size; - flb_kube_prop_pack(&props, &prop_buf, &prop_size); - msgpack_sbuffer_write(&mp_sbuf, prop_buf, prop_size); - flb_kube_prop_destroy(&props); - flb_free(prop_buf); - } - - if (api_buf != NULL) { - msgpack_unpacked_destroy(&api_result); - if (meta_found == FLB_TRUE) { - msgpack_unpacked_destroy(&meta_result); - } - } - - /* Set outgoing msgpack buffer */ - *out_buf = mp_sbuf.data; - *out_size = mp_sbuf.size; - - return 0; -} - -static inline int extract_meta(struct flb_kube *ctx, - const char *tag, int tag_len, - const char *data, size_t data_size, - struct flb_kube_meta *meta) -{ - int i; - size_t off = 0; - ssize_t n; - int kube_tag_len; - const char *kube_tag_str; - const char *container = NULL; - int container_found = FLB_FALSE; - int container_length = 0; - struct flb_regex_search result; - msgpack_unpacked mp_result; - msgpack_object root; - msgpack_object map; - msgpack_object key; - msgpack_object val; - - /* Reset meta context */ - memset(meta, '\0', sizeof(struct flb_kube_meta)); - - /* Journald */ - if (ctx->use_journal == FLB_TRUE) { - off = 0; - msgpack_unpacked_init(&mp_result); - while (msgpack_unpack_next(&mp_result, data, data_size, &off) == MSGPACK_UNPACK_SUCCESS) { - root = mp_result.data; - if (root.type != MSGPACK_OBJECT_ARRAY) { - continue; - } - - /* Lookup the CONTAINER_NAME key/value */ - map = root.via.array.ptr[1]; - for (i = 0; i < map.via.map.size; i++) { - key = map.via.map.ptr[i].key; - if (key.via.str.size != 14) { - continue; - } - - if (strncmp(key.via.str.ptr, "CONTAINER_NAME", 14) == 0) { - val = map.via.map.ptr[i].val; - container = val.via.str.ptr; - container_length = val.via.str.size; - container_found = FLB_TRUE; - break; - } - } - - if (container_found == FLB_TRUE) { - break; - } - } - - if (container_found == FLB_FALSE) { - msgpack_unpacked_destroy(&mp_result); - return -1; - } - n = flb_regex_do(ctx->regex, - container, container_length, - &result); - msgpack_unpacked_destroy(&mp_result); - } - else { - /* - * Lookup metadata using regular expression. In order to let the - * regex work we need to know before hand what's the Tag prefix - * set and make sure the adjustment can be done. - */ - kube_tag_len = flb_sds_len(ctx->kube_tag_prefix); - if (kube_tag_len + 1 >= tag_len) { - flb_plg_error(ctx->ins, "incoming record tag (%s) is shorter " - "than kube_tag_prefix value (%s), skip filter", - tag, ctx->kube_tag_prefix); - return -1; - } - kube_tag_str = tag + kube_tag_len; - kube_tag_len = tag_len - kube_tag_len; - - n = flb_regex_do(ctx->regex, kube_tag_str, kube_tag_len, &result); - } - - if (n <= 0) { - flb_plg_warn(ctx->ins, "invalid pattern for given tag %s", tag); - return -1; - } - - /* Parse the regex results */ - flb_regex_parse(ctx->regex, &result, cb_results, meta); - - /* Compose API server cache key */ - if (meta->podname && meta->namespace) { - /* calculate estimated buffer size */ - n = meta->namespace_len + 1 + meta->podname_len + 1; - if (meta->container_name) { - n += meta->container_name_len + 1; - } - if (ctx->cache_use_docker_id && meta->docker_id) { - n += meta->docker_id_len + 1; - } - meta->cache_key = flb_malloc(n); - if (!meta->cache_key) { - flb_errno(); - return -1; - } - - /* Copy namespace */ - memcpy(meta->cache_key, meta->namespace, meta->namespace_len); - off = meta->namespace_len; - - /* Separator */ - meta->cache_key[off++] = ':'; - - /* Copy podname */ - memcpy(meta->cache_key + off, meta->podname, meta->podname_len); - off += meta->podname_len; - - if (meta->container_name) { - /* Separator */ - meta->cache_key[off++] = ':'; - memcpy(meta->cache_key + off, meta->container_name, meta->container_name_len); - off += meta->container_name_len; - } - - if (ctx->cache_use_docker_id && meta->docker_id) { - /* Separator */ - meta->cache_key[off++] = ':'; - memcpy(meta->cache_key + off, meta->docker_id, meta->docker_id_len); - off += meta->docker_id_len; - } - - meta->cache_key[off] = '\0'; - meta->cache_key_len = off; - } - else { - meta->cache_key = NULL; - meta->cache_key_len = 0; - } - - return 0; -} - -/* - * Given a fixed meta data (namespace and podname), get API server information - * and merge buffers. - */ -static int get_and_merge_meta(struct flb_kube *ctx, struct flb_kube_meta *meta, - char **out_buf, size_t *out_size) -{ - int ret; - char *api_buf; - size_t api_size; - - if (ctx->use_tag_for_meta) { - ret = merge_meta_from_tag(ctx, meta, out_buf, out_size); - return ret; - } - else if (ctx->use_kubelet) { - ret = get_pods_from_kubelet(ctx, meta->namespace, meta->podname, - &api_buf, &api_size); - } - else { - ret = get_api_server_info(ctx, meta->namespace, meta->podname, - &api_buf, &api_size); - } - if (ret == -1) { - return -1; - } - - ret = merge_meta(meta, ctx, - api_buf, api_size, - out_buf, out_size); - - if (api_buf != NULL) { - flb_free(api_buf); - } - - return ret; -} - -/* - * Work around kubernetes/kubernetes/issues/78479 by waiting - * for DNS to start up. - */ -static int wait_for_dns(struct flb_kube *ctx) -{ - int i; - struct addrinfo *res; - struct addrinfo hints; - - memset(&hints, 0, sizeof(hints)); - hints.ai_family = AF_UNSPEC; - hints.ai_socktype = SOCK_STREAM; - - for (i = 0; i < ctx->dns_retries; i++) { - if (getaddrinfo(ctx->api_host, NULL, &hints, &res) == 0) { - freeaddrinfo(res); - return 0; - } - flb_plg_info(ctx->ins, "host: %s Wait %i secs until DNS starts up (%i/%i)", - ctx->api_host, ctx->dns_wait_time, i + 1, ctx->dns_retries); - sleep(ctx->dns_wait_time); - } - return -1; -} - -static int flb_kube_network_init(struct flb_kube *ctx, struct flb_config *config) -{ - int io_type = FLB_IO_TCP; - - ctx->upstream = NULL; - - if (ctx->api_https == FLB_TRUE) { - if (!ctx->tls_ca_path && !ctx->tls_ca_file) { - ctx->tls_ca_file = flb_strdup(FLB_KUBE_CA); - } - ctx->tls = flb_tls_create(FLB_TLS_CLIENT_MODE, - ctx->tls_verify, - ctx->tls_debug, - ctx->tls_vhost, - ctx->tls_ca_path, - ctx->tls_ca_file, - NULL, NULL, NULL); - if (!ctx->tls) { - return -1; - } - - io_type = FLB_IO_TLS; - } - - /* Create an Upstream context */ - ctx->upstream = flb_upstream_create(config, - ctx->api_host, - ctx->api_port, - io_type, - ctx->tls); - if (!ctx->upstream) { - /* note: if ctx->tls.context is set, it's destroyed upon context exit */ - flb_plg_debug(ctx->ins, "kube network init create upstream failed"); - return -1; - } - - /* Remove async flag from upstream */ - flb_stream_disable_async_mode(&ctx->upstream->base); - - return 0; -} - -/* Initialize local context */ -int flb_kube_meta_init(struct flb_kube *ctx, struct flb_config *config) -{ - int ret; - char *meta_buf; - size_t meta_size; - - if (ctx->dummy_meta == FLB_TRUE) { - flb_plg_warn(ctx->ins, "using Dummy Metadata"); - return 0; - } - - if (ctx->use_tag_for_meta) { - flb_plg_info(ctx->ins, "no network access required (OK)"); - return 0; - } - - /* Init network */ - flb_kube_network_init(ctx, config); - - /* Gather local info */ - ret = get_local_pod_info(ctx); - if (ret == FLB_TRUE && !ctx->use_tag_for_meta) { - flb_plg_info(ctx->ins, "local POD info OK"); - - ret = wait_for_dns(ctx); - if (ret == -1) { - flb_plg_warn(ctx->ins, "could not resolve %s", ctx->api_host); - return -1; - } - - if (ctx->use_kubelet) { - /* Gather info from Kubelet */ - flb_plg_info(ctx->ins, "testing connectivity with Kubelet..."); - ret = get_pods_from_kubelet(ctx, ctx->namespace, ctx->podname, - &meta_buf, &meta_size); - } - else { - /* Gather info from API server */ - flb_plg_info(ctx->ins, "testing connectivity with API server..."); - ret = get_api_server_info(ctx, ctx->namespace, ctx->podname, - &meta_buf, &meta_size); - } - if (ret == -1) { - if (!ctx->podname) { - flb_plg_warn(ctx->ins, "could not get meta for local POD"); - } - else { - flb_plg_warn(ctx->ins, "could not get meta for POD %s", - ctx->podname); - } - return -1; - } - flb_plg_info(ctx->ins, "connectivity OK"); - flb_free(meta_buf); - } - else { - flb_plg_info(ctx->ins, "Fluent Bit not running in a POD"); - } - - return 0; -} - -int flb_kube_dummy_meta_get(char **out_buf, size_t *out_size) -{ - int len; - time_t t; - char stime[32]; - struct tm result; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - - t = time(NULL); - localtime_r(&t, &result); -#ifdef FLB_SYSTEM_WINDOWS - asctime_s(stime, sizeof(stime), &result); -#else - asctime_r(&result, stime); -#endif - len = strlen(stime) - 1; - - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - msgpack_pack_map(&mp_pck, 1); - msgpack_pack_str(&mp_pck, 5 /* dummy */ ); - msgpack_pack_str_body(&mp_pck, "dummy", 5); - msgpack_pack_str(&mp_pck, len); - msgpack_pack_str_body(&mp_pck, stime, len); - - *out_buf = mp_sbuf.data; - *out_size = mp_sbuf.size; - - return 0; -} - -int flb_kube_meta_get(struct flb_kube *ctx, - const char *tag, int tag_len, - const char *data, size_t data_size, - const char **out_buf, size_t *out_size, - struct flb_kube_meta *meta, - struct flb_kube_props *props) -{ - int id; - int ret; - const char *hash_meta_buf; - char *tmp_hash_meta_buf; - size_t off = 0; - size_t hash_meta_size; - msgpack_unpacked result; - - /* Get metadata from tag or record (cache key is the important one) */ - ret = extract_meta(ctx, tag, tag_len, data, data_size, meta); - if (ret != 0) { - return -1; - } - - /* Check if we have some data associated to the cache key */ - ret = flb_hash_table_get(ctx->hash_table, - meta->cache_key, meta->cache_key_len, - (void *) &hash_meta_buf, &hash_meta_size); - if (ret == -1) { - /* Retrieve API server meta and merge with local meta */ - ret = get_and_merge_meta(ctx, meta, - &tmp_hash_meta_buf, &hash_meta_size); - if (ret == -1) { - *out_buf = NULL; - *out_size = 0; - return 0; - } - - id = flb_hash_table_add(ctx->hash_table, - meta->cache_key, meta->cache_key_len, - tmp_hash_meta_buf, hash_meta_size); - if (id >= 0) { - /* - * Release the original buffer created on extract_meta() as a new - * copy have been generated into the hash table, then re-set - * the outgoing buffer and size. - */ - flb_free(tmp_hash_meta_buf); - flb_hash_table_get_by_id(ctx->hash_table, id, meta->cache_key, - &hash_meta_buf, &hash_meta_size); - } - } - - /* - * The retrieved buffer may have two serialized items: - * - * [0] = kubernetes metadata (annotations, labels) - * [1] = Annotation properties - * - * note: annotation properties are optional. - */ - msgpack_unpacked_init(&result); - - /* Unpack to get the offset/bytes of the first item */ - msgpack_unpack_next(&result, hash_meta_buf, hash_meta_size, &off); - - /* Set the pointer and proper size for the caller */ - *out_buf = hash_meta_buf; - *out_size = off; - - /* A new unpack_next() call will succeed If annotation properties exists */ - ret = msgpack_unpack_next(&result, hash_meta_buf, hash_meta_size, &off); - if (ret == MSGPACK_UNPACK_SUCCESS) { - /* Unpack the remaining data into properties structure */ - flb_kube_prop_unpack(props, - hash_meta_buf + *out_size, - hash_meta_size - *out_size); - } - msgpack_unpacked_destroy(&result); - - return 0; -} - -int flb_kube_meta_release(struct flb_kube_meta *meta) -{ - int r = 0; - - if (meta->namespace) { - flb_free(meta->namespace); - r++; - } - - if (meta->podname) { - flb_free(meta->podname); - r++; - } - - if (meta->container_name) { - flb_free(meta->container_name); - r++; - } - - if (meta->docker_id) { - flb_free(meta->docker_id); - r++; - } - - if (meta->container_hash) { - flb_free(meta->container_hash); - r++; - } - - if (meta->container_image) { - flb_free(meta->container_image); - r++; - } - - if (meta->cache_key) { - flb_free(meta->cache_key); - } - - return r; -} diff --git a/fluent-bit/plugins/filter_kubernetes/kube_meta.h b/fluent-bit/plugins/filter_kubernetes/kube_meta.h deleted file mode 100644 index fb0278afc..000000000 --- a/fluent-bit/plugins/filter_kubernetes/kube_meta.h +++ /dev/null @@ -1,69 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_FILTER_KUBE_META_H -#define FLB_FILTER_KUBE_META_H - -#include "kube_props.h" - -struct flb_kube; - -struct flb_kube_meta { - int fields; - - int namespace_len; - int podname_len; - int cache_key_len; - int container_name_len; - int docker_id_len; - int container_hash_len; - int container_image_len; - - char *namespace; - char *podname; - char *container_name; - char *container_image; - char *docker_id; - - char *container_hash; /* set only on Systemd mode */ - - char *cache_key; -}; - -/* Constant Kubernetes paths */ -#define FLB_KUBE_NAMESPACE "/var/run/secrets/kubernetes.io/serviceaccount/namespace" -#define FLB_KUBE_TOKEN "/var/run/secrets/kubernetes.io/serviceaccount/token" -#define FLB_KUBE_CA "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" -#define FLB_KUBE_API_HOST "kubernetes.default.svc" -#define FLB_KUBE_API_PORT 443 -#define FLB_KUBE_API_FMT "/api/v1/namespaces/%s/pods/%s" -#define FLB_KUBELET_PODS "/pods" - -int flb_kube_meta_init(struct flb_kube *ctx, struct flb_config *config); -int flb_kube_meta_fetch(struct flb_kube *ctx); -int flb_kube_dummy_meta_get(char **out_buf, size_t *out_size); -int flb_kube_meta_get(struct flb_kube *ctx, - const char *tag, int tag_len, - const char *data, size_t data_size, - const char **out_buf, size_t *out_size, - struct flb_kube_meta *meta, - struct flb_kube_props *props); -int flb_kube_meta_release(struct flb_kube_meta *meta); - -#endif diff --git a/fluent-bit/plugins/filter_kubernetes/kube_property.c b/fluent-bit/plugins/filter_kubernetes/kube_property.c deleted file mode 100644 index 4399d692c..000000000 --- a/fluent-bit/plugins/filter_kubernetes/kube_property.c +++ /dev/null @@ -1,360 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include - -#include "kube_conf.h" -#include "kube_meta.h" -#include "kube_property.h" - -#define FLB_KUBE_PROP_PARSER "parser" -#define FLB_KUBE_PROP_PARSER_LEN (sizeof(FLB_KUBE_PROP_PARSER) - 1) -#define FLB_KUBE_PROP_EXCLUDE "exclude" -#define FLB_KUBE_PROP_EXCLUDE_LEN (sizeof(FLB_KUBE_PROP_EXCLUDE) - 1) - -static inline int prop_cmp(const char *key, size_t keylen, - const char *property, size_t proplen) -{ - return proplen >= keylen && strncmp(key, property, keylen) == 0; -} - -static inline void prop_not_allowed(const char *prop, struct flb_kube_meta *meta, - struct flb_kube *ctx) -{ - flb_plg_warn(ctx->ins, "annotation '%s' not allowed " - "(ns='%s' pod_name='%s')", - prop, meta->namespace, meta->podname); -} - -/* Property: parser */ -static int prop_set_parser(struct flb_kube *ctx, struct flb_kube_meta *meta, - int is_container_specific, int stream, - const char *val_buf, size_t val_len, - struct flb_kube_props *props) -{ - char *tmp; - struct flb_parser *parser; - - /* Parser property must be allowed by k8s-logging.parser */ - if (ctx->k8s_logging_parser == FLB_FALSE) { - prop_not_allowed("fluentbit.io/parser", meta, ctx); - return -1; - } - - /* Check the parser exists */ - tmp = flb_strndup(val_buf, val_len); - if (!tmp) { - flb_errno(); - return -1; - } - - /* Get parser context */ - parser = flb_parser_get(tmp, ctx->config); - if (!parser) { - flb_plg_warn(ctx->ins, "annotation parser '%s' not found " - "(ns='%s' pod_name='%s', container_name='%s')", - tmp, meta->namespace, meta->podname, meta->container_name); - flb_free(tmp); - return -1; - } - - /* Save the parser in the properties context */ - if ((stream == FLB_KUBE_PROP_NO_STREAM || - stream == FLB_KUBE_PROP_STREAM_STDOUT) && - (is_container_specific == FLB_TRUE || - props->stdout_parser == FLB_KUBE_PROP_UNDEF)) { - props->stdout_parser = flb_sds_create(tmp); - } - if ((stream == FLB_KUBE_PROP_NO_STREAM || - stream == FLB_KUBE_PROP_STREAM_STDERR) && - (is_container_specific == FLB_TRUE || - props->stderr_parser == FLB_KUBE_PROP_UNDEF)) { - props->stderr_parser = flb_sds_create(tmp); - } - - flb_free(tmp); - - return 0; -} - -static int prop_set_exclude(struct flb_kube *ctx, struct flb_kube_meta *meta, - int is_container_specific, int stream, - const char *val_buf, size_t val_len, - struct flb_kube_props *props) -{ - char *tmp; - int exclude; - - /* Exclude property must be allowed by k8s-logging.exclude */ - if (ctx->k8s_logging_exclude == FLB_FALSE) { - prop_not_allowed("fluentbit.io/exclude", meta, ctx); - return -1; - } - - /* Get the bool value */ - tmp = flb_strndup(val_buf, val_len); - if (!tmp) { - flb_errno(); - return -1; - } - - exclude = flb_utils_bool(tmp) == FLB_TRUE ? - FLB_KUBE_PROP_TRUE : FLB_KUBE_PROP_FALSE; - - /* Save the exclude property in the context */ - if ((stream == FLB_KUBE_PROP_NO_STREAM || - stream == FLB_KUBE_PROP_STREAM_STDOUT) && - (is_container_specific == FLB_TRUE || - props->stdout_exclude == FLB_KUBE_PROP_UNDEF)) { - props->stdout_exclude = exclude; - } - if ((stream == FLB_KUBE_PROP_NO_STREAM || - stream == FLB_KUBE_PROP_STREAM_STDERR) && - (is_container_specific == FLB_TRUE || - props->stderr_exclude == FLB_KUBE_PROP_UNDEF)) { - props->stderr_exclude = exclude; - } - - flb_free(tmp); - - return 0; -} - -int flb_kube_prop_set(struct flb_kube *ctx, struct flb_kube_meta *meta, - const char *prop, int prop_len, - const char *val_buf, size_t val_len, - struct flb_kube_props *props) -{ - /* - * Property can take the following forms: - * applies to streams stdout and stderr of every pod's containers - * - applies to streams stdout and stderr of a specific pod's container - * _stdout applies to stream stdout of every pod's containers - * _stderr applies to stream stderr of every pod's containers - * _stdout- applies to stream stdout of a specific pod's container - * _stderr- applies to stream stderr of a specific pod's container - */ - const char *cur = prop; - size_t len = prop_len; - const char *container = NULL; - size_t container_len = 0; - int stream = FLB_KUBE_PROP_NO_STREAM; - int (*function)(struct flb_kube *ctx, struct flb_kube_meta *meta, - int is_container_specific, int stream, - const char *val_buf, size_t val_len, - struct flb_kube_props *props); - - if (prop_cmp(FLB_KUBE_PROP_PARSER, FLB_KUBE_PROP_PARSER_LEN, prop, prop_len)) { - function = prop_set_parser; - cur += FLB_KUBE_PROP_PARSER_LEN; - } - else if (prop_cmp(FLB_KUBE_PROP_EXCLUDE, FLB_KUBE_PROP_EXCLUDE_LEN, prop, prop_len)) { - function = prop_set_exclude; - cur += FLB_KUBE_PROP_EXCLUDE_LEN; - } - else { - flb_plg_warn(ctx->ins, "unknown annotation 'fluentbit.io/%.*s' " - "(ns='%s' pod_name='%s')", - prop_len, prop, meta->namespace, meta->podname); - return -1; - } - - len = prop_len - (cur - prop); - - if (prop_cmp("_", 1, cur, len)) { - cur++; - len--; - - if (prop_cmp("stdout", sizeof("stdout") - 1, cur, len)) { - stream = FLB_KUBE_PROP_STREAM_STDOUT; - cur += sizeof("stdout") - 1; - } - else if (prop_cmp("stderr", sizeof("stderr") - 1, cur, len)) { - stream = FLB_KUBE_PROP_STREAM_STDERR; - cur += sizeof("stderr") - 1; - } - else { - flb_plg_warn(ctx->ins, "invalid stream in annotation " - "'fluentbit.io/%.*s' (ns='%s' pod_name='%s')", - prop_len, prop, meta->namespace, meta->podname); - return -1; - } - - len = prop_len - (cur - prop); - } - - if (prop_cmp("-", 1, cur, len)) { - cur++; - len--; - - if (len == 0) { - flb_plg_warn(ctx->ins, "invalid container in annotation " - "'fluentbit.io/%.*s' (ns='%s' pod_name='%s')", - prop_len, prop, meta->namespace, meta->podname); - return -1; - } - - container = cur; - container_len = len; - len = 0; - } - - if (len > 0) { - flb_plg_warn(ctx->ins, "invalid annotation 'fluentbit.io/%.*s' " - "(ns='%s' pod_name='%s')", - prop_len, prop, meta->namespace, meta->podname); - return -1; - } - - /* If the property is for a specific container, and this is not - * that container, bail out - */ - if (container && strncmp(container, meta->container_name, container_len)) { - return 0; - } - - return function(ctx, meta, - (container ? FLB_TRUE : FLB_FALSE), stream, - val_buf, val_len, props); -} - -int flb_kube_prop_pack(struct flb_kube_props *props, - void **out_buf, size_t *out_size) -{ - int size; - msgpack_packer pck; - msgpack_sbuffer sbuf; - - /* Number of fields in props structure */ - size = FLB_KUBE_NUMBER_OF_PROPS; - - /* Create msgpack buffer */ - msgpack_sbuffer_init(&sbuf); - msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); - - /* Main array */ - msgpack_pack_array(&pck, size); - - /* Index 0: FLB_KUBE_PROPS_STDOUT_PARSER */ - if (props->stdout_parser) { - msgpack_pack_str(&pck, flb_sds_len(props->stdout_parser)); - msgpack_pack_str_body(&pck, props->stdout_parser, flb_sds_len(props->stdout_parser)); - } - else { - msgpack_pack_nil(&pck); - } - - /* Index 1: FLB_KUBE_PROPS_STDERR_PARSER */ - if (props->stderr_parser) { - msgpack_pack_str(&pck, flb_sds_len(props->stderr_parser)); - msgpack_pack_str_body(&pck, props->stderr_parser, flb_sds_len(props->stderr_parser)); - } - else { - msgpack_pack_nil(&pck); - } - - /* Index 2: FLB_KUBE_PROPS_STDOUT_EXCLUDE */ - if (props->stdout_exclude == FLB_KUBE_PROP_TRUE) { - msgpack_pack_true(&pck); - } - else { - msgpack_pack_false(&pck); - } - - /* Index 3: FLB_KUBE_PROPS_STDERR_EXCLUDE */ - if (props->stderr_exclude == FLB_KUBE_PROP_TRUE) { - msgpack_pack_true(&pck); - } - else { - msgpack_pack_false(&pck); - } - - /* Set outgoing msgpack buffer */ - *out_buf = sbuf.data; - *out_size = sbuf.size; - - return 0; -} - -int flb_kube_prop_unpack(struct flb_kube_props *props, - const char *buf, size_t size) -{ - int ret; - size_t off = 0; - msgpack_object o; - msgpack_object root; - msgpack_unpacked result; - - memset(props, '\0', sizeof(struct flb_kube_props)); - - msgpack_unpacked_init(&result); - ret = msgpack_unpack_next(&result, buf, size, &off); - if (ret == MSGPACK_UNPACK_PARSE_ERROR) { - msgpack_unpacked_destroy(&result); - return -1; - } - root = result.data; - - /* Index 0: stdout_parser */ - o = root.via.array.ptr[FLB_KUBE_PROPS_STDOUT_PARSER]; - if (o.type == MSGPACK_OBJECT_NIL) { - props->stdout_parser = NULL; - } - else { - props->stdout_parser = flb_sds_create_len(o.via.str.ptr, o.via.str.size); - } - - /* Index 1: stderr_parser */ - o = root.via.array.ptr[FLB_KUBE_PROPS_STDERR_PARSER]; - if (o.type == MSGPACK_OBJECT_NIL) { - props->stderr_parser = NULL; - } - else { - props->stderr_parser = flb_sds_create_len(o.via.str.ptr, o.via.str.size); - } - - /* Index 2: stdout_exclude */ - o = root.via.array.ptr[FLB_KUBE_PROPS_STDOUT_EXCLUDE]; - props->stdout_exclude = o.via.boolean; - - /* Index 3: stderr_exclude */ - o = root.via.array.ptr[FLB_KUBE_PROPS_STDERR_EXCLUDE]; - props->stderr_exclude = o.via.boolean; - - msgpack_unpacked_destroy(&result); - return 0; -} - -/* Destroy any resource held by a props element */ -void flb_kube_prop_destroy(struct flb_kube_props *props) -{ - if (props->stdout_parser) { - flb_sds_destroy(props->stdout_parser); - props->stdout_parser = NULL; - } - if (props->stderr_parser) { - flb_sds_destroy(props->stderr_parser); - props->stderr_parser = NULL; - } -} diff --git a/fluent-bit/plugins/filter_kubernetes/kube_property.h b/fluent-bit/plugins/filter_kubernetes/kube_property.h deleted file mode 100644 index c2c8503b6..000000000 --- a/fluent-bit/plugins/filter_kubernetes/kube_property.h +++ /dev/null @@ -1,40 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_FILTER_KUBE_PROP_H -#define FLB_FILTER_KUBE_PROP_H - -#include "kube_meta.h" -#include "kube_props.h" - -#define FLB_KUBE_PROP_NO_STREAM 0 -#define FLB_KUBE_PROP_STREAM_STDOUT 1 -#define FLB_KUBE_PROP_STREAM_STDERR 2 -#define FLB_KUBE_PROP_STREAM_UNKNOWN 3 - -int flb_kube_prop_set(struct flb_kube *ctx, struct flb_kube_meta *meta, - const char *prop, int prop_len, - const char *val_buf, size_t val_len, - struct flb_kube_props *props); -int flb_kube_prop_pack(struct flb_kube_props *props, - void **out_buf, size_t *out_size); -int flb_kube_prop_unpack(struct flb_kube_props *props, const char *buf, size_t size); -void flb_kube_prop_destroy(struct flb_kube_props *props); - -#endif diff --git a/fluent-bit/plugins/filter_kubernetes/kube_props.h b/fluent-bit/plugins/filter_kubernetes/kube_props.h deleted file mode 100644 index 79388e43c..000000000 --- a/fluent-bit/plugins/filter_kubernetes/kube_props.h +++ /dev/null @@ -1,44 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_FILTER_KUBE_PROPS_H -#define FLB_FILTER_KUBE_PROPS_H - -#include -#include - -/* Property structure/array index */ -#define FLB_KUBE_PROPS_STDOUT_PARSER 0 -#define FLB_KUBE_PROPS_STDERR_PARSER 1 -#define FLB_KUBE_PROPS_STDOUT_EXCLUDE 2 -#define FLB_KUBE_PROPS_STDERR_EXCLUDE 3 -#define FLB_KUBE_NUMBER_OF_PROPS 4 - -#define FLB_KUBE_PROP_UNDEF 0 -#define FLB_KUBE_PROP_FALSE 1 -#define FLB_KUBE_PROP_TRUE 2 - -struct flb_kube_props { - flb_sds_t stdout_parser; /* suggested parser for stdout */ - flb_sds_t stderr_parser; /* suggested parser for stderr */ - int stdout_exclude; /* bool: exclude stdout logs ? */ - int stderr_exclude; /* bool: exclude stderr logs ? */ -}; - -#endif diff --git a/fluent-bit/plugins/filter_kubernetes/kube_regex.c b/fluent-bit/plugins/filter_kubernetes/kube_regex.c deleted file mode 100644 index e530ecf02..000000000 --- a/fluent-bit/plugins/filter_kubernetes/kube_regex.c +++ /dev/null @@ -1,43 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include "kube_conf.h" -#include "kube_regex.h" - -int flb_kube_regex_init(struct flb_kube *ctx) -{ - /* If a custom parser is not set, use the defaults */ - if (!ctx->parser) { - if (ctx->use_journal == FLB_TRUE) { - ctx->regex = flb_regex_create(KUBE_JOURNAL_TO_REGEX); - } - else { - ctx->regex = flb_regex_create(KUBE_TAG_TO_REGEX); - } - } - - if (!ctx->regex) { - return -1; - } - - return 0; -} diff --git a/fluent-bit/plugins/filter_kubernetes/kube_regex.h b/fluent-bit/plugins/filter_kubernetes/kube_regex.h deleted file mode 100644 index ae648fd74..000000000 --- a/fluent-bit/plugins/filter_kubernetes/kube_regex.h +++ /dev/null @@ -1,31 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_FILTER_KUBE_REGEX_H -#define FLB_FILTER_KUBE_REGEX_H - -#include "kube_conf.h" - -#define KUBE_TAG_TO_REGEX "(?[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?[^_]+)_(?.+)-(?[a-z0-9]{64})\\.log$" - -#define KUBE_JOURNAL_TO_REGEX "^(?[^_]+)_(?[^\\._]+)(\\.(?[^_]+))?_(?[^_]+)_(?[^_]+)_[^_]+_[^_]+$" - -int flb_kube_regex_init(struct flb_kube *ctx); - -#endif diff --git a/fluent-bit/plugins/filter_kubernetes/kubernetes.c b/fluent-bit/plugins/filter_kubernetes/kubernetes.c deleted file mode 100644 index f54e08483..000000000 --- a/fluent-bit/plugins/filter_kubernetes/kubernetes.c +++ /dev/null @@ -1,1000 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "kube_conf.h" -#include "kube_meta.h" -#include "kube_regex.h" -#include "kube_property.h" - -#include -#include - -/* Merge status used by merge_log_handler() */ -#define MERGE_NONE 0 /* merge unescaped string in temporary buffer */ -#define MERGE_PARSED 1 /* merge parsed string (log_buf) */ -#define MERGE_MAP 2 /* merge direct binary object (v) */ - -static int get_stream(msgpack_object_map map) -{ - int i; - msgpack_object k; - msgpack_object v; - - for (i = 0; i < map.size; i++) { - k = map.ptr[i].key; - v = map.ptr[i].val; - - if (k.type == MSGPACK_OBJECT_STR && - strncmp(k.via.str.ptr, "stream", k.via.str.size) == 0) { - if (strncmp(v.via.str.ptr, "stdout", v.via.str.size) == 0) { - return FLB_KUBE_PROP_STREAM_STDOUT; - } - else if (strncmp(v.via.str.ptr, "stderr", v.via.str.size) == 0) { - return FLB_KUBE_PROP_STREAM_STDERR; - } - else { - return FLB_KUBE_PROP_STREAM_UNKNOWN; - } - } - } - - return FLB_KUBE_PROP_NO_STREAM; -} - -static int value_trim_size(msgpack_object o) -{ - int i; - int size = o.via.str.size; - - for (i = size - 1; i > 0; i--) { - if (o.via.str.ptr[i] == '\n') { - size -= 1; - continue; - } - - if (o.via.str.ptr[i - 1] == '\\' && - (o.via.str.ptr[i] == 'n' || o.via.str.ptr[i] == 'r')) { - size -= 2; - i--; - } - else { - break; - } - } - - return size; -} - -static int merge_log_handler(msgpack_object o, - struct flb_parser *parser, - void **out_buf, size_t *out_size, - struct flb_time *log_time, - struct flb_kube *ctx) -{ - int ret; - int new_size; - int root_type; - int records = 0; - char *tmp; - - /* Reset vars */ - *out_buf = NULL; - *out_size = 0; - - /* Allocate more space if required */ - if (o.via.str.size >= ctx->unesc_buf_size) { - new_size = o.via.str.size + 1; - tmp = flb_realloc(ctx->unesc_buf, new_size); - if (tmp) { - ctx->unesc_buf = tmp; - ctx->unesc_buf_size = new_size; - } - else { - flb_errno(); - return -1; - } - } - - /* Copy the string value and append the required NULL byte */ - ctx->unesc_buf_len = (int) o.via.str.size; - memcpy(ctx->unesc_buf, o.via.str.ptr, o.via.str.size); - ctx->unesc_buf[ctx->unesc_buf_len] = '\0'; - - ret = -1; - - /* Parser set by Annotation */ - if (parser) { - ret = flb_parser_do(parser, ctx->unesc_buf, ctx->unesc_buf_len, - out_buf, out_size, log_time); - if (ret >= 0) { - if (flb_time_to_nanosec(log_time) == 0L) { - flb_time_get(log_time); - } - return MERGE_PARSED; - } - } - else if (ctx->merge_parser) { /* Custom parser 'merge_parser' option */ - ret = flb_parser_do(ctx->merge_parser, - ctx->unesc_buf, ctx->unesc_buf_len, - out_buf, out_size, log_time); - if (ret >= 0) { - if (flb_time_to_nanosec(log_time) == 0L) { - flb_time_get(log_time); - } - return MERGE_PARSED; - } - } - else { /* Default JSON parser */ - ret = flb_pack_json_recs(ctx->unesc_buf, ctx->unesc_buf_len, - (char **) out_buf, out_size, &root_type, - &records, NULL); - if (ret == 0 && root_type != FLB_PACK_JSON_OBJECT) { - flb_plg_debug(ctx->ins, "could not merge JSON, root_type=%i", - root_type); - flb_free(*out_buf); - return MERGE_NONE; - } - - if (ret == 0 && records != 1) { - flb_plg_debug(ctx->ins, - "could not merge JSON, invalid number of records: %i", - records); - flb_free(*out_buf); - return MERGE_NONE; - } - } - - if (ret == -1) { - return MERGE_NONE; - } - - return MERGE_PARSED; -} - -static int cb_kube_init(struct flb_filter_instance *f_ins, - struct flb_config *config, - void *data) -{ - int ret; - struct flb_kube *ctx; - (void) data; - - /* Create configuration context */ - ctx = flb_kube_conf_create(f_ins, config); - if (!ctx) { - return -1; - } - - /* Initialize regex context */ - ret = flb_kube_regex_init(ctx); - if (ret == -1) { - flb_kube_conf_destroy(ctx); - return -1; - } - - /* Set context */ - flb_filter_set_context(f_ins, ctx); - - /* - * Get Kubernetes Metadata: we gather this at the beginning - * as we need this information to process logs in Kubernetes - * environment, otherwise the service should not start. - */ - flb_kube_meta_init(ctx, config); - - return 0; -} - -static int pack_map_content(struct flb_log_event_encoder *log_encoder, - msgpack_object source_map, - const char *kube_buf, size_t kube_size, - struct flb_kube_meta *meta, - struct flb_time *time_lookup, - struct flb_parser *parser, - struct flb_kube *ctx) -{ - int append_original_objects; - int scope_opened; - int ret; - int i; - int map_size = 0; - int merge_status = -1; - int log_index = -1; - int log_buf_entries = 0; - size_t off = 0; - void *log_buf = NULL; - size_t log_size = 0; - msgpack_unpacked result; - msgpack_object k; - msgpack_object v; - msgpack_object root; - struct flb_time log_time; - - /* Original map size */ - map_size = source_map.via.map.size; - - /* If merge_log is enabled, we need to lookup the 'log' field */ - if (ctx->merge_log == FLB_TRUE) { - for (i = 0; i < map_size; i++) { - k = source_map.via.map.ptr[i].key; - - /* Validate 'log' field */ - if (k.via.str.size == 3 && - strncmp(k.via.str.ptr, "log", 3) == 0) { - log_index = i; - break; - } - } - } - - /* reset */ - flb_time_zero(&log_time); - - /* - * If a log_index exists, the application log content inside the - * Docker JSON map is a escaped string. Proceed to reserve a temporary - * buffer and create an unescaped version. - */ - if (log_index != -1) { - v = source_map.via.map.ptr[log_index].val; - if (v.type == MSGPACK_OBJECT_MAP) { - /* This is the easiest way, no extra processing required */ - merge_status = MERGE_MAP; - } - else if (v.type == MSGPACK_OBJECT_STR) { - merge_status = merge_log_handler(v, parser, - &log_buf, &log_size, - &log_time, ctx); - } - } - - /* Append record timestamp */ - if (merge_status == MERGE_PARSED) { - if (flb_time_to_nanosec(&log_time) == 0L) { - ret = flb_log_event_encoder_set_timestamp( - log_encoder, time_lookup); - } - else { - ret = flb_log_event_encoder_set_timestamp( - log_encoder, &log_time); - } - } - else { - ret = flb_log_event_encoder_set_timestamp( - log_encoder, time_lookup); - } - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - return -1; - } - - /* If a merged status exists, check the number of entries to merge */ - if (log_index != -1) { - if (merge_status == MERGE_PARSED) { - off = 0; - msgpack_unpacked_init(&result); - msgpack_unpack_next(&result, log_buf, log_size, &off); - root = result.data; - if (root.type == MSGPACK_OBJECT_MAP) { - log_buf_entries = root.via.map.size; - } - msgpack_unpacked_destroy(&result); - } - else if (merge_status == MERGE_MAP) { - /* object 'v' represents the original binary log */ - log_buf_entries = v.via.map.size; - } - } - - if ((merge_status == MERGE_PARSED || merge_status == MERGE_MAP) && - ctx->keep_log == FLB_FALSE) { - } - - /* Original map */ - for (i = 0; - i < map_size && - ret == FLB_EVENT_ENCODER_SUCCESS; - i++) { - k = source_map.via.map.ptr[i].key; - v = source_map.via.map.ptr[i].val; - - /* - * If log_index is set, means a merge log is a requirement but - * will depend on merge_status. If the parsing failed we cannot - * merge so we keep the 'log' key/value. - */ - append_original_objects = FLB_FALSE; - - if (log_index == i) { - if (ctx->keep_log == FLB_TRUE) { - if (merge_status == MERGE_NONE || merge_status == MERGE_PARSED){ - ret = flb_log_event_encoder_append_body_values( - log_encoder, - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&k), - FLB_LOG_EVENT_STRING_VALUE(ctx->unesc_buf, - ctx->unesc_buf_len)); - } - else { - append_original_objects = FLB_TRUE; - } - } - else if (merge_status == MERGE_NONE) { - append_original_objects = FLB_TRUE; - } - } - else { - append_original_objects = FLB_TRUE; - } - - if (append_original_objects) { - ret = flb_log_event_encoder_append_body_values( - log_encoder, - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&k), - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&v)); - } - } - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - return -2; - } - - scope_opened = FLB_FALSE; - /* Merge Log */ - if (log_index != -1) { - if (merge_status == MERGE_PARSED) { - if (ctx->merge_log_key && log_buf_entries > 0) { - ret = flb_log_event_encoder_append_body_string( - log_encoder, - ctx->merge_log_key, - flb_sds_len(ctx->merge_log_key)); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_body_begin_map(log_encoder); - } - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - return -3; - } - - scope_opened = FLB_TRUE; - } - - off = 0; - msgpack_unpacked_init(&result); - msgpack_unpack_next(&result, log_buf, log_size, &off); - root = result.data; - - for (i = 0; - i < log_buf_entries && - ret == FLB_EVENT_ENCODER_SUCCESS; - i++) { - k = root.via.map.ptr[i].key; - - ret = flb_log_event_encoder_append_body_msgpack_object( - log_encoder, &k); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - return -4; - } - - v = root.via.map.ptr[i].val; - - /* - * If this is the last string value, trim any remaining - * break line or return carrier character. - */ - if (v.type == MSGPACK_OBJECT_STR && - ctx->merge_log_trim == FLB_TRUE) { - ret = flb_log_event_encoder_append_body_string( - log_encoder, - (char *) v.via.str.ptr, - value_trim_size(v)); - } - else { - ret = flb_log_event_encoder_append_body_msgpack_object( - log_encoder, &v); - } - } - - msgpack_unpacked_destroy(&result); - - flb_free(log_buf); - - if (scope_opened && ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_body_commit_map(log_encoder); - } - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - return -5; - } - } - else if (merge_status == MERGE_MAP) { - msgpack_object map; - - if (ctx->merge_log_key && log_buf_entries > 0) { - ret = flb_log_event_encoder_append_body_string( - log_encoder, - ctx->merge_log_key, - flb_sds_len(ctx->merge_log_key)); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_body_begin_map(log_encoder); - } - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - return -6; - } - - scope_opened = FLB_TRUE; - } - - map = source_map.via.map.ptr[log_index].val; - for (i = 0; - i < map.via.map.size && - ret == FLB_EVENT_ENCODER_SUCCESS; - i++) { - k = map.via.map.ptr[i].key; - v = map.via.map.ptr[i].val; - - ret = flb_log_event_encoder_append_body_values( - log_encoder, - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&k), - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&v)); - } - - if (scope_opened && ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_body_commit_map(log_encoder); - } - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - return -7; - } - } - } - - /* Kubernetes */ - if (kube_buf && kube_size > 0) { - ret = flb_log_event_encoder_append_body_cstring( - log_encoder, - "kubernetes"); - - off = 0; - msgpack_unpacked_init(&result); - msgpack_unpack_next(&result, kube_buf, kube_size, &off); - - if (kube_size != off) { - /* This buffer should contain a single map and we shouldn't - * have to unpack it in order to ensure that we are appending - * a single map but considering that the current code only - * appends the first entry without taking any actions I think - * we should warn the user if there is more than one entry in - * it so in the future we can remove the unpack code and just - * use flb_log_event_encoder_append_body_raw_msgpack with - * kube_size. - */ - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_raw_msgpack(log_encoder, - (char *) kube_buf, off); - } - - msgpack_unpacked_destroy(&result); - } - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - return -8; - } - - return 0; -} - -static int cb_kube_filter(const void *data, size_t bytes, - const char *tag, int tag_len, - void **out_buf, size_t *out_bytes, - struct flb_filter_instance *f_ins, - struct flb_input_instance *i_ins, - void *filter_context, - struct flb_config *config) -{ - int ret; - size_t pre = 0; - size_t off = 0; - char *dummy_cache_buf = NULL; - const char *cache_buf = NULL; - size_t cache_size = 0; - msgpack_object map; - struct flb_parser *parser = NULL; - struct flb_kube *ctx = filter_context; - struct flb_kube_meta meta = {0}; - struct flb_kube_props props = {0}; - struct flb_log_event_encoder log_encoder; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - (void) f_ins; - (void) i_ins; - (void) config; - - if (ctx->use_journal == FLB_FALSE || ctx->dummy_meta == FLB_TRUE) { - if (ctx->dummy_meta == FLB_TRUE) { - ret = flb_kube_dummy_meta_get(&dummy_cache_buf, &cache_size); - cache_buf = dummy_cache_buf; - } - else { - /* Check if we have some cached metadata for the incoming events */ - ret = flb_kube_meta_get(ctx, - tag, tag_len, - data, bytes, - &cache_buf, &cache_size, &meta, &props); - } - if (ret == -1) { - return FLB_FILTER_NOTOUCH; - } - } - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - flb_kube_meta_release(&meta); - flb_kube_prop_destroy(&props); - - return FLB_FILTER_NOTOUCH; - } - - ret = flb_log_event_encoder_init(&log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event encoder initialization error : %d", ret); - - flb_log_event_decoder_destroy(&log_decoder); - flb_kube_meta_release(&meta); - flb_kube_prop_destroy(&props); - - return FLB_FILTER_NOTOUCH; - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - off = log_decoder.offset; - /* - * Journal entries can be origined by different Pods, so we are forced - * to parse and check it metadata. - * - * note: when the source is in_tail the situation is different since all - * records passed to the filter have a unique source log file. - */ - if (ctx->use_journal == FLB_TRUE && ctx->dummy_meta == FLB_FALSE) { - ret = flb_kube_meta_get(ctx, - tag, tag_len, - (char *) data + pre, off - pre, - &cache_buf, &cache_size, &meta, &props); - if (ret == -1) { - continue; - } - - pre = off; - } - - parser = NULL; - - switch (get_stream(log_event.body->via.map)) { - case FLB_KUBE_PROP_STREAM_STDOUT: - { - if (props.stdout_exclude == FLB_TRUE) { - /* Skip this record */ - if (ctx->use_journal == FLB_TRUE) { - flb_kube_meta_release(&meta); - flb_kube_prop_destroy(&props); - } - continue; - } - if (props.stdout_parser != NULL) { - parser = flb_parser_get(props.stdout_parser, config); - } - } - break; - case FLB_KUBE_PROP_STREAM_STDERR: - { - if (props.stderr_exclude == FLB_TRUE) { - /* Skip this record */ - if (ctx->use_journal == FLB_TRUE) { - flb_kube_meta_release(&meta); - flb_kube_prop_destroy(&props); - } - continue; - } - if (props.stderr_parser != NULL) { - parser = flb_parser_get(props.stderr_parser, config); - } - } - break; - default: - { - if (props.stdout_exclude == props.stderr_exclude && - props.stderr_exclude == FLB_TRUE) { - continue; - } - if (props.stdout_parser == props.stderr_parser && - props.stderr_parser != NULL) { - parser = flb_parser_get(props.stdout_parser, config); - } - } - break; - } - - /* get records map */ - map = *log_event.body; - - ret = flb_log_event_encoder_begin_record(&log_encoder); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - break; - } - - ret = pack_map_content(&log_encoder, - map, - cache_buf, cache_size, - &meta, &log_event.timestamp, parser, ctx); - if (ret != 0) { - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - if (ctx->dummy_meta == FLB_TRUE) { - flb_free(dummy_cache_buf); - } - - flb_kube_meta_release(&meta); - flb_kube_prop_destroy(&props); - - return FLB_FILTER_NOTOUCH; - } - - ret = flb_log_event_encoder_commit_record(&log_encoder); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_log_event_encoder_rollback_record(&log_encoder); - - break; - } - - if (ctx->use_journal == FLB_TRUE) { - flb_kube_meta_release(&meta); - flb_kube_prop_destroy(&props); - } - } - - /* Release meta fields */ - if (ctx->use_journal == FLB_FALSE) { - flb_kube_meta_release(&meta); - flb_kube_prop_destroy(&props); - } - - if (ctx->dummy_meta == FLB_TRUE) { - flb_free(dummy_cache_buf); - } - - *out_buf = log_encoder.output_buffer; - *out_bytes = log_encoder.output_length; - - flb_log_event_encoder_claim_internal_buffer_ownership(&log_encoder); - - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - return FLB_FILTER_MODIFIED; -} - -static int cb_kube_exit(void *data, struct flb_config *config) -{ - struct flb_kube *ctx; - - ctx = data; - flb_kube_conf_destroy(ctx); - - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - - /* Buffer size for HTTP Client when reading responses from API Server */ - { - FLB_CONFIG_MAP_SIZE, "buffer_size", "32K", - 0, FLB_TRUE, offsetof(struct flb_kube, buffer_size), - "buffer size to receive response from API server", - }, - - /* TLS: set debug 'level' */ - { - FLB_CONFIG_MAP_INT, "tls.debug", "0", - 0, FLB_TRUE, offsetof(struct flb_kube, tls_debug), - "set TLS debug level: 0 (no debug), 1 (error), " - "2 (state change), 3 (info) and 4 (verbose)" - }, - - /* TLS: enable verification */ - { - FLB_CONFIG_MAP_BOOL, "tls.verify", "true", - 0, FLB_TRUE, offsetof(struct flb_kube, tls_verify), - "enable or disable verification of TLS peer certificate" - }, - - /* TLS: set tls.vhost feature */ - { - FLB_CONFIG_MAP_STR, "tls.vhost", NULL, - 0, FLB_TRUE, offsetof(struct flb_kube, tls_vhost), - "set optional TLS virtual host" - }, - - /* Merge structured record as independent keys */ - { - FLB_CONFIG_MAP_BOOL, "merge_log", "false", - 0, FLB_TRUE, offsetof(struct flb_kube, merge_log), - "merge 'log' key content as individual keys" - }, - - /* Optional parser for 'log' key content */ - { - FLB_CONFIG_MAP_STR, "merge_parser", NULL, - 0, FLB_FALSE, 0, - "specify a 'parser' name to parse the 'log' key content" - }, - - /* New key name to merge the structured content of 'log' */ - { - FLB_CONFIG_MAP_STR, "merge_log_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_kube, merge_log_key), - "set the 'key' name where the content of 'key' will be placed. Only " - "used if the option 'merge_log' is enabled" - }, - - /* On merge, trim field values (remove possible ending \n or \r) */ - { - FLB_CONFIG_MAP_BOOL, "merge_log_trim", "true", - 0, FLB_TRUE, offsetof(struct flb_kube, merge_log_trim), - "remove ending '\\n' or '\\r' characters from the log content" - }, - - /* Keep original log key after successful merging/parsing */ - { - FLB_CONFIG_MAP_BOOL, "keep_log", "true", - 0, FLB_TRUE, offsetof(struct flb_kube, keep_log), - "keep original log content if it was successfully parsed and merged" - }, - - /* Full Kubernetes API server URL */ - { - FLB_CONFIG_MAP_STR, "kube_url", "https://kubernetes.default.svc", - 0, FLB_FALSE, 0, - "Kubernetes API server URL" - }, - - /* - * If set, meta-data load will be attempted from files in this dir, - * falling back to API if not existing. - */ - { - FLB_CONFIG_MAP_STR, "kube_meta_preload_cache_dir", NULL, - 0, FLB_TRUE, offsetof(struct flb_kube, meta_preload_cache_dir), - "set directory with metadata files" - }, - - /* Kubernetes TLS: CA file */ - { - FLB_CONFIG_MAP_STR, "kube_ca_file", FLB_KUBE_CA, - 0, FLB_TRUE, offsetof(struct flb_kube, tls_ca_file), - "Kubernetes TLS CA file" - }, - - /* Kubernetes TLS: CA certs path */ - { - FLB_CONFIG_MAP_STR, "kube_ca_path", NULL, - 0, FLB_TRUE, offsetof(struct flb_kube, tls_ca_path), - "Kubernetes TLS ca path" - }, - - /* Kubernetes Tag prefix */ - { - FLB_CONFIG_MAP_STR, "kube_tag_prefix", FLB_KUBE_TAG_PREFIX, - 0, FLB_TRUE, offsetof(struct flb_kube, kube_tag_prefix), - "prefix used in tag by the input plugin" - }, - - /* Kubernetes Token file */ - { - FLB_CONFIG_MAP_STR, "kube_token_file", FLB_KUBE_TOKEN, - 0, FLB_TRUE, offsetof(struct flb_kube, token_file), - "Kubernetes authorization token file" - }, - - /* Kubernetes Token command */ - { - FLB_CONFIG_MAP_STR, "kube_token_command", NULL, - 0, FLB_FALSE, 0, - "command to get Kubernetes authorization token" - }, - - /* Include Kubernetes Labels in the final record ? */ - { - FLB_CONFIG_MAP_BOOL, "labels", "true", - 0, FLB_TRUE, offsetof(struct flb_kube, labels), - "include Kubernetes labels on every record" - }, - - /* Include Kubernetes Annotations in the final record ? */ - { - FLB_CONFIG_MAP_BOOL, "annotations", "true", - 0, FLB_TRUE, offsetof(struct flb_kube, annotations), - "include Kubernetes annotations on every record" - }, - - /* - * The Application may 'propose' special configuration keys - * to the logging agent (Fluent Bit) through the annotations - * set in the Pod definition, e.g: - * - * "annotations": { - * "logging": {"parser": "apache"} - * } - * - * As of now, Fluent Bit/filter_kubernetes supports the following - * options under the 'logging' map value: - * - * - k8s-logging.parser: propose Fluent Bit to parse the content - * using the pre-defined parser in the - * value (e.g: apache). - * - k8s-logging.exclude: Fluent Bit allows Pods to exclude themselves - * - * By default all options are disabled, so each option needs to - * be enabled manually. - */ - { - FLB_CONFIG_MAP_BOOL, "k8s-logging.parser", "false", - 0, FLB_TRUE, offsetof(struct flb_kube, k8s_logging_parser), - "allow Pods to suggest a parser" - }, - { - FLB_CONFIG_MAP_BOOL, "k8s-logging.exclude", "false", - 0, FLB_TRUE, offsetof(struct flb_kube, k8s_logging_exclude), - "allow Pods to exclude themselves from the logging pipeline" - }, - - /* Use Systemd Journal mode ? */ - { - FLB_CONFIG_MAP_BOOL, "use_journal", "false", - 0, FLB_TRUE, offsetof(struct flb_kube, use_journal), - "use Journald (Systemd) mode" - }, - - /* Custom Tag Regex */ - { - FLB_CONFIG_MAP_STR, "regex_parser", NULL, - 0, FLB_FALSE, 0, - "optional regex parser to extract metadata from container name or container log file name" - }, - - /* Generate dummy metadata (only for test/dev purposes) */ - { - FLB_CONFIG_MAP_BOOL, "dummy_meta", "false", - 0, FLB_TRUE, offsetof(struct flb_kube, dummy_meta), - "use 'dummy' metadata, do not talk to API server" - }, - - /* - * Poll DNS status to mitigate unreliable network issues. - * See fluent/fluent-bit/2144. - */ - { - FLB_CONFIG_MAP_INT, "dns_retries", "6", - 0, FLB_TRUE, offsetof(struct flb_kube, dns_retries), - "dns lookup retries N times until the network start working" - }, - - { - FLB_CONFIG_MAP_TIME, "dns_wait_time", "30", - 0, FLB_TRUE, offsetof(struct flb_kube, dns_wait_time), - "dns interval between network status checks" - }, - /* Fetch K8s meta when docker_id has changed */ - { - FLB_CONFIG_MAP_BOOL, "cache_use_docker_id", "false", - 0, FLB_TRUE, offsetof(struct flb_kube, cache_use_docker_id), - "fetch K8s meta when docker_id is changed" - }, - - { - FLB_CONFIG_MAP_BOOL, "use_tag_for_meta", "false", - 0, FLB_TRUE, offsetof(struct flb_kube, use_tag_for_meta), - "use tag associated to retrieve metadata instead of kube-server" - }, - - /* - * Enable the feature for using kubelet to get pods information - */ - { - FLB_CONFIG_MAP_BOOL, "use_kubelet", "false", - 0, FLB_TRUE, offsetof(struct flb_kube, use_kubelet), - "use kubelet to get metadata instead of kube-server" - }, - /* - * The kubelet host for /pods endpoint, default is 127.0.0.1 - * Will only check when "use_kubelet" config is set to true - */ - { - FLB_CONFIG_MAP_STR, "kubelet_host", "127.0.0.1", - 0, FLB_TRUE, offsetof(struct flb_kube, kubelet_host), - "kubelet host to connect with when using kubelet" - }, - /* - * The kubelet port for /pods endpoint, default is 10250 - * Will only check when "use_kubelet" config is set to true - */ - { - FLB_CONFIG_MAP_INT, "kubelet_port", "10250", - 0, FLB_TRUE, offsetof(struct flb_kube, kubelet_port), - "kubelet port to connect with when using kubelet" - }, - { - FLB_CONFIG_MAP_TIME, "kube_token_ttl", "10m", - 0, FLB_TRUE, offsetof(struct flb_kube, kube_token_ttl), - "kubernetes token ttl, until it is reread from the token file. Default: 10m" - }, - /* - * Set TTL for K8s cached metadata - */ - { - FLB_CONFIG_MAP_TIME, "kube_meta_cache_ttl", "0", - 0, FLB_TRUE, offsetof(struct flb_kube, kube_meta_cache_ttl), - "configurable TTL for K8s cached metadata. " - "By default, it is set to 0 which means TTL for cache entries is disabled and " - "cache entries are evicted at random when capacity is reached. " - "In order to enable this option, you should set the number to a time interval. " - "For example, set this value to 60 or 60s and cache entries " - "which have been created more than 60s will be evicted" - }, - /* EOF */ - {0} -}; - -struct flb_filter_plugin filter_kubernetes_plugin = { - .name = "kubernetes", - .description = "Filter to append Kubernetes metadata", - .cb_init = cb_kube_init, - .cb_filter = cb_kube_filter, - .cb_exit = cb_kube_exit, - .config_map = config_map, - .flags = 0 -}; diff --git a/fluent-bit/plugins/filter_log_to_metrics/CMakeLists.txt b/fluent-bit/plugins/filter_log_to_metrics/CMakeLists.txt deleted file mode 100644 index bc52b4c7f..000000000 --- a/fluent-bit/plugins/filter_log_to_metrics/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -if(NOT FLB_METRICS) - message(FATAL_ERROR "Log Metrics filter plugin requires FLB_METRICS=On.") -endif() - -set(src - log_to_metrics.c) - -FLB_PLUGIN(filter_log_to_metrics "${src}" "") diff --git a/fluent-bit/plugins/filter_log_to_metrics/log_to_metrics.c b/fluent-bit/plugins/filter_log_to_metrics/log_to_metrics.c deleted file mode 100644 index a61e4827f..000000000 --- a/fluent-bit/plugins/filter_log_to_metrics/log_to_metrics.c +++ /dev/null @@ -1,965 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "log_to_metrics.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -static char kubernetes_label_keys[NUMBER_OF_KUBERNETES_LABELS][16] = - { "namespace_name", - "pod_name", - "container_name", - "docker_id", - "pod_id" - }; - -static void delete_rules(struct log_to_metrics_ctx *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - struct grep_rule *rule; - - mk_list_foreach_safe(head, tmp, &ctx->rules) { - rule = mk_list_entry(head, struct grep_rule, _head); - flb_sds_destroy(rule->field); - flb_free(rule->regex_pattern); - flb_ra_destroy(rule->ra); - flb_regex_destroy(rule->regex); - mk_list_del(&rule->_head); - flb_free(rule); - } -} - -static int log_to_metrics_destroy(struct log_to_metrics_ctx *ctx) -{ - int i; - if (!ctx) { - return 0; - } - if(ctx->histogram_buckets){ - cmt_histogram_buckets_destroy(ctx->histogram_buckets); - } - - if (ctx->cmt) { - cmt_destroy(ctx->cmt); - } - - delete_rules(ctx); - - if (ctx->label_accessors != NULL) { - for (i = 0; i < MAX_LABEL_COUNT; i++) { - flb_free(ctx->label_accessors[i]); - } - flb_free(ctx->label_accessors); - } - if (ctx->label_keys != NULL) { - for (i = 0; i < MAX_LABEL_COUNT; i++) { - flb_free(ctx->label_keys[i]); - } - flb_free(ctx->label_keys); - } - flb_free(ctx->buckets); - flb_free(ctx->bucket_counter); - flb_free(ctx->label_counter); - flb_free(ctx); - return 0; -} - -static int set_rules(struct log_to_metrics_ctx *ctx, - struct flb_filter_instance *f_ins) -{ - flb_sds_t tmp; - struct mk_list *head; - struct mk_list *split; - struct flb_split_entry *sentry; - struct flb_kv *kv; - struct grep_rule *rule; - - /* Iterate all filter properties */ - mk_list_foreach(head, &f_ins->properties) { - kv = mk_list_entry(head, struct flb_kv, _head); - - /* Create a new rule */ - rule = flb_malloc(sizeof(struct grep_rule)); - if (!rule) { - flb_errno(); - return -1; - } - - /* Get the type */ - if (strcasecmp(kv->key, "regex") == 0) { - rule->type = GREP_REGEX; - } - else if (strcasecmp(kv->key, "exclude") == 0) { - rule->type = GREP_EXCLUDE; - } - else { - flb_free(rule); - continue; - } - - /* As a value we expect a pair of field name and a regular expression */ - split = flb_utils_split(kv->val, ' ', 1); - if (mk_list_size(split) != 2) { - flb_plg_error(ctx->ins, - "invalid regex, expected field and regular expression"); - delete_rules(ctx); - flb_free(rule); - flb_utils_split_free(split); - return -1; - } - - /* Get first value (field) */ - sentry = mk_list_entry_first(split, struct flb_split_entry, _head); - if (*sentry->value == '$') { - rule->field = flb_sds_create_len(sentry->value, sentry->len); - } - else { - rule->field = flb_sds_create_size(sentry->len + 2); - tmp = flb_sds_cat(rule->field, "$", 1); - rule->field = tmp; - - tmp = flb_sds_cat(rule->field, sentry->value, sentry->len); - rule->field = tmp; - } - - /* Get remaining content (regular expression) */ - sentry = mk_list_entry_last(split, struct flb_split_entry, _head); - rule->regex_pattern = flb_strndup(sentry->value, sentry->len); - if (rule->regex_pattern == NULL) { - flb_errno(); - delete_rules(ctx); - flb_free(rule); - flb_utils_split_free(split); - return -1; - } - - /* Release split */ - flb_utils_split_free(split); - - /* Create a record accessor context for this rule */ - rule->ra = flb_ra_create(rule->field, FLB_FALSE); - if (!rule->ra) { - flb_plg_error(ctx->ins, "invalid record accessor? '%s'", - rule->field); - delete_rules(ctx); - flb_free(rule); - return -1; - } - - /* Convert string to regex pattern */ - rule->regex = flb_regex_create(rule->regex_pattern); - if (!rule->regex) { - flb_plg_error(ctx->ins, "could not compile regex pattern '%s'", - rule->regex_pattern); - delete_rules(ctx); - flb_free(rule); - return -1; - } - - /* Link to parent list */ - mk_list_add(&rule->_head, &ctx->rules); - } - - return 0; -} - -/* Given a msgpack record, do some filter action based on the defined rules */ -static inline int grep_filter_data(msgpack_object map, - struct log_to_metrics_ctx *ctx) -{ - ssize_t ret; - struct mk_list *head; - struct grep_rule *rule; - - /* For each rule, validate against map fields */ - mk_list_foreach(head, &ctx->rules) { - rule = mk_list_entry(head, struct grep_rule, _head); - - ret = flb_ra_regex_match(rule->ra, map, rule->regex, NULL); - if (ret <= 0) { /* no match */ - if (rule->type == GREP_REGEX) { - return GREP_RET_EXCLUDE; - } - } - else { - if (rule->type == GREP_EXCLUDE) { - return GREP_RET_EXCLUDE; - } - else { - return GREP_RET_KEEP; - } - } - } - - return GREP_RET_KEEP; -} - -static int set_labels(struct log_to_metrics_ctx *ctx, - char **label_accessors, - char **label_keys, - int *label_counter, - struct flb_filter_instance *f_ins) -{ - - struct mk_list *head; - struct mk_list *split; - flb_sds_t tmp; - struct flb_kv *kv; - struct flb_split_entry *sentry; - int counter = 0; - int i; - if (MAX_LABEL_COUNT < NUMBER_OF_KUBERNETES_LABELS){ - flb_errno(); - return -1; - } - if (ctx->kubernetes_mode){ - for (i = 0; i < NUMBER_OF_KUBERNETES_LABELS; i++){ - snprintf(label_keys[i], MAX_LABEL_LENGTH - 1, "%s", - kubernetes_label_keys[i]); - } - counter = NUMBER_OF_KUBERNETES_LABELS; - } - - /* Iterate all filter properties */ - mk_list_foreach(head, &f_ins->properties) { - kv = mk_list_entry(head, struct flb_kv, _head); - - if (counter >= MAX_LABEL_COUNT) { - return MAX_LABEL_COUNT; - } - - if (strcasecmp(kv->key, "label_field") == 0) { - snprintf(label_accessors[counter], MAX_LABEL_LENGTH - 1, "%s", kv->val); - snprintf(label_keys[counter], MAX_LABEL_LENGTH - 1, "%s", kv->val); - counter++; - } - else if (strcasecmp(kv->key, "add_label") == 0) { - split = flb_utils_split(kv->val, ' ', 1); - if (mk_list_size(split) != 2) { - flb_plg_error(ctx->ins, "invalid label, expected name and key"); - flb_utils_split_free(split); - return -1; - } - - sentry = mk_list_entry_first(split, struct flb_split_entry, _head); - tmp = flb_sds_create_len(sentry->value, sentry->len); - snprintf(label_keys[counter], MAX_LABEL_LENGTH - 1, "%s", tmp); - flb_sds_destroy(tmp); - - sentry = mk_list_entry_last(split, struct flb_split_entry, _head); - tmp = flb_sds_create_len(sentry->value, sentry->len); - snprintf(label_accessors[counter], MAX_LABEL_LENGTH - 1, "%s", tmp); - flb_sds_destroy(tmp); - counter++; - - flb_utils_split_free(split); - } - else { - continue; - } - } - *label_counter = counter; - return counter; -} - -static int convert_double(char *str, double *value) -{ - char *endptr = str; - int valid = 1; - int i = 0; - /* input validation */ - for (i = 0; str[i] != '\0'; i++) { - if (!(str[i]>='0') && !(str[i] <= '9') && str[i] != '.' - && str[i] != '-' && str[i] != '+') { - valid = 0; - break; - } - } - /* convert to double */ - if (valid) { - *value = strtod(str, &endptr); - if (str == endptr) { - valid = 0; - } - } - return valid; -} - -static void sort_doubles_ascending(double *arr, int size) -{ - int i, j; - double tmp; - - for (i = 0; i < size - 1; i++) { - for (j = 0; j < size - i - 1; j++) { - if (arr[j] > arr[j + 1]) { - tmp = arr[j]; - arr[j] = arr[j + 1]; - arr[j + 1] = tmp; - } - } - } -} -static int set_buckets(struct log_to_metrics_ctx *ctx, - struct flb_filter_instance *f_ins) -{ - - struct mk_list *head; - struct flb_kv *kv; - double parsed_double = 0.0; - int counter = 0; - int valid = 1; - - /* Iterate filter properties to get count of buckets to allocate memory */ - mk_list_foreach(head, &f_ins->properties) { - kv = mk_list_entry(head, struct flb_kv, _head); - - if (strcasecmp(kv->key, "bucket") != 0) { - continue; - } - counter++; - } - /* Allocate the memory for buckets */ - ctx->buckets = (double *) flb_malloc(counter * sizeof(double)); - /* Set the buckets */ - counter = 0; - mk_list_foreach(head, &f_ins->properties) { - kv = mk_list_entry(head, struct flb_kv, _head); - - if (strcasecmp(kv->key, "bucket") != 0) { - continue; - } - valid = convert_double(kv->val, &parsed_double); - if(!valid){ - flb_error("Error during conversion"); - return -1; - } - else{ - ctx->buckets[counter++] = parsed_double; - } - } - *ctx->bucket_counter = counter; - sort_doubles_ascending(ctx->buckets, counter); - return 0; -} - -static int fill_labels(struct log_to_metrics_ctx *ctx, char **label_values, - char kubernetes_label_values - [NUMBER_OF_KUBERNETES_LABELS][MAX_LABEL_LENGTH], - char **label_accessors, int label_counter, msgpack_object map) -{ - int label_iterator_start = 0; - int i; - struct flb_record_accessor *ra = NULL; - struct flb_ra_value *rval = NULL; - - if (label_counter == 0 && !ctx->kubernetes_mode){ - return 0; - } - if (MAX_LABEL_COUNT < NUMBER_OF_KUBERNETES_LABELS){ - flb_errno(); - return -1; - } - if (ctx->kubernetes_mode){ - for (i = 0; i < NUMBER_OF_KUBERNETES_LABELS; i++){ - if (kubernetes_label_keys[i] == NULL){ - return -1; - } - snprintf(label_values[i], MAX_LABEL_LENGTH - 1, "%s", - kubernetes_label_values[i]); - } - label_iterator_start = NUMBER_OF_KUBERNETES_LABELS; - } - - for (i = label_iterator_start; i < label_counter; i++){ - ra = flb_ra_create(label_accessors[i], FLB_TRUE); - if (!ra) { - flb_warn("invalid record accessor key, aborting"); - break; - } - - rval = flb_ra_get_value_object(ra, map); - if (!rval) { - /* Set value to empty string, so the value will be dropped in Cmetrics*/ - label_values[i][0] = '\0'; - } else if (rval->type == FLB_RA_STRING) { - snprintf(label_values[i], MAX_LABEL_LENGTH - 1, "%s", - rval->val.string); - } - else if (rval->type == FLB_RA_FLOAT) { - snprintf(label_values[i], MAX_LABEL_LENGTH - 1, "%f", - rval->val.f64); - } - else if (rval->type == FLB_RA_INT) { - snprintf(label_values[i], MAX_LABEL_LENGTH - 1, "%ld", - (long)rval->val.i64); - } - else { - flb_warn("cannot convert given value to metric"); - break; - } - if (rval){ - flb_ra_key_value_destroy(rval); - rval = NULL; - } - if (ra){ - flb_ra_destroy(ra); - ra = NULL; - } - } - return label_counter; -} - -static int cb_log_to_metrics_init(struct flb_filter_instance *f_ins, - struct flb_config *config, void *data) -{ - int ret; - struct log_to_metrics_ctx *ctx; - flb_sds_t tmp; - char metric_description[MAX_METRIC_LENGTH]; - char metric_name[MAX_METRIC_LENGTH]; - char value_field[MAX_METRIC_LENGTH]; - struct flb_input_instance *input_ins; - int label_count; - int i; - /* Create context */ - ctx = flb_malloc(sizeof(struct log_to_metrics_ctx)); - if (!ctx) { - flb_errno(); - return -1; - } - if (flb_filter_config_map_set(f_ins, ctx) < 0) { - flb_errno(); - flb_plg_error(f_ins, "configuration error"); - flb_free(ctx); - return -1; - } - mk_list_init(&ctx->rules); - - ctx->ins = f_ins; - - /* Load rules */ - ret = set_rules(ctx, f_ins); - if (ret == -1) { - flb_free(ctx); - return -1; - } - - /* Set the context */ - flb_filter_set_context(f_ins, ctx); - - /* Set buckets for histogram */ - ctx->bucket_counter = NULL; - ctx->histogram_buckets = NULL; - ctx->buckets = NULL; - ctx->bucket_counter = flb_malloc(sizeof(int)); - if(set_buckets(ctx, f_ins) != 0) - { - flb_plg_error(f_ins, "Setting buckets failed"); - log_to_metrics_destroy(ctx); - return -1; - } - - ctx->label_accessors = NULL; - ctx->label_accessors = (char **) flb_malloc(MAX_LABEL_COUNT * sizeof(char *)); - for (i = 0; i < MAX_LABEL_COUNT; i++) { - ctx->label_accessors[i] = flb_malloc(MAX_LABEL_LENGTH * sizeof(char)); - } - /* Set label keys */ - ctx->label_keys = NULL; - ctx->label_keys = (char **) flb_malloc(MAX_LABEL_COUNT * sizeof(char *)); - for (i = 0; i < MAX_LABEL_COUNT; i++) { - ctx->label_keys[i] = flb_malloc(MAX_LABEL_LENGTH * sizeof(char)); - } - ctx->label_counter = NULL; - ctx->label_counter = flb_malloc(sizeof(int)); - label_count = set_labels(ctx, ctx->label_accessors, ctx->label_keys, ctx->label_counter, f_ins); - if (label_count < 0){ - log_to_metrics_destroy(ctx); - return -1; - } - - /* Check metric tag */ - if (ctx->tag == NULL || strlen(ctx->tag) == 0) { - flb_plg_error(f_ins, "Metric tag is not set"); - log_to_metrics_destroy(ctx); - return -1; - } - - /* Check property metric mode */ - ctx->mode = 0; - tmp = (char *)flb_filter_get_property("metric_mode", f_ins); - if (tmp != NULL) { - if (strcasecmp(tmp, FLB_LOG_TO_METRICS_COUNTER_STR) == 0) { - ctx->mode = FLB_LOG_TO_METRICS_COUNTER; - } - else if (strcasecmp(tmp, FLB_LOG_TO_METRICS_GAUGE_STR) == 0) { - ctx->mode = FLB_LOG_TO_METRICS_GAUGE; - } - else if (strcasecmp(tmp, FLB_LOG_TO_METRICS_HISTOGRAM_STR) == 0) { - ctx->mode = FLB_LOG_TO_METRICS_HISTOGRAM; - } - else { - flb_plg_error(f_ins, - "invalid 'mode' value. Only " - "'counter', 'gauge' or " - "'histogram' types are allowed"); - log_to_metrics_destroy(ctx); - return -1; - } - } - else { - flb_plg_error(f_ins, "configuration property not set"); - log_to_metrics_destroy(ctx); - return -1; - } - - /* Check property metric name */ - if (ctx->metric_name == NULL || strlen(ctx->metric_name) == 0) { - flb_plg_error(f_ins, "metric_name is not set"); - log_to_metrics_destroy(ctx); - return -1; - } - snprintf(metric_name, sizeof(metric_name) - 1, "%s", ctx->metric_name); - - /* Check property metric description */ - if (ctx->metric_description == NULL || - strlen(ctx->metric_description) == 0) { - flb_plg_error(f_ins, "metric_description is not set"); - log_to_metrics_destroy(ctx); - return -1; - } - snprintf(metric_description, sizeof(metric_description) - 1, "%s", - ctx->metric_description); - - /* Value field only needed for modes gauge and histogram */ - if (ctx->mode > 0) { - if (ctx->value_field == NULL || strlen(ctx->value_field) == 0) { - flb_plg_error(f_ins, "value_field is not set"); - log_to_metrics_destroy(ctx); - return -1; - } - snprintf(value_field, sizeof(value_field) - 1, "%s", - ctx->value_field); - } - - - /* Check if buckets are defined for histogram, if not assume defaults */ - if (ctx->mode == FLB_LOG_TO_METRICS_HISTOGRAM ){ - if (ctx->bucket_counter == 0){ - flb_plg_error(f_ins, - "buckets are not set for histogram." - "Will use defaults: 0.005, 0.01, 0.025, " - "0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0"); - ctx->histogram_buckets = cmt_histogram_buckets_default_create(); - } - else{ - ctx->histogram_buckets = cmt_histogram_buckets_create_size( - ctx->buckets, *ctx->bucket_counter); - } - } - - - /* create the metric */ - ctx->cmt = NULL; - ctx->cmt = cmt_create(); - - /* Depending on mode create different types of cmetrics metrics */ - switch (ctx->mode) { - case FLB_LOG_TO_METRICS_COUNTER: - ctx->c = cmt_counter_create(ctx->cmt, "log_metric", "counter", - metric_name, metric_description, - label_count, ctx->label_keys); - break; - case FLB_LOG_TO_METRICS_GAUGE: - ctx->g = cmt_gauge_create(ctx->cmt, "log_metric", "gauge", - metric_name, metric_description, - label_count, ctx->label_keys); - break; - case FLB_LOG_TO_METRICS_HISTOGRAM: - ctx->h = cmt_histogram_create(ctx->cmt, "log_metric", "histogram", - metric_name, metric_description, - ctx->histogram_buckets, - label_count, ctx->label_keys); - break; - default: - flb_plg_error(f_ins, "unsupported mode"); - log_to_metrics_destroy(ctx); - return -1; - } - - input_ins = flb_input_new(config, "emitter", NULL, FLB_FALSE); - if (!input_ins) { - flb_plg_error(f_ins, "cannot create metrics emitter instance"); - log_to_metrics_destroy(ctx); - return -1; - } - - /* Set the storage type for emitter */ - ret = flb_input_set_property(input_ins, "storage.type", "memory"); - if (ret == -1) { - flb_plg_error(f_ins, "cannot set storage type for emitter instance"); - log_to_metrics_destroy(ctx); - return -1; - } - - /* Initialize emitter plugin */ - ret = flb_input_instance_init(input_ins, config); - if (ret == -1) { - flb_errno(); - flb_plg_error(f_ins, "cannot initialize metrics emitter instance."); - log_to_metrics_destroy(ctx); - return -1; - } - - ret = flb_storage_input_create(config->cio, input_ins); - if (ret == -1) { - flb_plg_error(ctx->ins, "cannot initialize storage for metrics stream"); - log_to_metrics_destroy(ctx); - return -1; - } - ctx->input_ins = input_ins; - - return 0; -} - -static int cb_log_to_metrics_filter(const void *data, size_t bytes, - const char *tag, int tag_len, - void **out_buf, size_t *out_size, - struct flb_filter_instance *f_ins, - struct flb_input_instance *i_ins, void *context, - struct flb_config *config) -{ - int ret; - msgpack_unpacked result; - msgpack_object map; - msgpack_object root; - size_t off = 0; - msgpack_sbuffer tmp_sbuf; - msgpack_packer tmp_pck; - uint64_t ts; - struct log_to_metrics_ctx *ctx = context; - struct flb_ra_value *rval = NULL; - struct flb_record_accessor *ra = NULL; - char fmt[MAX_LABEL_LENGTH]; - char **label_values = NULL; - int label_count = 0; - int i; - double gauge_value = 0; - double histogram_value = 0; - char kubernetes_label_values - [NUMBER_OF_KUBERNETES_LABELS][MAX_LABEL_LENGTH]; - - /* Create temporary msgpack buffer */ - msgpack_sbuffer_init(&tmp_sbuf); - msgpack_packer_init(&tmp_pck, &tmp_sbuf, msgpack_sbuffer_write); - - /* Iterate each item array and apply rules and generate metric values */ - msgpack_unpacked_init(&result); - while (msgpack_unpack_next(&result, data, bytes, &off) == - MSGPACK_UNPACK_SUCCESS) { - root = result.data; - if (root.type != MSGPACK_OBJECT_ARRAY) { - continue; - } - - /* get time and map */ - map = root.via.array.ptr[1]; - - ret = grep_filter_data(map, context); - if (ret == GREP_RET_KEEP) { - ts = cfl_time_now(); - if(ctx->kubernetes_mode){ - for(i = 0; i < NUMBER_OF_KUBERNETES_LABELS; i++){ - if (kubernetes_label_keys[i] == NULL){ - flb_error("error during kubernetes label processing. " - "Skipping labels."); - ctx->label_counter = 0; - break; - } - snprintf(fmt, MAX_LABEL_LENGTH - 1, "$kubernetes['%s']", - kubernetes_label_keys[i]); - ra = flb_ra_create(fmt, FLB_TRUE); - if (!ra) { - flb_error("invalid record accessor key, aborting"); - break; - } - rval = flb_ra_get_value_object(ra, map); - if (!rval) { - flb_error("given value field is empty or not " - "existent: %s. Skipping labels.", fmt); - ctx->label_counter = 0; - } - else if (rval->type != FLB_RA_STRING) { - flb_plg_error(f_ins, - "cannot access label %s", kubernetes_label_keys[i]); - break; - } - else { - snprintf(kubernetes_label_values[i], - MAX_LABEL_LENGTH - 1, "%s", rval->val.string); - } - if (rval){ - flb_ra_key_value_destroy(rval); - rval = NULL; - } - if (ra){ - flb_ra_destroy(ra); - ra = NULL; - } - } - } - if (ctx->label_counter > 0){ - /* Fill optional labels */ - label_values = flb_malloc(MAX_LABEL_COUNT * sizeof(char *)); - for (i = 0; i < MAX_LABEL_COUNT; i++) { - label_values[i] = flb_malloc(MAX_LABEL_LENGTH * - sizeof(char)); - } - - label_count = fill_labels(ctx, label_values, - kubernetes_label_values, ctx->label_accessors, - *ctx->label_counter, map); - if (label_count != *ctx->label_counter){ - label_count = 0; - } - } - - /* Calculating and setting metric depending on the mode */ - switch (ctx->mode) { - case FLB_LOG_TO_METRICS_COUNTER: - ret = cmt_counter_inc(ctx->c, ts, label_count, - label_values); - break; - - case FLB_LOG_TO_METRICS_GAUGE: - ra = flb_ra_create(ctx->value_field, FLB_TRUE); - if (!ra) { - flb_error("invalid record accessor key, aborting"); - break; - } - - rval = flb_ra_get_value_object(ra, map); - - if (!rval) { - flb_warn("given value field is empty or not existent"); - break; - } - if (rval->type == FLB_RA_STRING) { - sscanf(rval->val.string, "%lf", &gauge_value); - } - else if (rval->type == FLB_RA_FLOAT) { - gauge_value = rval->val.f64; - } - else if (rval->type == FLB_RA_INT) { - gauge_value = (double)rval->val.i64; - } - else { - flb_plg_error(f_ins, - "cannot convert given value to metric"); - break; - } - - ret = cmt_gauge_set(ctx->g, ts, gauge_value, - label_count, label_values); - if (rval) { - flb_ra_key_value_destroy(rval); - rval = NULL; - } - if (ra) { - flb_ra_destroy(ra); - ra = NULL; - } - break; - - case FLB_LOG_TO_METRICS_HISTOGRAM: - ra = flb_ra_create(ctx->value_field, FLB_TRUE); - if (!ra) { - flb_error("invalid record accessor key, aborting"); - break; - } - - rval = flb_ra_get_value_object(ra, map); - - if (!rval) { - flb_warn("given value field is empty or not existent"); - break; - } - if (rval->type == FLB_RA_STRING) { - sscanf(rval->val.string, "%lf", &histogram_value); - } - else if (rval->type == FLB_RA_FLOAT) { - histogram_value = rval->val.f64; - } - else if (rval->type == FLB_RA_INT) { - histogram_value = (double)rval->val.i64; - } - else { - flb_plg_error(f_ins, - "cannot convert given value to metric"); - break; - } - - ret = cmt_histogram_observe(ctx->h, ts, histogram_value, - label_count, label_values); - if (rval) { - flb_ra_key_value_destroy(rval); - rval = NULL; - } - if (ra) { - flb_ra_destroy(ra); - ra = NULL; - } - break; - default: - flb_plg_error(f_ins, "unsupported mode"); - log_to_metrics_destroy(ctx); - return -1; - } - - ret = flb_input_metrics_append(ctx->input_ins, ctx->tag, strlen(ctx->tag), ctx->cmt); - - if (ret != 0) { - flb_plg_error(ctx->ins, "could not append metrics"); - } - - /* Cleanup */ - msgpack_unpacked_destroy(&result); - if (label_values != NULL){ - for (i = 0; i < MAX_LABEL_COUNT; i++) { - if (label_values[i] != NULL){ - flb_free(label_values[i]); - } - } - flb_free(label_values); - } - } - else if (ret == GREP_RET_EXCLUDE) { - /* Do nothing */ - } - } - /* Cleanup */ - msgpack_unpacked_destroy(&result); - msgpack_sbuffer_destroy(&tmp_sbuf); - - /* Do not modify message stream */ - return FLB_FILTER_NOTOUCH; -} - -static int cb_log_to_metrics_exit(void *data, struct flb_config *config) -{ - struct log_to_metrics_ctx *ctx = data; - - return log_to_metrics_destroy(ctx); -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "regex", NULL, - FLB_CONFIG_MAP_MULT, FLB_FALSE, 0, - "Optional filter for records in which the content of KEY " - "matches the regular expression." - }, - { - FLB_CONFIG_MAP_STR, "exclude", NULL, - FLB_CONFIG_MAP_MULT, FLB_FALSE, 0, - "Optional filter for records in which the content of KEY " - "does not matches the regular expression." - }, - { - FLB_CONFIG_MAP_STR, "metric_mode", "counter", - FLB_FALSE, FLB_TRUE, - offsetof(struct log_to_metrics_ctx, mode), - "Mode selector. Values counter, gauge," - " or histogram. Summary is not supported" - }, - { - FLB_CONFIG_MAP_STR, "value_field", NULL, - FLB_FALSE, FLB_TRUE, - offsetof(struct log_to_metrics_ctx, value_field), - "Numeric field to use for gauge or histogram" - }, - { - FLB_CONFIG_MAP_STR, "metric_name", NULL, - FLB_FALSE, FLB_TRUE, - offsetof(struct log_to_metrics_ctx, metric_name), - "Name of metric" - }, - { - FLB_CONFIG_MAP_STR, "metric_description", NULL, - FLB_FALSE, FLB_TRUE, - offsetof(struct log_to_metrics_ctx, metric_description), - "Help text for metric" - }, - { - FLB_CONFIG_MAP_BOOL, "kubernetes_mode", "false", - 0, FLB_TRUE, offsetof(struct log_to_metrics_ctx, kubernetes_mode), - "Enable kubernetes log metric fields" - }, - { - FLB_CONFIG_MAP_STR, "add_label", NULL, - FLB_CONFIG_MAP_MULT, FLB_FALSE, 0, - "Add a label to the metric by supporting record accessor pattern" - }, - { - FLB_CONFIG_MAP_STR, "label_field", NULL, - FLB_CONFIG_MAP_MULT, FLB_FALSE, 0, - "Specify message field that should be included in the metric" - }, - { - FLB_CONFIG_MAP_STR, "bucket", NULL, - FLB_CONFIG_MAP_MULT, FLB_FALSE, 0, - "Specify bucket for histogram metric" - }, - { - FLB_CONFIG_MAP_STR, "tag", NULL, - FLB_FALSE, FLB_TRUE, - offsetof(struct log_to_metrics_ctx, tag), - "Metric Tag" - }, - {0} -}; - -struct flb_filter_plugin filter_log_to_metrics_plugin = { - .name = "log_to_metrics", - .description = "generate log derived metrics", - .cb_init = cb_log_to_metrics_init, - .cb_filter = cb_log_to_metrics_filter, - .cb_exit = cb_log_to_metrics_exit, - .config_map = config_map, - .flags = 0}; diff --git a/fluent-bit/plugins/filter_log_to_metrics/log_to_metrics.h b/fluent-bit/plugins/filter_log_to_metrics/log_to_metrics.h deleted file mode 100644 index 6edb5ab30..000000000 --- a/fluent-bit/plugins/filter_log_to_metrics/log_to_metrics.h +++ /dev/null @@ -1,85 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_FILTER_LOG_TO_METRICS_H -#define FLB_FILTER_LOG_TO_METRICS_H - -#include -#include -#include -#include - -/* rule types */ -#define GREP_REGEX 1 -#define GREP_EXCLUDE 2 - -/* actions */ -#define GREP_RET_KEEP 0 -#define GREP_RET_EXCLUDE 1 - -/* modes */ -#define FLB_LOG_TO_METRICS_COUNTER_STR "counter" -#define FLB_LOG_TO_METRICS_GAUGE_STR "gauge" -#define FLB_LOG_TO_METRICS_HISTOGRAM_STR "histogram" - - -#define FLB_LOG_TO_METRICS_COUNTER 0 -#define FLB_LOG_TO_METRICS_GAUGE 1 -#define FLB_LOG_TO_METRICS_HISTOGRAM 2 - -#define NUMBER_OF_KUBERNETES_LABELS 5 -#define MAX_LABEL_LENGTH 253 -#define MAX_METRIC_LENGTH 253 -#define MAX_LABEL_COUNT 32 - - -struct log_to_metrics_ctx -{ - struct mk_list rules; - struct flb_filter_instance *ins; - int mode; - flb_sds_t metric_name; - flb_sds_t metric_description; - struct cmt *cmt; - struct flb_input_instance *input_ins; - flb_sds_t value_field; - struct cmt_counter *c; - struct cmt_gauge *g; - struct cmt_histogram *h; - struct cmt_histogram_buckets *histogram_buckets; - char **label_accessors; - char **label_keys; - int *label_counter; - bool kubernetes_mode; - flb_sds_t tag; - int *bucket_counter; - double *buckets; -}; - -struct grep_rule -{ - int type; - flb_sds_t field; - char *regex_pattern; - struct flb_regex *regex; - struct flb_record_accessor *ra; - struct mk_list _head; -}; - -#endif diff --git a/fluent-bit/plugins/filter_lua/CMakeLists.txt b/fluent-bit/plugins/filter_lua/CMakeLists.txt deleted file mode 100644 index 2812e3622..000000000 --- a/fluent-bit/plugins/filter_lua/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -set(src - lua_config.c - lua.c) - -if(MSVC) - FLB_PLUGIN(filter_lua "${src}" "") -else() - FLB_PLUGIN(filter_lua "${src}" "m") -endif() - -if(FLB_FILTER_LUA_USE_MPACK) - add_definitions(-DFLB_FILTER_LUA_USE_MPACK) -endif() diff --git a/fluent-bit/plugins/filter_lua/lua.c b/fluent-bit/plugins/filter_lua/lua.c deleted file mode 100644 index bb7bb566a..000000000 --- a/fluent-bit/plugins/filter_lua/lua.c +++ /dev/null @@ -1,713 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "fluent-bit/flb_mem.h" -#include "lua.h" -#include "lua_config.h" -#include "mpack/mpack.h" - -static int cb_lua_init(struct flb_filter_instance *f_ins, - struct flb_config *config, - void *data) -{ - int err; - int ret; - (void) data; - struct lua_filter *ctx; - struct flb_luajit *lj; - - /* Create context */ - ctx = lua_config_create(f_ins, config); - if (!ctx) { - flb_error("[filter_lua] filter cannot be loaded"); - return -1; - } - - /* Create LuaJIT state/vm */ - lj = flb_luajit_create(config); - if (!lj) { - lua_config_destroy(ctx); - return -1; - } - ctx->lua = lj; - - if (ctx->enable_flb_null) { - flb_lua_enable_flb_null(lj->state); - } - - /* Lua script source code */ - if (ctx->code) { - ret = flb_luajit_load_buffer(ctx->lua, - ctx->code, flb_sds_len(ctx->code), - "fluentbit.lua"); - } - else { - /* Load Script / file path*/ - ret = flb_luajit_load_script(ctx->lua, ctx->script); - } - - if (ret == -1) { - lua_config_destroy(ctx); - return -1; - } - - err = lua_pcall(ctx->lua->state, 0, 0, 0); - if (err != 0) { - flb_error("[luajit] invalid lua content, error=%d: %s", - err, lua_tostring(lj->state, -1)); - lua_pop(lj->state, 1); - lua_config_destroy(ctx); - return -1; - } - - - if (flb_lua_is_valid_func(ctx->lua->state, ctx->call) != FLB_TRUE) { - flb_plg_error(ctx->ins, "function %s is not found", ctx->call); - lua_config_destroy(ctx); - return -1; - } - - /* Initialize packing buffer */ - ctx->packbuf = flb_sds_create_size(1024); - if (!ctx->packbuf) { - flb_error("[filter_lua] failed to allocate packbuf"); - return -1; - } - - /* Set context */ - flb_filter_set_context(f_ins, ctx); - - return 0; -} - -#ifdef FLB_FILTER_LUA_USE_MPACK - -#pragma message "This code does not support the new log event encoding format" - -static void mpack_buffer_flush(mpack_writer_t* writer, const char* buffer, size_t count) -{ - struct lua_filter *ctx = writer->context; - flb_sds_cat_safe(&ctx->packbuf, buffer, count); -} - -static void pack_result_mpack(lua_State *l, - mpack_writer_t *writer, - struct flb_lua_l2c_config *l2cc, - struct flb_time *t) -{ - int i; - int len; - - if (lua_type(l, -1) != LUA_TTABLE) { - return; - } - - len = flb_lua_arraylength(l); - if (len > 0) { - /* record split */ - for (i = 1; i <= len; i++) { - /* write array tag */ - mpack_write_tag(writer, mpack_tag_array(2)); - /* write header tag */ - mpack_write_tag(writer, mpack_tag_array(2)); - /* write timestamp */ - flb_time_append_to_mpack(writer, t, 0); - /* write metadata */ - mpack_write_tag(writer, mpack_tag_map(0)); - /* get the subrecord */ - lua_rawgeti(l, -1, i); - /* convert */ - flb_lua_tompack(l, writer, 0, l2cc); - lua_pop(l, 1); - } - } - else { - /* write array tag */ - mpack_write_tag(writer, mpack_tag_array(2)); - /* write header tag */ - mpack_write_tag(writer, mpack_tag_array(2)); - /* write timestamp */ - flb_time_append_to_mpack(writer, t, 0); - /* write metadata */ - mpack_write_tag(writer, mpack_tag_map(0)); - /* convert */ - flb_lua_tompack(l, writer, 0, l2cc); - } - /* pop */ - lua_pop(l, 1); -} - -static int cb_lua_filter_mpack(const void *data, size_t bytes, - const char *tag, int tag_len, - void **out_buf, size_t *out_bytes, - struct flb_filter_instance *f_ins, - struct flb_input_instance *i_ins, - void *filter_context, - struct flb_config *config) -{ - (void) i_ins; - int ret; - struct flb_time t_orig; - struct flb_time t; - struct lua_filter *ctx = filter_context; - double ts = 0; - int l_code; - double l_timestamp; - char *outbuf; - char writebuf[1024]; - mpack_writer_t writer; - - flb_sds_len_set(ctx->packbuf, 0); - mpack_reader_t reader; - mpack_reader_init_data(&reader, data, bytes); - - while (bytes > 0) { - /* Save record start */ - const char *record_start = reader.data; - size_t record_size = 0; - - /* This is a hack, in order to have this thing work - * we rely on flb_time_pop_from_mpack skipping the - * metadata map. - */ - - /* Get timestamp */ - if (flb_time_pop_from_mpack(&t, &reader)) { - /* failed to parse */ - return FLB_FILTER_NOTOUCH; - } - t_orig = t; - - /* Prepare function call, pass 3 arguments, expect 3 return values */ - lua_getglobal(ctx->lua->state, ctx->call); - lua_pushstring(ctx->lua->state, tag); - - /* Timestamp */ - if (ctx->time_as_table == FLB_TRUE) { - flb_lua_pushtimetable(ctx->lua->state, &t); - } - else { - ts = flb_time_to_double(&t); - lua_pushnumber(ctx->lua->state, ts); - } - - if (flb_lua_pushmpack(ctx->lua->state, &reader)) { - return FLB_FILTER_NOTOUCH; - } - record_size = reader.data - record_start; - bytes -= record_size; - - if (ctx->protected_mode) { - ret = lua_pcall(ctx->lua->state, 3, 3, 0); - if (ret != 0) { - flb_plg_error(ctx->ins, "error code %d: %s", - ret, lua_tostring(ctx->lua->state, -1)); - lua_pop(ctx->lua->state, 1); - return FLB_FILTER_NOTOUCH; - } - } - else { - lua_call(ctx->lua->state, 3, 3); - } - - /* Returned values are on the stack in the following order: - * -1: table/record - * -2: timestamp - * -3: code - * since we will process code first, then timestamp then record, - * we need to swap - * - * use lua_insert to put the table/record on the bottom */ - lua_insert(ctx->lua->state, -3); - /* now swap timestamp with code */ - lua_insert(ctx->lua->state, -2); - - /* check code */ - l_code = (int) lua_tointeger(ctx->lua->state, -1); - lua_pop(ctx->lua->state, 1); - - if (l_code == -1) { /* Skip record */ - lua_pop(ctx->lua->state, 2); - continue; - } - else if (l_code == 0) { /* Keep record, copy original to packbuf */ - flb_sds_cat_safe(&ctx->packbuf, record_start, record_size); - lua_pop(ctx->lua->state, 2); - continue; - } - else if (l_code != 1 && l_code != 2) {/* Unexpected return code, keep original content */ - flb_sds_cat_safe(&ctx->packbuf, record_start, record_size); - lua_pop(ctx->lua->state, 2); - flb_plg_error(ctx->ins, "unexpected Lua script return code %i, " - "original record will be kept." , l_code); - continue; - } - - /* process record timestamp */ - l_timestamp = ts; - if (ctx->time_as_table == FLB_TRUE) { - if (lua_type(ctx->lua->state, -1) == LUA_TTABLE) { - /* Retrieve seconds */ - lua_getfield(ctx->lua->state, -1, "sec"); - t.tm.tv_sec = lua_tointeger(ctx->lua->state, -1); - lua_pop(ctx->lua->state, 1); - - /* Retrieve nanoseconds */ - lua_getfield(ctx->lua->state, -1, "nsec"); - t.tm.tv_nsec = lua_tointeger(ctx->lua->state, -1); - lua_pop(ctx->lua->state, 2); - } - else { - flb_plg_error(ctx->ins, "invalid lua timestamp type returned"); - t = t_orig; - } - } - else { - l_timestamp = (double) lua_tonumber(ctx->lua->state, -1); - lua_pop(ctx->lua->state, 1); - } - - if (l_code == 1) { - if (ctx->time_as_table == FLB_FALSE) { - flb_time_from_double(&t, l_timestamp); - } - } - else if (l_code == 2) { - /* Keep the timestamp */ - t = t_orig; - } - - /* process the record table */ - /* initialize writer and set packbuf as context */ - mpack_writer_init(&writer, writebuf, sizeof(writebuf)); - mpack_writer_set_context(&writer, ctx); - mpack_writer_set_flush(&writer, mpack_buffer_flush); - /* write the result */ - pack_result_mpack(ctx->lua->state, &writer, &ctx->l2cc, &t); - /* flush the writer */ - mpack_writer_flush_message(&writer); - mpack_writer_destroy(&writer); - } - - if (flb_sds_len(ctx->packbuf) == 0) { - /* All records are removed */ - *out_buf = NULL; - *out_bytes = 0; - return FLB_FILTER_MODIFIED; - } - - /* allocate outbuf that contains the modified chunks */ - outbuf = flb_malloc(flb_sds_len(ctx->packbuf)); - if (!outbuf) { - flb_plg_error(ctx->ins, "failed to allocate outbuf"); - return FLB_FILTER_NOTOUCH; - } - memcpy(outbuf, ctx->packbuf, flb_sds_len(ctx->packbuf)); - /* link new buffer */ - *out_buf = outbuf; - *out_bytes = flb_sds_len(ctx->packbuf); - - return FLB_FILTER_MODIFIED; -} - -#else - -static int pack_record(struct lua_filter *ctx, - struct flb_log_event_encoder *log_encoder, - struct flb_time *ts, - msgpack_object *metadata, - msgpack_object *body) -{ - int ret; - - ret = flb_log_event_encoder_begin_record(log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_timestamp(log_encoder, ts); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS && metadata != NULL) { - ret = flb_log_event_encoder_set_metadata_from_msgpack_object( - log_encoder, metadata); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_body_from_msgpack_object( - log_encoder, body); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(log_encoder); - } - - return ret; -} - -static int pack_result (struct lua_filter *ctx, struct flb_time *ts, - msgpack_object *metadata, - struct flb_log_event_encoder *log_encoder, - char *data, size_t bytes) -{ - int ret; - size_t index = 0; - size_t off = 0; - msgpack_object *entry; - msgpack_unpacked result; - - msgpack_unpacked_init(&result); - - ret = msgpack_unpack_next(&result, data, bytes, &off); - - if (ret != MSGPACK_UNPACK_SUCCESS) { - msgpack_unpacked_destroy(&result); - - return FLB_FALSE; - } - - if (result.data.type == MSGPACK_OBJECT_MAP) { - ret = pack_record(ctx, log_encoder, - ts, metadata, &result.data); - - msgpack_unpacked_destroy(&result); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - return FLB_FALSE; - } - - return FLB_TRUE; - } - else if (result.data.type == MSGPACK_OBJECT_ARRAY) { - for (index = 0 ; index < result.data.via.array.size ; index++) { - entry = &result.data.via.array.ptr[index]; - - if (entry->type == MSGPACK_OBJECT_MAP) { - ret = pack_record(ctx, log_encoder, - ts, metadata, entry); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - msgpack_unpacked_destroy(&result); - - return FLB_FALSE; - } - } - else { - msgpack_unpacked_destroy(&result); - - return FLB_FALSE; - } - } - - msgpack_unpacked_destroy(&result); - - return FLB_TRUE; - } - - msgpack_unpacked_destroy(&result); - - return FLB_FALSE; -} - -static int cb_lua_filter(const void *data, size_t bytes, - const char *tag, int tag_len, - void **out_buf, size_t *out_bytes, - struct flb_filter_instance *f_ins, - struct flb_input_instance *i_ins, - void *filter_context, - struct flb_config *config) -{ - int ret; - double ts = 0; - struct flb_time t_orig; - struct flb_time t; - struct lua_filter *ctx = filter_context; - /* Lua return values */ - int l_code; - double l_timestamp; - msgpack_packer data_pck; - msgpack_sbuffer data_sbuf; - struct flb_log_event_encoder log_encoder; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - (void) f_ins; - (void) i_ins; - (void) config; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return FLB_FILTER_NOTOUCH; - } - - ret = flb_log_event_encoder_init(&log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event encoder initialization error : %d", ret); - - flb_log_event_decoder_destroy(&log_decoder); - - return FLB_FILTER_NOTOUCH; - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - msgpack_sbuffer_init(&data_sbuf); - msgpack_packer_init(&data_pck, &data_sbuf, msgpack_sbuffer_write); - - /* Get timestamp */ - flb_time_copy(&t, &log_event.timestamp); - flb_time_copy(&t_orig, &log_event.timestamp); - - /* Prepare function call, pass 3 arguments, expect 3 return values */ - lua_getglobal(ctx->lua->state, ctx->call); - lua_pushstring(ctx->lua->state, tag); - - /* Timestamp */ - if (ctx->time_as_table == FLB_TRUE) { - flb_lua_pushtimetable(ctx->lua->state, &t); - } - else { - ts = flb_time_to_double(&t); - lua_pushnumber(ctx->lua->state, ts); - } - - flb_lua_pushmsgpack(ctx->lua->state, log_event.body); - if (ctx->protected_mode) { - ret = lua_pcall(ctx->lua->state, 3, 3, 0); - if (ret != 0) { - flb_plg_error(ctx->ins, "error code %d: %s", - ret, lua_tostring(ctx->lua->state, -1)); - lua_pop(ctx->lua->state, 1); - - msgpack_sbuffer_destroy(&data_sbuf); - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - return FLB_FILTER_NOTOUCH; - } - } - else { - lua_call(ctx->lua->state, 3, 3); - } - - /* Initialize Return values */ - l_code = 0; - l_timestamp = ts; - - flb_lua_tomsgpack(ctx->lua->state, &data_pck, 0, &ctx->l2cc); - lua_pop(ctx->lua->state, 1); - - /* Lua table */ - if (ctx->time_as_table == FLB_TRUE) { - if (lua_type(ctx->lua->state, -1) == LUA_TTABLE) { - /* Retrieve seconds */ - lua_getfield(ctx->lua->state, -1, "sec"); - t.tm.tv_sec = lua_tointeger(ctx->lua->state, -1); - lua_pop(ctx->lua->state, 1); - - /* Retrieve nanoseconds */ - lua_getfield(ctx->lua->state, -1, "nsec"); - t.tm.tv_nsec = lua_tointeger(ctx->lua->state, -1); - lua_pop(ctx->lua->state, 2); - } - else { - flb_plg_error(ctx->ins, "invalid lua timestamp type returned"); - t = t_orig; - } - } - else { - l_timestamp = (double) lua_tonumber(ctx->lua->state, -1); - lua_pop(ctx->lua->state, 1); - } - - l_code = (int) lua_tointeger(ctx->lua->state, -1); - lua_pop(ctx->lua->state, 1); - - if (l_code == -1) { /* Skip record */ - msgpack_sbuffer_destroy(&data_sbuf); - continue; - } - else if (l_code == 1 || l_code == 2) { /* Modified, pack new data */ - if (l_code == 1) { - if (ctx->time_as_table == FLB_FALSE) { - flb_time_from_double(&t, l_timestamp); - } - } - else if (l_code == 2) { - /* Keep the timestamp */ - t = t_orig; - } - - ret = pack_result(ctx, &t, log_event.metadata, &log_encoder, - data_sbuf.data, data_sbuf.size); - - if (ret == FLB_FALSE) { - flb_plg_error(ctx->ins, "invalid table returned at %s(), %s", - ctx->call, ctx->script); - msgpack_sbuffer_destroy(&data_sbuf); - - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - return FLB_FILTER_NOTOUCH; - } - } - else { /* Unexpected return code, keep original content */ - /* Code 0 means Keep record, so we don't emit the warning */ - if (l_code != 0) { - flb_plg_error(ctx->ins, - "unexpected Lua script return code %i, " - "original record will be kept." , l_code); - } - - ret = flb_log_event_encoder_emit_raw_record( - &log_encoder, - log_decoder.record_base, - log_decoder.record_length); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event encoder error : %d", ret); - } - } - - msgpack_sbuffer_destroy(&data_sbuf); - } - - if (ret == FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA) { - ret = FLB_EVENT_ENCODER_SUCCESS; - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - *out_buf = log_encoder.output_buffer; - *out_bytes = log_encoder.output_length; - - ret = FLB_FILTER_MODIFIED; - - flb_log_event_encoder_claim_internal_buffer_ownership(&log_encoder); - } - else { - flb_plg_error(ctx->ins, - "Log event encoder error : %d", ret); - - ret = FLB_FILTER_NOTOUCH; - } - - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - return ret; -} -#endif - -static int cb_lua_exit(void *data, struct flb_config *config) -{ - struct lua_filter *ctx; - - ctx = data; - flb_luajit_destroy(ctx->lua); - lua_config_destroy(ctx); - - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "script", NULL, - 0, FLB_FALSE, 0, - "The path of lua script." - }, - { - FLB_CONFIG_MAP_STR, "code", NULL, - 0, FLB_FALSE, 0, - "String that contains the Lua script source code" - }, - { - FLB_CONFIG_MAP_STR, "call", NULL, - 0, FLB_TRUE, offsetof(struct lua_filter, call), - "Lua function name that will be triggered to do filtering." - }, - { - FLB_CONFIG_MAP_STR, "type_int_key", NULL, - 0, FLB_FALSE, 0, - "If these keys are matched, the fields are converted to integer. " - "If more than one key, delimit by space." - }, - { - FLB_CONFIG_MAP_STR, "type_array_key", NULL, - 0, FLB_FALSE, 0, - "If these keys are matched, the fields are converted to array. " - "If more than one key, delimit by space." - }, - { - FLB_CONFIG_MAP_BOOL, "protected_mode", "true", - 0, FLB_TRUE, offsetof(struct lua_filter, protected_mode), - "If enabled, Lua script will be executed in protected mode. " - "It prevents to crash when invalid Lua script is executed." - }, - { - FLB_CONFIG_MAP_BOOL, "time_as_table", "false", - 0, FLB_TRUE, offsetof(struct lua_filter, time_as_table), - "If enabled, Fluent-bit will pass the timestamp as a Lua table " - "with keys \"sec\" for seconds since epoch and \"nsec\" for nanoseconds." - }, - { - FLB_CONFIG_MAP_BOOL, "enable_flb_null", "false", - 0, FLB_TRUE, offsetof(struct lua_filter, enable_flb_null), - "If enabled, null will be converted to flb_null in Lua. " - "It is useful to prevent removing key/value " - "since nil is a special value to remove key value from map in Lua." - }, - - {0} -}; - -struct flb_filter_plugin filter_lua_plugin = { - .name = "lua", - .description = "Lua Scripting Filter", - .cb_init = cb_lua_init, -#ifdef FLB_FILTER_LUA_USE_MPACK - .cb_filter = cb_lua_filter_mpack, -#else - .cb_filter = cb_lua_filter, -#endif - .cb_exit = cb_lua_exit, - .config_map = config_map, - .flags = 0 -}; diff --git a/fluent-bit/plugins/filter_lua/lua_config.c b/fluent-bit/plugins/filter_lua/lua_config.c deleted file mode 100644 index f0c154196..000000000 --- a/fluent-bit/plugins/filter_lua/lua_config.c +++ /dev/null @@ -1,206 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "lua_config.h" - -#include -#include -#include - -struct lua_filter *lua_config_create(struct flb_filter_instance *ins, - struct flb_config *config) -{ - int ret; - char *tmp_key; - char buf[PATH_MAX]; - const char *script = NULL; - const char *tmp = NULL; - (void) config; - struct stat st; - struct lua_filter *lf; - struct mk_list *split = NULL; - struct mk_list *head = NULL; - struct mk_list *tmp_list= NULL; - struct flb_lua_l2c_type *l2c = NULL; - struct flb_split_entry *sentry = NULL; - - /* Allocate context */ - lf = flb_calloc(1, sizeof(struct lua_filter)); - if (!lf) { - flb_errno(); - return NULL; - } - ret = flb_filter_config_map_set(ins, (void*)lf); - if (ret < 0) { - flb_errno(); - flb_plg_error(ins, "configuration error"); - flb_free(lf); - return NULL; - } - - mk_list_init(&lf->l2cc.l2c_types); - lf->ins = ins; - lf->script = NULL; - - /* config: code */ - tmp = flb_filter_get_property("code", ins); - if (tmp) { - lf->code = flb_sds_create(tmp); - } - else { - /* Config: script */ - script = flb_filter_get_property("script", ins); - if (!script) { - flb_plg_error(lf->ins, "no script path defined"); - flb_free(lf); - return NULL; - } - - /* Compose path */ - ret = stat(script, &st); - if (ret == -1 && errno == ENOENT) { - if (script[0] == '/') { - flb_plg_error(lf->ins, "cannot access script '%s'", script); - flb_free(lf); - return NULL; - } - - if (config->conf_path) { - snprintf(buf, sizeof(buf) - 1, "%s%s", - config->conf_path, script); - script = buf; - } - } - - /* Validate script path */ - ret = access(script, R_OK); - if (ret == -1) { - flb_plg_error(lf->ins, "cannot access script '%s'", script); - flb_free(lf); - return NULL; - } - - lf->script = flb_sds_create(script); - if (!lf->script) { - flb_plg_error(lf->ins, "could not allocate string"); - flb_free(lf); - return NULL; - } - } - - if (!lf->call) { - flb_plg_error(lf->ins, "function name defined by 'call' is not set"); - lua_config_destroy(lf); - return NULL; - } - - lf->buffer = flb_sds_create_size(LUA_BUFFER_CHUNK); - if (!lf->buffer) { - flb_plg_error(lf->ins, "could not allocate decode buffer"); - lua_config_destroy(lf); - return NULL; - } - - lf->l2cc.l2c_types_num = 0; - tmp = flb_filter_get_property("type_int_key", ins); - if (tmp) { - split = flb_utils_split(tmp, ' ', FLB_LUA_L2C_TYPES_NUM_MAX); - mk_list_foreach_safe(head, tmp_list, split) { - l2c = flb_malloc(sizeof(struct flb_lua_l2c_type)); - - sentry = mk_list_entry(head, struct flb_split_entry, _head); - - tmp_key = flb_strndup(sentry->value, sentry->len); - l2c->key = flb_sds_create(tmp_key); - l2c->type = FLB_LUA_L2C_TYPE_INT; - flb_free(tmp_key); - - mk_list_add(&l2c->_head, &lf->l2cc.l2c_types); - lf->l2cc.l2c_types_num++; - } - flb_utils_split_free(split); - } - - tmp = flb_filter_get_property("type_array_key", ins); - if (tmp) { - split = flb_utils_split(tmp, ' ', FLB_LUA_L2C_TYPES_NUM_MAX); - mk_list_foreach_safe(head, tmp_list, split) { - l2c = flb_malloc(sizeof(struct flb_lua_l2c_type)); - - sentry = mk_list_entry(head, struct flb_split_entry, _head); - - tmp_key = flb_strndup(sentry->value, sentry->len); - l2c->key = flb_sds_create(tmp_key); - l2c->type = FLB_LUA_L2C_TYPE_ARRAY; - flb_free(tmp_key); - - mk_list_add(&l2c->_head, &lf->l2cc.l2c_types); - lf->l2cc.l2c_types_num++; - } - flb_utils_split_free(split); - } - - return lf; -} - -void lua_config_destroy(struct lua_filter *lf) -{ - struct mk_list *tmp_list = NULL; - struct mk_list *head = NULL; - struct flb_lua_l2c_type *l2c = NULL; - - if (!lf) { - return; - } - - if (lf->code) { - flb_sds_destroy(lf->code); - } - - if (lf->script) { - flb_sds_destroy(lf->script); - } - - if (lf->buffer) { - flb_sds_destroy(lf->buffer); - } - - mk_list_foreach_safe(head, tmp_list, &lf->l2cc.l2c_types) { - l2c = mk_list_entry(head, struct flb_lua_l2c_type, _head); - if (l2c) { - if (l2c->key) { - flb_sds_destroy(l2c->key); - } - mk_list_del(&l2c->_head); - flb_free(l2c); - } - } - - flb_sds_destroy(lf->packbuf); - flb_free(lf); -} diff --git a/fluent-bit/plugins/filter_lua/lua_config.h b/fluent-bit/plugins/filter_lua/lua_config.h deleted file mode 100644 index af8d6f128..000000000 --- a/fluent-bit/plugins/filter_lua/lua_config.h +++ /dev/null @@ -1,49 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_LUA_CONFIG_H -#define FLB_LUA_CONFIG_H - -#include -#include -#include -#include -#include - -#define LUA_BUFFER_CHUNK 1024 * 8 /* 8K should be enough to get started */ - -struct lua_filter { - flb_sds_t code; /* lua script source code */ - flb_sds_t script; /* lua script path */ - flb_sds_t call; /* function name */ - flb_sds_t buffer; /* json dec buffer */ - int protected_mode; /* exec lua function in protected mode */ - int time_as_table; /* timestamp as a Lua table */ - int enable_flb_null; /* Use flb_null in Lua */ - struct flb_lua_l2c_config l2cc; /* lua -> C config */ - struct flb_luajit *lua; /* state context */ - struct flb_filter_instance *ins; /* filter instance */ - flb_sds_t packbuf; /* dynamic buffer used for mpack write */ -}; - -struct lua_filter *lua_config_create(struct flb_filter_instance *ins, - struct flb_config *config); -void lua_config_destroy(struct lua_filter *lf); - -#endif diff --git a/fluent-bit/plugins/filter_modify/CMakeLists.txt b/fluent-bit/plugins/filter_modify/CMakeLists.txt deleted file mode 100644 index e63bf25b2..000000000 --- a/fluent-bit/plugins/filter_modify/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - modify.c) - -FLB_PLUGIN(filter_modify "${src}" "") diff --git a/fluent-bit/plugins/filter_modify/modify.c b/fluent-bit/plugins/filter_modify/modify.c deleted file mode 100644 index 22d2e21c0..000000000 --- a/fluent-bit/plugins/filter_modify/modify.c +++ /dev/null @@ -1,1659 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "modify.h" - -#include -#include - -static void condition_free(struct modify_condition *condition) -{ - if (condition == NULL) { - return; - } - - if (condition->a) { - flb_sds_destroy(condition->a); - } - if (condition->b) { - flb_free(condition->b); - } - if (condition->raw_k) { - flb_free(condition->raw_k); - } - if (condition->raw_v) { - flb_free(condition->raw_v); - } - - if (condition->a_regex) { - flb_regex_destroy(condition->a_regex); - } - if (condition->b_regex) { - flb_regex_destroy(condition->b_regex); - } - if (condition->ra_a) { - flb_ra_destroy(condition->ra_a); - condition->ra_a = NULL; - } - if (!mk_list_entry_is_orphan(&condition->_head)) { - mk_list_del(&condition->_head); - } - flb_free(condition); -} - -static void rule_free(struct modify_rule *rule) -{ - if (rule == NULL) { - return; - } - - if (rule->key) { - flb_free(rule->key); - } - if (rule->val) { - flb_free(rule->val); - } - if (rule->raw_k) { - flb_free(rule->raw_k); - } - if (rule->raw_v) { - flb_free(rule->raw_v); - } - if (rule->key_regex) { - flb_regex_destroy(rule->key_regex); - } - if (rule->val_regex) { - flb_regex_destroy(rule->val_regex); - } - if (!mk_list_entry_is_orphan(&rule->_head)) { - mk_list_del(&rule->_head); - } - flb_free(rule); -} - -static void teardown(struct filter_modify_ctx *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - - struct modify_rule *rule; - struct modify_condition *condition; - - mk_list_foreach_safe(head, tmp, &ctx->conditions) { - condition = mk_list_entry(head, struct modify_condition, _head); - condition_free(condition); - } - - mk_list_foreach_safe(head, tmp, &ctx->rules) { - rule = mk_list_entry(head, struct modify_rule, _head); - rule_free(rule); - } -} - -static void helper_pack_string(struct filter_modify_ctx *ctx, - msgpack_packer *packer, const char *str, - int len) -{ - - if (str == NULL) { - flb_plg_error(ctx->ins, "helper_pack_string : NULL passed"); - msgpack_pack_nil(packer); - } - else { - msgpack_pack_str(packer, len); - msgpack_pack_str_body(packer, str, len); - } -} - -static int setup(struct filter_modify_ctx *ctx, - struct flb_filter_instance *f_ins, struct flb_config *config) -{ - struct mk_list *head; - struct mk_list *split; - struct flb_kv *kv; - struct flb_split_entry *sentry; - struct modify_rule *rule = NULL; - struct modify_condition *condition; - - int list_size; - - // Split list - // - Arg 1 is condition? - // --> Setup Condition - // - Malloc Condition - // - Switch list size - // --> Setup Rule - // - Malloc Rule - // - Switch list size - - if (flb_filter_config_map_set(f_ins, ctx) < 0) { - flb_errno(); - flb_plg_error(f_ins, "configuration error"); - return -1; - } - - mk_list_foreach(head, &f_ins->properties) { - kv = mk_list_entry(head, struct flb_kv, _head); - - split = flb_utils_split_quoted(kv->val, ' ', 3); - list_size = mk_list_size(split); - - // Conditions are, - // CONDITION CONDITIONTYPE VAL_A VAL_B - - if (list_size == 0 || list_size > 3) { - flb_plg_error(ctx->ins, "Invalid config for %s", kv->key); - teardown(ctx); - flb_utils_split_free(split); - return -1; - } - else if (strcasecmp(kv->key, "condition") == 0) { - - // - // Build a condition - // - - condition = flb_calloc(1, sizeof(struct modify_condition)); - if (!condition) { - flb_errno(); - flb_plg_error(ctx->ins, "Unable to allocate memory for " - "condition"); - teardown(ctx); - flb_utils_split_free(split); - return -1; - } - - condition->a_is_regex = false; - condition->b_is_regex = false; - condition->ra_a = NULL; - condition->raw_k = flb_strndup(kv->key, flb_sds_len(kv->key)); - if (condition->raw_k == NULL) { - flb_errno(); - flb_plg_error(ctx->ins, "Unable to allocate memory for " - "condition->raw_k"); - teardown(ctx); - condition_free(condition); - flb_utils_split_free(split); - return -1; - } - condition->raw_v = flb_strndup(kv->val, flb_sds_len(kv->val)); - if (condition->raw_v == NULL) { - flb_errno(); - flb_plg_error(ctx->ins, "Unable to allocate memory for " - "condition->raw_v"); - teardown(ctx); - condition_free(condition); - flb_utils_split_free(split); - return -1; - } - - sentry = - mk_list_entry_first(split, struct flb_split_entry, _head); - - if (strcasecmp(sentry->value, "key_exists") == 0) { - condition->conditiontype = KEY_EXISTS; - } - else if (strcasecmp(sentry->value, "key_does_not_exist") == 0) { - condition->conditiontype = KEY_DOES_NOT_EXIST; - } - else if (strcasecmp(sentry->value, "a_key_matches") == 0) { - condition->conditiontype = A_KEY_MATCHES; - condition->a_is_regex = true; - } - else if (strcasecmp(sentry->value, "no_key_matches") == 0) { - condition->conditiontype = NO_KEY_MATCHES; - condition->a_is_regex = true; - } - else if (strcasecmp(sentry->value, "key_value_equals") == 0) { - condition->conditiontype = KEY_VALUE_EQUALS; - } - else if (strcasecmp(sentry->value, "key_value_does_not_equal") == - 0) { - condition->conditiontype = KEY_VALUE_DOES_NOT_EQUAL; - } - else if (strcasecmp(sentry->value, "key_value_matches") == 0) { - condition->conditiontype = KEY_VALUE_MATCHES; - condition->b_is_regex = true; - } - else if (strcasecmp(sentry->value, "key_value_does_not_match") == - 0) { - condition->conditiontype = KEY_VALUE_DOES_NOT_MATCH; - condition->b_is_regex = true; - } - else if (strcasecmp - (sentry->value, - "matching_keys_have_matching_values") == 0) { - condition->conditiontype = MATCHING_KEYS_HAVE_MATCHING_VALUES; - condition->a_is_regex = true; - condition->b_is_regex = true; - } - else if (strcasecmp - (sentry->value, - "matching_keys_do_not_have_matching_values") == 0) { - condition->conditiontype = - MATCHING_KEYS_DO_NOT_HAVE_MATCHING_VALUES; - condition->a_is_regex = true; - condition->b_is_regex = true; - } - else { - flb_plg_error(ctx->ins, "Invalid config for %s : %s", - kv->key, kv->val); - teardown(ctx); - condition_free(condition); - flb_utils_split_free(split); - return -1; - } - - sentry = - mk_list_entry_next(&sentry->_head, struct flb_split_entry, - _head, split); - condition->a = flb_sds_create_len(sentry->value, sentry->len); - condition->a_len = sentry->len; - condition->ra_a = flb_ra_create(condition->a, FLB_FALSE); - if (list_size == 3) { - sentry = - mk_list_entry_last(split, struct flb_split_entry, _head); - condition->b = flb_strndup(sentry->value, sentry->len); - if (condition->b == NULL) { - flb_errno(); - flb_plg_error(ctx->ins, "Unable to allocate memory for " - "condition->b"); - teardown(ctx); - condition_free(condition); - flb_utils_split_free(split); - return -1; - } - condition->b_len = sentry->len; - } - else { - condition->b = NULL; - condition->b_len = 0; - } - - if (condition->a_is_regex) { - if (condition->a_len < 1) { - flb_plg_error(ctx->ins, "Unable to create regex for " - "condition %s %s", - condition->raw_k, condition->raw_v); - teardown(ctx); - condition_free(condition); - flb_utils_split_free(split); - return -1; - } - else { - flb_plg_debug(ctx->ins, "Creating regex for condition A : " - "%s %s : %s", - condition->raw_k, condition->raw_v, - condition->a); - condition->a_regex = - flb_regex_create(condition->a); - } - } - - if (condition->b_is_regex) { - if (condition->b_len < 1) { - flb_plg_error(ctx->ins, "Unable to create regex " - "for condition %s %s", - condition->raw_k, condition->raw_v); - teardown(ctx); - condition_free(condition); - flb_utils_split_free(split); - return -1; - } - else { - flb_plg_debug(ctx->ins, "Creating regex for condition B : %s " - "%s : %s", - condition->raw_k, condition->raw_v, condition->b); - condition->b_regex = - flb_regex_create(condition->b); - } - } - - flb_utils_split_free(split); - - mk_list_add(&condition->_head, &ctx->conditions); - ctx->conditions_cnt++; - } - else { - - // - // Build a rule - // - - rule = flb_calloc(1, sizeof(struct modify_rule)); - if (!rule) { - flb_plg_error(ctx->ins, "Unable to allocate memory for rule"); - teardown(ctx); - flb_utils_split_free(split); - return -1; - } - - rule->key_is_regex = false; - rule->val_is_regex = false; - rule->raw_k = flb_strndup(kv->key, flb_sds_len(kv->key)); - if (rule->raw_k == NULL) { - flb_errno(); - flb_plg_error(ctx->ins, "Unable to allocate memory for rule->raw_k"); - teardown(ctx); - rule_free(rule); - flb_utils_split_free(split); - return -1; - } - rule->raw_v = flb_strndup(kv->val, flb_sds_len(kv->val)); - if (rule->raw_v == NULL) { - flb_errno(); - flb_plg_error(ctx->ins, "Unable to allocate memory for rule->raw_v"); - teardown(ctx); - rule_free(rule); - flb_utils_split_free(split); - return -1; - } - - sentry = - mk_list_entry_first(split, struct flb_split_entry, _head); - rule->key = flb_strndup(sentry->value, sentry->len); - if (rule->key == NULL) { - flb_errno(); - flb_plg_error(ctx->ins, "Unable to allocate memory for rule->key"); - teardown(ctx); - rule_free(rule); - flb_utils_split_free(split); - return -1; - } - rule->key_len = sentry->len; - - sentry = mk_list_entry_last(split, struct flb_split_entry, _head); - rule->val = flb_strndup(sentry->value, sentry->len); - if (rule->val == NULL) { - flb_errno(); - flb_plg_error(ctx->ins, "Unable to allocate memory for rule->val"); - teardown(ctx); - rule_free(rule); - flb_utils_split_free(split); - return -1; - } - rule->val_len = sentry->len; - - flb_utils_split_free(split); - - if (list_size == 1) { - if (strcasecmp(kv->key, "remove") == 0) { - rule->ruletype = REMOVE; - } - else if (strcasecmp(kv->key, "remove_wildcard") == 0) { - rule->ruletype = REMOVE_WILDCARD; - } - else if (strcasecmp(kv->key, "remove_regex") == 0) { - rule->ruletype = REMOVE_REGEX; - rule->key_is_regex = true; - } - else if (strcasecmp(kv->key, "move_to_start") == 0) { - rule->ruletype = MOVE_TO_START; - } - else if (strcasecmp(kv->key, "move_to_end") == 0) { - rule->ruletype = MOVE_TO_END; - } - else { - flb_plg_error(ctx->ins, "Invalid operation %s : %s in " - "configuration", kv->key, kv->val); - teardown(ctx); - rule_free(rule); - return -1; - } - } - else if (list_size == 2) { - if (strcasecmp(kv->key, "rename") == 0) { - rule->ruletype = RENAME; - } - else if (strcasecmp(kv->key, "hard_rename") == 0) { - rule->ruletype = HARD_RENAME; - } - else if (strcasecmp(kv->key, "add") == 0) { - rule->ruletype = ADD; - } - else if (strcasecmp(kv->key, "add_if_not_present") == 0) { - flb_plg_info(ctx->ins, "DEPRECATED : Operation " - "'add_if_not_present' has been replaced " - "by 'add'."); - rule->ruletype = ADD; - } - else if (strcasecmp(kv->key, "set") == 0) { - rule->ruletype = SET; - } - else if (strcasecmp(kv->key, "copy") == 0) { - rule->ruletype = COPY; - } - else if (strcasecmp(kv->key, "hard_copy") == 0) { - rule->ruletype = HARD_COPY; - } - else { - flb_plg_error(ctx->ins, "Invalid operation %s : %s in " - "configuration", kv->key, kv->val); - teardown(ctx); - rule_free(rule); - return -1; - } - } - - if (rule->key_is_regex && rule->key_len == 0) { - flb_plg_error(ctx->ins, "Unable to create regex for rule %s %s", - rule->raw_k, rule->raw_v); - teardown(ctx); - rule_free(rule); - return -1; - } - else { - rule->key_regex = - flb_regex_create(rule->key); - if (rule->key_regex == NULL) { - flb_plg_error(ctx->ins, "Unable to create regex(key) from %s", - rule->key); - teardown(ctx); - rule_free(rule); - return -1; - } - } - - if (rule->val_is_regex && rule->val_len == 0) { - flb_plg_error(ctx->ins, "Unable to create regex for rule %s %s", - rule->raw_k, rule->raw_v); - teardown(ctx); - rule_free(rule); - return -1; - } - else { - rule->val_regex = - flb_regex_create(rule->val); - if (rule->val_regex == NULL) { - flb_plg_error(ctx->ins, "Unable to create regex(val) from %s", - rule->val); - teardown(ctx); - rule_free(rule); - return -1; - } - } - - mk_list_add(&rule->_head, &ctx->rules); - ctx->rules_cnt++; - } - - } - - flb_plg_debug(ctx->ins, "Initialized modify filter with %d conditions " - "and %d rules", - ctx->conditions_cnt, ctx->rules_cnt); - return 0; -} - - -/* Regex matchers */ -static inline bool helper_msgpack_object_matches_regex(msgpack_object * obj, - struct flb_regex - *regex) -{ - int len; - const char *key; - - if (obj->type == MSGPACK_OBJECT_BIN) { - return false; - } - else if (obj->type == MSGPACK_OBJECT_STR) { - key = obj->via.str.ptr; - len = obj->via.str.size; - } - else if (obj->type == MSGPACK_OBJECT_BOOLEAN) { - if (obj->via.boolean) { - key = "true"; - len = 4; - } - else { - key = "false"; - len = 5; - } - } - else { - return false; - } - - return flb_regex_match(regex, (unsigned char *) key, len) > 0; -} - -static inline bool kv_key_matches_regex(msgpack_object_kv * kv, - struct flb_regex *regex) -{ - return helper_msgpack_object_matches_regex(&kv->key, regex); -} - -static inline bool kv_val_matches_regex(msgpack_object_kv * kv, - struct flb_regex *regex) -{ - return helper_msgpack_object_matches_regex(&kv->val, regex); -} - -static inline bool kv_key_matches_regex_rule_key(msgpack_object_kv * kv, - struct modify_rule *rule) -{ - return kv_key_matches_regex(kv, rule->key_regex); -} - -static inline bool kv_key_does_not_match_regex_rule_key(msgpack_object_kv * - kv, - struct modify_rule - *rule) -{ - return !kv_key_matches_regex_rule_key(kv, rule); -} - -static inline int map_count_keys_matching_regex(msgpack_object * map, - struct flb_regex *regex) -{ - int i; - int count = 0; - - for (i = 0; i < map->via.map.size; i++) { - if (kv_key_matches_regex(&map->via.map.ptr[i], regex)) { - count++; - } - } - return count; -} - - -/* - * Wildcard matchers - */ - -static inline bool helper_msgpack_object_matches_wildcard(msgpack_object * - obj, char *str, - int len) -{ - const char *key; - - if (obj->type == MSGPACK_OBJECT_BIN) { - key = obj->via.bin.ptr; - } - else if (obj->type == MSGPACK_OBJECT_STR) { - key = obj->via.str.ptr; - } - else { - return false; - } - - return (strncmp(str, key, len) == 0); -} - -static inline bool kv_key_matches_wildcard(msgpack_object_kv * kv, - char *str, int len) -{ - return helper_msgpack_object_matches_wildcard(&kv->key, str, len); -} - -static inline bool kv_key_matches_wildcard_rule_key(msgpack_object_kv * kv, - struct modify_rule *rule) -{ - return kv_key_matches_wildcard(kv, rule->key, rule->key_len); -} - -static inline bool kv_key_does_not_match_wildcard_rule_key(msgpack_object_kv * - kv, - struct modify_rule - *rule) -{ - return !kv_key_matches_wildcard_rule_key(kv, rule); -} - -static inline int map_count_keys_matching_wildcard(msgpack_object * map, - char *str, int len) -{ - int i; - int count = 0; - - for (i = 0; i < map->via.map.size; i++) { - if (kv_key_matches_wildcard(&map->via.map.ptr[i], str, len)) { - count++; - } - } - return count; -} - -// -// String matchers -// - -static inline bool helper_msgpack_object_matches_str(msgpack_object * obj, - char *str, int len) -{ - - const char *key; - int klen; - - if (obj->type == MSGPACK_OBJECT_BIN) { - key = obj->via.bin.ptr; - klen = obj->via.bin.size; - } - else if (obj->type == MSGPACK_OBJECT_STR) { - key = obj->via.str.ptr; - klen = obj->via.str.size; - } - else { - return false; - } - - return ((len == klen) && (strncmp(str, key, klen) == 0) - ); -} - -static inline bool kv_key_matches_str(msgpack_object_kv * kv, - char *str, int len) -{ - return helper_msgpack_object_matches_str(&kv->key, str, len); -} - -static inline bool kv_key_matches_str_rule_key(msgpack_object_kv * kv, - struct modify_rule *rule) -{ - return kv_key_matches_str(kv, rule->key, rule->key_len); -} - -static inline bool kv_key_does_not_match_str_rule_key(msgpack_object_kv * kv, - struct modify_rule - *rule) -{ - return !kv_key_matches_str_rule_key(kv, rule); -} - -static inline bool kv_key_matches_str_rule_val(msgpack_object_kv * kv, - struct modify_rule *rule) -{ - return kv_key_matches_str(kv, rule->val, rule->val_len); -} - -static inline int map_count_keys_matching_str(msgpack_object * map, - char *str, int len) -{ - int i; - int count = 0; - - for (i = 0; i < map->via.map.size; i++) { - if (kv_key_matches_str(&map->via.map.ptr[i], str, len)) { - count++; - } - } - return count; -} - -static inline void map_pack_each(msgpack_packer * packer, - msgpack_object * map) -{ - int i; - - for (i = 0; i < map->via.map.size; i++) { - msgpack_pack_object(packer, map->via.map.ptr[i].key); - msgpack_pack_object(packer, map->via.map.ptr[i].val); - } -} - -static inline void map_pack_each_fn(msgpack_packer * packer, - msgpack_object * map, - struct modify_rule *rule, - bool(*f) (msgpack_object_kv * kv, - struct modify_rule * rule) - ) -{ - int i; - - for (i = 0; i < map->via.map.size; i++) { - if ((*f) (&map->via.map.ptr[i], rule)) { - msgpack_pack_object(packer, map->via.map.ptr[i].key); - msgpack_pack_object(packer, map->via.map.ptr[i].val); - } - } -} - -static inline bool evaluate_condition_KEY_EXISTS(msgpack_object * map, - struct modify_condition - *condition) -{ - msgpack_object *skey = NULL; - msgpack_object *okey = NULL; - msgpack_object *oval = NULL; - - flb_ra_get_kv_pair(condition->ra_a, *map, &skey, &okey, &oval); - if (skey == NULL || okey == NULL || oval == NULL) { - return false; - } - return true; -} - -static inline bool evaluate_condition_KEY_DOES_NOT_EXIST(msgpack_object * map, - struct - modify_condition - *condition) -{ - return !evaluate_condition_KEY_EXISTS(map, condition); -} - -static inline bool evaluate_condition_A_KEY_MATCHES(msgpack_object * map, - struct modify_condition - *condition) -{ - return (map_count_keys_matching_regex(map, condition->a_regex) > 0); -} - -static inline bool evaluate_condition_NO_KEY_MATCHES(msgpack_object * map, - struct - modify_condition - *condition) -{ - return !evaluate_condition_A_KEY_MATCHES(map, condition); -} - -static inline bool evaluate_condition_KEY_VALUE_EQUALS(struct filter_modify_ctx *ctx, - msgpack_object * map, - struct - modify_condition - *condition) -{ - msgpack_object *skey = NULL; - msgpack_object *okey = NULL; - msgpack_object *oval = NULL; - bool ret = false; - - flb_ra_get_kv_pair(condition->ra_a, *map, &skey, &okey, &oval); - if (skey == NULL || okey == NULL || oval == NULL) { - return false; - } - ret = helper_msgpack_object_matches_str(oval, condition->b, condition->b_len); - if (ret) { - flb_plg_debug(ctx->ins, "Match for condition KEY_VALUE_EQUALS %s", - condition->b); - } - return ret; -} - -static inline -bool evaluate_condition_KEY_VALUE_DOES_NOT_EQUAL(struct filter_modify_ctx *ctx, - msgpack_object - *map, - struct - modify_condition - *condition) -{ - if (!evaluate_condition_KEY_EXISTS(map, condition)) { - return false; - } - return !evaluate_condition_KEY_VALUE_EQUALS(ctx, map, condition); -} - -static inline bool evaluate_condition_KEY_VALUE_MATCHES(struct filter_modify_ctx *ctx, - msgpack_object *map, - struct - modify_condition - *condition) -{ - msgpack_object *skey = NULL; - msgpack_object *okey = NULL; - msgpack_object *oval = NULL; - bool ret = false; - - flb_ra_get_kv_pair(condition->ra_a, *map, &skey, &okey, &oval); - if (skey == NULL || okey == NULL || oval == NULL) { - return false; - } - ret = helper_msgpack_object_matches_regex(oval, condition->b_regex); - if (ret) { - flb_plg_debug(ctx->ins, "Match for condition KEY_VALUE_MATCHES " - "%s", condition->b); - } - return ret; -} - -static inline -bool evaluate_condition_KEY_VALUE_DOES_NOT_MATCH(struct filter_modify_ctx *ctx, - msgpack_object - * map, - struct - modify_condition - *condition) -{ - if (!evaluate_condition_KEY_EXISTS(map, condition)) { - return false; - } - return !evaluate_condition_KEY_VALUE_MATCHES(ctx, map, condition); -} - -static inline bool -evaluate_condition_MATCHING_KEYS_HAVE_MATCHING_VALUES(struct filter_modify_ctx *ctx, - msgpack_object *map, - struct modify_condition - *condition) -{ - int i; - bool match = true; - msgpack_object_kv *kv; - - for (i = 0; i < map->via.map.size; i++) { - kv = &map->via.map.ptr[i]; - if (kv_key_matches_regex(kv, condition->a_regex)) { - if (!kv_val_matches_regex(kv, condition->b_regex)) { - flb_plg_debug(ctx->ins, "Match MISSED for condition " - "MATCHING_KEYS_HAVE_MATCHING_VALUES %s", - condition->b); - match = false; - break; - } - } - } - return match; -} - -static inline bool -evaluate_condition_MATCHING_KEYS_DO_NOT_HAVE_MATCHING_VALUES(struct filter_modify_ctx *ctx, - msgpack_object * - map, - struct - modify_condition - *condition) -{ - return !evaluate_condition_MATCHING_KEYS_HAVE_MATCHING_VALUES(ctx, - map, - condition); -} - -static inline bool evaluate_condition(struct filter_modify_ctx *ctx, - msgpack_object * map, - struct modify_condition *condition) -{ - switch (condition->conditiontype) { - case KEY_EXISTS: - return evaluate_condition_KEY_EXISTS(map, condition); - case KEY_DOES_NOT_EXIST: - return evaluate_condition_KEY_DOES_NOT_EXIST(map, condition); - case A_KEY_MATCHES: - return evaluate_condition_A_KEY_MATCHES(map, condition); - case NO_KEY_MATCHES: - return evaluate_condition_NO_KEY_MATCHES(map, condition); - case KEY_VALUE_EQUALS: - return evaluate_condition_KEY_VALUE_EQUALS(ctx, map, condition); - case KEY_VALUE_DOES_NOT_EQUAL: - return evaluate_condition_KEY_VALUE_DOES_NOT_EQUAL(ctx, map, condition); - case KEY_VALUE_MATCHES: - return evaluate_condition_KEY_VALUE_MATCHES(ctx, map, condition); - case KEY_VALUE_DOES_NOT_MATCH: - return evaluate_condition_KEY_VALUE_DOES_NOT_MATCH(ctx, map, condition); - case MATCHING_KEYS_HAVE_MATCHING_VALUES: - return evaluate_condition_MATCHING_KEYS_HAVE_MATCHING_VALUES(ctx, - map, - condition); - case MATCHING_KEYS_DO_NOT_HAVE_MATCHING_VALUES: - return - evaluate_condition_MATCHING_KEYS_DO_NOT_HAVE_MATCHING_VALUES(ctx, - map, - condition); - default: - flb_plg_warn(ctx->ins, "Unknown conditiontype for condition %s : %s, " - "assuming result FAILED TO MEET CONDITION", - condition->raw_k, condition->raw_v); - } - return false; -} - -static inline bool evaluate_conditions(msgpack_object * map, - struct filter_modify_ctx *ctx) -{ - bool ok = true; - - struct mk_list *tmp; - struct mk_list *head; - struct modify_condition *condition; - - mk_list_foreach_safe(head, tmp, &ctx->conditions) { - condition = mk_list_entry(head, struct modify_condition, _head); - if (!evaluate_condition(ctx, map, condition)) { - flb_plg_debug(ctx->ins, "Condition not met : %s", - condition->raw_v); - ok = false; - } - } - - return ok; -} - -static inline int apply_rule_RENAME(struct filter_modify_ctx *ctx, - msgpack_packer *packer, - msgpack_object *map, - struct modify_rule *rule) -{ - int i; - - int match_keys = - map_count_keys_matching_str(map, rule->key, rule->key_len); - int conflict_keys = - map_count_keys_matching_str(map, rule->val, rule->val_len); - - if (match_keys == 0) { - flb_plg_debug(ctx->ins, "Rule RENAME %s TO %s : No keys matching %s " - "found, not applying rule", - rule->key, rule->val, rule->key); - return FLB_FILTER_NOTOUCH; - } - else if (conflict_keys > 0) { - flb_plg_debug(ctx->ins, "Rule RENAME %s TO %s : Existing key %s found, " - "not applying rule", - rule->key, rule->val, rule->key); - return FLB_FILTER_NOTOUCH; - } - else { - msgpack_pack_map(packer, map->via.map.size); - for (i = 0; i < map->via.map.size; i++) { - if (kv_key_matches_str_rule_key(&map->via.map.ptr[i], rule)) { - helper_pack_string(ctx, packer, rule->val, rule->val_len); - } - else { - msgpack_pack_object(packer, map->via.map.ptr[i].key); - } - msgpack_pack_object(packer, map->via.map.ptr[i].val); - } - return FLB_FILTER_MODIFIED; - } -} - -static inline int apply_rule_HARD_RENAME(struct filter_modify_ctx *ctx, - msgpack_packer *packer, - msgpack_object *map, - struct modify_rule *rule) -{ - int i; - - int match_keys = - map_count_keys_matching_str(map, rule->key, rule->key_len); - int conflict_keys = - map_count_keys_matching_str(map, rule->val, rule->val_len); - msgpack_object_kv *kv; - - if (match_keys == 0) { - flb_plg_debug(ctx->ins, "Rule HARD_RENAME %s TO %s : No keys matching " - "%s found, not applying rule", - rule->key, rule->val, rule->key); - return FLB_FILTER_NOTOUCH; - } - else if (conflict_keys == 0) { - msgpack_pack_map(packer, map->via.map.size); - for (i = 0; i < map->via.map.size; i++) { - kv = &map->via.map.ptr[i]; - if (kv_key_matches_str_rule_key(kv, rule)) { - helper_pack_string(ctx, packer, rule->val, rule->val_len); - } - else { - msgpack_pack_object(packer, kv->key); - } - msgpack_pack_object(packer, kv->val); - } - return FLB_FILTER_MODIFIED; - } - else { - msgpack_pack_map(packer, map->via.map.size - conflict_keys); - - for (i = 0; i < map->via.map.size; i++) { - kv = &map->via.map.ptr[i]; - // If this kv->key matches rule->val it's a conflict source key and - // will be skipped - if (!kv_key_matches_str_rule_val(kv, rule)) { - if (kv_key_matches_str_rule_key(kv, rule)) { - helper_pack_string(ctx, packer, rule->val, rule->val_len); - } - else { - msgpack_pack_object(packer, kv->key); - } - - msgpack_pack_object(packer, kv->val); - } - } - return FLB_FILTER_MODIFIED; - } -} - -static inline int apply_rule_COPY(struct filter_modify_ctx *ctx, - msgpack_packer *packer, - msgpack_object *map, - struct modify_rule *rule) -{ - int match_keys = - map_count_keys_matching_str(map, rule->key, rule->key_len); - int conflict_keys = - map_count_keys_matching_str(map, rule->val, rule->val_len); - int i; - msgpack_object_kv *kv; - - if (match_keys < 1) { - flb_plg_debug(ctx->ins, "Rule COPY %s TO %s : No keys matching %s " - "found, not applying rule", - rule->key, rule->val, rule->key); - return FLB_FILTER_NOTOUCH; - } - else if (match_keys > 1) { - flb_plg_debug(ctx->ins, "Rule COPY %s TO %s : Multiple keys matching " - "%s found, not applying rule", - rule->key, rule->val, rule->key); - return FLB_FILTER_NOTOUCH; - } - else if (conflict_keys > 0) { - flb_plg_debug(ctx->ins, "Rule COPY %s TO %s : Existing keys matching " - "target %s found, not applying rule", - rule->key, rule->val, rule->key); - return FLB_FILTER_NOTOUCH; - } - else { - msgpack_pack_map(packer, map->via.map.size + 1); - for (i = 0; i < map->via.map.size; i++) { - kv = &map->via.map.ptr[i]; - - msgpack_pack_object(packer, kv->key); - msgpack_pack_object(packer, kv->val); - - if (kv_key_matches_str_rule_key(kv, rule)) { - helper_pack_string(ctx, packer, rule->val, rule->val_len); - msgpack_pack_object(packer, kv->val); - } - } - return FLB_FILTER_MODIFIED; - } -} - -static inline int apply_rule_HARD_COPY(struct filter_modify_ctx *ctx, - msgpack_packer *packer, - msgpack_object *map, - struct modify_rule *rule) -{ - int i; - - int match_keys = - map_count_keys_matching_str(map, rule->key, rule->key_len); - int conflict_keys = - map_count_keys_matching_str(map, rule->val, rule->val_len); - msgpack_object_kv *kv; - - if (match_keys < 1) { - flb_plg_debug(ctx->ins, "Rule HARD_COPY %s TO %s : No keys matching %s " - "found, not applying rule", - rule->key, rule->val, rule->key); - return FLB_FILTER_NOTOUCH; - } - else if (match_keys > 1) { - flb_plg_warn(ctx->ins, "Rule HARD_COPY %s TO %s : Multiple keys " - "matching %s found, not applying rule", - rule->key, rule->val, rule->key); - return FLB_FILTER_NOTOUCH; - } - else if (conflict_keys > 1) { - flb_plg_warn(ctx->ins, "Rule HARD_COPY %s TO %s : Multiple target keys " - "matching %s found, not applying rule", - rule->key, rule->val, rule->val); - return FLB_FILTER_NOTOUCH; - } - else if (conflict_keys == 0) { - msgpack_pack_map(packer, map->via.map.size + 1); - for (i = 0; i < map->via.map.size; i++) { - kv = &map->via.map.ptr[i]; - msgpack_pack_object(packer, kv->key); - msgpack_pack_object(packer, kv->val); - - // This is our copy - if (kv_key_matches_str_rule_key(kv, rule)) { - helper_pack_string(ctx, packer, rule->val, rule->val_len); - msgpack_pack_object(packer, kv->val); - } - } - return FLB_FILTER_MODIFIED; - } - else { - msgpack_pack_map(packer, map->via.map.size); - - for (i = 0; i < map->via.map.size; i++) { - kv = &map->via.map.ptr[i]; - - // Skip the conflict key, we will create a new one - if (!kv_key_matches_str_rule_val(kv, rule)) { - msgpack_pack_object(packer, kv->key); - msgpack_pack_object(packer, kv->val); - - // This is our copy - if (kv_key_matches_str_rule_key(kv, rule)) { - helper_pack_string(ctx, packer, rule->val, rule->val_len); - msgpack_pack_object(packer, kv->val); - } - } - } - - return FLB_FILTER_MODIFIED; - } -} - -static inline int apply_rule_ADD(struct filter_modify_ctx *ctx, - msgpack_packer *packer, - msgpack_object *map, - struct modify_rule *rule) -{ - if (map_count_keys_matching_str(map, rule->key, rule->key_len) == 0) { - msgpack_pack_map(packer, map->via.map.size + 1); - map_pack_each(packer, map); - helper_pack_string(ctx, packer, rule->key, rule->key_len); - helper_pack_string(ctx, packer, rule->val, rule->val_len); - return FLB_FILTER_MODIFIED; - } - else { - flb_plg_debug(ctx->ins, "Rule ADD %s : this key already exists, " - "skipping", rule->key); - return FLB_FILTER_NOTOUCH; - } -} - -static inline int apply_rule_SET(struct filter_modify_ctx *ctx, - msgpack_packer * packer, - msgpack_object * map, - struct modify_rule *rule) -{ - int matches = map_count_keys_matching_str(map, rule->key, rule->key_len); - - msgpack_pack_map(packer, map->via.map.size - matches + 1); - - if (matches == 0) { - map_pack_each(packer, map); - helper_pack_string(ctx, packer, rule->key, rule->key_len); - helper_pack_string(ctx, packer, rule->val, rule->val_len); - } - else { - map_pack_each_fn(packer, map, rule, - kv_key_does_not_match_str_rule_key); - helper_pack_string(ctx, packer, rule->key, rule->key_len); - helper_pack_string(ctx, packer, rule->val, rule->val_len); - } - - return FLB_FILTER_MODIFIED; -} - -static inline int apply_rule_REMOVE(msgpack_packer *packer, - msgpack_object *map, - struct modify_rule *rule) -{ - int matches = map_count_keys_matching_str(map, rule->key, rule->key_len); - - if (matches == 0) { - return FLB_FILTER_NOTOUCH; - } - else { - msgpack_pack_map(packer, map->via.map.size - matches); - map_pack_each_fn(packer, map, rule, - kv_key_does_not_match_str_rule_key); - return FLB_FILTER_MODIFIED; - } -} - -static inline int apply_rule_REMOVE_WILDCARD(msgpack_packer * packer, - msgpack_object * map, - struct modify_rule *rule) -{ - int matches = - map_count_keys_matching_wildcard(map, rule->key, rule->key_len); - - if (matches == 0) { - return FLB_FILTER_NOTOUCH; - } - else { - msgpack_pack_map(packer, map->via.map.size - matches); - map_pack_each_fn(packer, map, rule, - kv_key_does_not_match_wildcard_rule_key); - return FLB_FILTER_MODIFIED; - } -} - -static inline int apply_rule_REMOVE_REGEX(msgpack_packer * packer, - msgpack_object * map, - struct modify_rule *rule) -{ - int matches = map_count_keys_matching_regex(map, rule->key_regex); - - if (matches == 0) { - return FLB_FILTER_NOTOUCH; - } - else { - msgpack_pack_map(packer, map->via.map.size - matches); - map_pack_each_fn(packer, map, rule, - kv_key_does_not_match_regex_rule_key); - return FLB_FILTER_MODIFIED; - } -} - -static inline int apply_rule_MOVE_TO_END(struct filter_modify_ctx *ctx, - msgpack_packer *packer, - msgpack_object *map, - struct modify_rule *rule) -{ - - int match_keys = - map_count_keys_matching_wildcard(map, rule->key, rule->key_len); - - if (match_keys == 0) { - return FLB_FILTER_NOTOUCH; - } - else { - msgpack_pack_map(packer, map->via.map.size); - map_pack_each_fn(packer, map, rule, - kv_key_does_not_match_wildcard_rule_key); - map_pack_each_fn(packer, map, rule, - kv_key_matches_wildcard_rule_key); - return FLB_FILTER_MODIFIED; - } -} - -static inline int apply_rule_MOVE_TO_START(struct filter_modify_ctx *ctx, - msgpack_packer *packer, - msgpack_object *map, - struct modify_rule *rule) -{ - - int match_keys = - map_count_keys_matching_wildcard(map, rule->key, rule->key_len); - - if (match_keys == 0) { - return FLB_FILTER_NOTOUCH; - } - else { - msgpack_pack_map(packer, map->via.map.size); - map_pack_each_fn(packer, map, rule, - kv_key_matches_wildcard_rule_key); - map_pack_each_fn(packer, map, rule, - kv_key_does_not_match_wildcard_rule_key); - return FLB_FILTER_MODIFIED; - } -} - -static inline int apply_modifying_rule(struct filter_modify_ctx *ctx, - msgpack_packer *packer, - msgpack_object *map, - struct modify_rule *rule) -{ - switch (rule->ruletype) { - case RENAME: - return apply_rule_RENAME(ctx, packer, map, rule); - case HARD_RENAME: - return apply_rule_HARD_RENAME(ctx, packer, map, rule); - case ADD: - return apply_rule_ADD(ctx, packer, map, rule); - case SET: - return apply_rule_SET(ctx, packer, map, rule); - case REMOVE: - return apply_rule_REMOVE(packer, map, rule); - case REMOVE_WILDCARD: - return apply_rule_REMOVE_WILDCARD(packer, map, rule); - case REMOVE_REGEX: - return apply_rule_REMOVE_REGEX(packer, map, rule); - case COPY: - return apply_rule_COPY(ctx, packer, map, rule); - case HARD_COPY: - return apply_rule_HARD_COPY(ctx, packer, map, rule); - case MOVE_TO_START: - return apply_rule_MOVE_TO_START(ctx, packer, map, rule); - case MOVE_TO_END: - return apply_rule_MOVE_TO_END(ctx, packer, map, rule); - default: - flb_plg_warn(ctx->ins, "Unknown ruletype for rule with key %s, ignoring", - rule->key); - } - return FLB_FILTER_NOTOUCH; -} - - - -static inline int apply_modifying_rules( - struct flb_log_event_encoder *log_encoder, - struct flb_log_event *log_event, - struct filter_modify_ctx *ctx) -{ - int ret; - int records_in; - msgpack_object map; - struct modify_rule *rule; - msgpack_sbuffer sbuffer; - msgpack_packer in_packer; - msgpack_unpacker unpacker; - msgpack_unpacked unpacked; - int initial_buffer_size = 1024 * 8; - int new_buffer_size = 0; - struct mk_list *tmp; - struct mk_list *head; - bool has_modifications = false; - - map = *log_event->body; - records_in = map.via.map.size; - - if (!evaluate_conditions(&map, ctx)) { - flb_plg_debug(ctx->ins, "Conditions not met, not touching record"); - return 0; - } - - msgpack_sbuffer_init(&sbuffer); - msgpack_packer_init(&in_packer, &sbuffer, msgpack_sbuffer_write); - msgpack_unpacked_init(&unpacked); - - if (!msgpack_unpacker_init(&unpacker, initial_buffer_size)) { - flb_plg_error(ctx->ins, "Unable to allocate memory for unpacker, aborting"); - return -1; - } - - mk_list_foreach_safe(head, tmp, &ctx->rules) { - rule = mk_list_entry(head, struct modify_rule, _head); - - msgpack_sbuffer_clear(&sbuffer); - - if (apply_modifying_rule(ctx, &in_packer, &map, rule) != - FLB_FILTER_NOTOUCH) { - - has_modifications = true; - new_buffer_size = sbuffer.size * 2; - - if (msgpack_unpacker_buffer_capacity(&unpacker) < new_buffer_size) { - if (!msgpack_unpacker_reserve_buffer - (&unpacker, new_buffer_size)) { - flb_plg_error(ctx->ins, "Unable to re-allocate memory for " - "unpacker, aborting"); - return -1; - } - } - - memcpy(msgpack_unpacker_buffer(&unpacker), sbuffer.data, - sbuffer.size); - msgpack_unpacker_buffer_consumed(&unpacker, sbuffer.size); - - msgpack_unpacker_next(&unpacker, &unpacked); - - if (unpacked.data.type == MSGPACK_OBJECT_MAP) { - map = unpacked.data; - } - else { - flb_plg_error(ctx->ins, "Expected MSGPACK_MAP, this is not a " - "valid return value, skipping"); - } - } - } - - if (has_modifications) { - ret = flb_log_event_encoder_begin_record(log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_timestamp( - log_encoder, &log_event->timestamp); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_metadata_from_msgpack_object( - log_encoder, log_event->metadata); - } - - flb_plg_trace(ctx->ins, "Input map size %d elements, output map size " - "%d elements", records_in, map.via.map.size); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_body_from_msgpack_object( - log_encoder, &map); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(log_encoder); - } - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, "log event encoding error : %d", ret); - - flb_log_event_encoder_rollback_record(log_encoder); - - has_modifications = FLB_FALSE; - } - } - - msgpack_unpacked_destroy(&unpacked); - msgpack_unpacker_destroy(&unpacker); - msgpack_sbuffer_destroy(&sbuffer); - - return has_modifications ? 1 : 0; - -} - -static int cb_modify_init(struct flb_filter_instance *f_ins, - struct flb_config *config, void *data) -{ - struct filter_modify_ctx *ctx; - - // Create context - ctx = flb_malloc(sizeof(struct filter_modify_ctx)); - if (!ctx) { - flb_errno(); - return -1; - } - mk_list_init(&ctx->conditions); - mk_list_init(&ctx->rules); - ctx->ins = f_ins; - ctx->rules_cnt = 0; - ctx->conditions_cnt = 0; - - if (setup(ctx, f_ins, config) < 0) { - flb_free(ctx); - return -1; - } - - // Set context - flb_filter_set_context(f_ins, ctx); - return 0; -} - -static int cb_modify_filter(const void *data, size_t bytes, - const char *tag, int tag_len, - void **out_buf, size_t * out_size, - struct flb_filter_instance *f_ins, - struct flb_input_instance *i_ins, - void *context, struct flb_config *config) -{ - struct flb_log_event_encoder log_encoder; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - struct filter_modify_ctx *ctx = context; - int modifications = 0; - int total_modifications = 0; - int ret; - - (void) f_ins; - (void) i_ins; - (void) config; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return FLB_FILTER_NOTOUCH; - } - - ret = flb_log_event_encoder_init(&log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event encoder initialization error : %d", ret); - - flb_log_event_decoder_destroy(&log_decoder); - - return FLB_FILTER_NOTOUCH; - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - modifications = - apply_modifying_rules(&log_encoder, &log_event, ctx); - - if (modifications == 0) { - /* not matched, so copy original event. */ - ret = flb_log_event_encoder_emit_raw_record( - &log_encoder, - log_decoder.record_base, - log_decoder.record_length); - } - - total_modifications += modifications; - } - - if(total_modifications > 0) { - if (ret == FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA && - log_decoder.offset == bytes) { - ret = FLB_EVENT_ENCODER_SUCCESS; - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - *out_buf = log_encoder.output_buffer; - *out_size = log_encoder.output_length; - - ret = FLB_FILTER_MODIFIED; - - flb_log_event_encoder_claim_internal_buffer_ownership(&log_encoder); - } - else { - flb_plg_error(ctx->ins, - "Log event encoder error : %d", ret); - - ret = FLB_FILTER_NOTOUCH; - } - } - else { - ret = FLB_FILTER_NOTOUCH; - } - - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - return ret; -} - -static int cb_modify_exit(void *data, struct flb_config *config) -{ - struct filter_modify_ctx *ctx = data; - - teardown(ctx); - flb_free(ctx); - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "Set", NULL, - FLB_CONFIG_MAP_MULT, FLB_FALSE, 0, - "Add a key/value pair with key KEY and value VALUE. " - "If KEY already exists, this field is overwritten." - }, - { - FLB_CONFIG_MAP_STR, "Add", NULL, - FLB_CONFIG_MAP_MULT, FLB_FALSE, 0, - "Add a key/value pair with key KEY and value VALUE if KEY does not exist" - }, - { - FLB_CONFIG_MAP_STR, "Remove", NULL, - FLB_CONFIG_MAP_MULT, FLB_FALSE, 0, - "Remove a key/value pair with key KEY if it exists" - }, - { - FLB_CONFIG_MAP_STR, "Remove_wildcard", NULL, - FLB_CONFIG_MAP_MULT, FLB_FALSE, 0, - "Remove all key/value pairs with key matching wildcard KEY" - }, - { - FLB_CONFIG_MAP_STR, "Remove_regex", NULL, - FLB_CONFIG_MAP_MULT, FLB_FALSE, 0, - "Remove all key/value pairs with key matching regexp KEY" - }, - { - FLB_CONFIG_MAP_STR, "Move_To_Start", NULL, - FLB_CONFIG_MAP_MULT, FLB_FALSE, 0, - "Move key/value pairs with keys matching KEY to the start of the message" - }, - { - FLB_CONFIG_MAP_STR, "Move_To_End", NULL, - FLB_CONFIG_MAP_MULT, FLB_FALSE, 0, - "Move key/value pairs with keys matching KEY to the end of the message" - }, - { - FLB_CONFIG_MAP_STR, "Rename", NULL, - FLB_CONFIG_MAP_MULT, FLB_FALSE, 0, - "Rename a key/value pair with key KEY to RENAMED_KEY " - "if KEY exists AND RENAMED_KEY does not exist" - }, - { - FLB_CONFIG_MAP_STR, "Hard_Rename", NULL, - FLB_CONFIG_MAP_MULT, FLB_FALSE, 0, - "Rename a key/value pair with key KEY to RENAMED_KEY if KEY exists. " - "If RENAMED_KEY already exists, this field is overwritten" - }, - { - FLB_CONFIG_MAP_STR, "Copy", NULL, - FLB_CONFIG_MAP_MULT, FLB_FALSE, 0, - "Copy a key/value pair with key KEY to COPIED_KEY " - "if KEY exists AND COPIED_KEY does not exist" - }, - { - FLB_CONFIG_MAP_STR, "Hard_copy", NULL, - FLB_CONFIG_MAP_MULT, FLB_FALSE, 0, - "Copy a key/value pair with key KEY to COPIED_KEY if KEY exists. " - "If COPIED_KEY already exists, this field is overwritten" - }, - { - FLB_CONFIG_MAP_STR, "Condition", NULL, - FLB_CONFIG_MAP_MULT, FLB_FALSE, 0, - "Set the condition to modify. Key_exists, Key_does_not_exist, A_key_matches, " - "No_key_matches, Key_value_equals, Key_value_does_not_equal, Key_value_matches, " - "Key_value_does_not_match, Matching_keys_have_matching_values " - "and Matching_keys_do_not_have_matching_values are supported." - }, - {0} -}; - -struct flb_filter_plugin filter_modify_plugin = { - .name = "modify", - .description = "modify records by applying rules", - .cb_init = cb_modify_init, - .cb_filter = cb_modify_filter, - .cb_exit = cb_modify_exit, - .config_map = config_map, - .flags = 0 -}; diff --git a/fluent-bit/plugins/filter_modify/modify.h b/fluent-bit/plugins/filter_modify/modify.h deleted file mode 100644 index 92c590e01..000000000 --- a/fluent-bit/plugins/filter_modify/modify.h +++ /dev/null @@ -1,96 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_FILTER_MODIFY_H -#define FLB_FILTER_MODIFY_H - -#include -#include -#include -#include - -enum FLB_FILTER_MODIFY_RULETYPE { - RENAME, - HARD_RENAME, - ADD, - SET, - REMOVE, - REMOVE_WILDCARD, - REMOVE_REGEX, - COPY, - HARD_COPY, - MOVE_TO_START, - MOVE_TO_END -}; - -enum FLB_FILTER_MODIFY_CONDITIONTYPE { - KEY_EXISTS, - KEY_DOES_NOT_EXIST, - A_KEY_MATCHES, - NO_KEY_MATCHES, - KEY_VALUE_EQUALS, - KEY_VALUE_DOES_NOT_EQUAL, - KEY_VALUE_MATCHES, - KEY_VALUE_DOES_NOT_MATCH, - MATCHING_KEYS_HAVE_MATCHING_VALUES, - MATCHING_KEYS_DO_NOT_HAVE_MATCHING_VALUES -}; - -struct filter_modify_ctx -{ - int rules_cnt; - struct mk_list rules; - int conditions_cnt; - struct mk_list conditions; - struct flb_filter_instance *ins; -}; - -struct modify_rule -{ - enum FLB_FILTER_MODIFY_RULETYPE ruletype; - int key_len; - int val_len; - char *key; - char *val; - bool key_is_regex; - bool val_is_regex; - struct flb_regex *key_regex; - struct flb_regex *val_regex; - char *raw_k; - char *raw_v; - struct mk_list _head; -}; - -struct modify_condition -{ - enum FLB_FILTER_MODIFY_CONDITIONTYPE conditiontype; - int a_len; - int b_len; - flb_sds_t a; - char *b; - bool a_is_regex; - bool b_is_regex; - struct flb_regex *a_regex; - struct flb_regex *b_regex; - struct flb_record_accessor *ra_a; - char *raw_k; - char *raw_v; - struct mk_list _head; -}; -#endif diff --git a/fluent-bit/plugins/filter_multiline/CMakeLists.txt b/fluent-bit/plugins/filter_multiline/CMakeLists.txt deleted file mode 100644 index 5b677c50f..000000000 --- a/fluent-bit/plugins/filter_multiline/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -set(src - ml.c - ml_concat.c) - -FLB_PLUGIN(filter_multiline "${src}" "") diff --git a/fluent-bit/plugins/filter_multiline/ml.c b/fluent-bit/plugins/filter_multiline/ml.c deleted file mode 100644 index b63282628..000000000 --- a/fluent-bit/plugins/filter_multiline/ml.c +++ /dev/null @@ -1,931 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "ml.h" -#include "ml_concat.h" - -static struct ml_stream *get_by_id(struct ml_ctx *ctx, uint64_t stream_id); - -/* Create an emitter input instance */ -static int emitter_create(struct ml_ctx *ctx) -{ - int ret; - struct flb_input_instance *ins; - - ret = flb_input_name_exists(ctx->emitter_name, ctx->config); - if (ret == FLB_TRUE) { - flb_plg_error(ctx->ins, "emitter_name '%s' already exists", - ctx->emitter_name); - return -1; - } - - ins = flb_input_new(ctx->config, "emitter", NULL, FLB_FALSE); - if (!ins) { - flb_plg_error(ctx->ins, "cannot create emitter instance"); - return -1; - } - - /* Set the alias name */ - ret = flb_input_set_property(ins, "alias", ctx->emitter_name); - if (ret == -1) { - flb_plg_warn(ctx->ins, - "cannot set emitter_name, using fallback name '%s'", - ins->name); - } - - /* Set the emitter_mem_buf_limit */ - if(ctx->emitter_mem_buf_limit > 0) { - ins->mem_buf_limit = ctx->emitter_mem_buf_limit; - } - - /* Set the storage type */ - ret = flb_input_set_property(ins, "storage.type", - ctx->emitter_storage_type); - if (ret == -1) { - flb_plg_error(ctx->ins, "cannot set storage.type"); - } - - /* Initialize emitter plugin */ - ret = flb_input_instance_init(ins, ctx->config); - if (ret == -1) { - flb_plg_error(ctx->ins, "cannot initialize emitter instance '%s'", - ins->name); - flb_input_instance_exit(ins, ctx->config); - flb_input_instance_destroy(ins); - return -1; - } - -#ifdef FLB_HAVE_METRICS - /* Override Metrics title */ - ret = flb_metrics_title(ctx->emitter_name, ins->metrics); - if (ret == -1) { - flb_plg_warn(ctx->ins, "cannot set metrics title, using fallback name %s", - ins->name); - } -#endif - - /* Storage context */ - ret = flb_storage_input_create(ctx->config->cio, ins); - if (ret == -1) { - flb_plg_error(ctx->ins, "cannot initialize storage for stream '%s'", - ctx->emitter_name); - flb_input_instance_exit(ins, ctx->config); - flb_input_instance_destroy(ins); - return -1; - } - ctx->ins_emitter = ins; - return 0; -} - -static int multiline_load_parsers(struct ml_ctx *ctx) -{ - int ret; - struct mk_list *head; - struct mk_list *head_p; - struct flb_config_map_val *mv; - struct flb_slist_entry *val = NULL; - struct flb_ml_parser_ins *parser_i; - - if (!ctx->multiline_parsers) { - return -1; - } - - /* - * Iterate all 'multiline.parser' entries. Every entry is considered - * a group which can have multiple multiline parser instances. - */ - flb_config_map_foreach(head, mv, ctx->multiline_parsers) { - mk_list_foreach(head_p, mv->val.list) { - val = mk_list_entry(head_p, struct flb_slist_entry, _head); - - /* Create an instance of the defined parser */ - parser_i = flb_ml_parser_instance_create(ctx->m, val->str); - if (!parser_i) { - return -1; - } - - /* Always override parent parser values */ - if (ctx->key_content) { - ret = flb_ml_parser_instance_set(parser_i, - "key_content", - ctx->key_content); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not override 'key_content'"); - return -1; - } - } - } - } - - return 0; -} - -static int flush_callback(struct flb_ml_parser *parser, - struct flb_ml_stream *mst, - void *data, char *buf_data, size_t buf_size) -{ - int ret; - struct ml_ctx *ctx = data; - struct ml_stream *stream; - - if (ctx->debug_flush) { - flb_ml_flush_stdout(parser, mst, data, buf_data, buf_size); - } - - if (ctx->use_buffer == FLB_FALSE) { - /* Append incoming record to our msgpack context buffer */ - msgpack_sbuffer_write(&ctx->mp_sbuf, buf_data, buf_size); - return 0; - - } else { /* buffered mode */ - stream = get_by_id(ctx, mst->id); - if (!stream) { - flb_plg_error(ctx->ins, "Could not find tag to re-emit from stream %s", - mst->name); - return -1; - } - - /* Emit record with original tag */ - flb_plg_trace(ctx->ins, "emitting from %s to %s", stream->input_name, stream->tag); - ret = in_emitter_add_record(stream->tag, flb_sds_len(stream->tag), buf_data, buf_size, - ctx->ins_emitter); - - return ret; - } -} - -static int cb_ml_init(struct flb_filter_instance *ins, - struct flb_config *config, - void *data) -{ - int ret; - struct ml_ctx *ctx; - flb_sds_t tmp; - flb_sds_t emitter_name = NULL; - int len; - uint64_t stream_id; - (void) config; - - ctx = flb_calloc(1, sizeof(struct ml_ctx)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->ins = ins; - ctx->debug_flush = FLB_FALSE; - ctx->config = config; - ctx->timer_created = FLB_FALSE; - - /* - * Config map is not yet set at this point in the code - * user must explicitly set buffer to false to turn it off - */ - ctx->use_buffer = FLB_TRUE; - tmp = (char *) flb_filter_get_property("buffer", ins); - if (tmp) { - ctx->use_buffer = flb_utils_bool(tmp); - } - ctx->partial_mode = FLB_FALSE; - tmp = (char *) flb_filter_get_property("mode", ins); - if (tmp != NULL) { - if (strcasecmp(tmp, FLB_MULTILINE_MODE_PARTIAL_MESSAGE) == 0) { - ctx->partial_mode = FLB_TRUE; - } else if (strcasecmp(tmp, FLB_MULTILINE_MODE_PARSER) == 0) { - ctx->partial_mode = FLB_FALSE; - } else { - flb_plg_error(ins, "'Mode' must be '%s' or '%s'", - FLB_MULTILINE_MODE_PARTIAL_MESSAGE, - FLB_MULTILINE_MODE_PARSER); - return -1; - } - } - - if (ctx->partial_mode == FLB_TRUE && ctx->use_buffer == FLB_FALSE) { - flb_plg_error(ins, "'%s' 'Mode' requires 'Buffer' to be 'On'", - FLB_MULTILINE_MODE_PARTIAL_MESSAGE); - } - - if (ctx->use_buffer == FLB_FALSE) { - /* Init buffers */ - msgpack_sbuffer_init(&ctx->mp_sbuf); - msgpack_packer_init(&ctx->mp_pck, &ctx->mp_sbuf, msgpack_sbuffer_write); - } else { - /* - * Emitter name: every buffered multiline instance needs an emitter input plugin, - * with that one is able to emit records. We use a unique instance so we - * can use the metrics interface. - * - * If not set, we define an emitter name - * - * Validate if the emitter_name has been set before to check with the - * config map. If is not set, do a manual set of the property, so we let the - * config map handle the memory allocation. - */ - tmp = (char *) flb_filter_get_property("emitter_name", ins); - if (!tmp) { - emitter_name = flb_sds_create_size(64); - if (!emitter_name) { - flb_free(ctx); - return -1; - } - - tmp = flb_sds_printf(&emitter_name, "emitter_for_%s", - flb_filter_name(ins)); - if (!tmp) { - flb_plg_error(ins, "cannot compose emitter_name"); - flb_sds_destroy(emitter_name); - flb_free(ctx); - return -1; - } - - flb_filter_set_property(ins, "emitter_name", emitter_name); - flb_plg_info(ins, "created emitter: %s", emitter_name); - flb_sds_destroy(emitter_name); - } - } - - /* Load the config map */ - ret = flb_filter_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return -1; - } - - /* Set plugin context */ - flb_filter_set_context(ins, ctx); - - if (ctx->key_content == NULL && ctx->partial_mode == FLB_TRUE) { - flb_plg_error(ins, "'Mode' '%s' requires 'multiline.key_content'", - FLB_MULTILINE_MODE_PARTIAL_MESSAGE); - flb_free(ctx); - return -1; - } - - if (ctx->partial_mode == FLB_FALSE && mk_list_size(ctx->multiline_parsers) == 0) { - flb_plg_error(ins, "The default 'Mode' '%s' requires at least one 'multiline.parser'", - FLB_MULTILINE_MODE_PARSER); - flb_free(ctx); - return -1; - } - - - if (ctx->use_buffer == FLB_TRUE) { - /* - * Emitter Storage Type: the emitter input plugin to be created by default - * uses memory buffer, this option allows to define a filesystem mechanism - * for new records created (only if the main service is also filesystem - * enabled). - * - * On this code we just validate the input type: 'memory' or 'filesystem'. - */ - tmp = ctx->emitter_storage_type; - if (strcasecmp(tmp, "memory") != 0 && strcasecmp(tmp, "filesystem") != 0) { - flb_plg_error(ins, "invalid 'emitter_storage.type' value. Only " - "'memory' or 'filesystem' types are allowed"); - flb_free(ctx); - return -1; - } - - /* Create the emitter context */ - ret = emitter_create(ctx); - if (ret == -1) { - flb_free(ctx); - return -1; - } - - /* Register a metric to count the number of emitted records */ -#ifdef FLB_HAVE_METRICS - ctx->cmt_emitted = cmt_counter_create(ins->cmt, - "fluentbit", "filter", "emit_records_total", - "Total number of emitted records", - 1, (char *[]) {"name"}); - - /* OLD api */ - flb_metrics_add(FLB_MULTILINE_METRIC_EMITTED, - "emit_records", ctx->ins->metrics); -#endif - } - - mk_list_init(&ctx->ml_streams); - mk_list_init(&ctx->split_message_packers); - - if (ctx->partial_mode == FLB_FALSE) { - /* Create multiline context */ - ctx->m = flb_ml_create(config, ctx->ins->name); - if (!ctx->m) { - /* - * we don't free the context since upon init failure, the exit - * callback will be triggered with our context set above. - */ - return -1; - } - - /* Load the parsers/config */ - ret = multiline_load_parsers(ctx); - if (ret == -1) { - return -1; - } - - if (ctx->use_buffer == FLB_TRUE) { - - ctx->m->flush_ms = ctx->flush_ms; - ret = flb_ml_auto_flush_init(ctx->m); - if (ret == -1) { - return -1; - } - } else { - /* Create a stream for this file */ - len = strlen(ins->name); - ret = flb_ml_stream_create(ctx->m, - ins->name, len, - flush_callback, ctx, - &stream_id); - if (ret != 0) { - flb_plg_error(ctx->ins, "could not create multiline stream"); - return -1; - } - ctx->stream_id = stream_id; - } - } - - return 0; -} - -void ml_stream_destroy(struct ml_stream *stream) -{ - if (!stream) { - return; - } - - if (stream->input_name) { - flb_sds_destroy(stream->input_name); - } - if (stream->tag) { - flb_sds_destroy(stream->tag); - } - flb_free(stream); - return; -} - -static struct ml_stream *get_by_id(struct ml_ctx *ctx, uint64_t stream_id) -{ - struct mk_list *tmp; - struct mk_list *head; - struct ml_stream *stream; - - mk_list_foreach_safe(head, tmp, &ctx->ml_streams) { - stream = mk_list_entry(head, struct ml_stream, _head); - if (stream->stream_id == stream_id) { - return stream; - } - } - - return NULL; -} - -static struct ml_stream *get_or_create_stream(struct ml_ctx *ctx, - struct flb_input_instance *i_ins, - const char *tag, int tag_len) -{ - uint64_t stream_id; - struct mk_list *tmp; - struct mk_list *head; - struct ml_stream *stream; - flb_sds_t stream_name; - flb_sds_t tmp_sds; - int name_check; - int tag_check; - int len; - int ret; - - mk_list_foreach_safe(head, tmp, &ctx->ml_streams) { - stream = mk_list_entry(head, struct ml_stream, _head); - name_check = strcmp(stream->input_name, i_ins->name); - tag_check = strcmp(stream->tag, tag); - if (tag_check == 0 && name_check == 0) { - flb_plg_trace(ctx->ins, "using stream %s_%s", stream->input_name, stream->tag); - return stream; - } - } - - /* create a new stream */ - - stream_name = flb_sds_create_size(64); - - tmp_sds = flb_sds_printf(&stream_name, "%s_%s", i_ins->name, tag); - if (!tmp_sds) { - flb_errno(); - flb_sds_destroy(stream_name); - return NULL; - } - stream_name = tmp_sds; - - stream = flb_calloc(1, sizeof(struct ml_stream)); - if (!stream) { - flb_errno(); - flb_sds_destroy(stream_name); - return NULL; - } - - tmp_sds = flb_sds_create(tag); - if (!tmp) { - flb_errno(); - flb_sds_destroy(stream_name); - ml_stream_destroy(stream); - return NULL; - } - stream->tag = tmp_sds; - - tmp_sds = flb_sds_create(i_ins->name); - if (!tmp_sds) { - flb_errno(); - flb_sds_destroy(stream_name); - ml_stream_destroy(stream); - return NULL; - } - stream->input_name = tmp_sds; - - /* Create an flb_ml_stream for this stream */ - flb_plg_info(ctx->ins, "created new multiline stream for %s", stream_name); - len = flb_sds_len(stream_name); - ret = flb_ml_stream_create(ctx->m, - stream_name, len, - flush_callback, ctx, - &stream_id); - if (ret != 0) { - flb_plg_error(ctx->ins, "could not create multiline stream for %s", - stream_name); - flb_sds_destroy(stream_name); - ml_stream_destroy(stream); - return NULL; - } - stream->stream_id = stream_id; - mk_list_add(&stream->_head, &ctx->ml_streams); - flb_plg_debug(ctx->ins, "Created new ML stream for %s", stream_name); - flb_sds_destroy(stream_name); - - return stream; -} - -static void partial_timer_cb(struct flb_config *config, void *data) -{ - struct ml_ctx *ctx = data; - (void) config; - struct mk_list *tmp; - struct mk_list *head; - struct split_message_packer *packer; - unsigned long long now; - unsigned long long diff; - int ret; - - now = ml_current_timestamp(); - - mk_list_foreach_safe(head, tmp, &ctx->split_message_packers) { - packer = mk_list_entry(head, struct split_message_packer, _head); - - diff = now - packer->last_write_time; - if (diff <= ctx->flush_ms) { - continue; - } - - mk_list_del(&packer->_head); - ml_split_message_packer_complete(packer); - /* re-emit record with original tag */ - if (packer->log_encoder.output_buffer != NULL && - packer->log_encoder.output_length > 0) { - - flb_plg_trace(ctx->ins, "emitting from %s to %s", packer->input_name, packer->tag); - ret = in_emitter_add_record(packer->tag, flb_sds_len(packer->tag), - packer->log_encoder.output_buffer, - packer->log_encoder.output_length, - ctx->ins_emitter); - if (ret < 0) { - /* this shouldn't happen in normal execution */ - flb_plg_warn(ctx->ins, - "Couldn't send concatenated record of size %zu " - "bytes to in_emitter %s", - packer->log_encoder.output_length, - ctx->ins_emitter->name); - } - } - ml_split_message_packer_destroy(packer); - } -} - -static int ml_filter_partial(const void *data, size_t bytes, - const char *tag, int tag_len, - void **out_buf, size_t *out_bytes, - struct flb_filter_instance *f_ins, - struct flb_input_instance *i_ins, - void *filter_context, - struct flb_config *config) -{ - int ret; - struct ml_ctx *ctx = filter_context; - msgpack_sbuffer tmp_sbuf; - msgpack_packer tmp_pck; - int partial_records = 0; - int return_records = 0; - int partial = FLB_FALSE; - int is_last_partial = FLB_FALSE; - struct split_message_packer *packer; - char *partial_id_str = NULL; - size_t partial_id_size = 0; - struct flb_sched *sched; - struct flb_log_event_encoder log_encoder; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - (void) f_ins; - (void) config; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return FLB_FILTER_NOTOUCH; - } - - ret = flb_log_event_encoder_init(&log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event encoder initialization error : %d", ret); - - flb_log_event_decoder_destroy(&log_decoder); - - return FLB_FILTER_NOTOUCH; - } - - /* - * create a timer that will run periodically and check if pending buffers - * have expired - * this is created once on the first flush - */ - if (ctx->timer_created == FLB_FALSE) { - flb_plg_debug(ctx->ins, - "Creating flush timer with frequency %dms", - ctx->flush_ms); - - sched = flb_sched_ctx_get(); - - ret = flb_sched_timer_cb_create(sched, FLB_SCHED_TIMER_CB_PERM, - ctx->flush_ms / 2, partial_timer_cb, - ctx, NULL); - if (ret < 0) { - flb_plg_error(ctx->ins, "Failed to create flush timer"); - } else { - ctx->timer_created = FLB_TRUE; - } - } - - /* - * Create temporary msgpack buffer - * for non-partial messages which are passed on as-is - */ - msgpack_sbuffer_init(&tmp_sbuf); - msgpack_packer_init(&tmp_pck, &tmp_sbuf, msgpack_sbuffer_write); - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - partial = ml_is_partial(log_event.body); - if (partial == FLB_TRUE) { - partial_records++; - ret = ml_get_partial_id(log_event.body, &partial_id_str, &partial_id_size); - if (ret == -1) { - flb_plg_warn(ctx->ins, "Could not find partial_id but partial_message key is FLB_TRUE for record with tag %s", tag); - /* handle this record as non-partial */ - partial_records--; - goto pack_non_partial; - } - packer = ml_get_packer(&ctx->split_message_packers, tag, - i_ins->name, partial_id_str, partial_id_size); - if (packer == NULL) { - flb_plg_trace(ctx->ins, "Found new partial record with tag %s", tag); - packer = ml_create_packer(tag, i_ins->name, partial_id_str, partial_id_size, - log_event.body, ctx->key_content, &log_event.timestamp); - if (packer == NULL) { - flb_plg_warn(ctx->ins, "Could not create packer for partial record with tag %s", tag); - /* handle this record as non-partial */ - partial_records--; - goto pack_non_partial; - } - mk_list_add(&packer->_head, &ctx->split_message_packers); - } - ret = ml_split_message_packer_write(packer, log_event.body, ctx->key_content); - if (ret < 0) { - flb_plg_warn(ctx->ins, "Could not append content for partial record with tag %s", tag); - /* handle this record as non-partial */ - partial_records--; - goto pack_non_partial; - } - is_last_partial = ml_is_partial_last(log_event.body); - if (is_last_partial == FLB_TRUE) { - /* emit the record in this filter invocation */ - return_records++; - ml_split_message_packer_complete(packer); - ml_append_complete_record(packer, &log_encoder); - mk_list_del(&packer->_head); - ml_split_message_packer_destroy(packer); - } - } else { - -pack_non_partial: - return_records++; - /* record passed from filter as-is */ - - ret = flb_log_event_encoder_emit_raw_record( - &log_encoder, - log_decoder.record_base, - log_decoder.record_length); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event encoder initialization error : %d", ret); - } - } - } - - if (partial_records == 0) { - /* if no records were partial, we didn't modify the chunk */ - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - msgpack_sbuffer_destroy(&tmp_sbuf); - - return FLB_FILTER_NOTOUCH; - } else if (return_records > 0) { - /* some new records can be returned now, return a new buffer */ - if (log_encoder.output_length > 0) { - *out_buf = log_encoder.output_buffer; - *out_bytes = log_encoder.output_length; - - ret = FLB_FILTER_MODIFIED; - - flb_log_event_encoder_claim_internal_buffer_ownership(&log_encoder); - } - else { - ret = FLB_FILTER_NOTOUCH; - } - - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - return ret; - } else { - /* no records to return right now, free buffer */ - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - msgpack_sbuffer_destroy(&tmp_sbuf); - } - - return FLB_FILTER_MODIFIED; -} - -static int cb_ml_filter(const void *data, size_t bytes, - const char *tag, int tag_len, - void **out_buf, size_t *out_bytes, - struct flb_filter_instance *f_ins, - struct flb_input_instance *i_ins, - void *filter_context, - struct flb_config *config) -{ - size_t tmp_size; - char *tmp_buf; - struct flb_log_event_decoder decoder; - struct ml_stream *stream; - struct flb_log_event event; - int ret; - struct ml_ctx *ctx; - - (void) f_ins; - (void) config; - - ctx = (struct ml_ctx *) filter_context; - - if (i_ins == ctx->ins_emitter) { - flb_plg_trace(ctx->ins, "not processing records from the emitter"); - return FLB_FILTER_NOTOUCH; - } - - /* 'partial_message' mode */ - if (ctx->partial_mode == FLB_TRUE) { - return ml_filter_partial(data, bytes, tag, tag_len, - out_buf, out_bytes, - f_ins, i_ins, - filter_context, config); - } - - /* 'parser' mode */ - if (ctx->use_buffer == FLB_FALSE) { - /* reset mspgack size content */ - ctx->mp_sbuf.size = 0; - - /* process records */ - flb_log_event_decoder_init(&decoder, (char *) data, bytes); - - while (flb_log_event_decoder_next(&decoder, &event) == - FLB_EVENT_DECODER_SUCCESS) { - ret = flb_ml_append_event(ctx->m, ctx->stream_id, &event); - - if (ret != 0) { - flb_plg_debug(ctx->ins, - "could not append object from tag: %s", tag); - } - } - - flb_log_event_decoder_destroy(&decoder); - - /* flush all pending data (there is no auto-flush when unbuffered) */ - flb_ml_flush_pending_now(ctx->m); - - if (ctx->mp_sbuf.size > 0) { - /* - * If the filter will report a new set of records because the - * original data was modified, we make a copy to a new memory - * area, since the buffer might be invalidated in the filter - * chain. - */ - - tmp_buf = flb_malloc(ctx->mp_sbuf.size); - if (!tmp_buf) { - flb_errno(); - return FLB_FILTER_NOTOUCH; - } - tmp_size = ctx->mp_sbuf.size; - memcpy(tmp_buf, ctx->mp_sbuf.data, tmp_size); - *out_buf = tmp_buf; - *out_bytes = tmp_size; - ctx->mp_sbuf.size = 0; - - return FLB_FILTER_MODIFIED; - } - - /* unlikely to happen.. but just in case */ - return FLB_FILTER_NOTOUCH; - - } else { /* buffered mode */ - stream = get_or_create_stream(ctx, i_ins, tag, tag_len); - - if (!stream) { - flb_plg_error(ctx->ins, "Could not find or create ML stream for %s", tag); - return FLB_FILTER_NOTOUCH; - } - - /* process records */ - flb_log_event_decoder_init(&decoder, (char *) data, bytes); - - while (flb_log_event_decoder_next(&decoder, &event) == - FLB_EVENT_DECODER_SUCCESS) { - ret = flb_ml_append_event(ctx->m, stream->stream_id, &event); - - if (ret != 0) { - flb_plg_debug(ctx->ins, - "could not append object from tag: %s", tag); - } - } - - flb_log_event_decoder_destroy(&decoder); - - /* - * always returned modified, which will be 0 records, since the emitter takes - * all records. - */ - return FLB_FILTER_MODIFIED; - } -} - -static int cb_ml_exit(void *data, struct flb_config *config) -{ - struct ml_ctx *ctx = data; - struct mk_list *tmp; - struct mk_list *head; - struct ml_stream *stream; - - if (!ctx) { - return 0; - } - - if (ctx->m) { - flb_ml_destroy(ctx->m); - } - - mk_list_foreach_safe(head, tmp, &ctx->ml_streams) { - stream = mk_list_entry(head, struct ml_stream, _head); - mk_list_del(&stream->_head); - ml_stream_destroy(stream); - } - - msgpack_sbuffer_destroy(&ctx->mp_sbuf); - flb_free(ctx); - - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_BOOL, "debug_flush", "false", - 0, FLB_TRUE, offsetof(struct ml_ctx, debug_flush), - "enable debugging for concatenation flush to stdout" - }, - - { - FLB_CONFIG_MAP_BOOL, "buffer", "true", - 0, FLB_TRUE, offsetof(struct ml_ctx, use_buffer), - "Enable buffered mode. In buffered mode, the filter can concatenate " - "multilines from inputs that ingest records one by one (ex: Forward), " - "rather than in chunks, re-emitting them into the beggining of the " - "pipeline using the in_emitter instance. " - "With buffer off, this filter will not work with most inputs, except tail." - }, - - { - FLB_CONFIG_MAP_STR, "mode", "parser", - 0, FLB_TRUE, offsetof(struct ml_ctx, mode), - "Mode can be 'parser' for regex concat, or 'partial_message' to " - "concat split docker logs." - }, - - { - FLB_CONFIG_MAP_INT, "flush_ms", "2000", - 0, FLB_TRUE, offsetof(struct ml_ctx, flush_ms), - "Flush time for pending multiline records" - }, - - /* Multiline Core Engine based API */ - { - FLB_CONFIG_MAP_CLIST, "multiline.parser", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct ml_ctx, multiline_parsers), - "specify one or multiple multiline parsers: docker, cri, go, java, etc." - }, - - { - FLB_CONFIG_MAP_STR, "multiline.key_content", NULL, - 0, FLB_TRUE, offsetof(struct ml_ctx, key_content), - "specify the key name that holds the content to process." - }, - - /* emitter config */ - { - FLB_CONFIG_MAP_STR, "emitter_name", NULL, - FLB_FALSE, FLB_TRUE, offsetof(struct ml_ctx, emitter_name), - NULL - }, - { - FLB_CONFIG_MAP_STR, "emitter_storage.type", "memory", - FLB_FALSE, FLB_TRUE, offsetof(struct ml_ctx, emitter_storage_type), - NULL - }, - { - FLB_CONFIG_MAP_SIZE, "emitter_mem_buf_limit", FLB_MULTILINE_MEM_BUF_LIMIT_DEFAULT, - FLB_FALSE, FLB_TRUE, offsetof(struct ml_ctx, emitter_mem_buf_limit), - "set a memory buffer limit to restrict memory usage of emitter" - }, - - /* EOF */ - {0} -}; - -struct flb_filter_plugin filter_multiline_plugin = { - .name = "multiline", - .description = "Concatenate multiline messages", - .cb_init = cb_ml_init, - .cb_filter = cb_ml_filter, - .cb_exit = cb_ml_exit, - .config_map = config_map, - .flags = 0 -}; diff --git a/fluent-bit/plugins/filter_multiline/ml.h b/fluent-bit/plugins/filter_multiline/ml.h deleted file mode 100644 index 973346007..000000000 --- a/fluent-bit/plugins/filter_multiline/ml.h +++ /dev/null @@ -1,87 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_FILTER_MULTILINE_H -#define FLB_FILTER_MULTILINE_H - -#include -#include "ml_concat.h" - -#define FLB_MULTILINE_MEM_BUF_LIMIT_DEFAULT "10M" -#define FLB_MULTILINE_METRIC_EMITTED 200 -#define FLB_MULTILINE_MODE_PARTIAL_MESSAGE "partial_message" -#define FLB_MULTILINE_MODE_PARSER "parser" - -/* - * input instance + tag is the unique identifier - * for a multiline stream - * TODO: implement clean up of streams that haven't been used recently - */ -struct ml_stream { - flb_sds_t tag; - flb_sds_t input_name; - uint64_t stream_id; - - struct mk_list _head; -}; - -struct ml_ctx { - int debug_flush; - int use_buffer; - flb_sds_t key_content; - flb_sds_t mode; - - /* packaging buffers */ - msgpack_sbuffer mp_sbuf; /* temporary msgpack buffer */ - msgpack_packer mp_pck; /* temporary msgpack packer */ - - /* Multiline core engine */ - uint64_t stream_id; - struct flb_ml *m; - struct mk_list *multiline_parsers; - int flush_ms; - - int timer_created; - - int partial_mode; - - struct mk_list ml_streams; - - struct mk_list split_message_packers; - - struct flb_filter_instance *ins; - - /* emitter */ - flb_sds_t emitter_name; /* emitter input plugin name */ - flb_sds_t emitter_storage_type; /* emitter storage type */ - size_t emitter_mem_buf_limit; /* Emitter buffer limit */ - struct flb_input_instance *ins_emitter; /* emitter input plugin instance */ - struct flb_config *config; /* Fluent Bit context */ - -#ifdef FLB_HAVE_METRICS - struct cmt_counter *cmt_emitted; -#endif -}; - -/* Register external function to emit records, check 'plugins/in_emitter' */ -int in_emitter_add_record(const char *tag, int tag_len, - const char *buf_data, size_t buf_size, - struct flb_input_instance *in); - -#endif diff --git a/fluent-bit/plugins/filter_multiline/ml_concat.c b/fluent-bit/plugins/filter_multiline/ml_concat.c deleted file mode 100644 index 27121f48d..000000000 --- a/fluent-bit/plugins/filter_multiline/ml_concat.c +++ /dev/null @@ -1,473 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2021 The Fluent Bit Authors - * Copyright (C) 2015-2018 Treasure Data Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "ml_concat.h" - -msgpack_object_kv *ml_get_key(msgpack_object *map, char *check_for_key) -{ - int i; - char *key_str = NULL; - size_t key_str_size = 0; - msgpack_object_kv *kv; - msgpack_object key; - int check_key = FLB_FALSE; - - kv = map->via.map.ptr; - - for(i=0; i < map->via.map.size; i++) { - check_key = FLB_FALSE; - - key = (kv+i)->key; - if (key.type == MSGPACK_OBJECT_BIN) { - key_str = (char *) key.via.bin.ptr; - key_str_size = key.via.bin.size; - check_key = FLB_TRUE; - } - if (key.type == MSGPACK_OBJECT_STR) { - key_str = (char *) key.via.str.ptr; - key_str_size = key.via.str.size; - check_key = FLB_TRUE; - } - - if (check_key == FLB_TRUE) { - if (strncmp(check_for_key, key_str, key_str_size) == 0) { - return (kv+i); - } - } - } - return NULL; -} - -int ml_is_partial(msgpack_object *map) -{ - char *val_str = NULL; - msgpack_object_kv *kv; - msgpack_object val; - - kv = ml_get_key(map, FLB_MULTILINE_PARTIAL_MESSAGE_KEY); - - if (kv == NULL) { - return FLB_FALSE; - } - - val = kv->val; - if (val.type == MSGPACK_OBJECT_BIN) { - val_str = (char *) val.via.bin.ptr; - } - if (val.type == MSGPACK_OBJECT_STR) { - val_str = (char *) val.via.str.ptr; - } - - if (strncasecmp("true", val_str, 4) == 0) { - return FLB_TRUE; - } - return FLB_FALSE; -} - -int ml_is_partial_last(msgpack_object *map) -{ - char *val_str = NULL; - msgpack_object_kv *kv; - msgpack_object val; - - kv = ml_get_key(map, FLB_MULTILINE_PARTIAL_LAST_KEY); - - if (kv == NULL) { - return FLB_FALSE; - } - - val = kv->val; - if (val.type == MSGPACK_OBJECT_BIN) { - val_str = (char *) val.via.bin.ptr; - } - if (val.type == MSGPACK_OBJECT_STR) { - val_str = (char *) val.via.str.ptr; - } - - if (strncasecmp("true", val_str, 4) == 0) { - return FLB_TRUE; - } - return FLB_FALSE; -} - -int ml_get_partial_id(msgpack_object *map, - char **partial_id_str, - size_t *partial_id_size) -{ - char *val_str = NULL; - size_t val_str_size = 0; - msgpack_object_kv *kv; - msgpack_object val; - - kv = ml_get_key(map, FLB_MULTILINE_PARTIAL_ID_KEY); - - if (kv == NULL) { - return -1; - } - - val = kv->val; - if (val.type == MSGPACK_OBJECT_BIN) { - val_str = (char *) val.via.bin.ptr; - val_str_size = val.via.bin.size; - } - if (val.type == MSGPACK_OBJECT_STR) { - val_str = (char *) val.via.str.ptr; - val_str_size = val.via.str.size; - } - - *partial_id_str = val_str; - *partial_id_size = val_str_size; - - return 0; -} - -struct split_message_packer *ml_get_packer(struct mk_list *packers, const char *tag, - char *input_name, - char *partial_id_str, size_t partial_id_size) -{ - struct mk_list *tmp; - struct mk_list *head; - struct split_message_packer *packer; - int name_check; - int tag_check; - int id_check; - - - mk_list_foreach_safe(head, tmp, packers) { - packer = mk_list_entry(head, struct split_message_packer, _head); - id_check = strncmp(packer->partial_id, partial_id_str, partial_id_size); - if (id_check != 0) { - continue; - } - name_check = strcmp(packer->input_name, input_name); - if (name_check != 0) { - continue; - } - tag_check = strcmp(packer->tag, tag); - if (tag_check == 0) { - return packer; - } - } - - return NULL; -} - -struct split_message_packer *ml_create_packer(const char *tag, char *input_name, - char *partial_id_str, size_t partial_id_size, - msgpack_object *map, char *multiline_key_content, - struct flb_time *tm) -{ - struct split_message_packer *packer; - msgpack_object_kv *kv; - msgpack_object_kv *split_kv; - flb_sds_t tmp; - int i; - char *key_str = NULL; - size_t key_str_size = 0; - msgpack_object key; - int check_key = FLB_FALSE; - size_t len; - int ret; - - packer = flb_calloc(1, sizeof(struct split_message_packer)); - if (!packer) { - flb_errno(); - return NULL; - } - - tmp = flb_sds_create(input_name); - if (!tmp) { - flb_errno(); - flb_free(packer); - return NULL; - } - packer->input_name = tmp; - - tmp = flb_sds_create(tag); - if (!tmp) { - flb_errno(); - ml_split_message_packer_destroy(packer); - return NULL; - } - packer->tag = tmp; - - tmp = flb_sds_create_len(partial_id_str, partial_id_size); - if (!tmp) { - flb_errno(); - ml_split_message_packer_destroy(packer); - return NULL; - } - packer->partial_id = tmp; - - packer->buf = flb_sds_create_size(FLB_MULTILINE_PARTIAL_BUF_SIZE); - if (!packer->buf) { - flb_errno(); - ml_split_message_packer_destroy(packer); - return NULL; - } - - ret = flb_log_event_encoder_init(&packer->log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_error("[partial message concat] Log event encoder initialization error : %d", ret); - - ml_split_message_packer_destroy(packer); - - return NULL; - } - - /* get the key that is split */ - split_kv = ml_get_key(map, multiline_key_content); - if (split_kv == NULL) { - flb_error("[partial message concat] Could not find key %s in record", multiline_key_content); - ml_split_message_packer_destroy(packer); - return NULL; - } - - ret = flb_log_event_encoder_begin_record(&packer->log_encoder); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_error("[partial message concat] Log event encoder error : %d", ret); - - ml_split_message_packer_destroy(packer); - - return NULL; - } - - /* write all of the keys except the split one and the partial metadata */ - ret = flb_log_event_encoder_set_timestamp( - &packer->log_encoder, tm); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_error("[partial message concat] Log event encoder error : %d", ret); - - ml_split_message_packer_destroy(packer); - - return NULL; - } - - kv = map->via.map.ptr; - - for(i=0; - i < map->via.map.size && - ret == FLB_EVENT_ENCODER_SUCCESS; - i++) { - if ((kv+i) == split_kv) { - continue; - } - - key = (kv+i)->key; - if (key.type == MSGPACK_OBJECT_BIN) { - key_str = (char *) key.via.bin.ptr; - key_str_size = key.via.bin.size; - check_key = FLB_TRUE; - } - if (key.type == MSGPACK_OBJECT_STR) { - key_str = (char *) key.via.str.ptr; - key_str_size = key.via.str.size; - check_key = FLB_TRUE; - } - - len = FLB_MULTILINE_PARTIAL_PREFIX_LEN; - if (key_str_size < len) { - len = key_str_size; - } - - if (check_key == FLB_TRUE) { - if (strncmp(FLB_MULTILINE_PARTIAL_PREFIX, key_str, len) == 0) { - /* don't pack the partial keys */ - continue; - } - } - - ret = flb_log_event_encoder_append_body_values( - &packer->log_encoder, - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&kv[i].key), - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&kv[i].val)); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - /* write split kv last, so we can append to it later as needed */ - ret = flb_log_event_encoder_append_body_msgpack_object( - &packer->log_encoder, - &split_kv->key); - } - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_error("[partial message concat] Log event encoder error : %d", ret); - - ml_split_message_packer_destroy(packer); - - return NULL; - } - - return packer; -} - -unsigned long long ml_current_timestamp() { - struct flb_time te; - flb_time_get(&te); - return flb_time_to_nanosec(&te) / 1000000LL; -} - -int ml_split_message_packer_write(struct split_message_packer *packer, - msgpack_object *map, char *multiline_key_content) -{ - char *val_str = NULL; - size_t val_str_size = 0; - msgpack_object_kv *kv; - msgpack_object val; - - kv = ml_get_key(map, multiline_key_content); - - if (kv == NULL) { - flb_error("[partial message concat] Could not find key %s in record", multiline_key_content); - return -1; - } - - val = kv->val; - if (val.type == MSGPACK_OBJECT_BIN) { - val_str = (char *) val.via.bin.ptr; - val_str_size = val.via.bin.size; - } else if (val.type == MSGPACK_OBJECT_STR) { - val_str = (char *) val.via.str.ptr; - val_str_size = val.via.str.size; - } else { - return -1; - } - - flb_sds_cat_safe(&packer->buf, val_str, val_str_size); - packer->last_write_time = ml_current_timestamp(); - - return 0; -} - -void ml_split_message_packer_complete(struct split_message_packer *packer) -{ - flb_log_event_encoder_append_body_string(&packer->log_encoder, - packer->buf, - flb_sds_len(packer->buf)); - - flb_log_event_encoder_commit_record(&packer->log_encoder); -} - -void ml_append_complete_record(struct split_message_packer *packer, - struct flb_log_event_encoder *log_encoder) -{ - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - int ret; - - ret = flb_log_event_decoder_init( - &log_decoder, - packer->log_encoder.output_buffer, - packer->log_encoder.output_length); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_error("[partial message concat] Log event decoder error : %d", - ret); - - return; - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - ret = flb_log_event_encoder_begin_record(log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_timestamp( - log_encoder, - &log_event.timestamp); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_metadata_from_msgpack_object( - log_encoder, - log_event.metadata); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_body_from_msgpack_object( - log_encoder, - log_event.body); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(log_encoder); - } - else { - flb_log_event_encoder_rollback_record(log_encoder); - - break; - } - } - - if (ret == FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA && - log_decoder.offset == packer->log_encoder.output_length) { - ret = FLB_EVENT_ENCODER_SUCCESS; - } - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_error("[partial message concat] Log event encoder error : %d", - ret); - - return; - } - - flb_log_event_decoder_destroy(&log_decoder); -} - -void ml_split_message_packer_destroy(struct split_message_packer *packer) -{ - if (!packer) { - return; - } - - if (packer->tag) { - flb_sds_destroy(packer->tag); - } - if (packer->buf) { - flb_sds_destroy(packer->buf); - } - if (packer->input_name) { - flb_sds_destroy(packer->input_name); - } - if (packer->partial_id) { - flb_sds_destroy(packer->partial_id); - } - - flb_log_event_encoder_destroy(&packer->log_encoder); - - flb_free(packer); -} - diff --git a/fluent-bit/plugins/filter_multiline/ml_concat.h b/fluent-bit/plugins/filter_multiline/ml_concat.h deleted file mode 100644 index a2d3a79e1..000000000 --- a/fluent-bit/plugins/filter_multiline/ml_concat.h +++ /dev/null @@ -1,84 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2021 The Fluent Bit Authors - * Copyright (C) 2015-2018 Treasure Data Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_FILTER_MULTILINE_CONCAT_H -#define FLB_FILTER_MULTILINE_CONCAT_H - -#include -#include -#include - -#define FLB_MULTILINE_MEM_BUF_LIMIT_DEFAULT "10M" -#define FLB_MULTILINE_METRIC_EMITTED 200 -/* docker logs are split at 16KB */ -#define FLB_MULTILINE_PARTIAL_BUF_SIZE 24000 - -/* - * Long term these keys could be made user configurable - * But everyone who's asking for this right now wants it for split - * Docker logs, which has a set series of keys - */ -#define FLB_MULTILINE_PARTIAL_PREFIX "partial_" -#define FLB_MULTILINE_PARTIAL_PREFIX_LEN 8 -#define FLB_MULTILINE_PARTIAL_MESSAGE_KEY "partial_message" -#define FLB_MULTILINE_PARTIAL_ID_KEY "partial_id" -#define FLB_MULTILINE_PARTIAL_LAST_KEY "partial_last" - -struct split_message_packer { - flb_sds_t tag; - flb_sds_t input_name; - flb_sds_t partial_id; - - /* packaging buffers */ - // msgpack_sbuffer mp_sbuf; /* temporary msgpack buffer */ - // msgpack_packer mp_pck; /* temporary msgpack packer */ - struct flb_log_event_encoder log_encoder; - - flb_sds_t buf; - - /* used to flush buffers that have been pending for more than flush_ms */ - unsigned long long last_write_time; - - struct mk_list _head; -}; - -msgpack_object_kv *ml_get_key(msgpack_object *map, char *check_for_key); -int ml_is_partial(msgpack_object *map); -int ml_is_partial_last(msgpack_object *map); -int ml_get_partial_id(msgpack_object *map, - char **partial_id_str, - size_t *partial_id_size); -struct split_message_packer *ml_get_packer(struct mk_list *packers, const char *tag, - char *input_name, - char *partial_id_str, size_t partial_id_size); -struct split_message_packer *ml_create_packer(const char *tag, char *input_name, - char *partial_id_str, size_t partial_id_size, - msgpack_object *map, char *multiline_key_content, - struct flb_time *tm); -int ml_split_message_packer_write(struct split_message_packer *packer, - msgpack_object *map, char *multiline_key_content); -void ml_split_message_packer_complete(struct split_message_packer *packer); -void ml_split_message_packer_destroy(struct split_message_packer *packer); -void ml_append_complete_record(struct split_message_packer *packer, - struct flb_log_event_encoder *log_encoder); -unsigned long long ml_current_timestamp(); - - -#endif diff --git a/fluent-bit/plugins/filter_nest/CMakeLists.txt b/fluent-bit/plugins/filter_nest/CMakeLists.txt deleted file mode 100644 index a78f591b2..000000000 --- a/fluent-bit/plugins/filter_nest/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - nest.c) - -FLB_PLUGIN(filter_nest "${src}" "") diff --git a/fluent-bit/plugins/filter_nest/nest.c b/fluent-bit/plugins/filter_nest/nest.c deleted file mode 100644 index 96bcc2974..000000000 --- a/fluent-bit/plugins/filter_nest/nest.c +++ /dev/null @@ -1,761 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "nest.h" - -#include -#include - - -static void teardown(struct filter_nest_ctx *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - - struct filter_nest_wildcard *wildcard; - - flb_free(ctx->prefix); - flb_free(ctx->key); - - mk_list_foreach_safe(head, tmp, &ctx->wildcards) { - wildcard = mk_list_entry(head, struct filter_nest_wildcard, _head); - flb_free(wildcard->key); - mk_list_del(&wildcard->_head); - flb_free(wildcard); - } - -} - -static int configure(struct filter_nest_ctx *ctx, - struct flb_filter_instance *f_ins, - struct flb_config *config) -{ - - struct mk_list *head; - struct flb_kv *kv; - struct filter_nest_wildcard *wildcard; - - char *operation_nest = "nest"; - char *operation_lift = "lift"; - - ctx->key = NULL; - ctx->key_len = 0; - ctx->prefix = NULL; - ctx->prefix_len = 0; - ctx->remove_prefix = false; - ctx->add_prefix = false; - - if (flb_filter_config_map_set(f_ins, ctx) < 0) { - flb_plg_error(f_ins, "unable to load configuration"); - return -1; - } - - mk_list_foreach(head, &f_ins->properties) { - kv = mk_list_entry(head, struct flb_kv, _head); - - if (strcasecmp(kv->key, "operation") == 0) { - if (strncmp(kv->val, operation_nest, 4) == 0) { - ctx->operation = NEST; - } - else if (strncmp(kv->val, operation_lift, 4) == 0) { - ctx->operation = LIFT; - } - else { - flb_plg_error(ctx->ins, "Key \"operation\" has invalid value " - "'%s'. Expected 'nest' or 'lift'\n", - kv->val); - return -1; - } - } - else if (strcasecmp(kv->key, "wildcard") == 0) { - wildcard = flb_malloc(sizeof(struct filter_nest_wildcard)); - if (!wildcard) { - flb_plg_error(ctx->ins, "Unable to allocate memory for " - "wildcard"); - flb_free(wildcard); - return -1; - } - - wildcard->key = flb_strndup(kv->val, flb_sds_len(kv->val)); - if (wildcard->key == NULL) { - flb_errno(); - flb_free(wildcard); - return -1; - } - wildcard->key_len = flb_sds_len(kv->val); - - if (wildcard->key[wildcard->key_len - 1] == '*') { - wildcard->key_is_dynamic = true; - wildcard->key_len--; - } - else { - wildcard->key_is_dynamic = false; - } - - mk_list_add(&wildcard->_head, &ctx->wildcards); - ctx->wildcards_cnt++; - - } - else if (strcasecmp(kv->key, "nest_under") == 0) { - ctx->key = flb_strdup(kv->val); - ctx->key_len = flb_sds_len(kv->val); - } - else if (strcasecmp(kv->key, "nested_under") == 0) { - ctx->key = flb_strdup(kv->val); - ctx->key_len = flb_sds_len(kv->val); - } - else if (strcasecmp(kv->key, "prefix_with") == 0) { - ctx->prefix = flb_strdup(kv->val); - ctx->prefix_len = flb_sds_len(kv->val); - ctx->add_prefix = true; - } - else if (strcasecmp(kv->key, "add_prefix") == 0) { - ctx->prefix = flb_strdup(kv->val); - ctx->prefix_len = flb_sds_len(kv->val); - ctx->add_prefix = true; - } - else if (strcasecmp(kv->key, "remove_prefix") == 0) { - ctx->prefix = flb_strdup(kv->val); - ctx->prefix_len = flb_sds_len(kv->val); - ctx->remove_prefix = true; - } else { - flb_plg_error(ctx->ins, "Invalid configuration key '%s'", kv->key); - return -1; - } - } - - /* Sanity checks */ - if (ctx->remove_prefix && ctx->add_prefix) { - flb_plg_error(ctx->ins, "Add_prefix and Remove_prefix are exclusive"); - return -1; - } - - if ((ctx->operation != NEST) && - (ctx->operation != LIFT)) { - flb_plg_error(ctx->ins, "Operation can only be NEST or LIFT"); - return -1; - } - - if ((ctx->remove_prefix || ctx->add_prefix) && ctx->prefix == 0) { - flb_plg_error(ctx->ins, "A prefix has to be specified for prefix add " - "or remove operations"); - return -1; - } - - return 0; - -} - -static void helper_pack_string_remove_prefix( - struct flb_log_event_encoder *log_encoder, - struct filter_nest_ctx *ctx, - const char *str, - int len) -{ - if (strncmp(str, ctx->prefix, ctx->prefix_len) == 0) { - flb_log_event_encoder_append_body_string( - log_encoder, - (char *) &str[ctx->prefix_len], - len - ctx->prefix_len); - } - else { - /* Key does not contain specified prefix */ - flb_log_event_encoder_append_body_string( - log_encoder, (char *) str, len); - } -} - -static void helper_pack_string_add_prefix(struct flb_log_event_encoder *log_encoder, - struct filter_nest_ctx *ctx, - const char *str, - int len) -{ - flb_log_event_encoder_append_body_values( - log_encoder, - FLB_LOG_EVENT_STRING_LENGTH_VALUE(ctx->prefix_len + len), - FLB_LOG_EVENT_STRING_BODY_VALUE(ctx->prefix, ctx->prefix_len), - FLB_LOG_EVENT_STRING_BODY_VALUE(str, len)); -} - -static inline void map_pack_each_fn(struct flb_log_event_encoder *log_encoder, - msgpack_object * map, - struct filter_nest_ctx *ctx, - bool(*f) (msgpack_object_kv * kv, - struct filter_nest_ctx * ctx)) -{ - int i; - int ret; - - ret = FLB_EVENT_ENCODER_SUCCESS; - for (i = 0; - i < map->via.map.size && - ret == FLB_EVENT_ENCODER_SUCCESS; - i++) { - if ((*f) (&map->via.map.ptr[i], ctx)) { - ret = flb_log_event_encoder_append_body_values( - log_encoder, - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE( - &map->via.map.ptr[i].key), - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE( - &map->via.map.ptr[i].val)); - } - } -} - -static inline void map_transform_and_pack_each_fn(struct flb_log_event_encoder *log_encoder, - msgpack_object * map, - struct filter_nest_ctx *ctx, - bool(*f) (msgpack_object_kv * kv, - struct filter_nest_ctx * ctx) - ) -{ - int i; - int ret; - msgpack_object *key; - - ret = FLB_EVENT_ENCODER_SUCCESS; - for (i = 0; - i < map->via.map.size && - ret == FLB_EVENT_ENCODER_SUCCESS ; - i++) { - if ((*f) (&map->via.map.ptr[i], ctx)) { - key = &map->via.map.ptr[i].key; - - if (ctx->add_prefix) { - helper_pack_string_add_prefix(log_encoder, ctx, key->via.str.ptr, key->via.str.size); - } - else if (ctx->remove_prefix) { - helper_pack_string_remove_prefix(log_encoder, ctx, key->via.str.ptr, key->via.str.size); - } - else { - ret = flb_log_event_encoder_append_body_msgpack_object( - log_encoder, key); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_msgpack_object( - log_encoder, &map->via.map.ptr[i].val); - } - } - } -} - -static inline int map_count_fn(msgpack_object * map, - struct filter_nest_ctx *ctx, - bool(*f) (msgpack_object_kv * kv, - struct filter_nest_ctx * ctx) - ) -{ - int i; - int count = 0; - - for (i = 0; i < map->via.map.size; i++) { - if ((*f) (&map->via.map.ptr[i], ctx)) { - count++; - } - } - return count; -} - -static inline bool is_kv_to_nest(msgpack_object_kv * kv, - struct filter_nest_ctx *ctx) -{ - - const char *key; - int klen; - - msgpack_object *obj = &kv->key; - - struct mk_list *tmp; - struct mk_list *head; - struct filter_nest_wildcard *wildcard; - - if (obj->type == MSGPACK_OBJECT_BIN) { - key = obj->via.bin.ptr; - klen = obj->via.bin.size; - } - else if (obj->type == MSGPACK_OBJECT_STR) { - key = obj->via.str.ptr; - klen = obj->via.str.size; - } - else { - /* If the key is not something we can match on, leave it alone */ - return false; - } - - mk_list_foreach_safe(head, tmp, &ctx->wildcards) { - wildcard = mk_list_entry(head, struct filter_nest_wildcard, _head); - - if (wildcard->key_is_dynamic) { - /* This will positively match "ABC123" with prefix "ABC*" */ - if (strncmp(key, wildcard->key, wildcard->key_len) == 0) { - return true; - } - } - else { - /* This will positively match "ABC" with prefix "ABC" */ - if ((wildcard->key_len == klen) && - (strncmp(key, wildcard->key, klen) == 0) - ) { - return true; - } - } - } - - return false; - -} - -static inline bool is_not_kv_to_nest(msgpack_object_kv * kv, - struct filter_nest_ctx *ctx) -{ - return !is_kv_to_nest(kv, ctx); -} - -static inline bool is_kv_to_lift(msgpack_object_kv * kv, - struct filter_nest_ctx *ctx) -{ - - const char *key; - char *tmp; - int klen; - bool match; - - msgpack_object *obj = &kv->key; - - if (obj->type == MSGPACK_OBJECT_BIN) { - key = obj->via.bin.ptr; - klen = obj->via.bin.size; - } - else if (obj->type == MSGPACK_OBJECT_STR) { - key = obj->via.str.ptr; - klen = obj->via.str.size; - } - else { - /* If the key is not something we can match on, leave it alone */ - return false; - } - - match = ((ctx->key_len == klen) && - (strncmp(key, ctx->key, klen) == 0)); - - if (match && (kv->val.type != MSGPACK_OBJECT_MAP)) { - tmp = flb_malloc(klen + 1); - if (!tmp) { - flb_errno(); - return false; - } - memcpy(tmp, key, klen); - tmp[klen] = '\0'; - flb_plg_warn(ctx->ins, "Value of key '%s' is not a map. " - "Will not attempt to lift from here", - tmp); - flb_free(tmp); - return false; - } - else { - return match; - } -} - -static inline bool is_not_kv_to_lift(msgpack_object_kv * kv, - struct filter_nest_ctx *ctx) -{ - return !is_kv_to_lift(kv, ctx); -} - -static inline int count_items_to_lift(msgpack_object * map, - struct filter_nest_ctx *ctx) -{ - int i; - int count = 0; - msgpack_object_kv *kv; - - for (i = 0; i < map->via.map.size; i++) { - kv = &map->via.map.ptr[i]; - if (is_kv_to_lift(kv, ctx)) { - count = count + kv->val.via.map.size; - } - } - return count; -} - -static inline void pack_map( - struct flb_log_event_encoder *log_encoder, - msgpack_object * map, - struct filter_nest_ctx *ctx) -{ - int i; - int ret; - msgpack_object *key; - - ret = FLB_EVENT_ENCODER_SUCCESS; - - for (i = 0; - i < map->via.map.size && - ret == FLB_EVENT_ENCODER_SUCCESS ; - i++) { - key = &map->via.map.ptr[i].key; - - if (ctx->add_prefix) { - helper_pack_string_add_prefix(log_encoder, ctx, key->via.str.ptr, key->via.str.size); - } - else if (ctx->remove_prefix) { - helper_pack_string_remove_prefix(log_encoder, ctx, key->via.str.ptr, key->via.str.size); - } - else { - ret = flb_log_event_encoder_append_body_msgpack_object(log_encoder, key); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_msgpack_object(log_encoder, - &map->via.map.ptr[i].val); - } - } -} - -static inline void map_lift_each_fn(struct flb_log_event_encoder *log_encoder, - msgpack_object * map, - struct filter_nest_ctx *ctx, - bool(*f) (msgpack_object_kv * kv, - struct filter_nest_ctx * ctx) - ) -{ - int i; - msgpack_object_kv *kv; - - for (i = 0; i < map->via.map.size; i++) { - kv = &map->via.map.ptr[i]; - if ((*f) (kv, ctx)) { - pack_map(log_encoder, &kv->val, ctx); - } - } -} - -static inline int apply_lifting_rules(struct flb_log_event_encoder *log_encoder, - struct flb_log_event *log_event, - struct filter_nest_ctx *ctx) -{ - int ret; - msgpack_object map = *log_event->body; - - int items_to_lift = map_count_fn(&map, ctx, &is_kv_to_lift); - - if (items_to_lift == 0) { - flb_plg_debug(ctx->ins, "Lift : No match found for %s", ctx->key); - return 0; - } - - /* - * New items at top level = - * current size - * - number of maps to lift - * + number of element inside maps to lift - */ - int toplevel_items = - (map.via.map.size - items_to_lift) + count_items_to_lift(&map, ctx); - - flb_plg_debug(ctx->ins, "Lift : Outer map size is %d, will be %d, " - "lifting %d record(s)", - map.via.map.size, toplevel_items, items_to_lift); - - ret = flb_log_event_encoder_begin_record(log_encoder); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - return -1; - } - - ret = flb_log_event_encoder_set_timestamp( - log_encoder, &log_event->timestamp); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - return -2; - } - - ret = flb_log_event_encoder_set_metadata_from_msgpack_object( - log_encoder, log_event->metadata); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - return -3; - } - - /* Pack all current top-level items excluding the key keys */ - map_pack_each_fn(log_encoder, &map, ctx, &is_not_kv_to_lift); - - /* Lift and pack all elements in key keys */ - map_lift_each_fn(log_encoder, &map, ctx, &is_kv_to_lift); - - ret = flb_log_event_encoder_commit_record(log_encoder); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - return -2; - } - - return 1; -} - -static inline int apply_nesting_rules(struct flb_log_event_encoder *log_encoder, - struct flb_log_event *log_event, - struct filter_nest_ctx *ctx) -{ - int ret; - msgpack_object map = *log_event->body; - - size_t items_to_nest = map_count_fn(&map, ctx, &is_kv_to_nest); - - if (items_to_nest == 0) { - flb_plg_debug(ctx->ins, "no match found for %s", ctx->prefix); - return 0; - } - - size_t toplevel_items = (map.via.map.size - items_to_nest + 1); - - flb_plg_trace(ctx->ins, "outer map size is %d, will be %lu, nested " - "map size will be %lu", - map.via.map.size, toplevel_items, items_to_nest); - - ret = flb_log_event_encoder_begin_record(log_encoder); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - return -1; - } - - ret = flb_log_event_encoder_set_timestamp( - log_encoder, &log_event->timestamp); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - return -2; - } - - ret = flb_log_event_encoder_set_metadata_from_msgpack_object( - log_encoder, log_event->metadata); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - return -3; - } - - /* - * Record array item 2/2 - * Create a new map with toplevel items +1 for nested map - */ - map_pack_each_fn(log_encoder, &map, ctx, &is_not_kv_to_nest); - - /* Pack the nested map key */ - ret = flb_log_event_encoder_append_body_string( - log_encoder, ctx->key, ctx->key_len); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - return -4; - } - - /* Create the nest map value */ - ret = flb_log_event_encoder_body_begin_map(log_encoder); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - return -5; - } - - /* Pack the nested items */ - map_transform_and_pack_each_fn(log_encoder, &map, ctx, &is_kv_to_nest); - - ret = flb_log_event_encoder_commit_record(log_encoder); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - return -6; - } - - return 1; -} - -static int cb_nest_init(struct flb_filter_instance *f_ins, - struct flb_config *config, void *data) -{ - struct filter_nest_ctx *ctx; - - ctx = flb_malloc(sizeof(struct filter_nest_ctx)); - if (!ctx) { - flb_errno(); - return -1; - } - mk_list_init(&ctx->wildcards); - ctx->ins = f_ins; - ctx->wildcards_cnt = 0; - - if (configure(ctx, f_ins, config) < 0) { - flb_free(ctx); - return -1; - } - - flb_filter_set_context(f_ins, ctx); - return 0; -} - -static int cb_nest_filter(const void *data, size_t bytes, - const char *tag, int tag_len, - void **out_buf, size_t * out_size, - struct flb_filter_instance *f_ins, - struct flb_input_instance *i_ins, - void *context, struct flb_config *config) -{ - struct flb_log_event_encoder log_encoder; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - struct filter_nest_ctx *ctx = context; - int modified_records = 0; - int ret; - - (void) f_ins; - (void) i_ins; - (void) config; - - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return FLB_FILTER_NOTOUCH; - } - - ret = flb_log_event_encoder_init(&log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event encoder initialization error : %d", ret); - - flb_log_event_decoder_destroy(&log_decoder); - - return FLB_FILTER_NOTOUCH; - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - modified_records = 0; - - if (ctx->operation == NEST) { - modified_records = - apply_nesting_rules(&log_encoder, &log_event, ctx); - } - else { - modified_records = - apply_lifting_rules(&log_encoder, &log_event, ctx); - } - - if (modified_records == 0) { - ret = flb_log_event_encoder_emit_raw_record( - &log_encoder, - log_decoder.record_base, - log_decoder.record_length); - } - } - - if (ret == FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA && - log_decoder.offset == bytes) { - ret = FLB_EVENT_ENCODER_SUCCESS; - } - - if (log_encoder.output_length > 0) { - *out_buf = log_encoder.output_buffer; - *out_size = log_encoder.output_length; - - ret = FLB_FILTER_MODIFIED; - - flb_log_event_encoder_claim_internal_buffer_ownership(&log_encoder); - } - else { - flb_plg_error(ctx->ins, - "Log event encoder error : %d", ret); - - ret = FLB_FILTER_NOTOUCH; - } - - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - return ret; -} - -static int cb_nest_exit(void *data, struct flb_config *config) -{ - struct filter_nest_ctx *ctx = data; - - teardown(ctx); - flb_free(ctx); - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "Operation", NULL, - 0, FLB_FALSE, 0, - "Select the operation nest or lift" - }, - { - FLB_CONFIG_MAP_STR, "Wildcard", NULL, - FLB_CONFIG_MAP_MULT, FLB_FALSE, 0, - "Nest records which field matches the wildcard" - }, - { - FLB_CONFIG_MAP_STR, "Nest_under", NULL, - 0, FLB_FALSE, 0, - "Nest records matching the Wildcard under this key" - }, - { - FLB_CONFIG_MAP_STR, "Nested_under", NULL, - 0, FLB_FALSE, 0, - "Lift records nested under the Nested_under key" - }, - { - FLB_CONFIG_MAP_STR, "Add_prefix", NULL, - 0, FLB_FALSE, 0, - "Prefix affected keys with this string" - }, - { - FLB_CONFIG_MAP_STR, "Remove_prefix", NULL, - 0, FLB_FALSE, 0, - "Remove prefix from affected keys if it matches this string" - }, - {0} -}; - -struct flb_filter_plugin filter_nest_plugin = { - .name = "nest", - .description = "nest events by specified field values", - .cb_init = cb_nest_init, - .cb_filter = cb_nest_filter, - .cb_exit = cb_nest_exit, - .config_map = config_map, - .flags = 0 -}; diff --git a/fluent-bit/plugins/filter_nest/nest.h b/fluent-bit/plugins/filter_nest/nest.h deleted file mode 100644 index 47c35ee1e..000000000 --- a/fluent-bit/plugins/filter_nest/nest.h +++ /dev/null @@ -1,55 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_FILTER_NEST_H -#define FLB_FILTER_NEST_H - -#include -#include - -enum FILTER_NEST_OPERATION { - NEST, - LIFT -}; - -struct filter_nest_ctx -{ - enum FILTER_NEST_OPERATION operation; - char *key; - int key_len; - char *prefix; - int prefix_len; - // nest - struct mk_list wildcards; - int wildcards_cnt; - bool remove_prefix; - // lift - bool add_prefix; - struct flb_filter_instance *ins; -}; - -struct filter_nest_wildcard -{ - char *key; - int key_len; - bool key_is_dynamic; - struct mk_list _head; -}; - -#endif diff --git a/fluent-bit/plugins/filter_nightfall/CMakeLists.txt b/fluent-bit/plugins/filter_nightfall/CMakeLists.txt deleted file mode 100644 index d3535d15a..000000000 --- a/fluent-bit/plugins/filter_nightfall/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(src - nightfall.c - nightfall_api.c - ) - -FLB_PLUGIN(filter_nightfall "${src}" "") diff --git a/fluent-bit/plugins/filter_nightfall/nightfall.c b/fluent-bit/plugins/filter_nightfall/nightfall.c deleted file mode 100644 index 3e37d2a0b..000000000 --- a/fluent-bit/plugins/filter_nightfall/nightfall.c +++ /dev/null @@ -1,654 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2021 The Fluent Bit Authors - * Copyright (C) 2015-2018 Treasure Data Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "nightfall.h" -#include "nightfall_api.h" - -static int redact_array_fields(msgpack_packer *new_rec_pk, int *to_redact_index, - msgpack_object_array *to_redact, struct nested_obj *cur, - struct mk_list *stack, char *should_pop); -static int redact_map_fields(msgpack_packer *new_rec_pk, int *to_redact_index, - msgpack_object_array *to_redact, struct nested_obj *cur, - struct mk_list *stack, char *should_pop); -static void maybe_redact_field(msgpack_packer *new_rec_pk, msgpack_object *field, - msgpack_object_array *to_redact, int *to_redact_i, - int byte_offset); - -static int cb_nightfall_init(struct flb_filter_instance *f_ins, - struct flb_config *config, - void *data) -{ - struct flb_filter_nightfall *ctx = NULL; - int ret; - - /* Create context */ - ctx = flb_calloc(1, sizeof(struct flb_filter_nightfall)); - if (!ctx) { - flb_errno(); - return -1; - } - - ctx->ins = f_ins; - - /* Populate context with config map defaults and incoming properties */ - ret = flb_filter_config_map_set(f_ins, (void *) ctx); - if (ret == -1) { - flb_plg_error(f_ins, "configuration error"); - flb_free(ctx); - return -1; - } - - if (ctx->sampling_rate <= 0 || ctx->sampling_rate > 1) { - flb_plg_error(f_ins, "invalid sampling rate, must be (0,1]"); - flb_free(ctx); - return -1; - } - - if (ctx->nightfall_api_key == NULL) { - flb_plg_error(f_ins, "invalid Nightfall API key"); - flb_free(ctx); - return -1; - } - - if (ctx->policy_id == NULL) { - flb_plg_error(f_ins, "invalid Nightfall policy ID"); - flb_free(ctx); - return -1; - } - - ctx->auth_header = flb_sds_create_size(42); - flb_sds_printf(&ctx->auth_header, - "Bearer %s", - ctx->nightfall_api_key); - - ctx->tls = flb_tls_create(FLB_TLS_CLIENT_MODE, - ctx->tls_verify, - ctx->tls_debug, - ctx->tls_vhost, - ctx->tls_ca_path, - NULL, - NULL, NULL, NULL); - if (!ctx->tls) { - flb_plg_error(f_ins, "tls initialization error"); - flb_free(ctx); - return -1; - } - - ctx->upstream = flb_upstream_create_url(config, - FLB_FILTER_NIGHTFALL_API_URL, - FLB_IO_TLS, - ctx->tls); - if (!ctx->upstream) { - flb_plg_error(ctx->ins, "connection initialization error"); - flb_free(ctx); - return -1; - } - - flb_stream_disable_async_mode(&ctx->upstream->base); - - flb_filter_set_context(f_ins, ctx); - - srand((unsigned int)time(NULL)); - return 0; -} - -static int redact_record(msgpack_object *data, char **to_redact_data, size_t *to_redact_size, - struct flb_time t, msgpack_sbuffer *new_rec) -{ - int ret; - struct mk_list stack; - struct nested_obj *cur; - struct nested_obj *new_obj; - struct mk_list *head; - struct mk_list *tmp; - - msgpack_sbuffer new_rec_sbuf; - msgpack_packer new_rec_pk; - - char should_pop = FLB_TRUE; - - int to_redact_index = 0; - msgpack_unpacked finding_list_unpacked; - size_t finding_list_off = 0; - msgpack_object_array to_redact; - - /* Convert to_redact_data to a msgpack_object_array */ - msgpack_unpacked_init(&finding_list_unpacked); - ret = msgpack_unpack_next(&finding_list_unpacked, *to_redact_data, *to_redact_size, - &finding_list_off); - if (ret == MSGPACK_UNPACK_SUCCESS) { - to_redact = finding_list_unpacked.data.via.array; - } - - mk_list_init(&stack); - - msgpack_sbuffer_init(&new_rec_sbuf); - msgpack_packer_init(&new_rec_pk, &new_rec_sbuf, msgpack_sbuffer_write); - - new_obj = flb_calloc(1, sizeof(struct nested_obj)); - new_obj->obj = data; - new_obj->cur_index = 0; - new_obj->start_at_val = FLB_FALSE; - mk_list_add(&new_obj->_head, &stack); - - if (data->type == MSGPACK_OBJECT_ARRAY) { - msgpack_pack_array(&new_rec_pk, data->via.array.size); - } - else if (data->type == MSGPACK_OBJECT_MAP) { - msgpack_pack_map(&new_rec_pk, data->via.map.size); - } - - /* - * Since logs can contain many levels of nested objects, use stack-based DFS here - * to build back and redact log. - */ - while (mk_list_is_empty(&stack) == -1) { - cur = mk_list_entry_last(&stack, struct nested_obj, _head); - should_pop = FLB_TRUE; - - switch(cur->obj->type) { - case MSGPACK_OBJECT_ARRAY: - ret = redact_array_fields(&new_rec_pk, &to_redact_index, &to_redact, cur, - &stack, &should_pop); - if (ret != 0) { - msgpack_unpacked_destroy(&finding_list_unpacked); - mk_list_foreach_safe(head, tmp, &stack) { - cur = mk_list_entry(head, struct nested_obj, _head); - mk_list_del(&cur->_head); - flb_free(cur); - } - return -1; - } - break; - case MSGPACK_OBJECT_MAP: - ret = redact_map_fields(&new_rec_pk, &to_redact_index, &to_redact, cur, - &stack, &should_pop); - if (ret != 0) { - msgpack_unpacked_destroy(&finding_list_unpacked); - mk_list_foreach_safe(head, tmp, &stack) { - cur = mk_list_entry(head, struct nested_obj, _head); - mk_list_del(&cur->_head); - flb_free(cur); - } - return -1; - } - break; - case MSGPACK_OBJECT_STR: - maybe_redact_field(&new_rec_pk, cur->obj, &to_redact, &to_redact_index, 0); - break; - case MSGPACK_OBJECT_POSITIVE_INTEGER: - maybe_redact_field(&new_rec_pk, cur->obj, &to_redact, &to_redact_index, 0); - break; - case MSGPACK_OBJECT_NEGATIVE_INTEGER: - maybe_redact_field(&new_rec_pk, cur->obj, &to_redact, &to_redact_index, 0); - break; - default: - msgpack_pack_object(&new_rec_pk, *cur->obj); - } - - if (should_pop) { - mk_list_del(&cur->_head); - flb_free(cur); - } - } - msgpack_unpacked_destroy(&finding_list_unpacked); - - *new_rec = new_rec_sbuf; - return 0; -} - -static int redact_array_fields(msgpack_packer *new_rec_pk, int *to_redact_index, - msgpack_object_array *to_redact, struct nested_obj *cur, - struct mk_list *stack, char *should_pop) -{ - msgpack_object *item; - struct nested_obj *new_obj; - int i; - - for (i = cur->cur_index; i < cur->obj->via.array.size; i++) { - item = &cur->obj->via.array.ptr[i]; - if (item->type == MSGPACK_OBJECT_MAP || item->type == MSGPACK_OBJECT_ARRAY) { - /* A nested object, so add to stack and return to DFS to process immediately */ - new_obj = flb_malloc(sizeof(struct nested_obj)); - if (!new_obj) { - flb_errno(); - return -1; - } - new_obj->obj = item; - new_obj->cur_index = 0; - new_obj->start_at_val = FLB_FALSE; - mk_list_add(&new_obj->_head, stack); - - if (item->type == MSGPACK_OBJECT_ARRAY) { - msgpack_pack_array(new_rec_pk, item->via.array.size); - } - else { - msgpack_pack_map(new_rec_pk, item->via.map.size); - } - - /* - * Since we are not done yet with the current array, increment the index that - * keeps track of progress and don't pop the current array so we can come - * back later. - */ - cur->cur_index = i + 1; - *should_pop = FLB_FALSE; - break; - } - else if (item->type == MSGPACK_OBJECT_STR || - item->type == MSGPACK_OBJECT_POSITIVE_INTEGER || - item->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { - /* - * A field that could potentially contain sensitive content, so we check - * if there were any findings associated with it - */ - maybe_redact_field(new_rec_pk, item, to_redact, to_redact_index, 0); - } - else { - /* Non scannable type, so just append as is. */ - msgpack_pack_object(new_rec_pk, *item); - } - } - - return 0; -} - -static int redact_map_fields(msgpack_packer *new_rec_pk, int *to_redact_index, - msgpack_object_array *to_redact, struct nested_obj *cur, - struct mk_list *stack, char *should_pop) -{ - msgpack_object *k; - msgpack_object *v; - struct nested_obj *new_obj; - int i; - - for (i = cur->cur_index; i < cur->obj->via.map.size; i++) { - k = &cur->obj->via.map.ptr[i].key; - if (!cur->start_at_val) { - /* Handle the key of this kv pair */ - if (k->type == MSGPACK_OBJECT_MAP || k->type == MSGPACK_OBJECT_ARRAY) { - /* A nested object, so add to stack and return to DFS to process immediately */ - new_obj = flb_malloc(sizeof(struct nested_obj)); - if (!new_obj) { - flb_errno(); - return -1; - } - new_obj->obj = k; - new_obj->cur_index = 0; - new_obj->start_at_val = FLB_FALSE; - mk_list_add(&new_obj->_head, stack); - - if (k->type == MSGPACK_OBJECT_ARRAY) { - msgpack_pack_array(new_rec_pk, k->via.array.size); - } - else { - msgpack_pack_map(new_rec_pk, k->via.map.size); - } - - /* - * Since we are not done yet with the current kv pair, don't increment - * the progress index and set flag so we know to start at the value later - */ - cur->cur_index = i; - cur->start_at_val = FLB_TRUE; - /* Set should_pop to false because we are not done with the current map */ - *should_pop = FLB_FALSE; - break; - } - else if (k->type == MSGPACK_OBJECT_STR || - k->type == MSGPACK_OBJECT_POSITIVE_INTEGER || - k->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { - /* - * A field that could potentially contain sensitive content, so we check - * if there were any findings associated with it - */ - maybe_redact_field(new_rec_pk, k, to_redact, to_redact_index, 0); - } - else { - /* Non scannable type, so just append as is. */ - msgpack_pack_object(new_rec_pk, *k); - } - } - - /* Handle the value of this kv pair */ - v = &cur->obj->via.map.ptr[i].val; - if (v->type == MSGPACK_OBJECT_MAP || v->type == MSGPACK_OBJECT_ARRAY) { - /* A nested object, so add to stack and return to DFS to process immediately */ - new_obj = flb_malloc(sizeof(struct nested_obj)); - if (!new_obj) { - flb_errno(); - return -1; - } - new_obj->obj = v; - new_obj->cur_index = 0; - new_obj->start_at_val = FLB_FALSE; - mk_list_add(&new_obj->_head, stack); - - if (v->type == MSGPACK_OBJECT_ARRAY) { - msgpack_pack_array(new_rec_pk, v->via.array.size); - } - else { - msgpack_pack_map(new_rec_pk, v->via.map.size); - } - - /* Increment here because we are done with this kv pair */ - cur->cur_index = i + 1; - cur->start_at_val = FLB_FALSE; - /* Set should_pop to false because we are not done with the current map */ - *should_pop = FLB_FALSE; - break; - } - else if (v->type == MSGPACK_OBJECT_STR || - v->type == MSGPACK_OBJECT_POSITIVE_INTEGER || - v->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { - if (k->type == MSGPACK_OBJECT_STR) { - /* - * When building the request to scan the log, keys that are strings are - * appended to the beginning of the value to provide more context when - * scanning in the format of " ", which is why we need to - * offset the length of the key plus a space when we do redaction on the - * value on its own. - */ - maybe_redact_field(new_rec_pk, v, to_redact, to_redact_index, - k->via.str.size + 1); - } - else { - maybe_redact_field(new_rec_pk, v, to_redact, to_redact_index, 0); - } - } - else { - msgpack_pack_object(new_rec_pk, *v); - } - } - - return 0; -} - -static void maybe_redact_field(msgpack_packer *new_rec_pk, msgpack_object *field, - msgpack_object_array *to_redact, int *to_redact_i, - int byte_offset) -{ - flb_sds_t cur_str; - msgpack_object_array content_range; - int64_t content_start; - int64_t content_end; - int i; - int64_t replace_i; - - /* - * Should not happen under normal circumstances as len of to_redact should be the - * same as the number of scannable fields (positive/negative ints, strings) in the - * event, but if that is the case just append the rest of the fields. - */ - if (*to_redact_i >= to_redact->size) { - msgpack_pack_object(new_rec_pk, *field); - return; - } - - /* - * Check if there was anything sensitive found for this field, if there wasn't we - * can leave it as is - */ - if (to_redact->ptr[*to_redact_i].via.array.size == 0) { - msgpack_pack_object(new_rec_pk, *field); - *to_redact_i = *to_redact_i + 1; - return; - } - - /* If field is an integer redact entire field */ - if (field->type == MSGPACK_OBJECT_POSITIVE_INTEGER || - field->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { - msgpack_pack_str_with_body(new_rec_pk, "******", 7); - *to_redact_i = *to_redact_i + 1; - return; - } - - /* If field is a string redact only the sensitive parts */ - cur_str = flb_sds_create_len(field->via.str.ptr, field->via.str.size); - for (i = 0; i < to_redact->ptr[*to_redact_i].via.array.size; i++) { - content_range = to_redact->ptr[*to_redact_i].via.array.ptr[i].via.array; - content_start = content_range.ptr[0].via.i64 - byte_offset; - if (content_start < 0) { - content_start = 0; - } - content_end = content_range.ptr[1].via.i64 - byte_offset; - for (replace_i = content_start; replace_i < content_end && - replace_i < flb_sds_len(cur_str); replace_i++) { - cur_str[replace_i] = '*'; - } - } - msgpack_pack_str_with_body(new_rec_pk, cur_str, flb_sds_len(cur_str)); - *to_redact_i = *to_redact_i + 1; - - flb_sds_destroy(cur_str); -} - -static int cb_nightfall_filter(const void *data, size_t bytes, - const char *tag, int tag_len, - void **out_buf, size_t *out_size, - struct flb_filter_instance *f_ins, - struct flb_input_instance *i_ins, - void *context, - struct flb_config *config) -{ - struct flb_filter_nightfall *ctx = context; - int ret; - char is_modified = FLB_FALSE; - - struct flb_time tmp = {0}; - - char *to_redact; - size_t to_redact_size; - char is_sensitive = FLB_FALSE; - - msgpack_sbuffer new_rec_sbuf; - struct flb_log_event_encoder log_encoder; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - (void) f_ins; - (void) i_ins; - (void) config; - - /* - * Generate a random double between 0 and 1, if it is over the sampling rate - * configured don't scan this log. - */ - if ((double)rand()/(double)RAND_MAX > ctx->sampling_rate) { - return FLB_FILTER_NOTOUCH; - } - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return FLB_FILTER_NOTOUCH; - } - - ret = flb_log_event_encoder_init(&log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event encoder initialization error : %d", ret); - - flb_log_event_decoder_destroy(&log_decoder); - - return FLB_FILTER_NOTOUCH; - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - ret = scan_log(ctx, log_event.body, &to_redact, &to_redact_size, &is_sensitive); - if (ret != 0) { - flb_plg_error(ctx->ins, "scanning error"); - - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - return FLB_FILTER_NOTOUCH; - } - - if (is_sensitive == FLB_TRUE) { - ret = redact_record(log_event.body, &to_redact, &to_redact_size, tmp, &new_rec_sbuf); - if (ret != 0) { - flb_plg_error(ctx->ins, "redaction error"); - flb_free(to_redact); - msgpack_sbuffer_destroy(&new_rec_sbuf); - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - return FLB_FILTER_NOTOUCH; - } - is_modified = FLB_TRUE; - } - - if (is_modified) { - ret = flb_log_event_encoder_begin_record(&log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_timestamp( - &log_encoder, &log_event.timestamp); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_metadata_from_msgpack_object( - &log_encoder, log_event.metadata); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_body_from_raw_msgpack( - &log_encoder, new_rec_sbuf.data, new_rec_sbuf.size); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record( - &log_encoder); - } - } - } - flb_free(to_redact); - - if (log_encoder.output_length > 0) { - *out_buf = log_encoder.output_buffer; - *out_size = log_encoder.output_length; - - ret = FLB_FILTER_MODIFIED; - - flb_log_event_encoder_claim_internal_buffer_ownership(&log_encoder); - } - else { - flb_plg_error(ctx->ins, - "Log event encoder error : %d", ret); - - ret = FLB_FILTER_NOTOUCH; - } - - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - return ret; -} - -static int cb_nightfall_exit(void *data, struct flb_config *config) -{ - struct flb_filter_nightfall *ctx = data; - - if (ctx == NULL) { - return 0; - } - if (ctx->upstream) { - flb_upstream_destroy(ctx->upstream); - } - if (ctx->tls) { - flb_tls_destroy(ctx->tls); - } - if (ctx->auth_header) { - flb_sds_destroy(ctx->auth_header); - } - flb_free(ctx); - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "nightfall_api_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_filter_nightfall, nightfall_api_key), - "The Nightfall API key to scan your logs with." - }, - { - FLB_CONFIG_MAP_STR, "policy_id", NULL, - 0, FLB_TRUE, offsetof(struct flb_filter_nightfall, policy_id), - "The Nightfall policy ID to scan your logs with." - }, - { - FLB_CONFIG_MAP_DOUBLE, "sampling_rate", "1", - 0, FLB_TRUE, offsetof(struct flb_filter_nightfall, sampling_rate), - "The sampling rate for scanning, must be (0,1]. 1 means all logs will be scanned." - }, - { - FLB_CONFIG_MAP_INT, "tls.debug", "0", - 0, FLB_TRUE, offsetof(struct flb_filter_nightfall, tls_debug), - "Set TLS debug level: 0 (no debug), 1 (error), " - "2 (state change), 3 (info) and 4 (verbose)" - }, - { - FLB_CONFIG_MAP_BOOL, "tls.verify", "true", - 0, FLB_TRUE, offsetof(struct flb_filter_nightfall, tls_verify), - "Enable or disable verification of TLS peer certificate" - }, - { - FLB_CONFIG_MAP_STR, "tls.vhost", NULL, - 0, FLB_TRUE, offsetof(struct flb_filter_nightfall, tls_vhost), - "Set optional TLS virtual host" - }, - { - FLB_CONFIG_MAP_STR, "tls.ca_path", NULL, - 0, FLB_TRUE, offsetof(struct flb_filter_nightfall, tls_ca_path), - "Path to root certificates on the system" - }, - {0} -}; - -struct flb_filter_plugin filter_nightfall_plugin = { - .name = "nightfall", - .description = "scans records for sensitive content", - .cb_init = cb_nightfall_init, - .cb_filter = cb_nightfall_filter, - .cb_exit = cb_nightfall_exit, - .config_map = config_map, - .flags = 0 -}; diff --git a/fluent-bit/plugins/filter_nightfall/nightfall.h b/fluent-bit/plugins/filter_nightfall/nightfall.h deleted file mode 100644 index e190c1d51..000000000 --- a/fluent-bit/plugins/filter_nightfall/nightfall.h +++ /dev/null @@ -1,57 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_FILTER_NIGHTFALL_H -#define FLB_FILTER_NIGHTFALL_H - -#include -#include - -struct nested_obj { - msgpack_object *obj; - int cur_index; - char start_at_val; - - struct mk_list _head; -}; - -struct payload { - msgpack_object *obj; - msgpack_object *key_to_scan_with; - - struct mk_list _head; -}; - -struct flb_filter_nightfall { - /* Config values */ - flb_sds_t nightfall_api_key; - flb_sds_t policy_id; - double sampling_rate; - int tls_debug; - int tls_verify; - char *tls_ca_path; - flb_sds_t tls_vhost; - - struct flb_tls *tls; - struct flb_upstream *upstream; - struct flb_filter_instance *ins; - flb_sds_t auth_header; -}; - -#endif diff --git a/fluent-bit/plugins/filter_nightfall/nightfall_api.c b/fluent-bit/plugins/filter_nightfall/nightfall_api.c deleted file mode 100644 index 91ecf7948..000000000 --- a/fluent-bit/plugins/filter_nightfall/nightfall_api.c +++ /dev/null @@ -1,536 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2021 The Fluent Bit Authors - * Copyright (C) 2015-2018 Treasure Data Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "nightfall_api.h" - -static int extract_array_fields(struct nested_obj *cur, struct mk_list *stack, - struct mk_list *payload_list, char *should_pop); -static int extract_map_fields(struct nested_obj *cur, struct mk_list *stack, - struct mk_list *payload_list, char *should_pop); - -static flb_sds_t build_request_body(struct flb_filter_nightfall *ctx, - msgpack_object *data) -{ - int ret; - struct mk_list stack; - struct nested_obj *cur; - struct nested_obj *new_obj; - - struct mk_list payload_list; - struct mk_list *head; - struct mk_list *tmp; - struct payload *pl; - - msgpack_sbuffer req_sbuf; - msgpack_packer req_pk; - flb_sds_t num_str; - int num_str_len; - flb_sds_t key_str; - flb_sds_t val_str; - flb_sds_t key_val_str; - int key_val_str_len; - flb_sds_t request_body; - - char should_pop = FLB_TRUE; - - new_obj = flb_malloc(sizeof(struct nested_obj)); - if (!new_obj) { - flb_errno(); - return NULL; - } - new_obj->obj = data; - new_obj->cur_index = 0; - new_obj->start_at_val = FLB_FALSE; - mk_list_init(&stack); - mk_list_add(&new_obj->_head, &stack); - - mk_list_init(&payload_list); - - /* - * Since logs can contain many levels of nested objects, use stack-based DFS here - * to extract scannable fields (positive/negative ints, strings) - */ - while (mk_list_is_empty(&stack) == -1) { - cur = mk_list_entry_last(&stack, struct nested_obj, _head); - should_pop = FLB_TRUE; - - switch (cur->obj->type) { - case MSGPACK_OBJECT_ARRAY: - ret = extract_array_fields(cur, &stack, &payload_list, &should_pop); - if (ret != 0) { - mk_list_foreach_safe(head, tmp, &stack) { - cur = mk_list_entry(head, struct nested_obj, _head); - mk_list_del(&cur->_head); - flb_free(cur); - } - mk_list_foreach_safe(head, tmp, &payload_list) { - pl = mk_list_entry(head, struct payload, _head); - mk_list_del(&pl->_head); - flb_free(pl); - } - return NULL; - } - break; - case MSGPACK_OBJECT_MAP: - ret = extract_map_fields(cur, &stack, &payload_list, &should_pop); - if (ret != 0) { - mk_list_foreach_safe(head, tmp, &stack) { - cur = mk_list_entry(head, struct nested_obj, _head); - mk_list_del(&cur->_head); - flb_free(cur); - } - mk_list_foreach_safe(head, tmp, &payload_list) { - pl = mk_list_entry(head, struct payload, _head); - mk_list_del(&pl->_head); - flb_free(pl); - } - return NULL; - } - break; - default: - break; - } - - if (should_pop) { - mk_list_del(&cur->_head); - flb_free(cur); - } - } - - msgpack_sbuffer_init(&req_sbuf); - msgpack_packer_init(&req_pk, &req_sbuf, msgpack_sbuffer_write); - - /* - * Build request according to schema at - * https://docs.nightfall.ai/reference/scanpayloadv3 - */ - msgpack_pack_map(&req_pk, 2); - msgpack_pack_str_with_body(&req_pk, "payload", 7); - msgpack_pack_array(&req_pk, mk_list_size(&payload_list)); - /* Initialize buf to hold string representation of numbers */ - num_str = flb_sds_create_size(21); - mk_list_foreach_safe(head, tmp, &payload_list) { - pl = mk_list_entry(head, struct payload, _head); - if (pl->obj->type == MSGPACK_OBJECT_STR) { - if (pl->key_to_scan_with != NULL) { - /* - * Payload is the value of a keyval pair with a string key that could - * provide context when scanning, so join them together and scan. - */ - val_str = flb_sds_create_len(pl->obj->via.str.ptr, - pl->obj->via.str.size); - key_str = flb_sds_create_len(pl->key_to_scan_with->via.str.ptr, - pl->key_to_scan_with->via.str.size); - key_val_str = flb_sds_create_size(pl->key_to_scan_with->via.str.size + - pl->obj->via.str.size + 2); - key_val_str_len = flb_sds_snprintf(&key_val_str, - flb_sds_alloc(key_val_str), - "%s %s", key_str, val_str); - msgpack_pack_str_with_body(&req_pk, key_val_str, key_val_str_len); - flb_sds_destroy(val_str); - flb_sds_destroy(key_str); - flb_sds_destroy(key_val_str); - } - else { - msgpack_pack_str_with_body(&req_pk, pl->obj->via.str.ptr, - pl->obj->via.str.size); - } - } - else { - if (pl->key_to_scan_with != NULL) { - /* - * Payload is the value of a keyval pair with a string key that could - * provide context when scanning, so join them together and scan. - */ - key_str = flb_sds_create_len(pl->key_to_scan_with->via.str.ptr, - pl->key_to_scan_with->via.str.size); - key_val_str = flb_sds_create_size(pl->key_to_scan_with->via.str.size + - num_str_len + 2); - num_str_len = flb_sds_snprintf(&num_str, flb_sds_alloc(num_str), - "%"PRIi64, pl->obj->via.i64); - key_val_str_len = flb_sds_snprintf(&key_val_str, - flb_sds_alloc(key_val_str), - "%s %s", key_str, num_str); - msgpack_pack_str_with_body(&req_pk, key_val_str, key_val_str_len); - flb_sds_destroy(key_str); - flb_sds_destroy(key_val_str); - } - else { - num_str_len = flb_sds_snprintf(&num_str, flb_sds_alloc(num_str), - "%"PRIi64, pl->obj->via.i64); - msgpack_pack_str_with_body(&req_pk, num_str, num_str_len); - } - } - mk_list_del(&pl->_head); - flb_free(pl); - } - msgpack_pack_str_with_body(&req_pk, "policyUUIDs", 11); - msgpack_pack_array(&req_pk, 1); - msgpack_pack_str_with_body(&req_pk, ctx->policy_id, 36); - - request_body = flb_msgpack_raw_to_json_sds(req_sbuf.data, req_sbuf.size); - - msgpack_sbuffer_destroy(&req_sbuf); - flb_sds_destroy(num_str); - - return request_body; -} - -static int extract_array_fields(struct nested_obj *cur, struct mk_list *stack, - struct mk_list *payload_list, char *should_pop) -{ - msgpack_object *item; - struct nested_obj *new_obj; - struct payload *pl; - int i; - - for (i = cur->cur_index; i < cur->obj->via.array.size; i++) { - item = &cur->obj->via.array.ptr[i]; - if (item->type == MSGPACK_OBJECT_MAP || item->type == MSGPACK_OBJECT_ARRAY) { - /* A nested object, so add to stack and return to DFS to process immediately */ - new_obj = flb_malloc(sizeof(struct nested_obj)); - if (!new_obj) { - flb_errno(); - return -1; - } - new_obj->obj = item; - new_obj->cur_index = 0; - new_obj->start_at_val = FLB_FALSE; - mk_list_add(&new_obj->_head, stack); - - /* - * Since we are not done yet with the current array, increment the index that - * keeps track of progress and don't pop the current array so we can come - * back later. - */ - cur->cur_index = i + 1; - *should_pop = FLB_FALSE; - break; - } - else if (item->type == MSGPACK_OBJECT_STR || - item->type == MSGPACK_OBJECT_POSITIVE_INTEGER || - item->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { - /* Field is a scannable type, so add to payload list to build request later */ - pl = flb_calloc(1, sizeof(struct payload)); - if (!pl) { - flb_errno(); - return -1; - } - pl->obj = item; - mk_list_add(&pl->_head, payload_list); - } - } - - return 0; -} - -static int extract_map_fields(struct nested_obj *cur, struct mk_list *stack, - struct mk_list *payload_list, char *should_pop) -{ - struct nested_obj *new_obj; - msgpack_object *k; - msgpack_object *v; - struct payload *pl; - int i; - - for (i = cur->cur_index; i < cur->obj->via.map.size; i++) { - k = &cur->obj->via.map.ptr[i].key; - if (!cur->start_at_val) { - /* Handle the key of this kv pair */ - if (k->type == MSGPACK_OBJECT_MAP || k->type == MSGPACK_OBJECT_ARRAY) { - /* A nested object, so add to stack and return to DFS to process immediately */ - new_obj = flb_malloc(sizeof(struct nested_obj)); - if (!new_obj) { - flb_errno(); - return -1; - } - new_obj->obj = k; - new_obj->cur_index = 0; - new_obj->start_at_val = FLB_FALSE; - mk_list_add(&new_obj->_head, stack); - - /* - * Since we are not done yet with the current kv pair, don't increment - * the progress index and set flag so we know to start at the value later - */ - cur->cur_index = i; - cur->start_at_val = FLB_TRUE; - /* Set should_pop to false because we are not done with the current map */ - *should_pop = FLB_FALSE; - break; - } - else if (k->type == MSGPACK_OBJECT_STR || - k->type == MSGPACK_OBJECT_POSITIVE_INTEGER || - k->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { - /* Field is a scannable type, so add to payload list to build request later */ - pl = flb_calloc(1, sizeof(struct payload)); - if (!pl) { - flb_errno(); - return -1; - } - pl->obj = k; - mk_list_add(&pl->_head, payload_list); - } - } - - /* Handle the value of this kv pair */ - v = &cur->obj->via.map.ptr[i].val; - if (v->type == MSGPACK_OBJECT_MAP || v->type == MSGPACK_OBJECT_ARRAY) { - /* A nested object, so add to stack and return to DFS to process immediately */ - new_obj = flb_malloc(sizeof(struct nested_obj)); - if (!new_obj) { - flb_errno(); - return -1; - } - new_obj->obj = v; - new_obj->cur_index = 0; - new_obj->start_at_val = FLB_FALSE; - mk_list_add(&new_obj->_head, stack); - - /* Increment here because we are done with this kv pair */ - cur->cur_index = i + 1; - cur->start_at_val = FLB_FALSE; - /* Set should_pop to false because we are not done with the current map */ - *should_pop = FLB_FALSE; - break; - } - else if (v->type == MSGPACK_OBJECT_STR || - v->type == MSGPACK_OBJECT_POSITIVE_INTEGER || - v->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { - /* Field is a scannable type, so add to payload list to build request later */ - pl = flb_calloc(1, sizeof(struct payload)); - if (!pl) { - flb_errno(); - return -1; - } - if (k->type == MSGPACK_OBJECT_STR) { - /* - * The key could provide more context for scanning so save it to scan - * with the val together. - */ - pl->key_to_scan_with = k; - } - pl->obj = v; - mk_list_add(&pl->_head, payload_list); - } - } - - return 0; -} - -static int get_map_val(msgpack_object m, char *key, msgpack_object *ret) -{ - msgpack_object_kv kv; - int i; - - if (m.type != MSGPACK_OBJECT_MAP) { - return -1; - } - for (i = 0; i < m.via.map.size; i++) { - kv = m.via.map.ptr[i]; - if (kv.key.via.str.size == strlen(key) && - !strncmp(kv.key.via.str.ptr, key, strlen(key))) { - *ret = kv.val; - return 0; - } - } - return -1; -} - -static int process_response(const char *resp, size_t resp_size, - char **to_redact, size_t *to_redact_size, - char *is_sensitive) -{ - int root_type; - char *buf; - size_t size; - msgpack_unpacked resp_unpacked; - size_t off = 0; - int ret; - int i, j, k; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - - msgpack_object resp_map; - msgpack_object findings_list; - msgpack_object findings; - msgpack_object finding; - msgpack_object location; - msgpack_object byteRange; - - /* Convert json response body to msgpack */ - ret = flb_pack_json(resp, resp_size, &buf, &size, &root_type, NULL); - if (ret != 0) { - flb_errno(); - return -1; - } - - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - msgpack_unpacked_init(&resp_unpacked); - - /* - * For every scannable field (positive/negative ints, strings) we sent to - * scan, Nightfall returns an array of finding objects that inform - * which portions of the field may be sensitive. We return those byte - * ranges here so we can do redaction later. - */ - ret = msgpack_unpack_next(&resp_unpacked, buf, size, &off); - if (ret == MSGPACK_UNPACK_SUCCESS) { - resp_map = resp_unpacked.data; - ret = get_map_val(resp_map, "findings", &findings_list); - if (ret != 0) { - msgpack_unpacked_destroy(&resp_unpacked); - msgpack_sbuffer_destroy(&mp_sbuf); - flb_free(buf); - flb_errno(); - return -1; - } - msgpack_pack_array(&mp_pck, findings_list.via.array.size); - - for (i = 0; i < findings_list.via.array.size; i++) { - findings = findings_list.via.array.ptr[i]; - msgpack_pack_array(&mp_pck, findings.via.array.size); - - if (!*is_sensitive && findings.via.array.size > 0) { - *is_sensitive = FLB_TRUE; - } - - for (j = 0; j < findings.via.array.size; j++) { - finding = findings.via.array.ptr[j]; - ret = get_map_val(finding, "location", &location); - if (ret != 0) { - msgpack_unpacked_destroy(&resp_unpacked); - msgpack_sbuffer_destroy(&mp_sbuf); - flb_free(buf); - flb_errno(); - return -1; - } - - ret = get_map_val(location, "byteRange", &byteRange); - if (ret != 0) { - msgpack_unpacked_destroy(&resp_unpacked); - msgpack_sbuffer_destroy(&mp_sbuf); - flb_free(buf); - flb_errno(); - return -1; - } - - msgpack_pack_array(&mp_pck, byteRange.via.map.size); - for (k = 0; k < byteRange.via.map.size; k++) { - msgpack_pack_int64(&mp_pck, byteRange.via.map.ptr[k].val.via.i64); - } - } - } - } - msgpack_unpacked_destroy(&resp_unpacked); - flb_free(buf); - - *to_redact = mp_sbuf.data; - *to_redact_size = mp_sbuf.size; - - return 0; -} - -/* Scans log for sensitive content and returns the locations of such content */ -int scan_log(struct flb_filter_nightfall *ctx, msgpack_object *data, - char **to_redact, size_t *to_redact_size, char *is_sensitive) -{ - struct flb_http_client *client; - struct flb_connection *u_conn; - - flb_sds_t body; - int ret; - size_t b_sent; - - body = build_request_body(ctx, data); - if (body == NULL) { - flb_plg_error(ctx->ins, "could not build request"); - return -1; - } - - u_conn = flb_upstream_conn_get(ctx->upstream); - if (!u_conn) { - flb_plg_error(ctx->ins, "connection initialization error"); - flb_sds_destroy(body); - return -1; - } - - /* Compose HTTP Client request */ - client = flb_http_client(u_conn, - FLB_HTTP_POST, "/v3/scan", - body, flb_sds_len(body), - FLB_FILTER_NIGHTFALL_API_HOST, 443, - NULL, 0); - - if (!client) { - flb_plg_error(ctx->ins, "could not create http client"); - flb_sds_destroy(body); - flb_upstream_conn_release(u_conn); - return -1; - } - - flb_http_buffer_size(client, 0); - - flb_http_add_header(client, "Authorization", 13, ctx->auth_header, 42); - flb_http_add_header(client, "User-Agent", 10, "Fluent-Bit", 10); - flb_http_add_header(client, "Content-Type", 12, "application/json", 16); - - /* Perform request */ - ret = flb_http_do(client, &b_sent); - flb_plg_info(ctx->ins, "Nightfall request http_do=%i, HTTP Status: %i", - ret, client->resp.status); - flb_sds_destroy(body); - - if (ret != 0 || client->resp.status != 200) { - if (client->resp.payload_size > 0) { - flb_plg_info(ctx->ins, "Nightfall request\n%s", - client->resp.payload); - } - flb_http_client_destroy(client); - flb_upstream_conn_release(u_conn); - return -1; - } - - ret = process_response(client->resp.payload, client->resp.payload_size, - to_redact, to_redact_size, is_sensitive); - if (ret != 0) { - flb_plg_error(ctx->ins, "could not process response"); - flb_http_client_destroy(client); - flb_upstream_conn_release(u_conn); - return -1; - } - - flb_http_client_destroy(client); - flb_upstream_conn_release(u_conn); - - return 0; -} diff --git a/fluent-bit/plugins/filter_nightfall/nightfall_api.h b/fluent-bit/plugins/filter_nightfall/nightfall_api.h deleted file mode 100644 index 0d1e6b647..000000000 --- a/fluent-bit/plugins/filter_nightfall/nightfall_api.h +++ /dev/null @@ -1,31 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_FILTER_NIGHTFALL_API_H -#define FLB_FILTER_NIGHTFALL_API_H - -#define FLB_FILTER_NIGHTFALL_API_URL "https://api.nightfall.ai/" -#define FLB_FILTER_NIGHTFALL_API_HOST "api.nightfall.ai" - -#include "nightfall.h" - -int scan_log(struct flb_filter_nightfall *ctx, msgpack_object *data, - char **to_redact, size_t *to_redact_size, char *is_sensitive); - -#endif diff --git a/fluent-bit/plugins/filter_parser/CMakeLists.txt b/fluent-bit/plugins/filter_parser/CMakeLists.txt deleted file mode 100644 index e9a2d8a7f..000000000 --- a/fluent-bit/plugins/filter_parser/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - filter_parser.c) - -FLB_PLUGIN(filter_parser "${src}" "") diff --git a/fluent-bit/plugins/filter_parser/filter_parser.c b/fluent-bit/plugins/filter_parser/filter_parser.c deleted file mode 100644 index 0f2dc039d..000000000 --- a/fluent-bit/plugins/filter_parser/filter_parser.c +++ /dev/null @@ -1,452 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "filter_parser.h" - -static int msgpackobj2char(msgpack_object *obj, - const char **ret_char, int *ret_char_size) -{ - int ret = -1; - - if (obj->type == MSGPACK_OBJECT_STR) { - *ret_char = obj->via.str.ptr; - *ret_char_size = obj->via.str.size; - ret = 0; - } - else if (obj->type == MSGPACK_OBJECT_BIN) { - *ret_char = obj->via.bin.ptr; - *ret_char_size = obj->via.bin.size; - ret = 0; - } - - return ret; -} - -static int add_parser(const char *parser, struct filter_parser_ctx *ctx, - struct flb_config *config) -{ - struct flb_parser *p; - struct filter_parser *fp; - - p = flb_parser_get(parser, config); - if (!p) { - return -1; - } - - fp = flb_malloc(sizeof(struct filter_parser)); - if (!fp) { - flb_errno(); - return -1; - } - - fp->parser = p; - mk_list_add(&fp->_head, &ctx->parsers); - return 0; -} - -static int delete_parsers(struct filter_parser_ctx *ctx) -{ - int c = 0; - struct mk_list *tmp; - struct mk_list *head; - struct filter_parser *fp; - - mk_list_foreach_safe(head, tmp, &ctx->parsers) { - fp = mk_list_entry(head, struct filter_parser, _head); - mk_list_del(&fp->_head); - flb_free(fp); - c++; - } - - return c; -} - -static int configure(struct filter_parser_ctx *ctx, - struct flb_filter_instance *f_ins, - struct flb_config *config) -{ - int ret; - struct mk_list *head; - struct flb_kv *kv; - - ctx->key_name = NULL; - ctx->reserve_data = FLB_FALSE; - ctx->preserve_key = FLB_FALSE; - mk_list_init(&ctx->parsers); - - if (flb_filter_config_map_set(f_ins, ctx) < 0) { - flb_errno(); - flb_plg_error(f_ins, "configuration error"); - return -1; - } - - if (ctx->key_name == NULL) { - flb_plg_error(ctx->ins, "missing 'key_name'"); - return -1; - } - ctx->key_name_len = flb_sds_len(ctx->key_name); - - /* Read all Parsers */ - mk_list_foreach(head, &f_ins->properties) { - kv = mk_list_entry(head, struct flb_kv, _head); - if (strcasecmp("parser", kv->key) != 0) { - continue; - } - ret = add_parser(kv->val, ctx, config); - if (ret == -1) { - flb_plg_error(ctx->ins, "requested parser '%s' not found", kv->val); - } - } - - if (mk_list_size(&ctx->parsers) == 0) { - flb_plg_error(ctx->ins, "Invalid 'parser'"); - return -1; - } - - return 0; -} - -static int cb_parser_init(struct flb_filter_instance *f_ins, - struct flb_config *config, - void *data) -{ - (void) f_ins; - (void) config; - (void) data; - - struct filter_parser_ctx *ctx = NULL; - - /* Create context */ - ctx = flb_malloc(sizeof(struct filter_parser_ctx)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->ins = f_ins; - - if ( configure(ctx, f_ins, config) < 0 ){ - flb_free(ctx); - return -1; - } - - flb_filter_set_context(f_ins, ctx); - - return 0; -} - -static int cb_parser_filter(const void *data, size_t bytes, - const char *tag, int tag_len, - void **ret_buf, size_t *ret_bytes, - struct flb_filter_instance *f_ins, - struct flb_input_instance *i_ins, - void *context, - struct flb_config *config) -{ - int continue_parsing; - struct filter_parser_ctx *ctx = context; - struct flb_time tm; - msgpack_object *obj; - - msgpack_object_kv *kv; - int i; - int ret = FLB_FILTER_NOTOUCH; - int parse_ret = -1; - int map_num; - const char *key_str; - int key_len; - const char *val_str; - int val_len; - char *out_buf; - size_t out_size; - struct flb_time parsed_time; - - msgpack_object_kv **append_arr = NULL; - size_t append_arr_len = 0; - int append_arr_i; - struct mk_list *head; - struct filter_parser *fp; - struct flb_log_event_encoder log_encoder; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - int encoder_result; - - (void) f_ins; - (void) i_ins; - (void) config; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return FLB_FILTER_NOTOUCH; - } - - ret = flb_log_event_encoder_init(&log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event encoder initialization error : %d", ret); - - flb_log_event_decoder_destroy(&log_decoder); - - return FLB_FILTER_NOTOUCH; - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - out_buf = NULL; - append_arr_i = 0; - - flb_time_copy(&tm, &log_event.timestamp); - obj = log_event.body; - - if (obj->type == MSGPACK_OBJECT_MAP) { - map_num = obj->via.map.size; - if (ctx->reserve_data) { - append_arr_len = obj->via.map.size; - append_arr = flb_calloc(append_arr_len, sizeof(msgpack_object_kv *)); - - if (append_arr == NULL) { - flb_errno(); - - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - return FLB_FILTER_NOTOUCH; - } - } - - continue_parsing = FLB_TRUE; - for (i = 0; i < map_num && continue_parsing; i++) { - kv = &obj->via.map.ptr[i]; - if (ctx->reserve_data) { - append_arr[append_arr_i] = kv; - append_arr_i++; - } - if ( msgpackobj2char(&kv->key, &key_str, &key_len) < 0 ) { - /* key is not string */ - continue; - } - if (key_len == ctx->key_name_len && - !strncmp(key_str, ctx->key_name, key_len)) { - if ( msgpackobj2char(&kv->val, &val_str, &val_len) < 0 ) { - /* val is not string */ - continue; - } - - /* Lookup parser */ - mk_list_foreach(head, &ctx->parsers) { - fp = mk_list_entry(head, struct filter_parser, _head); - - /* Reset time */ - flb_time_zero(&parsed_time); - - parse_ret = flb_parser_do(fp->parser, val_str, val_len, - (void **) &out_buf, &out_size, - &parsed_time); - if (parse_ret >= 0) { - /* - * If the parser succeeded we need to check the - * status of the parsed time. If the time was - * parsed successfully 'parsed_time' will be - * different than zero, if so, override the time - * holder with the new value, otherwise keep the - * original. - */ - if (flb_time_to_nanosec(&parsed_time) != 0L) { - flb_time_copy(&tm, &parsed_time); - } - - if (ctx->reserve_data) { - if (!ctx->preserve_key) { - append_arr_i--; - append_arr_len--; - append_arr[append_arr_i] = NULL; - } - } - else { - continue_parsing = FLB_FALSE; - } - break; - } - } - } - } - - encoder_result = flb_log_event_encoder_begin_record(&log_encoder); - - if (encoder_result == FLB_EVENT_ENCODER_SUCCESS) { - encoder_result = flb_log_event_encoder_set_timestamp( - &log_encoder, &tm); - } - - if (encoder_result == FLB_EVENT_ENCODER_SUCCESS) { - encoder_result = \ - flb_log_event_encoder_set_metadata_from_msgpack_object( - &log_encoder, log_event.metadata); - } - - if (out_buf != NULL) { - if (ctx->reserve_data) { - char *new_buf = NULL; - int new_size; - int ret; - ret = flb_msgpack_expand_map(out_buf, out_size, - append_arr, append_arr_len, - &new_buf, &new_size); - if (ret == -1) { - flb_plg_error(ctx->ins, "cannot expand map"); - - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - flb_free(append_arr); - - return FLB_FILTER_NOTOUCH; - } - - flb_free(out_buf); - out_buf = new_buf; - out_size = new_size; - } - - if (encoder_result == FLB_EVENT_ENCODER_SUCCESS) { - encoder_result = \ - flb_log_event_encoder_set_body_from_raw_msgpack( - &log_encoder, out_buf, out_size); - } - - flb_free(out_buf); - ret = FLB_FILTER_MODIFIED; - } - else { - /* re-use original data*/ - if (encoder_result == FLB_EVENT_ENCODER_SUCCESS) { - encoder_result = \ - flb_log_event_encoder_set_body_from_msgpack_object( - &log_encoder, log_event.body); - } - } - - if (encoder_result == FLB_EVENT_ENCODER_SUCCESS) { - encoder_result = flb_log_event_encoder_commit_record(&log_encoder); - } - - flb_free(append_arr); - append_arr = NULL; - } - else { - continue; - } - } - - if (log_encoder.output_length > 0) { - *ret_buf = log_encoder.output_buffer; - *ret_bytes = log_encoder.output_length; - - ret = FLB_FILTER_MODIFIED; - - flb_log_event_encoder_claim_internal_buffer_ownership(&log_encoder); - } - else { - flb_plg_error(ctx->ins, - "Log event encoder error : %d", ret); - - ret = FLB_FILTER_NOTOUCH; - } - - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - return ret; -} - - -static int cb_parser_exit(void *data, struct flb_config *config) -{ - struct filter_parser_ctx *ctx = data; - - if (!ctx) { - return 0; - } - - delete_parsers(ctx); - flb_free(ctx); - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "Key_Name", NULL, - 0, FLB_TRUE, offsetof(struct filter_parser_ctx, key_name), - "Specify field name in record to parse." - }, - { - FLB_CONFIG_MAP_STR, "Parser", NULL, - FLB_CONFIG_MAP_MULT, FLB_FALSE, 0, - "Specify the parser name to interpret the field. " - "Multiple Parser entries are allowed (one per line)." - }, - { - FLB_CONFIG_MAP_BOOL, "Preserve_Key", "false", - 0, FLB_TRUE, offsetof(struct filter_parser_ctx, preserve_key), - "Keep original Key_Name field in the parsed result. If false, the field will be removed." - }, - { - FLB_CONFIG_MAP_BOOL, "Reserve_Data", "false", - 0, FLB_TRUE, offsetof(struct filter_parser_ctx, reserve_data), - "Keep all other original fields in the parsed result. " - "If false, all other original fields will be removed." - }, - { - FLB_CONFIG_MAP_DEPRECATED, "Unescape_key", NULL, - 0, FLB_FALSE, 0, - "(deprecated)" - }, - {0} -}; - -struct flb_filter_plugin filter_parser_plugin = { - .name = "parser", - .description = "Parse events", - .cb_init = cb_parser_init, - .cb_filter = cb_parser_filter, - .cb_exit = cb_parser_exit, - .config_map = config_map, - .flags = 0 -}; diff --git a/fluent-bit/plugins/filter_parser/filter_parser.h b/fluent-bit/plugins/filter_parser/filter_parser.h deleted file mode 100644 index 72749310e..000000000 --- a/fluent-bit/plugins/filter_parser/filter_parser.h +++ /dev/null @@ -1,42 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_FILTER_PARSER_H -#define FLB_FILTER_PARSER_H - -#include -#include -#include -#include - -struct filter_parser { - struct flb_parser *parser; - struct mk_list _head; -}; - -struct filter_parser_ctx { - flb_sds_t key_name; - int key_name_len; - int reserve_data; - int preserve_key; - struct mk_list parsers; - struct flb_filter_instance *ins; -}; - -#endif /* FLB_FILTER_PARSER_H */ diff --git a/fluent-bit/plugins/filter_record_modifier/CMakeLists.txt b/fluent-bit/plugins/filter_record_modifier/CMakeLists.txt deleted file mode 100644 index d971fa203..000000000 --- a/fluent-bit/plugins/filter_record_modifier/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - filter_modifier.c) - -FLB_PLUGIN(filter_record_modifier "${src}" "") diff --git a/fluent-bit/plugins/filter_record_modifier/filter_modifier.c b/fluent-bit/plugins/filter_record_modifier/filter_modifier.c deleted file mode 100644 index d95066a19..000000000 --- a/fluent-bit/plugins/filter_record_modifier/filter_modifier.c +++ /dev/null @@ -1,531 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include "filter_modifier.h" - -#define PLUGIN_NAME "filter_record_modifier" - -static int config_allowlist_key(struct record_modifier_ctx *ctx, - struct mk_list *list) -{ - struct modifier_key *mod_key = NULL; - struct mk_list *head = NULL; - struct flb_config_map_val *mv = NULL; - - if (ctx == NULL || list == NULL) { - return -1; - } - - flb_config_map_foreach(head, mv, list) { - mod_key = flb_malloc(sizeof(struct modifier_key)); - if (!mod_key) { - flb_errno(); - continue; - } - mod_key->key = mv->val.str; - mod_key->key_len = flb_sds_len(mv->val.str); - if (mod_key->key[mod_key->key_len - 1] == '*') { - mod_key->dynamic_key = FLB_TRUE; - mod_key->key_len--; - } - else { - mod_key->dynamic_key = FLB_FALSE; - } - mk_list_add(&mod_key->_head, &ctx->allowlist_keys); - ctx->allowlist_keys_num++; - } - return 0; -} - -static int configure(struct record_modifier_ctx *ctx, - struct flb_filter_instance *f_ins) -{ - struct mk_list *head = NULL; - struct modifier_key *mod_key; - struct modifier_record *mod_record; - struct flb_config_map_val *mv; - struct flb_slist_entry *sentry = NULL; - - ctx->records_num = 0; - ctx->remove_keys_num = 0; - ctx->allowlist_keys_num = 0; - - if (flb_filter_config_map_set(f_ins, ctx) < 0) { - flb_errno(); - flb_plg_error(f_ins, "configuration error"); - return -1; - } - - /* Check 'Record' properties */ - flb_config_map_foreach(head, mv, ctx->records_map) { - mod_record = flb_malloc(sizeof(struct modifier_record)); - if (!mod_record) { - flb_errno(); - continue; - } - - if (mk_list_size(mv->val.list) != 2) { - flb_plg_error(ctx->ins, "invalid record parameters, " - "expects 'KEY VALUE'"); - flb_free(mod_record); - continue; - } - /* Get first value (field) */ - sentry = mk_list_entry_first(mv->val.list, struct flb_slist_entry, _head); - mod_record->key_len = flb_sds_len(sentry->str); - mod_record->key = flb_strndup(sentry->str, mod_record->key_len); - if (mod_record->key == NULL) { - flb_errno(); - flb_free(mod_record); - continue; - } - - sentry = mk_list_entry_last(mv->val.list, struct flb_slist_entry, _head); - mod_record->val_len = flb_sds_len(sentry->str); - mod_record->val = flb_strndup(sentry->str, mod_record->val_len); - if (mod_record->val == NULL) { - flb_errno(); - flb_free(mod_record->key); - flb_free(mod_record); - continue; - } - - mk_list_add(&mod_record->_head, &ctx->records); - ctx->records_num++; - } - /* Check "Remove_Key" properties */ - flb_config_map_foreach(head, mv, ctx->remove_keys_map) { - mod_key = flb_malloc(sizeof(struct modifier_key)); - if (!mod_key) { - flb_errno(); - continue; - } - mod_key->key = mv->val.str; - mod_key->key_len = flb_sds_len(mv->val.str); - if (mod_key->key[mod_key->key_len - 1] == '*') { - mod_key->dynamic_key = FLB_TRUE; - mod_key->key_len--; - } - else { - mod_key->dynamic_key = FLB_FALSE; - } - mk_list_add(&mod_key->_head, &ctx->remove_keys); - ctx->remove_keys_num++; - } - - /* Check "Allowlist_key" and "Whitelist_key" properties */ - config_allowlist_key(ctx, ctx->allowlist_keys_map); - config_allowlist_key(ctx, ctx->whitelist_keys_map); - - if (ctx->remove_keys_num > 0 && ctx->allowlist_keys_num > 0) { - flb_plg_error(ctx->ins, "remove_keys and allowlist_keys are exclusive " - "with each other."); - return -1; - } - return 0; -} - -static int delete_list(struct record_modifier_ctx *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - struct modifier_key *key; - struct modifier_record *record; - - mk_list_foreach_safe(head, tmp, &ctx->remove_keys) { - key = mk_list_entry(head, struct modifier_key, _head); - mk_list_del(&key->_head); - flb_free(key); - } - mk_list_foreach_safe(head, tmp, &ctx->allowlist_keys) { - key = mk_list_entry(head, struct modifier_key, _head); - mk_list_del(&key->_head); - flb_free(key); - } - mk_list_foreach_safe(head, tmp, &ctx->records) { - record = mk_list_entry(head, struct modifier_record, _head); - flb_free(record->key); - flb_free(record->val); - mk_list_del(&record->_head); - flb_free(record); - } - return 0; -} - - -static int cb_modifier_init(struct flb_filter_instance *f_ins, - struct flb_config *config, - void *data) -{ - struct record_modifier_ctx *ctx = NULL; - - /* Create context */ - ctx = flb_calloc(1, sizeof(struct record_modifier_ctx)); - if (!ctx) { - flb_errno(); - return -1; - } - mk_list_init(&ctx->records); - mk_list_init(&ctx->remove_keys); - mk_list_init(&ctx->allowlist_keys); - ctx->ins = f_ins; - - if ( configure(ctx, f_ins) < 0 ){ - delete_list(ctx); - flb_free(ctx); - return -1; - } - - flb_filter_set_context(f_ins, ctx); - - return 0; -} - -static int make_bool_map(struct record_modifier_ctx *ctx, msgpack_object *map, - bool_map_t *bool_map, int map_num) -{ - struct mk_list *tmp; - struct mk_list *head; - struct mk_list *check = NULL; - msgpack_object_kv *kv; - struct modifier_key *mod_key; - - char result; - char is_to_delete; - msgpack_object *key; - int ret = map_num; - int i; - - for (i=0; iremove_keys_num > 0) { - check = &(ctx->remove_keys); - is_to_delete = FLB_TRUE; - } - else if(ctx->allowlist_keys_num > 0) { - check = &(ctx->allowlist_keys); - is_to_delete = FLB_FALSE; - } - - if (check != NULL){ - kv = map->via.map.ptr; - for(i=0; ikey; - result = FLB_FALSE; - - mk_list_foreach_safe(head, tmp, check) { - mod_key = mk_list_entry(head, struct modifier_key, _head); - if (key->via.bin.size != mod_key->key_len && - key->via.str.size != mod_key->key_len && - mod_key->dynamic_key == FLB_FALSE) { - continue; - } - if (key->via.bin.size < mod_key->key_len && - key->via.str.size < mod_key->key_len && - mod_key->dynamic_key == FLB_TRUE) { - continue; - } - if ((key->type == MSGPACK_OBJECT_BIN && - !strncasecmp(key->via.bin.ptr, mod_key->key, - mod_key->key_len)) || - (key->type == MSGPACK_OBJECT_STR && - !strncasecmp(key->via.str.ptr, mod_key->key, - mod_key->key_len)) - ) { - result = FLB_TRUE; - break; - } - } - if (result == is_to_delete) { - bool_map[i] = TO_BE_REMOVED; - ret--; - } - } - } - - return ret; -} - -static int create_uuid(struct record_modifier_ctx *ctx, char *uuid) -{ - int ret; - - if (uuid == NULL) { - return -1; - } - - ret = flb_utils_uuid_v4_gen(uuid); - if (ret < 0) { - flb_plg_error(ctx->ins, "failed to append uuid"); - return -1; - } - return 0; -} - -#define BOOL_MAP_LIMIT 65535 -static int cb_modifier_filter(const void *data, size_t bytes, - const char *tag, int tag_len, - void **out_buf, size_t *out_size, - struct flb_filter_instance *f_ins, - struct flb_input_instance *i_ins, - void *context, - struct flb_config *config) -{ - struct record_modifier_ctx *ctx = context; - char is_modified = FLB_FALSE; - int i; - int removed_map_num = 0; - int map_num = 0; - int ret; - char uuid[40] = {0}; - size_t uuid_len = 0; - bool_map_t *bool_map = NULL; - struct flb_time tm; - struct modifier_record *mod_rec; - msgpack_object *obj; - msgpack_object_kv *kv; - struct mk_list *tmp; - struct mk_list *head; - struct flb_log_event_encoder log_encoder; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - (void) f_ins; - (void) i_ins; - (void) config; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return FLB_FILTER_NOTOUCH; - } - - ret = flb_log_event_encoder_init(&log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event encoder initialization error : %d", ret); - - flb_log_event_decoder_destroy(&log_decoder); - - return FLB_FILTER_NOTOUCH; - } - - /* Iterate each item to know map number */ - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - map_num = 0; - removed_map_num = 0; - uuid_len = 0; - - if (bool_map != NULL) { - flb_free(bool_map); - bool_map = NULL; - } - - flb_time_copy(&tm, &log_event.timestamp); - obj = log_event.body; - - /* grep keys */ - if (obj->type == MSGPACK_OBJECT_MAP) { - map_num = obj->via.map.size; - if (map_num > BOOL_MAP_LIMIT) { - flb_plg_error(ctx->ins, "The number of elements exceeds limit %d", - BOOL_MAP_LIMIT); - return -1; - } - /* allocate map_num + guard byte */ - bool_map = flb_calloc(map_num+1, sizeof(bool_map_t)); - if (bool_map == NULL) { - flb_errno(); - return -1; - } - removed_map_num = make_bool_map(ctx, obj, - bool_map, obj->via.map.size); - } - else { - continue; - } - - if (removed_map_num != map_num) { - is_modified = FLB_TRUE; - } - - removed_map_num += ctx->records_num; - if (ctx->uuid_key) { - memset(&uuid[0], 0, sizeof(uuid)); - ret = create_uuid(ctx, &uuid[0]); - if (ret == 0) { - removed_map_num++; - uuid_len = strlen(&uuid[0]); - } - } - if (removed_map_num <= 0) { - continue; - } - - ret = flb_log_event_encoder_begin_record(&log_encoder); - - ret = flb_log_event_encoder_set_timestamp(&log_encoder, &tm); - - ret = flb_log_event_encoder_set_metadata_from_msgpack_object( - &log_encoder, log_event.metadata); - - kv = obj->via.map.ptr; - for(i=0; - bool_map[i] != TAIL_OF_ARRAY && - ret == FLB_EVENT_ENCODER_SUCCESS; - i++) { - if (bool_map[i] == TO_BE_REMAINED) { - ret = flb_log_event_encoder_append_body_values( - &log_encoder, - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&kv[i].key), - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&kv[i].val)); - } - } - - flb_free(bool_map); - bool_map = NULL; - - /* append record */ - if (ctx->records_num > 0) { - is_modified = FLB_TRUE; - - mk_list_foreach_safe(head, tmp, &ctx->records) { - mod_rec = mk_list_entry(head, struct modifier_record, _head); - - ret = flb_log_event_encoder_append_body_values( - &log_encoder, - FLB_LOG_EVENT_STRING_VALUE(mod_rec->key, mod_rec->key_len), - FLB_LOG_EVENT_STRING_VALUE(mod_rec->val, mod_rec->val_len)); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - break; - } - } - } - - if (uuid_len > 0) { - is_modified = FLB_TRUE; - - ret = flb_log_event_encoder_append_body_values( - &log_encoder, - FLB_LOG_EVENT_STRING_VALUE(ctx->uuid_key, flb_sds_len(ctx->uuid_key)), - FLB_LOG_EVENT_STRING_VALUE(&uuid[0], uuid_len)); - } - - flb_log_event_encoder_commit_record(&log_encoder); - } - - if (bool_map != NULL) { - flb_free(bool_map); - } - - if (is_modified && - log_encoder.output_length > 0) { - *out_buf = log_encoder.output_buffer; - *out_size = log_encoder.output_length; - - ret = FLB_FILTER_MODIFIED; - - flb_log_event_encoder_claim_internal_buffer_ownership(&log_encoder); - } - else { - ret = FLB_FILTER_NOTOUCH; - } - - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - return ret; -} - -static int cb_modifier_exit(void *data, struct flb_config *config) -{ - struct record_modifier_ctx *ctx = data; - - if (ctx != NULL) { - delete_list(ctx); - flb_free(ctx); - } - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_SLIST_2, "record", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct record_modifier_ctx, records_map), - "Append fields. This parameter needs key and value pair." - }, - - { - FLB_CONFIG_MAP_STR, "remove_key", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct record_modifier_ctx, remove_keys_map), - "If the key is matched, that field is removed." - }, - { - FLB_CONFIG_MAP_STR, "allowlist_key", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct record_modifier_ctx, allowlist_keys_map), - "If the key is not matched, that field is removed." - }, - { - FLB_CONFIG_MAP_STR, "whitelist_key", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct record_modifier_ctx, whitelist_keys_map), - "(Alias of allowlist_key)" - }, - - { - FLB_CONFIG_MAP_STR, "uuid_key", NULL, - 0, FLB_TRUE, offsetof(struct record_modifier_ctx, uuid_key), - "If set, the plugin generates uuid per record." - }, - - {0} -}; - -struct flb_filter_plugin filter_record_modifier_plugin = { - .name = "record_modifier", - .description = "modify record", - .cb_init = cb_modifier_init, - .cb_filter = cb_modifier_filter, - .cb_exit = cb_modifier_exit, - .config_map = config_map, - .flags = 0 -}; diff --git a/fluent-bit/plugins/filter_record_modifier/filter_modifier.h b/fluent-bit/plugins/filter_record_modifier/filter_modifier.h deleted file mode 100644 index b9d0818ef..000000000 --- a/fluent-bit/plugins/filter_record_modifier/filter_modifier.h +++ /dev/null @@ -1,68 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_FILTER_RECORD_MODIFIER_H -#define FLB_FILTER_RECORD_MODIFIER_H - -#include -#include -#include - -struct modifier_record { - char *key; - char *val; - int key_len; - int val_len; - struct mk_list _head; -}; - -struct modifier_key { - char *key; - int key_len; - int dynamic_key; - struct mk_list _head; -}; - -struct record_modifier_ctx { - int records_num; - int remove_keys_num; - int allowlist_keys_num; - - flb_sds_t uuid_key; - - /* config map */ - struct mk_list *records_map; - struct mk_list *remove_keys_map; - struct mk_list *allowlist_keys_map; - struct mk_list *whitelist_keys_map; - - struct mk_list records; - struct mk_list remove_keys; - struct mk_list allowlist_keys; - struct flb_filter_instance *ins; -}; - -typedef enum { - TO_BE_REMOVED = 0, - TO_BE_REMAINED = 1, - TAIL_OF_ARRAY = 2 -} bool_map_t; - - -#endif /* FLB_FILTER_RECORD_MODIFIER_H */ diff --git a/fluent-bit/plugins/filter_rewrite_tag/CMakeLists.txt b/fluent-bit/plugins/filter_rewrite_tag/CMakeLists.txt deleted file mode 100644 index 7b98aa218..000000000 --- a/fluent-bit/plugins/filter_rewrite_tag/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - rewrite_tag.c) - -FLB_PLUGIN(filter_rewrite_tag "${src}" "") diff --git a/fluent-bit/plugins/filter_rewrite_tag/rewrite_tag.c b/fluent-bit/plugins/filter_rewrite_tag/rewrite_tag.c deleted file mode 100644 index 5969d1582..000000000 --- a/fluent-bit/plugins/filter_rewrite_tag/rewrite_tag.c +++ /dev/null @@ -1,621 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "rewrite_tag.h" - -/* Create an emitter input instance */ -static int emitter_create(struct flb_rewrite_tag *ctx) -{ - int ret; - struct flb_input_instance *ins; - - ret = flb_input_name_exists(ctx->emitter_name, ctx->config); - if (ret == FLB_TRUE) { - flb_plg_error(ctx->ins, "emitter_name '%s' already exists", - ctx->emitter_name); - return -1; - } - - ins = flb_input_new(ctx->config, "emitter", NULL, FLB_FALSE); - if (!ins) { - flb_plg_error(ctx->ins, "cannot create emitter instance"); - return -1; - } - - /* Set the alias name */ - ret = flb_input_set_property(ins, "alias", ctx->emitter_name); - if (ret == -1) { - flb_plg_warn(ctx->ins, - "cannot set emitter_name, using fallback name '%s'", - ins->name); - } - - /* Set the emitter_mem_buf_limit */ - if(ctx->emitter_mem_buf_limit > 0) { - ins->mem_buf_limit = ctx->emitter_mem_buf_limit; - } - - /* Set the storage type */ - ret = flb_input_set_property(ins, "storage.type", - ctx->emitter_storage_type); - if (ret == -1) { - flb_plg_error(ctx->ins, "cannot set storage.type"); - } - - /* Initialize emitter plugin */ - ret = flb_input_instance_init(ins, ctx->config); - if (ret == -1) { - flb_plg_error(ctx->ins, "cannot initialize emitter instance '%s'", - ins->name); - flb_input_instance_exit(ins, ctx->config); - flb_input_instance_destroy(ins); - return -1; - } - -#ifdef FLB_HAVE_METRICS - /* Override Metrics title */ - ret = flb_metrics_title(ctx->emitter_name, ins->metrics); - if (ret == -1) { - flb_plg_warn(ctx->ins, "cannot set metrics title, using fallback name %s", - ins->name); - } -#endif - - /* Storage context */ - ret = flb_storage_input_create(ctx->config->cio, ins); - if (ret == -1) { - flb_plg_error(ctx->ins, "cannot initialize storage for stream '%s'", - ctx->emitter_name); - flb_input_instance_exit(ins, ctx->config); - flb_input_instance_destroy(ins); - return -1; - } - ctx->ins_emitter = ins; - return 0; -} - -/* - * Validate and prepare internal contexts based on the received - * config_map values. - */ -static int process_config(struct flb_rewrite_tag *ctx) -{ - struct mk_list *head; - struct flb_slist_entry *entry; - struct rewrite_rule *rule; - struct flb_config_map_val *val; - - if (!ctx->cm_rules) { - return -1; - } - - mk_list_foreach(head, ctx->cm_rules) { - /* - * When multiple entries are allowed in a config map, this becomes - * a list of struct flb_config_map_val. Every entry is linked in the - * 'mult' field - */ - val = mk_list_entry(head, struct flb_config_map_val, _head); - - /* Allocate a rule */ - rule = flb_malloc(sizeof(struct rewrite_rule)); - if (!rule) { - flb_errno(); - return -1; - } - - /* key */ - entry = flb_slist_entry_get(val->val.list, 0); - if (entry == NULL) { - flb_plg_error(ctx->ins, "failed to get entry"); - flb_free(rule); - return -1; - } - rule->ra_key = flb_ra_create(entry->str, FLB_FALSE); - if (!rule->ra_key) { - flb_plg_error(ctx->ins, "invalid record accessor key ? '%s'", - entry->str); - flb_free(rule); - return -1; - } - - /* regex */ - entry = flb_slist_entry_get(val->val.list, 1); - rule->regex = flb_regex_create(entry->str); - if (!rule->regex) { - flb_plg_error(ctx->ins, "could not compile regex pattern '%s'", - entry->str); - flb_ra_destroy(rule->ra_key); - flb_free(rule); - return -1; - } - - /* tag */ - entry = flb_slist_entry_get(val->val.list, 2); - rule->ra_tag = flb_ra_create(entry->str, FLB_FALSE); - - if (!rule->ra_tag) { - flb_plg_error(ctx->ins, "could not compose tag: %s", entry->str); - flb_ra_destroy(rule->ra_key); - flb_regex_destroy(rule->regex); - flb_free(rule); - return -1; - } - - /* keep record ? */ - entry = flb_slist_entry_get(val->val.list, 3); - rule->keep_record = flb_utils_bool(entry->str); - - /* Link new rule */ - mk_list_add(&rule->_head, &ctx->rules); - } - - if (mk_list_size(&ctx->rules) == 0) { - flb_plg_warn(ctx->ins, "no rules have defined"); - return 0; - } - - return 0; -} - -static int is_wildcard(char* match) -{ - size_t len; - size_t i; - - if (match == NULL) { - return 0; - } - len = strlen(match); - - /* '***' should be ignored. So we check every char. */ - for (i=0; imatch)) { - flb_plg_warn(ins, "'Match' may cause infinite loop."); - } - ctx->ins = ins; - ctx->config = config; - mk_list_init(&ctx->rules); - - /* - * Emitter name: every rewrite_tag instance needs an emitter input plugin, - * with that one is able to emit records. We use a unique instance so we - * can use the metrics interface. - * - * If not set, we define an emitter name - * - * Validate if the emitter_name has been set before to check with the - * config map. If is not set, do a manual set of the property, so we let the - * config map handle the memory allocation. - */ - tmp = (char *) flb_filter_get_property("emitter_name", ins); - if (!tmp) { - emitter_name = flb_sds_create_size(64); - if (!emitter_name) { - flb_free(ctx); - return -1; - } - - tmp = flb_sds_printf(&emitter_name, "emitter_for_%s", - flb_filter_name(ins)); - if (!tmp) { - flb_error("[filter rewrite_tag] cannot compose emitter_name"); - flb_sds_destroy(emitter_name); - flb_free(ctx); - return -1; - } - - flb_filter_set_property(ins, "emitter_name", emitter_name); - flb_sds_destroy(emitter_name); - } - - /* Set config_map properties in our local context */ - ret = flb_filter_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return -1; - } - - /* - * Emitter Storage Type: the emitter input plugin to be created by default - * uses memory buffer, this option allows to define a filesystem mechanism - * for new records created (only if the main service is also filesystem - * enabled). - * - * On this code we just validate the input type: 'memory' or 'filesystem'. - */ - tmp = ctx->emitter_storage_type; - if (strcasecmp(tmp, "memory") != 0 && strcasecmp(tmp, "filesystem") != 0) { - flb_plg_error(ins, "invalid 'emitter_storage.type' value. Only " - "'memory' or 'filesystem' types are allowed"); - flb_free(ctx); - return -1; - } - - /* Set plugin context */ - flb_filter_set_context(ins, ctx); - - /* Process the configuration */ - ret = process_config(ctx); - if (ret == -1) { - return -1; - } - - /* Create the emitter context */ - ret = emitter_create(ctx); - if (ret == -1) { - return -1; - } - - /* Register a metric to count the number of emitted records */ -#ifdef FLB_HAVE_METRICS - ctx->cmt_emitted = cmt_counter_create(ins->cmt, - "fluentbit", "filter", "emit_records_total", - "Total number of emitted records", - 1, (char *[]) {"name"}); - - /* OLD api */ - flb_metrics_add(FLB_RTAG_METRIC_EMITTED, - "emit_records", ctx->ins->metrics); -#endif - - return 0; -} - -static int ingest_inline(struct flb_rewrite_tag *ctx, - flb_sds_t out_tag, - const void *buf, size_t buf_size) -{ - struct flb_input_instance *input_instance; - struct flb_processor_unit *processor_unit; - struct flb_processor *processor; - int result; - - if (ctx->ins->parent_processor != NULL) { - processor_unit = (struct flb_processor_unit *) \ - ctx->ins->parent_processor; - processor = (struct flb_processor *) processor_unit->parent; - input_instance = (struct flb_input_instance *) processor->data; - - if (processor->source_plugin_type == FLB_PLUGIN_INPUT) { - result = flb_input_log_append_skip_processor_stages( - input_instance, - processor_unit->stage + 1, - out_tag, flb_sds_len(out_tag), - buf, buf_size); - - if (result == 0) { - return FLB_TRUE; - } - } - } - - return FLB_FALSE; -} - - -/* - * On given record, check if a rule applies or not to the map, if so, compose - * the new tag, emit the record and return FLB_TRUE, otherwise just return - * FLB_FALSE and the original record will remain. - */ -static int process_record(const char *tag, int tag_len, msgpack_object map, - const void *buf, size_t buf_size, int *keep, - struct flb_rewrite_tag *ctx, int *matched) -{ - int ret; - flb_sds_t out_tag; - struct mk_list *head; - struct rewrite_rule *rule = NULL; - struct flb_regex_search result = {0}; - - if (matched == NULL) { - return FLB_FALSE; - } - *matched = FLB_FALSE; - - mk_list_foreach(head, &ctx->rules) { - rule = mk_list_entry(head, struct rewrite_rule, _head); - if (rule) { - *keep = rule->keep_record; - } - ret = flb_ra_regex_match(rule->ra_key, map, rule->regex, &result); - if (ret < 0) { /* no match */ - rule = NULL; - continue; - } - - /* A record matched, just break and check 'rule' */ - break; - } - - if (!rule) { - return FLB_FALSE; - } - *matched = FLB_TRUE; - - /* Compose new tag */ - out_tag = flb_ra_translate(rule->ra_tag, (char *) tag, tag_len, map, &result); - - /* Release any capture info from 'results' */ - flb_regex_results_release(&result); - - /* Validate new outgoing tag */ - if (!out_tag) { - return FLB_FALSE; - } - - ret = ingest_inline(ctx, out_tag, buf, buf_size); - - if (!ret) { - /* Emit record with new tag */ - ret = in_emitter_add_record(out_tag, flb_sds_len(out_tag), buf, buf_size, - ctx->ins_emitter); - } - else { - ret = 0; - } - - /* Release the tag */ - flb_sds_destroy(out_tag); - - if (ret == -1) { - return FLB_FALSE; - } - - return FLB_TRUE; -} - -static int cb_rewrite_tag_filter(const void *data, size_t bytes, - const char *tag, int tag_len, - void **out_buf, size_t *out_bytes, - struct flb_filter_instance *f_ins, - struct flb_input_instance *i_ins, - void *filter_context, - struct flb_config *config) -{ - int keep; - int emitted_num = 0; - int is_matched = FLB_FALSE; - int is_emitted = FLB_FALSE; - size_t pre = 0; - size_t off = 0; -#ifdef FLB_HAVE_METRICS - uint64_t ts; - char *name; -#endif - msgpack_object map; - struct flb_rewrite_tag *ctx; - struct flb_log_event_encoder log_encoder; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - int ret; - - (void) config; - (void) i_ins; - - ctx = (struct flb_rewrite_tag *) filter_context; - -#ifdef FLB_HAVE_METRICS - ts = cfl_time_now(); - name = (char *) flb_filter_name(f_ins); -#endif - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return FLB_FILTER_NOTOUCH; - } - - ret = flb_log_event_encoder_init(&log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event encoder initialization error : %d", ret); - - flb_log_event_decoder_destroy(&log_decoder); - - return FLB_FILTER_NOTOUCH; - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - off = log_decoder.offset; - map = *log_event.body; - is_matched = FLB_FALSE; - /* - * Process the record according the defined rules. If it returns FLB_TRUE means - * the record was emitter with a different tag. - * - * If a record was emitted, the variable 'keep' will define if the record must - * be preserved or not. - */ - is_emitted = process_record(tag, tag_len, map, (char *) data + pre, off - pre, &keep, ctx, &is_matched); - if (is_emitted == FLB_TRUE) { - /* A record with the new tag was emitted */ - emitted_num++; - } - - /* - * Here we decide if the original record must be preserved or not: - * - * - record with new tag was emitted and the rule says it must be preserved - * - record was not emitted - */ - if (keep == FLB_TRUE || is_matched != FLB_TRUE) { - ret = flb_log_event_encoder_emit_raw_record( - &log_encoder, - log_decoder.record_base, - log_decoder.record_length); - } - - /* Adjust previous offset */ - pre = off; - } - - if (emitted_num == 0) { - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - return FLB_FILTER_NOTOUCH; - } -#ifdef FLB_HAVE_METRICS - else if (emitted_num > 0) { - cmt_counter_add(ctx->cmt_emitted, ts, emitted_num, - 1, (char *[]) {name}); - - /* OLD api */ - flb_metrics_sum(FLB_RTAG_METRIC_EMITTED, emitted_num, ctx->ins->metrics); - } -#endif - - if (ret == FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA && - log_decoder.offset == bytes) { - ret = FLB_EVENT_ENCODER_SUCCESS; - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - *out_buf = log_encoder.output_buffer; - *out_bytes = log_encoder.output_length; - - ret = FLB_FILTER_MODIFIED; - - flb_log_event_encoder_claim_internal_buffer_ownership(&log_encoder); - } - else { - flb_plg_error(ctx->ins, - "Log event encoder error : %d", ret); - - ret = FLB_FILTER_NOTOUCH; - } - - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - return ret; -} - -/* Destroy rules from context */ -static void destroy_rules(struct flb_rewrite_tag *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - struct rewrite_rule *rule; - - mk_list_foreach_safe(head, tmp, &ctx->rules) { - rule = mk_list_entry(head, struct rewrite_rule, _head); - flb_regex_destroy(rule->regex); - flb_ra_destroy(rule->ra_key); - flb_ra_destroy(rule->ra_tag); - mk_list_del(&rule->_head); - flb_free(rule); - } -} - -static int cb_rewrite_tag_exit(void *data, struct flb_config *config) -{ - struct flb_rewrite_tag *ctx = (struct flb_rewrite_tag *) data; - - if (!ctx) { - return 0; - } - - destroy_rules(ctx); - flb_free(ctx); - - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_SLIST_4, "rule", NULL, - FLB_TRUE, FLB_TRUE, offsetof(struct flb_rewrite_tag, cm_rules), - NULL - }, - { - FLB_CONFIG_MAP_STR, "emitter_name", NULL, - FLB_FALSE, FLB_TRUE, offsetof(struct flb_rewrite_tag, emitter_name), - NULL - }, - { - FLB_CONFIG_MAP_STR, "emitter_storage.type", "memory", - FLB_FALSE, FLB_TRUE, offsetof(struct flb_rewrite_tag, emitter_storage_type), - NULL - }, - { - FLB_CONFIG_MAP_SIZE, "emitter_mem_buf_limit", FLB_RTAG_MEM_BUF_LIMIT_DEFAULT, - FLB_FALSE, FLB_TRUE, offsetof(struct flb_rewrite_tag, emitter_mem_buf_limit), - "set a memory buffer limit to restrict memory usage of emitter" - }, - /* EOF */ - {0} -}; - -struct flb_filter_plugin filter_rewrite_tag_plugin = { - .name = "rewrite_tag", - .description = "Rewrite records tags", - .cb_init = cb_rewrite_tag_init, - .cb_filter = cb_rewrite_tag_filter, - .cb_exit = cb_rewrite_tag_exit, - .config_map = config_map, - .flags = 0 -}; diff --git a/fluent-bit/plugins/filter_rewrite_tag/rewrite_tag.h b/fluent-bit/plugins/filter_rewrite_tag/rewrite_tag.h deleted file mode 100644 index 1edcffd5c..000000000 --- a/fluent-bit/plugins/filter_rewrite_tag/rewrite_tag.h +++ /dev/null @@ -1,64 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_REWRITE_TAG_H -#define FLB_REWRITE_TAG_H - -#include -#include -#include -#include -#include - -#define FLB_RTAG_METRIC_EMITTED 200 -#define FLB_RTAG_MEM_BUF_LIMIT_DEFAULT "10M" - -/* Rewrite rule */ -struct rewrite_rule { - int keep_record; /* keep original record ? */ - struct flb_regex *regex; /* matching regex */ - struct flb_record_accessor *ra_key; /* key record accessor */ - struct flb_record_accessor *ra_tag; /* tag record accessor */ - struct mk_list _head; /* link to flb_rewrite_tag->rules */ -}; - -/* Plugin context */ -struct flb_rewrite_tag { - flb_sds_t emitter_name; /* emitter input plugin name */ - flb_sds_t emitter_storage_type; /* emitter storage type */ - size_t emitter_mem_buf_limit; /* Emitter buffer limit */ - struct mk_list rules; /* processed rules */ - struct mk_list *cm_rules; /* config_map rules (only strings) */ - struct flb_input_instance *ins_emitter; /* emitter input plugin instance */ - struct flb_filter_instance *ins; /* self-filter instance */ - struct flb_config *config; /* Fluent Bit context */ - -#ifdef FLB_HAVE_METRICS - struct cmt_counter *cmt_emitted; -#endif -}; - -/* Register external function to emit records, check 'plugins/in_emitter' */ -int in_emitter_add_record(const char *tag, int tag_len, - const char *buf_data, size_t buf_size, - struct flb_input_instance *in); -int in_emitter_get_collector_id(struct flb_input_instance *in); - - -#endif diff --git a/fluent-bit/plugins/filter_stdout/CMakeLists.txt b/fluent-bit/plugins/filter_stdout/CMakeLists.txt deleted file mode 100644 index 7c7794d72..000000000 --- a/fluent-bit/plugins/filter_stdout/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - stdout.c) - -FLB_PLUGIN(filter_stdout "${src}" "") diff --git a/fluent-bit/plugins/filter_stdout/stdout.c b/fluent-bit/plugins/filter_stdout/stdout.c deleted file mode 100644 index 1fb3fe040..000000000 --- a/fluent-bit/plugins/filter_stdout/stdout.c +++ /dev/null @@ -1,107 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include -#include -#include -#include -#include -#include - -static int cb_stdout_init(struct flb_filter_instance *f_ins, - struct flb_config *config, - void *data) -{ - (void) f_ins; - (void) config; - (void) data; - - if (flb_filter_config_map_set(f_ins, (void *)config) == -1) { - flb_plg_error(f_ins, "unable to load configuration"); - return -1; - } - return 0; -} - -static int cb_stdout_filter(const void *data, size_t bytes, - const char *tag, int tag_len, - void **out_buf, size_t *out_bytes, - struct flb_filter_instance *f_ins, - struct flb_input_instance *i_ins, - void *filter_context, - struct flb_config *config) -{ - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - size_t cnt; - int ret; - - (void) out_buf; - (void) out_bytes; - (void) f_ins; - (void) i_ins; - (void) filter_context; - (void) config; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(f_ins, - "Log event decoder initialization error : %d", ret); - - return FLB_FILTER_NOTOUCH; - } - - cnt = 0; - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - printf("[%zd] %s: [", cnt++, tag); - printf("%"PRIu32".%09lu, ", - (uint32_t) log_event.timestamp.tm.tv_sec, - log_event.timestamp.tm.tv_nsec); - msgpack_object_print(stdout, *log_event.metadata); - printf(", "); - msgpack_object_print(stdout, *log_event.body); - printf("]\n"); - } - - flb_log_event_decoder_destroy(&log_decoder); - - return FLB_FILTER_NOTOUCH; -} - -static struct flb_config_map config_map[] = { - /* EOF */ - {0} -}; - -struct flb_filter_plugin filter_stdout_plugin = { - .name = "stdout", - .description = "Filter events to STDOUT", - .cb_init = cb_stdout_init, - .cb_filter = cb_stdout_filter, - .cb_exit = NULL, - .config_map = config_map, - .flags = 0 -}; diff --git a/fluent-bit/plugins/filter_tensorflow/CMakeLists.txt b/fluent-bit/plugins/filter_tensorflow/CMakeLists.txt deleted file mode 100644 index 23bffe573..000000000 --- a/fluent-bit/plugins/filter_tensorflow/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(src - tensorflow.c) - -FLB_PLUGIN(filter_tensorflow "${src}" "") -target_include_directories(flb-plugin-filter_tensorflow PRIVATE ${Tensorflow_DIR}) -target_link_libraries(flb-plugin-filter_tensorflow -ltensorflowlite_c) diff --git a/fluent-bit/plugins/filter_tensorflow/tensorflow.c b/fluent-bit/plugins/filter_tensorflow/tensorflow.c deleted file mode 100644 index 3adf52ed0..000000000 --- a/fluent-bit/plugins/filter_tensorflow/tensorflow.c +++ /dev/null @@ -1,540 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include -#include -#include -#include -#include -#include - -#include "tensorflow/lite/c/c_api.h" -#include "tensorflow/lite/c/common.h" - -#include -#include -#include "tensorflow.h" - -#define MSGPACK_INTEGER(x) (x == MSGPACK_OBJECT_POSITIVE_INTEGER || \ - x == MSGPACK_OBJECT_NEGATIVE_INTEGER) -#define MSGPACK_FLOAT(x) (x == MSGPACK_OBJECT_FLOAT || \ - x == MSGPACK_OBJECT_FLOAT32) -#define MSGPACK_NUMBER(x) (MSGPACK_INTEGER(x) || MSGPACK_FLOAT(x)) - -void print_tensor_info(struct flb_tensorflow *ctx, const TfLiteTensor* tensor) -{ - int i; - TfLiteType type; - char tensor_info[128] = ""; - char tensor_dim[8]; - - type = TfLiteTensorType(tensor); - sprintf(tensor_info, "type: %s dimensions: {", TfLiteTypeGetName(type)); - for (i = 0; i < TfLiteTensorNumDims(tensor) - 1; i++) { - sprintf(tensor_dim, "%d, ", TfLiteTensorDim(tensor, i)); - strcat(tensor_info, tensor_dim); - } - sprintf(tensor_dim, "%d}", TfLiteTensorDim(tensor, i)); - strcat(tensor_info, tensor_dim); - - flb_plg_info(ctx->ins, "%s", tensor_info); -} - -void print_model_io(struct flb_tensorflow *ctx) -{ - int i; - int num; - const TfLiteTensor* tensor; - - /* Input information */ - num = TfLiteInterpreterGetInputTensorCount(ctx->interpreter); - for (i = 0; i < num; i++) { - tensor = TfLiteInterpreterGetInputTensor(ctx->interpreter, i); - flb_plg_info(ctx->ins, "===== input #%d =====", i + 1); - print_tensor_info(ctx, tensor); - } - - /* Output information */ - num = TfLiteInterpreterGetOutputTensorCount(ctx->interpreter); - for (i = 0; i < num; i++) { - tensor = TfLiteInterpreterGetOutputTensor(ctx->interpreter, i); - flb_plg_info(ctx->ins, "===== output #%d ====", i + 1); - print_tensor_info(ctx, tensor); - } -} - -void build_interpreter(struct flb_tensorflow *ctx, char* model_path) -{ - /* from c_api.h */ - ctx->model = TfLiteModelCreateFromFile(model_path); - ctx->interpreter_options = TfLiteInterpreterOptionsCreate(); - ctx->interpreter = TfLiteInterpreterCreate(ctx->model, ctx->interpreter_options); - TfLiteInterpreterAllocateTensors(ctx->interpreter); - - flb_plg_info(ctx->ins, "TensorFlow Lite interpreter created!"); - print_model_io(ctx); -} - -void inference(TfLiteInterpreter* interpreter, void* input_data, void* output_data, int input_buf_size, int output_buf_size) { - /* from c_api.h */ - TfLiteTensor* input_tensor = TfLiteInterpreterGetInputTensor(interpreter, 0); - TfLiteTensorCopyFromBuffer(input_tensor, input_data, input_buf_size); - - TfLiteInterpreterInvoke(interpreter); - - const TfLiteTensor* output_tensor = TfLiteInterpreterGetOutputTensor(interpreter, 0); - TfLiteTensorCopyToBuffer(output_tensor, output_data, output_buf_size); -} - -int allocateIOBuffer(struct flb_tensorflow *ctx, void** buf, TfLiteType type, int size) -{ - if (type == kTfLiteFloat32) { - *buf = (void*) flb_malloc(size * sizeof(float)); - } - else { - flb_plg_error(ctx->ins, "Tensor type (%d) is not currently supported!", type); - return -1; - } - - return 0; -} - -void flb_tensorflow_conf_destroy(struct flb_tensorflow *ctx) -{ - flb_sds_destroy(ctx->input_field); - - if (ctx->input) { - flb_free(ctx->input); - } - - if (ctx->output) { - flb_free(ctx->output); - } - - if (ctx->normalization_value) { - flb_free(ctx->normalization_value); - } - - /* delete TensorFlow model and interpreter */ - if (ctx->model) { - TfLiteModelDelete(ctx->model); - } - - TfLiteInterpreterOptionsDelete(ctx->interpreter_options); - TfLiteInterpreterDelete(ctx->interpreter); - - flb_free(ctx); -} - -static int cb_tensorflow_init(struct flb_filter_instance *f_ins, - struct flb_config *config, - void *data) -{ - int i; - int ret; - struct flb_tensorflow *ctx = NULL; - const char *tmp; - const TfLiteTensor* tensor; - - ctx = flb_calloc(1, sizeof(struct flb_tensorflow)); - if (!ctx) { - flb_errno(); - return -1; - } - - ret = flb_filter_config_map_set(f_ins, (void *) ctx); - if (ret == -1) { - flb_tensorflow_conf_destroy(ctx); - return -1; - } - - ctx->ins = f_ins; - - tmp = flb_filter_get_property("input_field", f_ins); - if (!tmp) { - flb_plg_error(ctx->ins, "input field is not defined!"); - flb_tensorflow_conf_destroy(ctx); - return -1; - } - - ctx->input_field = flb_sds_create(tmp); - - tmp = flb_filter_get_property("model_file", f_ins); - if (!tmp) { - flb_plg_error(ctx->ins, "TensorFlow Lite model file is not provided!"); - flb_tensorflow_conf_destroy(ctx); - return -1; - } - - if(access(tmp, F_OK) == -1) { - flb_plg_error(ctx->ins, "TensorFlow Lite model file %s not found!", tmp); - flb_tensorflow_conf_destroy(ctx); - return -1; - } - - build_interpreter(ctx, (char *) tmp); - - if (!ctx->interpreter) { - flb_plg_error(ctx->ins, "Error creating the interpreter"); - flb_tensorflow_conf_destroy(ctx); - return -1; - } - - /* calculate input information */ - ctx->input_size = 1; - tensor = TfLiteInterpreterGetInputTensor(ctx->interpreter, 0); - for (i = 0; i < TfLiteTensorNumDims(tensor); i++) { - ctx->input_size *= TfLiteTensorDim(tensor, i); - } - ctx->input_tensor_type = TfLiteTensorType(tensor); - if (allocateIOBuffer(ctx, &ctx->input, ctx->input_tensor_type, ctx->input_size) == -1) { - flb_tensorflow_conf_destroy(ctx); - return -1; - } - ctx->input_byte_size = TfLiteTensorByteSize(tensor); - - /* calculate output information */ - ctx->output_size = 1; - tensor = TfLiteInterpreterGetOutputTensor(ctx->interpreter, 0); - for (i = 0; i < TfLiteTensorNumDims(tensor); i++) { - ctx->output_size *= TfLiteTensorDim(tensor, i); - } - ctx->output_tensor_type = TfLiteTensorType(tensor); - if (allocateIOBuffer(ctx, &ctx->output, ctx->output_tensor_type, ctx->output_size) == -1) { - flb_tensorflow_conf_destroy(ctx); - return -1; - } - ctx->output_byte_size = TfLiteTensorByteSize(tensor); - - tmp = flb_filter_get_property("include_input_fields", f_ins); - if (!tmp) { - ctx->include_input_fields = FLB_TRUE; - } - else { - ctx->include_input_fields = flb_utils_bool(tmp); - } - - tmp = flb_filter_get_property("normalization_value", f_ins); - if (tmp) { - ctx->normalization_value = flb_malloc(sizeof(float)); - *ctx->normalization_value = atof(tmp); - } - - flb_filter_set_context(f_ins, ctx); - return 0; -} - -static int cb_tensorflow_filter(const void *data, size_t bytes, - const char *tag, int tag_len, - void **out_buf, size_t *out_bytes, - struct flb_filter_instance *f_ins, - struct flb_input_instance *i_ins, - void *filter_context, - struct flb_config *config) -{ - int i; - int j; - int input_data_type; - int map_size; - - msgpack_object map; - msgpack_object key; - msgpack_object value; - - struct flb_tensorflow* ctx; - - /* data pointers */ - float* dfloat; - - /* calculate inference time */ - clock_t start, end; - double inference_time; - - struct flb_log_event_encoder log_encoder; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - int ret; - - (void) out_buf; - (void) out_bytes; - (void) f_ins; - (void) i_ins; - - /* initializations */ - ctx = filter_context; - inference_time = 0; - start = clock(); - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return FLB_FILTER_NOTOUCH; - } - - ret = flb_log_event_encoder_init(&log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event encoder initialization error : %d", ret); - - flb_log_event_decoder_destroy(&log_decoder); - - return FLB_FILTER_NOTOUCH; - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - /* TODO check if msgpack type is map */ - map = *log_event.body; - map_size = map.via.map.size; - - for (i = 0; i < map_size; i++) { - key = map.via.map.ptr[i].key; - - if (flb_sds_cmp(ctx->input_field, (char *) key.via.str.ptr, key.via.str.size) != 0) { - continue; - } - - value = map.via.map.ptr[i].val; - if (value.type == MSGPACK_OBJECT_ARRAY) - { - int size = value.via.array.size; - if (size == 0) { - flb_plg_error(ctx->ins, "input data size has to be non-zero!"); - break; - } - - if (size != ctx->input_size) { - flb_plg_error(ctx->ins, "input data size doesn't match model's input size!"); - break; - } - - /* we only accept numbers inside input array */ - input_data_type = value.via.array.ptr[0].type; - if (!MSGPACK_NUMBER(input_data_type)) { - flb_plg_error(ctx->ins, "input data has to be of numerical type!"); - break; - } - - /* copy data from messagepack into the input buffer */ - /* tensor type: kTfLiteFloat32 */ - if (ctx->input_tensor_type == kTfLiteFloat32) { - if (sizeof(float) != sizeof(kTfLiteFloat32)) { - flb_plg_error(ctx->ins, "input tensor type (kTfLiteFloat32) doesn't match float size!"); - break; - } - - dfloat = (float *) ctx->input; - - if (MSGPACK_FLOAT(input_data_type)) { - for (i = 0; i < value.via.array.size; i++) { - dfloat[i] = value.via.array.ptr[i].via.f64; - } - } - else if (MSGPACK_INTEGER(input_data_type)) { - for (i = 0; i < value.via.array.size; i++) { - dfloat[i] = ((float) value.via.array.ptr[i].via.i64); - } - } - else { - flb_plg_error(ctx->ins, "input record type is not supported for a float32 input tensor!"); - break; - } - - if (ctx->normalization_value) { - for (i = 0; i < value.via.array.size; i++) { - dfloat[i] /= *ctx->normalization_value; - } - } - } - else { - flb_plg_error(ctx->ins, "input tensor type is not currently not supported!"); - break; - } - } - else if (value.type == MSGPACK_OBJECT_BIN) { - if (ctx->input_tensor_type == kTfLiteFloat32) { - dfloat = (float *) ctx->input; - - /* - * re:IEEE754 float size is 32 bits - * TODO: currently, the following assumes that the binrary - * string is the serialization of a string of characters (uint8_t). - * It is required to add other primitive data type encodings such as - * floating point numbers. - */ - if (ctx->input_byte_size != (value.via.bin.size << 2)) { - flb_plg_error(ctx->ins, "input data size (%d bytes * 4) doesn't" - "match model's input size (%d bytes)!", - value.via.bin.size, ctx->input_byte_size); - break; - } - - for (i = 0; i < value.via.bin.size; i++) { - dfloat[i] = ((float) value.via.bin.ptr[i]); - } - - if (ctx->normalization_value) { - for (i = 0; i < value.via.bin.size; i++) { - dfloat[i] /= *ctx->normalization_value; - } - } - } - } - else { - flb_plg_error(ctx->ins, "input data format is not currently supported!"); - break; - } - - /* run the inference */ - inference(ctx->interpreter, ctx->input, ctx->output, ctx->input_byte_size, ctx->output_byte_size); - - /* create output messagepack */ - end = clock(); - inference_time = ((double) (end - start)) / CLOCKS_PER_SEC; - - ret = flb_log_event_encoder_begin_record(&log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_timestamp( - &log_encoder, &log_event.timestamp); - } - - if (ctx->include_input_fields) { - for (j = 0; - j < map_size && - ret == FLB_EVENT_ENCODER_SUCCESS; - j++) { - ret = flb_log_event_encoder_append_body_values( - &log_encoder, - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&map.via.map.ptr[j].key), - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&map.via.map.ptr[j].val)); - } - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - &log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE("inference_time"), - FLB_LOG_EVENT_DOUBLE_VALUE(inference_time), - - FLB_LOG_EVENT_CSTRING_VALUE("output")); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_body_begin_array(&log_encoder); - } - - for (i=0; - i < ctx->output_size && - ret == FLB_EVENT_ENCODER_SUCCESS; - i++) { - if (ctx->output_tensor_type == kTfLiteFloat32) { - ret = flb_log_event_encoder_append_body_double( - &log_encoder, ((float*) ctx->output)[i]); - } - /* TODO: work out other types */ - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_body_commit_array(&log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_body_commit_record(&log_encoder); - } - else { - flb_log_event_encoder_body_rollback_record(&log_encoder); - } - - break; - } - } - - if (log_encoder.output_length > 0) { - *out_buf = log_encoder.output_buffer; - *out_bytes = log_encoder.output_length; - - ret = FLB_FILTER_MODIFIED; - - flb_log_event_encoder_claim_internal_buffer_ownership(&log_encoder); - } - else { - flb_plg_error(ctx->ins, - "Log event encoder error : %d", ret); - - ret = FLB_FILTER_NOTOUCH; - } - - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - return ret; -} - -static int cb_tensorflow_exit(void *data, struct flb_config *config) -{ - struct flb_tensorflow *ctx = data; - - flb_tensorflow_conf_destroy(ctx); - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "model_file", NULL, - 0, FLB_FALSE, 0, - "Address of the TensorFlow Lite model file (.tflite)" - }, - { - FLB_CONFIG_MAP_STR, "input_field", NULL, - 0, FLB_FALSE, 0, - "Input field name to use for inference." - }, - { - FLB_CONFIG_MAP_BOOL, "include_input_fields", "true", - 0, FLB_TRUE, offsetof(struct flb_tensorflow, include_input_fields), - "Include input field in the output of the filter." - }, - { - FLB_CONFIG_MAP_DOUBLE, "normalization_value", NULL, - 0, FLB_FALSE, 0, - "Divide input feature values to this value (e.g. divide image pixles by 255)." - }, - /* EOF */ - {0} -}; - -struct flb_filter_plugin filter_tensorflow_plugin = { - .name = "tensorflow", - .description = "TensorFlow Lite inference engine", - .cb_init = cb_tensorflow_init, - .cb_filter = cb_tensorflow_filter, - .cb_exit = cb_tensorflow_exit, - .config_map = config_map, - .flags = 0 -}; diff --git a/fluent-bit/plugins/filter_tensorflow/tensorflow.h b/fluent-bit/plugins/filter_tensorflow/tensorflow.h deleted file mode 100644 index 4c923cc5c..000000000 --- a/fluent-bit/plugins/filter_tensorflow/tensorflow.h +++ /dev/null @@ -1,46 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_FILTER_TENSORFLOW_H -#define FLB_FILTER_TENSORFLOW_H - -struct flb_tensorflow { - TfLiteModel* model; - TfLiteInterpreterOptions* interpreter_options; - TfLiteInterpreter* interpreter; - flb_sds_t input_field; - TfLiteType input_tensor_type; - TfLiteType output_tensor_type; - - /* IO buffer */ - void* input; - void* output; - int input_size; - int input_byte_size; - int output_size; - int output_byte_size; - - /* feature scaling/normalization */ - bool include_input_fields; - float* normalization_value; - - struct flb_filter_instance *ins; -}; - -#endif diff --git a/fluent-bit/plugins/filter_throttle/CMakeLists.txt b/fluent-bit/plugins/filter_throttle/CMakeLists.txt deleted file mode 100644 index adc7b8f4c..000000000 --- a/fluent-bit/plugins/filter_throttle/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(src - window.c - throttle.c - ) - -FLB_PLUGIN(filter_throttle "${src}" "") diff --git a/fluent-bit/plugins/filter_throttle/throttle.c b/fluent-bit/plugins/filter_throttle/throttle.c deleted file mode 100644 index 2a5ce29a4..000000000 --- a/fluent-bit/plugins/filter_throttle/throttle.c +++ /dev/null @@ -1,337 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "stdlib.h" - -#include "throttle.h" -#include "window.h" - -#include -#include - -pthread_mutex_t throttle_mut; - - -static bool apply_suffix (double *x, char suffix_char) -{ - int multiplier; - - switch (suffix_char) - { - case 0: - case 's': - multiplier = 1; - break; - case 'm': - multiplier = 60; - break; - case 'h': - multiplier = 60 * 60; - break; - case 'd': - multiplier = 60 * 60 * 24; - break; - default: - return false; - } - - *x *= multiplier; - - return true; -} - - -void *time_ticker(void *args) -{ - struct flb_time ftm; - long timestamp; - struct flb_filter_throttle_ctx *ctx = args; - - while (1) { - flb_time_get(&ftm); - timestamp = flb_time_to_double(&ftm); - pthread_mutex_lock(&throttle_mut); - window_add(ctx->hash, timestamp, 0); - - ctx->hash->current_timestamp = timestamp; - - if (ctx->print_status) { - flb_plg_info(ctx->ins, - "%ld: limit is %0.2f per %s with window size of %i, " - "current rate is: %i per interval", - timestamp, ctx->max_rate, ctx->slide_interval, - ctx->window_size, - ctx->hash->total / ctx->hash->size); - } - pthread_mutex_unlock(&throttle_mut); - /* sleep is a cancelable function */ - sleep(ctx->ticker_data.seconds); - } -} - -/* Given a msgpack record, do some filter action based on the defined rules */ -static inline int throttle_data(struct flb_filter_throttle_ctx *ctx) -{ - if ((ctx->hash->total / (double) ctx->hash->size) >= ctx->max_rate) { - return THROTTLE_RET_DROP; - } - - window_add(ctx->hash, ctx->hash->current_timestamp, 1); - - return THROTTLE_RET_KEEP; -} - -static int configure(struct flb_filter_throttle_ctx *ctx, struct flb_filter_instance *f_ins) -{ - int ret; - - ret = flb_filter_config_map_set(f_ins, ctx); - if (ret == -1) { - flb_plg_error(f_ins, "unable to load configuration"); - return -1; - } - if (ctx->max_rate <= 1.0) { - ctx->max_rate = strtod(THROTTLE_DEFAULT_RATE, NULL); - } - if (ctx->window_size <= 1) { - ctx->window_size = strtoul(THROTTLE_DEFAULT_WINDOW, NULL, 10); - } - - return 0; -} - -static int parse_duration(struct flb_filter_throttle_ctx *ctx, - const char *interval) -{ - double seconds = 0.0; - double s; - char *p; - - s = strtod(interval, &p); - if ( 0 >= s - /* No extra chars after the number and an optional s,m,h,d char. */ - || (*p && *(p+1)) - /* Check any suffix char and update S based on the suffix. */ - || ! apply_suffix (&s, *p)) - { - flb_plg_warn(ctx->ins, - "invalid time interval %s falling back to default: 1 " - "second", - interval); - } - - seconds += s; - return seconds; -} - -static int cb_throttle_init(struct flb_filter_instance *f_ins, - struct flb_config *config, - void *data) -{ - int ret; - struct flb_filter_throttle_ctx *ctx; - - pthread_mutex_init(&throttle_mut, NULL); - - /* Create context */ - ctx = flb_calloc(1, sizeof(struct flb_filter_throttle_ctx)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->ins = f_ins; - - /* parse plugin configuration */ - ret = configure(ctx, f_ins); - if (ret == -1) { - flb_free(ctx); - return -1; - } - - /* Set our context */ - flb_filter_set_context(f_ins, ctx); - - ctx->hash = window_create(ctx->window_size); - - ctx->ticker_data.seconds = parse_duration(ctx, ctx->slide_interval); - pthread_create(&ctx->ticker_data.thr, NULL, &time_ticker, ctx); - return 0; -} - -static int cb_throttle_filter(const void *data, size_t bytes, - const char *tag, int tag_len, - void **out_buf, size_t *out_size, - struct flb_filter_instance *f_ins, - struct flb_input_instance *i_ins, - void *context, - struct flb_config *config) -{ - int ret; - int old_size = 0; - int new_size = 0; - struct flb_log_event_encoder log_encoder; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - (void) f_ins; - (void) i_ins; - (void) config; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(f_ins, - "Log event decoder initialization error : %d", ret); - - return FLB_FILTER_NOTOUCH; - } - - ret = flb_log_event_encoder_init(&log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(f_ins, - "Log event encoder initialization error : %d", ret); - - flb_log_event_decoder_destroy(&log_decoder); - - return FLB_FILTER_NOTOUCH; - } - - /* Iterate each item array and apply rules */ - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - old_size++; - pthread_mutex_lock(&throttle_mut); - ret = throttle_data(context); - pthread_mutex_unlock(&throttle_mut); - - if (ret == THROTTLE_RET_KEEP) { - ret = flb_log_event_encoder_emit_raw_record( - &log_encoder, - &((char *) data)[log_decoder.previous_offset], - log_decoder.record_length); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - new_size++; - } - } - else if (ret == THROTTLE_RET_DROP) { - /* Do nothing */ - } - } - - /* we keep everything ? */ - if (old_size == new_size) { - /* Destroy the buffer to avoid more overhead */ - ret = FLB_FILTER_NOTOUCH; - } - else { - *out_buf = log_encoder.output_buffer; - *out_size = log_encoder.output_length; - - flb_log_event_encoder_claim_internal_buffer_ownership(&log_encoder); - - ret = FLB_FILTER_MODIFIED; - } - - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - return ret; -} - -static int cb_throttle_exit(void *data, struct flb_config *config) -{ - void *thr_res; - struct flb_filter_throttle_ctx *ctx = data; - - int s = pthread_cancel(ctx->ticker_data.thr); - if (s != 0) { - flb_plg_error(ctx->ins, "Unable to cancel ticker. Leaking context to avoid memory corruption."); - return 1; - } - - s = pthread_join(ctx->ticker_data.thr, &thr_res); - if (s != 0) { - flb_plg_error(ctx->ins, "Unable to join ticker. Leaking context to avoid memory corruption."); - return 1; - } - - if (thr_res != PTHREAD_CANCELED) { - flb_plg_error(ctx->ins, "Thread joined but was not canceled which is impossible."); - } - - flb_free(ctx->hash->table); - flb_free(ctx->hash); - flb_free(ctx); - return 0; -} - -static struct flb_config_map config_map[] = { - // rate - // window - // print_status - // interval - { - FLB_CONFIG_MAP_DOUBLE, "rate", THROTTLE_DEFAULT_RATE, - 0, FLB_TRUE, offsetof(struct flb_filter_throttle_ctx, max_rate), - "Set throttle rate" - }, - { - FLB_CONFIG_MAP_INT, "window", THROTTLE_DEFAULT_WINDOW, - 0, FLB_TRUE, offsetof(struct flb_filter_throttle_ctx, window_size), - "Set throttle window" - }, - { - FLB_CONFIG_MAP_BOOL, "print_status", THROTTLE_DEFAULT_STATUS, - 0, FLB_TRUE, offsetof(struct flb_filter_throttle_ctx, print_status), - "Set whether or not to print status information" - }, - { - FLB_CONFIG_MAP_STR, "interval", THROTTLE_DEFAULT_INTERVAL, - 0, FLB_TRUE, offsetof(struct flb_filter_throttle_ctx, slide_interval), - "Set the slide interval" - }, - /* EOF */ - {0} -}; - -struct flb_filter_plugin filter_throttle_plugin = { - .name = "throttle", - .description = "Throttle messages using sliding window algorithm", - .cb_init = cb_throttle_init, - .cb_filter = cb_throttle_filter, - .cb_exit = cb_throttle_exit, - .config_map = config_map, - .flags = 0 -}; diff --git a/fluent-bit/plugins/filter_throttle/throttle.h b/fluent-bit/plugins/filter_throttle/throttle.h deleted file mode 100644 index 30ca318c1..000000000 --- a/fluent-bit/plugins/filter_throttle/throttle.h +++ /dev/null @@ -1,56 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit Throttling - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_FILTER_THROTTLE_H -#define FLB_FILTER_THROTTLE_H - -#include -#include -#include - -/* actions */ -#define THROTTLE_RET_KEEP 0 -#define THROTTLE_RET_DROP 1 - -/* defaults */ -#define THROTTLE_DEFAULT_RATE "1" -#define THROTTLE_DEFAULT_WINDOW "5" -#define THROTTLE_DEFAULT_INTERVAL "1" -#define THROTTLE_DEFAULT_STATUS "false" - -struct ticker { - pthread_t thr; - double seconds; -}; - -struct flb_filter_throttle_ctx { - double max_rate; - unsigned int window_size; - const char *slide_interval; - int print_status; - - /* internal */ - struct throttle_window *hash; - struct flb_filter_instance *ins; - struct ticker ticker_data; -}; - - - -#endif diff --git a/fluent-bit/plugins/filter_throttle/window.c b/fluent-bit/plugins/filter_throttle/window.c deleted file mode 100644 index 75fcb492e..000000000 --- a/fluent-bit/plugins/filter_throttle/window.c +++ /dev/null @@ -1,97 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "window.h" -#include "throttle.h" - - -struct throttle_window *window_create(size_t size) { - struct throttle_window *tw; - - if (size <= 0) { - return NULL; - } - - tw = flb_malloc(sizeof(struct throttle_window)); - if (!tw) { - flb_errno(); - return NULL; - } - - tw->size = size; - tw->total = 0; - tw->current_timestamp = 0; - tw->max_index = -1; - tw->table = flb_calloc(size, sizeof(struct throttle_pane)); - if (!tw->table) { - flb_errno(); - flb_free(tw); - return NULL; - } - - return tw; -} - - -int window_get(struct throttle_window *tw, long timestamp) { - int i; - for (i=0; i< tw->size; i++ ) { - if (tw->table[i].timestamp == timestamp) { - return i; - } - } - return NOT_FOUND; -} - - -int window_add(struct throttle_window *tw, long timestamp, int val) { - int i, index, size; - int sum = 0; - tw->current_timestamp = timestamp; - - size = tw->size; - index = window_get(tw, timestamp); - - if (index == NOT_FOUND) { - if (size - 1 == tw->max_index) { - /* window must be shifted */ - tw->max_index = -1; - } - tw->max_index += 1; - tw->table[tw->max_index].timestamp= timestamp; - tw->table[tw->max_index].counter = val; - } else { - tw->table[index].counter += val; - } - - for (i=0; i < tw->size; i++ ) { - sum += tw->table[i].counter; - flb_debug("timestamp: %ld, value: %ld", - tw->table[i].timestamp, tw->table[i].counter); - } - tw->total = sum; - flb_debug("total: %i", tw->total); - return 0; -} diff --git a/fluent-bit/plugins/filter_throttle/window.h b/fluent-bit/plugins/filter_throttle/window.h deleted file mode 100644 index c7f392e07..000000000 --- a/fluent-bit/plugins/filter_throttle/window.h +++ /dev/null @@ -1,37 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define NOT_FOUND -1 - -struct throttle_pane { - long timestamp; - long counter; -}; - -struct throttle_window { - long current_timestamp; - unsigned size; - unsigned total; - pthread_mutex_t result_mutex; - int max_index; - struct throttle_pane *table; -}; - -struct throttle_window *window_create(size_t size); -int window_add(struct throttle_window *tw, long timestamp, int val); diff --git a/fluent-bit/plugins/filter_throttle_size/CMakeLists.txt b/fluent-bit/plugins/filter_throttle_size/CMakeLists.txt deleted file mode 100644 index 1cdf42b9d..000000000 --- a/fluent-bit/plugins/filter_throttle_size/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -set(src size_window.c throttle_size.c) - - FLB_PLUGIN(filter_throttle_size "${src}" "") diff --git a/fluent-bit/plugins/filter_throttle_size/size_window.c b/fluent-bit/plugins/filter_throttle_size/size_window.c deleted file mode 100644 index c3cfeb2e1..000000000 --- a/fluent-bit/plugins/filter_throttle_size/size_window.c +++ /dev/null @@ -1,226 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include - -#include "size_window.h" - -#ifdef _WIN32 -#include -#elif _M_X64 -#include -#else -#include -#endif - -/*This function create a new size throttling window named @name with @size number of panes. - The total amount of entries is 0 with timestamp set according to the current system time. - The name of the window is null terminated. The length of the name @name_lenght is used - for optimization when you use strings longer than the name you want. Otherwise use strlen(@name) - when such thing is not needed.*/ -struct throttle_size_window *size_window_create(const char *name, - unsigned name_length, - unsigned int size) -{ - struct throttle_size_window *stw; - struct flb_time ftm; - int i; - - if (size <= 0) { - return NULL; - } - - stw = flb_malloc(sizeof(struct throttle_size_window)); - if (!stw) { - flb_errno(); - return NULL; - } - - stw->size = size; - stw->total = 0; - stw->head = size - 1; - stw->tail = 0; - stw->table = flb_calloc(size, sizeof(struct throttle_size_pane)); - if (!stw->table) { - flb_errno(); - flb_free(stw); - return NULL; - } - - stw->name = flb_strndup(name, name_length); - - if (!stw->name) { - flb_errno(); - flb_free(stw->table); - flb_free(stw); - return NULL; - } - - flb_time_get(&ftm); - stw->timestamp = flb_time_to_double(&ftm); - - for (i = 0; i < size; i++) { - stw->table[i].timestamp = stw->timestamp; - stw->table[i].size = 0; - } - flb_debug - ("[filter_throttle_size] New size throttling window named \"%s\" was created.", - stw->name); - return stw; -} - -static inline void *create_lock() -{ -#ifdef _WIN32 - HANDLE lock = CreateMutex(NULL, // default security attributes - FALSE, // initially not owned - NULL); // unnamed mutex - if (lock == NULL) { - flb_error("CreateMutex error: %d\n", GetLastError()); - return NULL; - } - return lock; -#elif _M_X64 - HANDLE lock = CreateMutex(NULL, // default security attributes - FALSE, // initially not owned - NULL); // unnamed mutex - if (lock == NULL) { - flb_error("CreateMutex error: %d\n", GetLastError()); - return NULL; - } - return lock; -#else - pthread_mutex_t *lock = flb_malloc(sizeof(pthread_mutex_t)); - if (!lock) { - return NULL; - } - if (pthread_mutex_init(lock, NULL) != 0) { - flb_errno(); - return NULL; - } - return lock; -#endif -} - -void lock_throttle_size_table(struct throttle_size_table *ht) -{ -#ifdef _WIN32 - DWORD dwWaitResult = WaitForSingleObject(ht->lock, // handle to mutex - INFINITE); // no time-out interval - if (WAIT_ABANDONED == dwWaitResult) { - flb_warn - ("[filter_throttle_size]The thread got ownership of an abandoned mutex\nThe throttle_size_table is in an indeterminate state"); - } -#elif _M_X64 - DWORD dwWaitResult = WaitForSingleObject(ht->lock, // handle to mutex - INFINITE); // no time-out interval - if (WAIT_ABANDONED == dwWaitResult) { - flb_warn - ("[filter_throttle_size]The thread got ownership of an abandoned mutex\nThe throttle_size_table is in an indeterminate state"); - } -#else - pthread_mutex_lock(ht->lock); -#endif -} - -void unlock_throttle_size_table(struct throttle_size_table *ht) -{ -#ifdef _WIN32 - if (!ReleaseMutex(ht->lock)) { - flb_warn - ("[filter_throttle_size]Unable to release the ownership of throttle_size_table mutex!"); - } -#elif _M_X64 - if (!ReleaseMutex(ht->lock)) { - flb_warn - ("[filter_throttle_size]Unable to release the ownership of throttle_size_table mutex!"); - } -#else - pthread_mutex_unlock(ht->lock); -#endif -} - -static inline void destroy_throttle_size_table_lock(struct throttle_size_table - *ht) -{ -#ifdef _WIN32 - CloseHandle(ht->lock); -#elif _M_X64 - CloseHandle(ht->lock); -#else - pthread_mutex_destroy(ht->lock); - flb_free(ht->lock); -#endif -} - -struct throttle_size_table *create_throttle_size_table(size_t size) -{ - struct throttle_size_table *table; - table = flb_malloc(sizeof(struct throttle_size_table)); - if (!table) { - return NULL; - } - table->windows = - flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, size, - FLB_SIZE_WINDOW_HASH_MAX_ENTRIES); - if (!table->windows) { - flb_errno(); - flb_free(table); - return NULL; - } - table->lock = create_lock(); - if (!table->lock) { - flb_free(table); - return NULL; - } - return table; -} - -void destroy_throttle_size_table(struct throttle_size_table *ht) -{ - int i; - struct mk_list *tmp; - struct mk_list *head; - struct flb_hash_table_entry *entry; - struct flb_hash_table_chain *table; - - for (i = 0; i < ht->windows->size; i++) { - table = &ht->windows->table[i]; - mk_list_foreach_safe(head, tmp, &table->chains) { - entry = mk_list_entry(head, struct flb_hash_table_entry, _head); - free_stw_content((struct throttle_size_window *) entry->val); - mk_list_del(&entry->_head); - mk_list_del(&entry->_head_parent); - entry->table->count--; - ht->windows->total_count--; - flb_free(entry->key); - flb_free(entry->val); - flb_free(entry); - } - } - destroy_throttle_size_table_lock(ht); - flb_free(ht->windows->table); - flb_free(ht->windows); - flb_free(ht); -} diff --git a/fluent-bit/plugins/filter_throttle_size/size_window.h b/fluent-bit/plugins/filter_throttle_size/size_window.h deleted file mode 100644 index be661d6ec..000000000 --- a/fluent-bit/plugins/filter_throttle_size/size_window.h +++ /dev/null @@ -1,140 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_SIZE_WINDOW_H -#define FLB_SIZE_WINDOW_H - -#include - -#define FLB_SIZE_WINDOW_HASH_MAX_ENTRIES 100000 - -struct throttle_size_pane -{ - long timestamp; - unsigned long size; -}; - -struct throttle_size_window -{ - char *name; - unsigned size; - unsigned long total; - long timestamp; - int head; - int tail; - struct throttle_size_pane *table; -}; - -struct throttle_size_table -{ - struct flb_hash_table *windows; - void *lock; -}; - -struct throttle_size_table *create_throttle_size_table(); - -struct throttle_size_window *size_window_create(const char *name, - unsigned name_length, - unsigned int size); - -/*This function adds new pane on top of the pane stack by overwriting the oldes one - which @timestamp and load size of 0 bytes. The oldes pane's amount of load size - is subtracted of the total amount.*/ -inline static void add_new_pane(struct throttle_size_window *stw, - long timestamp) -{ - unsigned long tail_size = 0; - tail_size = stw->table[stw->tail].size; - if (stw->size - 1 == stw->head) { - /* the head will exceed the end of the inner array end must be put at the begging. */ - stw->head = -1; - } - stw->head += 1; - stw->table[stw->head].timestamp = timestamp; - stw->table[stw->head].size = 0; - stw->total -= tail_size; - if (stw->size - 1 == stw->tail) { - /* the tail will exceed the end of the inner array end must be put at the begging. */ - stw->tail = -1; - } - stw->tail += 1; -} - -/*This function adds @load to the latest pane which is on top of the pane stack. - @load is added to the total amount of the size throttling window. - If @load is not 0 then the size throttling window's timestamp will be updated to the - one which is on top of the pane stack(latest)*/ -inline static void add_load(struct throttle_size_window *stw, - unsigned long load) -{ - stw->table[stw->head].size += load; - stw->total += load; - if (load) { - stw->timestamp = stw->table[stw->head].timestamp; - } -} - -inline static void free_stw_content(struct throttle_size_window *stw) -{ - flb_free(stw->name); - flb_free(stw->table); -} - -inline static void free_stw(struct throttle_size_window *stw) -{ - free_stw_content(stw); - flb_free(stw); -} - -inline static struct throttle_size_window *find_throttle_size_window(struct - throttle_size_table - *table, - char - *name, - unsigned - name_length) -{ - char *window = NULL; - size_t out_size; - if (flb_hash_table_get(table->windows, name, name_length, - (const char **)&window, &out_size) >= 0) { - if (out_size < sizeof(struct throttle_size_window)) { - flb_error("Malformed data in size window hashtable"); - return NULL; - } - return (struct throttle_size_window *) window; - } - return NULL; -} - -inline static void add_throttle_size_window(struct throttle_size_table - *table, - struct throttle_size_window - *window) -{ - flb_hash_table_add(table->windows, window->name, strlen(window->name), - (char *) window, sizeof(struct throttle_size_window)); -} - -void destroy_throttle_size_table(struct throttle_size_table *table); - -void lock_throttle_size_table(struct throttle_size_table *ht); -void unlock_throttle_size_table(struct throttle_size_table *ht); - -#endif diff --git a/fluent-bit/plugins/filter_throttle_size/throttle_size.c b/fluent-bit/plugins/filter_throttle_size/throttle_size.c deleted file mode 100644 index 31abc3df3..000000000 --- a/fluent-bit/plugins/filter_throttle_size/throttle_size.c +++ /dev/null @@ -1,774 +0,0 @@ - /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "stdlib.h" -#include -#include - - -#include "throttle_size.h" - -#undef PLUGIN_NAME -#define PLUGIN_NAME "filter_throttle_size" -#define RELATIVE_ERROR 0.001 -#define KEY_DEPTH 20 -#define SPLIT_DELIMITER '|' - -struct field_key -{ - char *key; - int key_len; - struct mk_list _head; -}; - -static bool apply_suffix(double *x, char suffix_char) -{ - int multiplier; - - switch (suffix_char) { - case 0: - case 's': - multiplier = 1; - break; - case 'm': - multiplier = 60; - break; - case 'h': - multiplier = 60 * 60; - break; - case 'd': - multiplier = 60 * 60 * 24; - break; - default: - return false; - } - - *x *= multiplier; - - return true; -} - -/* - * add_new_pane_to_each will overides the oldes window pane with zero load and - * with new timestamp make it the newest. - */ -inline static void add_new_pane_to_each(struct throttle_size_table *ht, - double timestamp) -{ - struct mk_list *head; - struct flb_hash_table_entry *entry; - struct throttle_size_window *current_window; - struct flb_time ftm; - - if (!timestamp) { - flb_time_get(&ftm); - timestamp = flb_time_to_double(&ftm); - } - - mk_list_foreach(head, &ht->windows->entries) { - entry = mk_list_entry(head, struct flb_hash_table_entry, _head_parent); - current_window = (struct throttle_size_window *) (entry->val); - add_new_pane(current_window, timestamp); - flb_debug - ("[%s] Add new pane to \"%s\" window: timestamp: %ld, total %lu", - PLUGIN_NAME, current_window->name, - current_window->table[current_window->head].timestamp, - current_window->total); - } -} - -inline static void delete_older_than_n_seconds(struct throttle_size_table *ht, - long seconds, - double current_timestamp) -{ - int i; - struct mk_list *tmp; - struct mk_list *head; - struct flb_hash_table_entry *entry; - struct flb_hash_table_chain *table; - struct throttle_size_window *current_window; - struct flb_time ftm; - long time_treshold; - - if (!current_timestamp) { - flb_time_get(&ftm); - current_timestamp = flb_time_to_double(&ftm); - } - - time_treshold = current_timestamp - seconds; - for (i = 0; i < ht->windows->size; i++) { - table = &ht->windows->table[i]; - mk_list_foreach_safe(head, tmp, &table->chains) { - entry = mk_list_entry(head, struct flb_hash_table_entry, _head); - current_window = (struct throttle_size_window *) entry->val; - - if (time_treshold > current_window->timestamp) { - free_stw_content(current_window); - mk_list_del(&entry->_head); - mk_list_del(&entry->_head_parent); - entry->table->count--; - ht->windows->total_count--; - flb_free(entry->key); - flb_free(entry->val); - flb_free(entry); - flb_info - ("[%s] Window \"%s\" was deleted. CT%ld TT%ld T%ld ", - PLUGIN_NAME, current_window->name, - (long) current_timestamp, time_treshold, - current_window->timestamp); - } - } - } -} - -inline static void print_all(struct throttle_size_table *ht) -{ - struct mk_list *head; - struct flb_hash_table_entry *entry; - struct throttle_size_window *current_window; - - mk_list_foreach(head, &ht->windows->entries) { - entry = mk_list_entry(head, struct flb_hash_table_entry, _head_parent); - current_window = (struct throttle_size_window *) entry->val; - printf("[%s] Name %s\n", PLUGIN_NAME, current_window->name); - printf("[%s] Timestamp %ld\n", PLUGIN_NAME, - current_window->timestamp); - printf("[%s] Total %lu\n", PLUGIN_NAME, current_window->total); - printf("[%s] Rate %f\n", PLUGIN_NAME, - current_window->total / (double) current_window->size); - } -} - -void *size_time_ticker(void *args) -{ - struct flb_filter_throttle_size_ctx *ctx = args; - struct flb_time ftm; - long timestamp; - - while (!ctx->done) { - flb_time_get(&ftm); - timestamp = flb_time_to_double(&ftm); - - lock_throttle_size_table(ctx->hash); - add_new_pane_to_each(ctx->hash, timestamp); - delete_older_than_n_seconds(ctx->hash, - ctx->window_time_duration, timestamp); - if (ctx->print_status) { - print_all(ctx->hash); - } - unlock_throttle_size_table(ctx->hash); - - sleep(ctx->slide_interval); - } - - return NULL; -} - -/* Check if a msgpack type is either binary or string */ -static inline int is_valid_key(const msgpack_object key_as_msgpack) -{ - return key_as_msgpack.type == MSGPACK_OBJECT_BIN || - key_as_msgpack.type == MSGPACK_OBJECT_STR; -} - -/* - * If msgpack can be represented as string get_msgobject_as_str returns that - * representation - */ -static inline uint32_t get_msgobject_as_str(const msgpack_object msg, - char **out) -{ - if (msg.type == MSGPACK_OBJECT_STR) { - *out = (char *) msg.via.str.ptr; - return (uint32_t) msg.via.str.size; - } - if (msg.type == MSGPACK_OBJECT_BIN) { - *out = (char *) msg.via.bin.ptr; - return (uint32_t) msg.via.bin.size; - } - *out = NULL; - return (uint32_t) 0; -} - -static inline unsigned long get_msgpack_object_size(msgpack_object msg) -{ - int i; - unsigned long size = 0; - - switch (msg.type) { - case MSGPACK_OBJECT_STR: - return msg.via.str.size; - case MSGPACK_OBJECT_BIN: - return msg.via.bin.size; - case MSGPACK_OBJECT_MAP: - for (i = 0; i < msg.via.map.size; i++) { - size += get_msgpack_object_size(msg.via.map.ptr[i].key); - size += get_msgpack_object_size(msg.via.map.ptr[i].val); - } - return size; - case MSGPACK_OBJECT_ARRAY: - for (i = 0; i < msg.via.array.size; i++) { - size += get_msgpack_object_size(msg.via.array.ptr[i]); - } - return size; - default: - return 0; - }; - - return 0; -} - -/* - * get_value_of_msgpack_object_map_ search in msgpack_object map for @key - * and returns the value as msgpack_object if key is found or return NULL - * if not. This is helper function to get_value_of_msgpack_object_map - */ -static inline const msgpack_object *get_value_of_msgpack_object_map_(msgpack_object map, - struct field_key *key) -{ - int i; - int current_field_size; - char *current_field = NULL; - - /* Lookup target key/value */ - for (i = 0; i < map.via.map.size; i++) { - if (!is_valid_key(map.via.map.ptr[i].key)) { - continue; - } - - current_field_size = get_msgobject_as_str(map.via.map.ptr[i].key, ¤t_field); - if (key->key_len != current_field_size) { - continue; - } - - if (strncmp(key->key, current_field, current_field_size) != 0) { - continue; - } - - return &map.via.map.ptr[i].val; - } - - return NULL; -} - -/* - * get_value_of_msgpack_object_map search in msgpack_object map for @key and - * returns the value as msgpack_object if key is found or return NULL if - * not. @key is a list of strings representing the nested key. Each - * element in thje list represent the next element in depth. - */ -const msgpack_object *get_value_of_msgpack_object_map(msgpack_object map, - const struct mk_list *fields_name) -{ - struct mk_list *head = NULL; - struct field_key *field; - const msgpack_object *msg = ↦ - - mk_list_foreach(head, fields_name) { - field = mk_list_entry(head, struct field_key, _head); - msg = get_value_of_msgpack_object_map_(*msg, field); - if (msg == NULL) { - /* not found */ - flb_debug("Could not found field named %s", field->key); - return NULL; - } - } - - return msg; -} - -/* Given a msgpack record, do some filter action based on the defined rules */ -static inline int throttle_data_by_size(msgpack_object map, - struct flb_filter_throttle_size_ctx *ctx) -{ - char *name_field_str = NULL; - uint32_t name_field_size; - unsigned long load_size; - double current_rate; - struct throttle_size_window *window; - const msgpack_object *log_field; - const msgpack_object *name_field; - - if (ctx->name_fields_depth > 0) { - /* - * We are looking for a message with a specific field. The other will - * not be taken to account. - */ - name_field = get_value_of_msgpack_object_map(map, &ctx->name_fields); - if (name_field == NULL) { - /* We don't have such field so we keep the log */ - flb_plg_debug(ctx->ins, "the name field is missing, so we are keeping " - "the log"); - return throttle_size_RET_KEEP; - } - name_field_size = get_msgobject_as_str(*name_field, &name_field_str); - if (name_field_str == NULL) { - /* We don't have such field so we keep the log */ - flb_plg_info(ctx->ins, "the value of the name field is nether string " - "not binary format. The log will not be throttle"); - return throttle_size_RET_KEEP; - } - flb_plg_debug(ctx->ins, "field name found"); - } - else { - flb_plg_debug(ctx->ins, "using default field name. All log will be taken " - "to account"); - - /* take all logs into account */ - name_field_str = throttle_size_DEFAULT_NAME_FIELD; - name_field_size = strlen(throttle_size_DEFAULT_NAME_FIELD); - } - - if (ctx->log_fields_depth > 0) { - /* we are looking for specific field and we will take only its size */ - log_field = get_value_of_msgpack_object_map(map, &ctx->log_fields); - if (log_field == NULL) { - flb_plg_debug(ctx->ins, - "the log field is missing so we are keeping this log"); - return throttle_size_RET_KEEP; - } - flb_plg_debug(ctx->ins, "log field found"); - load_size = get_msgpack_object_size(*log_field); - } - else { - flb_plg_debug(ctx->ins, "using default log field name. All fields will be " - "taken into account"); - load_size = get_msgpack_object_size(map); - } - flb_plg_debug(ctx->ins, "load size is %lu", load_size); - - lock_throttle_size_table(ctx->hash); - - window = find_throttle_size_window(ctx->hash, name_field_str, name_field_size); - if (window == NULL) { - /* - * Since Fluent Bit works on one thread and there is no chance someone - * to create the same window so we can unlock the mutex to give it to the - * ticker. - */ - unlock_throttle_size_table(ctx->hash); - current_rate = load_size / (double) ctx->window_size; - if (current_rate - ctx->max_size_rate > RELATIVE_ERROR) { - flb_plg_info(ctx->ins, "load is too much for window \"%*.*s\". " - "The log record will be dropped", - name_field_size, name_field_str); - return throttle_size_RET_DROP; - } - - window = size_window_create(name_field_str, name_field_size, - ctx->window_size); - if (window == NULL) { - flb_plg_warn(ctx->ins, "not enough memory. Log will be kept.", - load_size); - return throttle_size_RET_KEEP; - } - - add_load(window, load_size); - flb_plg_debug(ctx->ins, "add %lu bytes to \"%s\" window: " - "timestamp: %ld, total %lu", - load_size, window->name, - window->table[window->head].timestamp, window->total); - lock_throttle_size_table(ctx->hash); - add_throttle_size_window(ctx->hash, window); - unlock_throttle_size_table(ctx->hash); - flb_plg_debug(ctx->ins, "new window named \"%s\" was added with load %lu.", - window->name, load_size); - flb_free(window); - } - else { - /* - * We found the wanted window and now we are going to make check and - * modify it if needed - */ - flb_plg_debug(ctx->ins, "current rate is %.2f for windoe \"%s\"", - ((window->total + load_size) / (double) window->size), - window->name); - - current_rate = (window->total + load_size) / (double) ctx->window_size; - - if (current_rate - ctx->max_size_rate > RELATIVE_ERROR) { - unlock_throttle_size_table(ctx->hash); - flb_plg_info(ctx->ins, "load is too much. The log %*.*s record " - "will be dropped.", - load_size, name_field_size, name_field_str); - return throttle_size_RET_DROP; - } - add_load(window, load_size); - flb_plg_debug(ctx->ins, "add %lu bytes to \"%s\" window: " - "timestamp: %ld, total %lu", load_size, window->name, - window->table[window->head].timestamp, window->total); - unlock_throttle_size_table(ctx->hash); - flb_plg_debug(ctx->ins, "load of %lu was added and the message was kept", - load_size); - } - - return throttle_size_RET_KEEP; -} - -/* - * load_field_key_list split @str into list of string representing the depth - * of a nested key. - * - * The split is base on SPLIT_DELIMITER - */ -static inline int load_field_key_list(char *str, struct mk_list *the_list, - size_t *list_size) -{ - struct mk_list *split; - struct mk_list *head = NULL; - struct field_key *fk; - struct flb_split_entry *entry; - - *list_size = 0; - mk_list_init(the_list); - - if (str != NULL) { - split = flb_utils_split(str, SPLIT_DELIMITER, KEY_DEPTH); - if (mk_list_size(split) < 1) { - return 0; - } - mk_list_foreach(head, split) { - fk = flb_malloc(sizeof(struct field_key)); - if (!fk) { - flb_errno(); - flb_utils_split_free(split); - return -1; - } - - entry = mk_list_entry(head, struct flb_split_entry, _head); - - fk->key = strndup(entry->value, entry->len); - fk->key_len = entry->len; - mk_list_add(&fk->_head, the_list); - (*list_size)++; - } - - flb_utils_split_free(split); - } - return 0; -} - -static int parse_duration(char *interval, int default_seconds, - struct flb_filter_throttle_size_ctx *ctx) -{ - double seconds = 0.0; - double s; - char *p; - - s = strtod(interval, &p); - if (0 >= s - /* No extra chars after the number and an optional s,m,h,d char. */ - || (*p && *(p + 1)) - /* Check any suffix char and update S based on the suffix. */ - || !apply_suffix(&s, *p)) { - flb_plg_warn(ctx->ins, "invalid time interval %s falling back to " - "default: %d second", - interval, default_seconds); - return default_seconds; - } - - seconds += s; - return seconds; -} - -static inline int configure(struct flb_filter_throttle_size_ctx *ctx, - struct flb_filter_instance *ins) -{ - const char *str = NULL; - double val = 0; - char *endp; - ssize_t bytes; - - ctx->name_fields_depth = 0; - - /* rate per second */ - str = flb_filter_get_property("rate", ins); - if (str) { - bytes = flb_utils_size_to_bytes(str); - if (bytes > 0) { - ctx->max_size_rate = (double) bytes; - } - else { - ctx->max_size_rate = throttle_size_DEFAULT_RATE; - } - } - else { - ctx->max_size_rate = throttle_size_DEFAULT_RATE; - } - - /* windows size */ - str = flb_filter_get_property("window", ins); - if (str != NULL && (val = strtoul(str, &endp, 10)) > 1) { - ctx->window_size = val; - } - else { - ctx->window_size = throttle_size_DEFAULT_WINDOW; - } - - /* print informational status */ - str = flb_filter_get_property("print_status", ins); - if (str != NULL) { - ctx->print_status = flb_utils_bool(str); - } - else { - ctx->print_status = throttle_size_DEFAULT_STATUS; - } - - /* sliding interval */ - str = flb_filter_get_property("interval", ins); - if (str != NULL) { - ctx->slide_interval = - parse_duration((char *) str, throttle_size_DEFAULT_INTERVAL, ctx); - } - else { - ctx->slide_interval = throttle_size_DEFAULT_INTERVAL; - } - - /* the field which size will be taken into account */ - str = flb_filter_get_property("log_field", ins); - if (load_field_key_list((char *) str, &ctx->log_fields, &ctx->log_fields_depth)) { - return -1; - } - - str = NULL; - - /* the field base on which new throttling window will be created */ - str = flb_filter_get_property("name_field", ins); - if (load_field_key_list((char *) str, &ctx->name_fields, &ctx->name_fields_depth)) { - return -1; - } - - /* - * The time after which the window will be delete when there is no log size - * recorded to it - */ - str = flb_filter_get_property("window_time_duration", ins); - if (str != NULL) { - ctx->window_time_duration = - parse_duration((char *) str, throttle_size_DEFAULT_WINDOW_DURATION, ctx); - } - else { - ctx->window_time_duration = throttle_size_DEFAULT_WINDOW_DURATION; - } - - /* Create the hash table of windows */ - str = flb_filter_get_property("hash_table_size", ins); - if (str != NULL && (val = strtoul(str, &endp, 10)) > 0) { - ctx->hash = create_throttle_size_table(val); - } - else { - ctx->hash = - create_throttle_size_table - (throttle_size_WINDOW_TABLE_DEFAULT_SIZE); - } - if (ctx->hash == NULL) { - flb_errno(); - return -1; - } - - return 0; -} - -static int cb_throttle_size_init(struct flb_filter_instance *ins, - struct flb_config *config, void *data) -{ - int ret; - struct flb_filter_throttle_size_ctx *ctx; - struct throttle_size_window *window; - - /* Create context */ - ctx = flb_calloc(1, sizeof(struct flb_filter_throttle_size_ctx)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->ins = ins; - - /* parse plugin configuration */ - ret = configure(ctx, ins); - if (ret == -1) { - flb_free(ctx); - return -1; - } - - if (ctx->window_time_duration < ctx->slide_interval * ctx->window_size) { - ctx->window_time_duration = ctx->slide_interval * ctx->window_size; - } - - /* - * if we specify "*" as a name field then all logs will be under - * the same window which we must make at initial time to save - * some checks later - */ - if (ctx->name_fields_depth == 0) { - window = size_window_create(throttle_size_DEFAULT_NAME_FIELD, - strlen(throttle_size_DEFAULT_NAME_FIELD), - ctx->window_size); - if (window == NULL) { - flb_free(ctx); - flb_errno(); - return -1; - } - add_throttle_size_window(ctx->hash, window); - flb_free(window); - } - - ctx->ticker_id = flb_malloc(sizeof(pthread_t)); - if (!ctx->ticker_id) { - flb_errno(); - return -1; - } - - ctx->done = false; - pthread_create((pthread_t *) ctx->ticker_id, NULL, &size_time_ticker, - ctx); - - /* Set our context */ - flb_filter_set_context(ins, ctx); - - return 0; -} - -static int cb_throttle_size_filter(const void *data, size_t bytes, - const char *tag, int tag_len, - void **out_buf, size_t * out_size, - struct flb_filter_instance *ins, - struct flb_input_instance *i_ins, - void *context, struct flb_config *config) -{ - int ret; - int old_size = 0; - int new_size = 0; - struct flb_log_event_encoder log_encoder; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - (void) ins; - (void) i_ins; - (void) config; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ins, - "Log event decoder initialization error : %d", ret); - - return FLB_FILTER_NOTOUCH; - } - - ret = flb_log_event_encoder_init(&log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ins, - "Log event encoder initialization error : %d", ret); - - flb_log_event_decoder_destroy(&log_decoder); - - return FLB_FILTER_NOTOUCH; - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - old_size++; - - ret = throttle_data_by_size(*log_event.body, context); - - if (ret == throttle_size_RET_KEEP) { - ret = flb_log_event_encoder_emit_raw_record( - &log_encoder, - log_decoder.record_base, - log_decoder.record_length); - - new_size++; - } - else if (ret == throttle_size_RET_DROP) { - /* Do nothing */ - } - } - - /* we keep everything ? */ - if (old_size == new_size) { - /* Destroy the buffer to avoid more overhead */ - ret = FLB_FILTER_NOTOUCH; - } - else { - *out_buf = log_encoder.output_buffer; - *out_size = log_encoder.output_length; - - flb_log_event_encoder_claim_internal_buffer_ownership(&log_encoder); - - ret = FLB_FILTER_MODIFIED; - } - - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - return ret; -} - -static void delete_field_key(struct mk_list *head) -{ - struct mk_list *curr = NULL, *n = NULL; - struct field_key *field; - - mk_list_foreach_safe(curr, n, head) { - field = mk_list_entry(curr, struct field_key, _head); - mk_list_del(curr); - flb_free(field->key); - flb_free(field); - } -} - -static int cb_throttle_size_exit(void *data, struct flb_config *config) -{ - struct flb_filter_throttle_size_ctx *ctx = data; - - ctx->done = true; - pthread_join(*(pthread_t *) ctx->ticker_id, NULL); - - flb_free(ctx->ticker_id); - destroy_throttle_size_table(ctx->hash); - delete_field_key(&ctx->log_fields); - delete_field_key(&ctx->name_fields); - flb_free(ctx); - - return 0; -} - -struct flb_filter_plugin filter_throttle_size_plugin = { - .name = "throttle_size", - .description = "Throttle messages by size using sliding window algorithm", - .cb_init = cb_throttle_size_init, - .cb_filter = cb_throttle_size_filter, - .cb_exit = cb_throttle_size_exit, - .flags = 0 -}; diff --git a/fluent-bit/plugins/filter_throttle_size/throttle_size.h b/fluent-bit/plugins/filter_throttle_size/throttle_size.h deleted file mode 100644 index 6a28e4613..000000000 --- a/fluent-bit/plugins/filter_throttle_size/throttle_size.h +++ /dev/null @@ -1,60 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_SIZE_FILTER_THROTTLE_H -#define FLB_SIZE_FILTER_THROTTLE_H - -/* actions */ -#define throttle_size_RET_KEEP 0 -#define throttle_size_RET_DROP 1 - -/* defaults */ -#define throttle_size_DEFAULT_RATE 1024*1024 /* bytes */ -#define throttle_size_DEFAULT_WINDOW 5 -#define throttle_size_DEFAULT_INTERVAL 1 -#define throttle_size_DEFAULT_STATUS FLB_FALSE; -#define throttle_size_DEFAULT_LOG_FIELD "*" -#define throttle_size_DEFAULT_NAME_FIELD "*" -#define throttle_size_DEFAULT_WINDOW_DURATION 60 -#define throttle_size_WINDOW_TABLE_DEFAULT_SIZE 256 - -#include "size_window.h" - -struct flb_filter_throttle_size_ctx -{ - int slide_interval; - int window_time_duration; - double max_size_rate; - unsigned int window_size; - size_t log_fields_depth; - size_t name_fields_depth; - void *ticker_id; - int print_status; - - volatile bool done; - - struct mk_list name_fields; - struct mk_list log_fields; - - /* internal */ - struct throttle_size_table *hash; - struct flb_filter_instance *ins; -}; - -#endif diff --git a/fluent-bit/plugins/filter_type_converter/CMakeLists.txt b/fluent-bit/plugins/filter_type_converter/CMakeLists.txt deleted file mode 100644 index 5938e1b6e..000000000 --- a/fluent-bit/plugins/filter_type_converter/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -set(src - type_converter.c) - -FLB_PLUGIN(filter_type_converter "${src}" "") - diff --git a/fluent-bit/plugins/filter_type_converter/type_converter.c b/fluent-bit/plugins/filter_type_converter/type_converter.c deleted file mode 100644 index 36422002c..000000000 --- a/fluent-bit/plugins/filter_type_converter/type_converter.c +++ /dev/null @@ -1,399 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "type_converter.h" - -static int delete_conv_entry(struct conv_entry *conv) -{ - if (conv == NULL) { - return 0; - } - - if (conv->from_key != NULL) { - flb_sds_destroy(conv->from_key); - conv->from_key = NULL; - } - if (conv->to_key != NULL) { - flb_sds_destroy(conv->to_key); - conv->to_key = NULL; - } - if (conv->rule != NULL) { - flb_typecast_rule_destroy(conv->rule); - } - if (conv->from_ra != NULL) { - flb_ra_destroy(conv->from_ra); - } - mk_list_del(&conv->_head); - flb_free(conv); - return 0; -} - -static int config_rule(struct type_converter_ctx *ctx, char* type_name, - struct flb_config_map_val *mv) -{ - struct conv_entry *entry = NULL; - struct flb_slist_entry *sentry = NULL; - - if (ctx == NULL || mv == NULL) { - return -1; - } - - entry = flb_calloc(1, sizeof(struct conv_entry)); - if (entry == NULL) { - flb_errno(); - return -1; - } - - entry->rule = NULL; - if (mk_list_size(mv->val.list) != 3) { - flb_plg_error(ctx->ins, "invalid record parameters, " - "expects 'from_key to_key type' %d", mk_list_size(mv->val.list)); - flb_free(entry); - return -1; - } - - /* from_key name */ - sentry = mk_list_entry_first(mv->val.list, struct flb_slist_entry, _head); - entry->from_key = flb_sds_create_len(sentry->str, flb_sds_len(sentry->str)); - - /* to_key name */ - sentry = mk_list_entry_next(&sentry->_head, struct flb_slist_entry, - _head, mv->val.list); - entry->to_key = flb_sds_create_len(sentry->str, flb_sds_len(sentry->str)); - - sentry = mk_list_entry_last(mv->val.list, struct flb_slist_entry, _head); - entry->rule = flb_typecast_rule_create(type_name, strlen(type_name), - sentry->str, - flb_sds_len(sentry->str)); - entry->from_ra = flb_ra_create(entry->from_key, FLB_FALSE); - if (entry->rule == NULL || entry->from_ra == NULL) { - flb_plg_error(ctx->ins, - "configuration error. ignore the key=%s", - entry->from_key); - delete_conv_entry(entry); - return -1; - } - - mk_list_add(&entry->_head, &ctx->conv_entries); - - return 0; -} - -static int configure(struct type_converter_ctx *ctx, - struct flb_filter_instance *f_ins) -{ - struct mk_list *head = NULL; - struct flb_config_map_val *mv = NULL; - - if (flb_filter_config_map_set(f_ins, ctx) < 0) { - flb_errno(); - flb_plg_error(f_ins, "configuration error"); - return -1; - } - - /* Create rules for each type */ - flb_config_map_foreach(head, mv, ctx->str_keys) { - config_rule(ctx, "string", mv); - } - flb_config_map_foreach(head, mv, ctx->int_keys) { - config_rule(ctx, "int", mv); - } - flb_config_map_foreach(head, mv, ctx->uint_keys) { - config_rule(ctx, "uint", mv); - } - flb_config_map_foreach(head, mv, ctx->float_keys) { - config_rule(ctx, "float", mv); - } - - if (mk_list_size(&ctx->conv_entries) == 0) { - flb_plg_error(ctx->ins, "no rules"); - return -1; - } - - return 0; -} - -static int delete_list(struct type_converter_ctx *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - struct conv_entry *conv; - - mk_list_foreach_safe(head, tmp, &ctx->conv_entries) { - conv = mk_list_entry(head, struct conv_entry, _head); - delete_conv_entry(conv); - } - return 0; -} - -static int cb_type_converter_init(struct flb_filter_instance *ins, - struct flb_config *config, - void *data) -{ - struct type_converter_ctx *ctx = NULL; - int ret = 0; - - ctx = flb_calloc(1, sizeof(struct type_converter_ctx)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->ins = ins; - mk_list_init(&ctx->conv_entries); - - ret = configure(ctx, ins); - if (ret < 0) { - flb_plg_error(ins, "configuration error"); - flb_free(ctx); - return -1; - } - /* set context */ - flb_filter_set_context(ins, ctx); - - return 0; -} - -static int cb_type_converter_filter(const void *data, size_t bytes, - const char *tag, int tag_len, - void **out_buf, size_t *out_bytes, - struct flb_filter_instance *f_ins, - struct flb_input_instance *i_ins, - void *filter_context, - struct flb_config *config) -{ - struct type_converter_ctx *ctx = filter_context; - struct flb_time tm; - int i; - int map_num; - int is_record_modified = FLB_FALSE; - int ret; - msgpack_sbuffer tmp_sbuf; - msgpack_packer tmp_pck; - msgpack_object *obj; - struct conv_entry *entry; - struct mk_list *tmp; - struct mk_list *head; - - msgpack_object *start_key; - msgpack_object *out_key; - msgpack_object *out_val; - struct flb_log_event_encoder log_encoder; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - (void) f_ins; - (void) i_ins; - (void) config; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(f_ins, - "Log event decoder initialization error : %d", ret); - - return FLB_FILTER_NOTOUCH; - } - - ret = flb_log_event_encoder_init(&log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(f_ins, - "Log event encoder initialization error : %d", ret); - - flb_log_event_decoder_destroy(&log_decoder); - - return FLB_FILTER_NOTOUCH; - } - - /* Create temporary msgpack buffer */ - msgpack_sbuffer_init(&tmp_sbuf); - msgpack_packer_init(&tmp_pck, &tmp_sbuf, msgpack_sbuffer_write); - - /* Iterate each item to know map number */ - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - - flb_time_copy(&tm, &log_event.timestamp); - obj = log_event.body; - - map_num = obj->via.map.size; - - ret = flb_log_event_encoder_begin_record(&log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_timestamp(&log_encoder, &tm); - } - - ret = flb_log_event_encoder_set_metadata_from_msgpack_object( - &log_encoder, - log_event.metadata); - - /* write original k/v */ - for (i = 0; - i < map_num && - ret == FLB_EVENT_ENCODER_SUCCESS; - i++) { - ret = flb_log_event_encoder_append_body_values( - &log_encoder, - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&obj->via.map.ptr[i].key), - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&obj->via.map.ptr[i].val)); - } - - mk_list_foreach_safe(head, tmp, &ctx->conv_entries) { - start_key = NULL; - out_key = NULL; - out_val = NULL; - - entry = mk_list_entry(head, struct conv_entry, _head); - ret = flb_ra_get_kv_pair(entry->from_ra, *obj, &start_key, &out_key, &out_val); - if (start_key == NULL || out_key == NULL || out_val == NULL) { - ret = FLB_EVENT_ENCODER_SUCCESS; - - continue; - } - - /* key is found. try to convert. */ - ret = flb_log_event_encoder_append_body_string( - &log_encoder, - entry->to_key, - flb_sds_len(entry->to_key)); - - ret = flb_typecast_pack(*out_val, entry->rule, &tmp_pck); - if (ret < 0) { - /* failed. try to write original val... */ - flb_plg_error(ctx->ins, "failed to convert. key=%s", entry->from_key); - - ret = flb_log_event_encoder_append_body_msgpack_object( - &log_encoder, - out_val); - - continue; - } - else { - ret = flb_log_event_encoder_append_body_raw_msgpack( - &log_encoder, - tmp_sbuf.data, tmp_sbuf.size); - - msgpack_sbuffer_clear(&tmp_sbuf); - } - - is_record_modified = FLB_TRUE; - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_log_event_encoder_commit_record(&log_encoder); - } - else { - flb_log_event_encoder_rollback_record(&log_encoder); - } - - } - msgpack_sbuffer_destroy(&tmp_sbuf); - - if (is_record_modified != FLB_TRUE) { - /* Destroy the buffer to avoid more overhead */ - flb_plg_trace(ctx->ins, "no touch"); - - ret = FLB_FILTER_NOTOUCH; - } - else { - if (ret == FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA && - log_decoder.offset == bytes) { - ret = FLB_EVENT_ENCODER_SUCCESS; - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - *out_buf = log_encoder.output_buffer; - *out_bytes = log_encoder.output_length; - - ret = FLB_FILTER_MODIFIED; - - flb_log_event_encoder_claim_internal_buffer_ownership(&log_encoder); - } - else { - flb_plg_error(ctx->ins, - "Log event encoder error : %d", ret); - - ret = FLB_FILTER_NOTOUCH; - } - } - - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - return ret; -} - -static int cb_type_converter_exit(void *data, struct flb_config *config) { - struct type_converter_ctx *ctx = data; - - if (ctx == NULL) { - return 0; - } - delete_list(ctx); - flb_free(ctx); - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_SLIST_3, "int_key", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct type_converter_ctx, int_keys), - "Convert integer to other type. e.g. int_key id id_str string" - }, - { - FLB_CONFIG_MAP_SLIST_3, "uint_key", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct type_converter_ctx, uint_keys), - "Convert unsinged integer to other type. e.g. uint_key id id_str string" - }, - { - FLB_CONFIG_MAP_SLIST_3, "float_key", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct type_converter_ctx, float_keys), - "Convert float to other type. e.g. float_key ratio id_str string" - }, - { - FLB_CONFIG_MAP_SLIST_3, "str_key", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct type_converter_ctx, str_keys), - "Convert string to other type. e.g. str_key id id_val integer" - }, - {0} -}; - - -struct flb_filter_plugin filter_type_converter_plugin = { - .name = "type_converter", - .description = "Data type converter", - .cb_init = cb_type_converter_init, - .cb_filter = cb_type_converter_filter, - .cb_exit = cb_type_converter_exit, - .config_map = config_map, - .flags = 0, -}; diff --git a/fluent-bit/plugins/filter_type_converter/type_converter.h b/fluent-bit/plugins/filter_type_converter/type_converter.h deleted file mode 100644 index cd7540351..000000000 --- a/fluent-bit/plugins/filter_type_converter/type_converter.h +++ /dev/null @@ -1,46 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_FILTER_TYPE_CONVERTER_H -#define FLB_FILTER_TYPE_CONVERTER_H - -#include -#include -#include -#include - -struct conv_entry { - flb_sds_t from_key; - struct flb_record_accessor *from_ra; - flb_sds_t to_key; - struct flb_typecast_rule *rule; - struct mk_list _head; -}; - -struct type_converter_ctx { - struct mk_list conv_entries; - struct flb_filter_instance *ins; - /* config maps */ - struct mk_list *int_keys; - struct mk_list *uint_keys; - struct mk_list *float_keys; - struct mk_list *str_keys; -}; - -#endif diff --git a/fluent-bit/plugins/filter_wasm/CMakeLists.txt b/fluent-bit/plugins/filter_wasm/CMakeLists.txt deleted file mode 100644 index e8e1f5ad4..000000000 --- a/fluent-bit/plugins/filter_wasm/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -set(WAMR_ROOT_DIR ../../${FLB_PATH_LIB_WASM_MICRO_RUNTIME}) -set(WASM_INCLUDE_DIRS - ${WAMR_ROOT_DIR}/core/iwasm/include - ) - -set(src - filter_wasm.c) - -FLB_PLUGIN(filter_wasm "${src}" "") -target_include_directories(flb-plugin-filter_wasm PRIVATE ${WASM_INCLUDE_DIRS}) -target_link_libraries(flb-plugin-filter_wasm flb-wasm-static vmlib-static) diff --git a/fluent-bit/plugins/filter_wasm/filter_wasm.c b/fluent-bit/plugins/filter_wasm/filter_wasm.c deleted file mode 100644 index be3adccd3..000000000 --- a/fluent-bit/plugins/filter_wasm/filter_wasm.c +++ /dev/null @@ -1,318 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "filter_wasm.h" - -/* cb_filter callback */ -static int cb_wasm_filter(const void *data, size_t bytes, - const char *tag, int tag_len, - void **out_buf, size_t *out_bytes, - struct flb_filter_instance *f_ins, - struct flb_input_instance *i_ins, - void *filter_context, - struct flb_config *config) -{ - int ret; - char *ret_val = NULL; - char *buf = NULL; - - size_t off = 0; - size_t last_off = 0; - size_t alloc_size = 0; - char *json_buf = NULL; - size_t json_size; - int root_type; - struct flb_wasm *wasm = NULL; - - struct flb_filter_wasm *ctx = filter_context; - struct flb_log_event_encoder log_encoder; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - (void) f_ins; - (void) i_ins; - (void) config; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return FLB_FILTER_NOTOUCH; - } - - ret = flb_log_event_encoder_init(&log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event encoder initialization error : %d", ret); - - flb_log_event_decoder_destroy(&log_decoder); - - return FLB_FILTER_NOTOUCH; - } - - wasm = flb_wasm_instantiate(config, ctx->wasm_path, ctx->accessible_dir_list, -1, -1, -1); - if (wasm == NULL) { - flb_plg_debug(ctx->ins, "instantiate wasm [%s] failed", ctx->wasm_path); - goto on_error; - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - off = log_decoder.offset; - alloc_size = (off - last_off) + 128; /* JSON is larger than msgpack */ - last_off = off; - - /* Encode as JSON from msgpack */ - buf = flb_msgpack_to_json_str(alloc_size, log_event.body); - - if (buf) { - /* Execute WASM program */ - ret_val = flb_wasm_call_function_format_json(wasm, ctx->wasm_function_name, - tag, tag_len, - log_event.timestamp, - buf, strlen(buf)); - - flb_free(buf); - } - else { - flb_plg_error(ctx->ins, "encode as JSON from msgpack is failed"); - - goto on_error; - } - - if (ret_val == NULL) { /* Skip record */ - flb_plg_debug(ctx->ins, "encode as JSON from msgpack is broken. Skip."); - continue; - } - - - if (strlen(ret_val) == 0) { /* Skip record */ - flb_plg_debug(ctx->ins, "WASM function returned empty string. Skip."); - flb_free(ret_val); - continue; - } - - ret = flb_log_event_encoder_begin_record(&log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_timestamp( - &log_encoder, &log_event.timestamp); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_metadata_from_msgpack_object( - &log_encoder, - log_event.metadata); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - /* Convert JSON payload to msgpack */ - ret = flb_pack_json(ret_val, strlen(ret_val), - &json_buf, &json_size, &root_type, NULL); - - if (ret == 0 && root_type == JSMN_OBJECT) { - /* JSON found, pack it msgpack representation */ - ret = flb_log_event_encoder_set_body_from_raw_msgpack( - &log_encoder, - json_buf, - json_size); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(&log_encoder); - } - else { - flb_log_event_encoder_rollback_record(&log_encoder); - } - } - else { - flb_plg_error(ctx->ins, "invalid JSON format. ret: %d, buf: %s", ret, ret_val); - - flb_log_event_encoder_rollback_record(&log_encoder); - } - } - else { - flb_log_event_encoder_rollback_record(&log_encoder); - } - - /* release 'ret_val' if it was allocated */ - if (ret_val != NULL) { - flb_free(ret_val); - } - - /* release 'json_buf' if it was allocated */ - if (json_buf != NULL) { - flb_free(json_buf); - } - } - - /* Teardown WASM context */ - flb_wasm_destroy(wasm); - - *out_buf = log_encoder.output_buffer; - *out_bytes = log_encoder.output_length; - - flb_log_event_encoder_claim_internal_buffer_ownership(&log_encoder); - - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - return FLB_FILTER_MODIFIED; - -on_error: - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - if (wasm != NULL) { - flb_wasm_destroy(wasm); - } - - return FLB_FILTER_NOTOUCH; -} - -/* read config file and*/ -static int filter_wasm_config_read(struct flb_filter_wasm *ctx, - struct flb_filter_instance *f_ins, - struct flb_config *config) -{ - int ret; - - ctx->ins = f_ins; - - /* Load the config map */ - ret = flb_filter_config_map_set(f_ins, (void *)ctx); - if (ret == -1) { - flb_plg_error(f_ins, "unable to load configuration"); - return -1; - } - - /* filepath setting */ - if (ctx->wasm_path == NULL) { - flb_plg_error(f_ins, "no WASM 'program path' was given"); - return -1; - } - - /* function_name setting */ - if (ctx->wasm_function_name == NULL) { - flb_plg_error(f_ins, "no WASM 'function name' was given"); - return -1; - } - - return 0; -} - -static void delete_wasm_config(struct flb_filter_wasm *ctx) -{ - if (!ctx) { - return; - } - - flb_free(ctx); -} - -/* Initialize plugin */ -static int cb_wasm_init(struct flb_filter_instance *f_ins, - struct flb_config *config, void *data) -{ - struct flb_filter_wasm *ctx = NULL; - int ret = -1; - - /* Allocate space for the configuration */ - ctx = flb_calloc(1, sizeof(struct flb_filter_wasm)); - if (!ctx) { - return -1; - } - - /* Initialize exec config */ - ret = filter_wasm_config_read(ctx, f_ins, config); - if (ret < 0) { - goto init_error; - } - - flb_wasm_init(config); - - /* Set context */ - flb_filter_set_context(f_ins, ctx); - return 0; - -init_error: - delete_wasm_config(ctx); - - return -1; -} - -static int cb_wasm_exit(void *data, struct flb_config *config) -{ - struct flb_filter_wasm *ctx = data; - - flb_wasm_destroy_all(config); - delete_wasm_config(ctx); - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "wasm_path", NULL, - 0, FLB_TRUE, offsetof(struct flb_filter_wasm, wasm_path), - "Set the wasm path to execute" - }, - { - FLB_CONFIG_MAP_CLIST, "accessible_paths", ".", - 0, FLB_TRUE, offsetof(struct flb_filter_wasm, accessible_dir_list), - "Specifying paths to be accessible from a WASM program." - "Default value is current working directory" - }, - { - FLB_CONFIG_MAP_STR, "function_name", NULL, - 0, FLB_TRUE, offsetof(struct flb_filter_wasm, wasm_function_name), - "Set the function name in wasm to execute" - }, - /* EOF */ - {0} -}; - -struct flb_filter_plugin filter_wasm_plugin = { - .name = "wasm", - .description = "WASM program filter", - .cb_init = cb_wasm_init, - .cb_filter = cb_wasm_filter, - .cb_exit = cb_wasm_exit, - .config_map = config_map -}; diff --git a/fluent-bit/plugins/filter_wasm/filter_wasm.h b/fluent-bit/plugins/filter_wasm/filter_wasm.h deleted file mode 100644 index 4aed048fb..000000000 --- a/fluent-bit/plugins/filter_wasm/filter_wasm.h +++ /dev/null @@ -1,41 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef FLB_FILTER_WASM_H -#define FLB_FILTER_WASM_H - -#include -#include -#include -#include -#include -#include -#include - -#include - -struct flb_filter_wasm { - flb_sds_t wasm_path; - struct mk_list *accessible_dir_list; /* list of directories to be - * accesible from WASM */ - flb_sds_t wasm_function_name; - struct flb_filter_instance *ins; - struct flb_wasm *wasm; -}; - -#endif /* FLB_FILTER_WASM_H */ diff --git a/fluent-bit/plugins/in_calyptia_fleet/CMakeLists.txt b/fluent-bit/plugins/in_calyptia_fleet/CMakeLists.txt deleted file mode 100644 index 593514d26..000000000 --- a/fluent-bit/plugins/in_calyptia_fleet/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - in_calyptia_fleet.c) - -FLB_PLUGIN(in_calyptia_fleet "${src}" "") diff --git a/fluent-bit/plugins/in_calyptia_fleet/in_calyptia_fleet.c b/fluent-bit/plugins/in_calyptia_fleet/in_calyptia_fleet.c deleted file mode 100644 index 3175b5645..000000000 --- a/fluent-bit/plugins/in_calyptia_fleet/in_calyptia_fleet.c +++ /dev/null @@ -1,1269 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -#define CALYPTIA_H_PROJECT "X-Project-Token" -#define CALYPTIA_H_CTYPE "Content-Type" -#define CALYPTIA_H_CTYPE_JSON "application/json" - -#define DEFAULT_INTERVAL_SEC "15" -#define DEFAULT_INTERVAL_NSEC "0" - -#define CALYPTIA_HOST "cloud-api.calyptia.com" -#define CALYPTIA_PORT "443" - -#ifndef _WIN32 -#define PATH_SEPARATOR "/" -#define DEFAULT_CONFIG_DIR "/tmp/calyptia-fleet" -#else -#define DEFAULT_CONFIG_DIR NULL -#define PATH_SEPARATOR "\\" -#endif - -struct flb_in_calyptia_fleet_config { - /* Time interval check */ - int interval_sec; - int interval_nsec; - - /* Grabbed from the cfg_path, used to check if configuration has - * has been updated. - */ - long config_timestamp; - - flb_sds_t api_key; - flb_sds_t fleet_id; - flb_sds_t fleet_name; - flb_sds_t machine_id; - flb_sds_t config_dir; - flb_sds_t cloud_host; - flb_sds_t cloud_port; - - flb_sds_t fleet_url; - - struct flb_input_instance *ins; /* plugin instance */ - struct flb_config *config; /* Fluent Bit context */ - - /* Networking */ - struct flb_upstream *u; - - int event_fd; - - int collect_fd; -}; - -static char *find_case_header(struct flb_http_client *cli, const char *header) -{ - char *ptr; - char *headstart; - - - headstart = strstr(cli->resp.data, "\r\n"); - - if (headstart == NULL) { - return NULL; - } - - /* Lookup the beginning of the header */ - for (ptr = headstart; ptr != NULL && ptr+2 < cli->resp.payload; ptr = strstr(ptr, "\r\n")) { - - if (ptr + 4 < cli->resp.payload && strcmp(ptr, "\r\n\r\n") == 0) { - return NULL; - } - - ptr+=2; - - /* no space left for header */ - if (ptr + strlen(header)+2 >= cli->resp.payload) { - return NULL; - } - - /* matched header and the delimiter */ - if (strncasecmp(ptr, header, strlen(header)) == 0) { - - if (ptr[strlen(header)] == ':' && ptr[strlen(header)+1] == ' ') { - return ptr; - } - } - } - - return NULL; -} - -/* Try to find a header value in the buffer. Copied from flb_http_client.c. */ -static int case_header_lookup(struct flb_http_client *cli, - const char *header, int header_len, - const char **out_val, int *out_len) -{ - char *ptr; - char *crlf; - char *end; - - if (!cli->resp.data) { - return -1; - } - - ptr = find_case_header(cli, header); - end = strstr(cli->resp.data, "\r\n\r\n"); - - if (!ptr) { - - if (end) { - /* The headers are complete but the header is not there */ - return -1; - } - - /* We need more data */ - return -1; - } - - /* Exclude matches in the body */ - if (end && ptr > end) { - return -1; - } - - /* Lookup CRLF (end of line \r\n) */ - crlf = strstr(ptr, "\r\n"); - - if (!crlf) { - return -1; - } - - /* sanity check that the header_len does not exceed the headers. */ - if (ptr + header_len + 2 > end) { - return -1; - } - - ptr += header_len + 2; - - *out_val = ptr; - *out_len = (crlf - ptr); - - return 0; -} - -struct reload_ctx { - flb_ctx_t *flb; - flb_sds_t cfg_path; -}; - -static flb_sds_t fleet_config_filename(struct flb_in_calyptia_fleet_config *ctx, char *fname) -{ - flb_sds_t cfgname; - - cfgname = flb_sds_create_size(4096); - - if (ctx->fleet_name != NULL) { - flb_sds_printf(&cfgname, - "%s" PATH_SEPARATOR "%s" PATH_SEPARATOR "%s" PATH_SEPARATOR "%s.ini", - ctx->config_dir, ctx->machine_id, ctx->fleet_name, fname); - } - else { - flb_sds_printf(&cfgname, - "%s" PATH_SEPARATOR "%s" PATH_SEPARATOR "%s" PATH_SEPARATOR "%s.ini", - ctx->config_dir, ctx->machine_id, ctx->fleet_id, fname); - } - - return cfgname; -} - -static flb_sds_t new_fleet_config_filename(struct flb_in_calyptia_fleet_config *ctx) -{ - return fleet_config_filename(ctx, "new"); -} - -static flb_sds_t cur_fleet_config_filename(struct flb_in_calyptia_fleet_config *ctx) -{ - return fleet_config_filename(ctx, "cur"); -} - -static flb_sds_t old_fleet_config_filename(struct flb_in_calyptia_fleet_config *ctx) -{ - return fleet_config_filename(ctx, "old"); -} - -static flb_sds_t time_fleet_config_filename(struct flb_in_calyptia_fleet_config *ctx, time_t t) -{ - char s_last_modified[32]; - - snprintf(s_last_modified, sizeof(s_last_modified)-1, "%d", (int)t); - return fleet_config_filename(ctx, s_last_modified); -} - -static int is_new_fleet_config(struct flb_in_calyptia_fleet_config *ctx, struct flb_config *cfg) -{ - flb_sds_t cfgnewname; - int ret = FLB_FALSE; - - - if (cfg->conf_path_file == NULL) { - return FLB_FALSE; - } - - cfgnewname = new_fleet_config_filename(ctx); - - if (strcmp(cfgnewname, cfg->conf_path_file) == 0) { - ret = FLB_TRUE; - } - - flb_sds_destroy(cfgnewname); - - return ret; -} - -static int is_cur_fleet_config(struct flb_in_calyptia_fleet_config *ctx, struct flb_config *cfg) -{ - flb_sds_t cfgcurname; - int ret = FLB_FALSE; - - - if (cfg->conf_path_file == NULL) { - return FLB_FALSE; - } - - cfgcurname = cur_fleet_config_filename(ctx); - - if (strcmp(cfgcurname, cfg->conf_path_file) == 0) { - ret = FLB_TRUE; - } - - flb_sds_destroy(cfgcurname); - - return ret; -} - -static int is_timestamped_fleet_config(struct flb_in_calyptia_fleet_config *ctx, struct flb_config *cfg) -{ - char *fname; - char *end; - long val; - - if (cfg->conf_path_file == NULL) { - return FLB_FALSE; - } - - fname = strrchr(cfg->conf_path_file, PATH_SEPARATOR[0]); - - if (fname == NULL) { - return FLB_FALSE; - } - - fname++; - - errno = 0; - val = strtol(fname, &end, 10); - - if ((errno == ERANGE && (val == LONG_MAX || val == LONG_MIN)) || - (errno != 0 && val == 0)) { - flb_errno(); - return FLB_FALSE; - } - - if (strcmp(end, ".ini") == 0) { - return FLB_TRUE; - } - - return FLB_FALSE; -} - -static int is_fleet_config(struct flb_in_calyptia_fleet_config *ctx, struct flb_config *cfg) -{ - if (cfg->conf_path_file == NULL) { - return FLB_FALSE; - } - - return is_new_fleet_config(ctx, cfg) || - is_cur_fleet_config(ctx, cfg) || - is_timestamped_fleet_config(ctx, cfg); -} - -static int exists_new_fleet_config(struct flb_in_calyptia_fleet_config *ctx) -{ - flb_sds_t cfgnewname; - int ret = FLB_FALSE; - - - cfgnewname = new_fleet_config_filename(ctx); - ret = access(cfgnewname, F_OK) == 0 ? FLB_TRUE : FLB_FALSE; - - flb_sds_destroy(cfgnewname); - return ret; -} - -static int exists_cur_fleet_config(struct flb_in_calyptia_fleet_config *ctx) -{ - flb_sds_t cfgcurname; - int ret = FLB_FALSE; - - - cfgcurname = cur_fleet_config_filename(ctx); - ret = access(cfgcurname, F_OK) == 0 ? FLB_TRUE : FLB_FALSE; - - flb_sds_destroy(cfgcurname); - return ret; -} - -static void *do_reload(void *data) -{ - struct reload_ctx *reload = (struct reload_ctx *)data; - - /* avoid reloading the current configuration... just use our new one! */ - flb_context_set(reload->flb); - reload->flb->config->enable_hot_reload = FLB_TRUE; - reload->flb->config->conf_path_file = reload->cfg_path; - - sleep(5); -#ifndef FLB_SYSTEM_WINDOWS - kill(getpid(), SIGHUP); -#else - GenerateConsoleCtrlEvent(1 /* CTRL_BREAK_EVENT_1 */, 0); -#endif - return NULL; -} - -static int test_config_is_valid(flb_sds_t cfgpath) -{ - struct flb_config *config; - struct flb_cf *conf; - int ret = FLB_FALSE; - - - config = flb_config_init(); - - if (config == NULL) { - goto config_init_error; - } - - conf = flb_cf_create(); - - if (conf == NULL) { - goto cf_create_error; - } - - conf = flb_cf_create_from_file(conf, cfgpath); - - if (conf == NULL) { - goto cf_create_from_file_error; - } - - if (flb_config_load_config_format(config, conf)) { - goto cf_load_config_format_error; - } - - if (flb_reload_property_check_all(config)) { - goto cf_property_check_error; - } - - ret = FLB_TRUE; - -cf_property_check_error: -cf_load_config_format_error: -cf_create_from_file_error: - flb_cf_destroy(conf); -cf_create_error: - flb_config_exit(config); -config_init_error: - return ret; -} - -static int execute_reload(struct flb_in_calyptia_fleet_config *ctx, flb_sds_t cfgpath) -{ - struct reload_ctx *reload; - pthread_t pth; - pthread_attr_t ptha; - flb_ctx_t *flb = flb_context_get(); - - if (ctx->collect_fd > 0) { - flb_input_collector_pause(ctx->collect_fd, ctx->ins); - } - - if (flb == NULL) { - flb_plg_error(ctx->ins, "unable to get fluent-bit context."); - - if (ctx->collect_fd > 0) { - flb_input_collector_resume(ctx->collect_fd, ctx->ins); - } - - return FLB_FALSE; - } - - /* fix execution in valgrind... - * otherwise flb_reload errors out with: - * [error] [reload] given flb context is NULL - */ - flb_plg_info(ctx->ins, "loading configuration from %s.", cfgpath); - - if (test_config_is_valid(cfgpath) == FLB_FALSE) { - flb_plg_error(ctx->ins, "unable to load configuration."); - - if (ctx->collect_fd > 0) { - flb_input_collector_resume(ctx->collect_fd, ctx->ins); - } - - return FLB_FALSE; - } - - reload = flb_calloc(1, sizeof(struct reload_ctx)); - reload->flb = flb; - reload->cfg_path = cfgpath; - - pthread_attr_init(&ptha); - pthread_attr_setdetachstate(&ptha, PTHREAD_CREATE_DETACHED); - pthread_create(&pth, &ptha, do_reload, reload); - - return FLB_TRUE; -} - -static char *tls_setting_string(int use_tls) -{ - if (use_tls) { - return "On"; - } - - return "Off"; -} - -static flb_sds_t parse_api_key_json(struct flb_in_calyptia_fleet_config *ctx, - char *payload, size_t size) -{ - int ret; - int out_size; - char *pack; - struct flb_pack_state pack_state; - size_t off = 0; - msgpack_unpacked result; - msgpack_object_kv *cur; - msgpack_object_str *key; - flb_sds_t project_id; - int idx = 0; - - /* Initialize packer */ - flb_pack_state_init(&pack_state); - - /* Pack JSON as msgpack */ - ret = flb_pack_json_state(payload, size, - &pack, &out_size, &pack_state); - flb_pack_state_reset(&pack_state); - - /* Handle exceptions */ - if (ret == FLB_ERR_JSON_PART) { - flb_plg_warn(ctx->ins, "JSON data is incomplete, skipping"); - return NULL; - } - else if (ret == FLB_ERR_JSON_INVAL) { - flb_plg_warn(ctx->ins, "invalid JSON message, skipping"); - return NULL; - } - else if (ret == -1) { - return NULL; - } - - msgpack_unpacked_init(&result); - while (msgpack_unpack_next(&result, pack, out_size, &off) == MSGPACK_UNPACK_SUCCESS) { - - if (result.data.type == MSGPACK_OBJECT_MAP) { - for (idx = 0; idx < result.data.via.map.size; idx++) { - cur = &result.data.via.map.ptr[idx]; - key = &cur->key.via.str; - - if (strncmp(key->ptr, "ProjectID", key->size) == 0) { - - if (cur->val.type != MSGPACK_OBJECT_STR) { - flb_plg_error(ctx->ins, "unable to find fleet by name"); - msgpack_unpacked_destroy(&result); - return NULL; - } - - project_id = flb_sds_create_len(cur->val.via.str.ptr, - cur->val.via.str.size); - msgpack_unpacked_destroy(&result); - flb_free(pack); - - return project_id; - } - } - } - } - - msgpack_unpacked_destroy(&result); - flb_free(pack); - - return NULL; -} - -static ssize_t parse_fleet_search_json(struct flb_in_calyptia_fleet_config *ctx, - char *payload, size_t size) -{ - int ret; - int out_size; - char *pack; - struct flb_pack_state pack_state; - size_t off = 0; - msgpack_unpacked result; - msgpack_object_array *results; - msgpack_object_kv *cur; - msgpack_object_str *key; - int idx = 0; - - /* Initialize packer */ - flb_pack_state_init(&pack_state); - - /* Pack JSON as msgpack */ - ret = flb_pack_json_state(payload, size, - &pack, &out_size, &pack_state); - flb_pack_state_reset(&pack_state); - - /* Handle exceptions */ - if (ret == FLB_ERR_JSON_PART) { - flb_plg_warn(ctx->ins, "JSON data is incomplete, skipping"); - return -1; - } - else if (ret == FLB_ERR_JSON_INVAL) { - flb_plg_warn(ctx->ins, "invalid JSON message, skipping"); - return -1; - } - else if (ret == -1) { - return -1; - } - - msgpack_unpacked_init(&result); - while (msgpack_unpack_next(&result, pack, out_size, &off) == MSGPACK_UNPACK_SUCCESS) { - - if (result.data.type == MSGPACK_OBJECT_ARRAY) { - results = &result.data.via.array; - - if (results->ptr[0].type == MSGPACK_OBJECT_MAP) { - - for (idx = 0; idx < results->ptr[0].via.map.size; idx++) { - cur = &results->ptr[0].via.map.ptr[idx]; - key = &cur->key.via.str; - - if (strncasecmp(key->ptr, "id", key->size) == 0) { - - if (cur->val.type != MSGPACK_OBJECT_STR) { - flb_plg_error(ctx->ins, "unable to find fleet by name"); - msgpack_unpacked_destroy(&result); - return -1; - } - - ctx->fleet_id = flb_sds_create_len(cur->val.via.str.ptr, - cur->val.via.str.size); - break; - } - break; - } - break; - } - } - } - - msgpack_unpacked_destroy(&result); - flb_free(pack); - - if (ctx->fleet_id == NULL) { - return -1; - } - - return 0; -} - -static int get_calyptia_fleet_id_by_name(struct flb_in_calyptia_fleet_config *ctx, - struct flb_connection *u_conn, - struct flb_config *config) -{ - struct flb_http_client *client; - flb_sds_t url; - flb_sds_t project_id; - unsigned char token[512] = {0}; - unsigned char encoded[256]; - size_t elen; - size_t tlen; - char *api_token_sep; - size_t b_sent; - int ret; - - api_token_sep = strchr(ctx->api_key, '.'); - - if (api_token_sep == NULL) { - return -1; - } - - elen = api_token_sep-ctx->api_key; - elen = elen + (4 - (elen % 4)); - - if (elen > sizeof(encoded)) { - flb_plg_error(ctx->ins, "API Token is too large"); - return -1; - } - - memset(encoded, '=', sizeof(encoded)); - memcpy(encoded, ctx->api_key, api_token_sep-ctx->api_key); - - ret = flb_base64_decode(token, sizeof(token)-1, &tlen, - encoded, elen); - - if (ret != 0) { - return ret; - } - - project_id = parse_api_key_json(ctx, (char *)token, tlen); - - if (project_id == NULL) { - return -1; - } - - url = flb_sds_create_size(4096); - flb_sds_printf(&url, "/v1/search?project_id=%s&resource=fleet&term=%s", - project_id, ctx->fleet_name); - - client = flb_http_client(u_conn, FLB_HTTP_GET, url, NULL, 0, - ctx->ins->host.name, ctx->ins->host.port, NULL, 0); - - if (!client) { - flb_plg_error(ctx->ins, "unable to create http client"); - return -1; - } - - flb_http_buffer_size(client, 8192); - - flb_http_add_header(client, - CALYPTIA_H_PROJECT, sizeof(CALYPTIA_H_PROJECT) - 1, - ctx->api_key, flb_sds_len(ctx->api_key)); - - ret = flb_http_do(client, &b_sent); - - if (ret != 0) { - flb_plg_error(ctx->ins, "http do error"); - return -1; - } - - if (client->resp.status != 200) { - flb_plg_error(ctx->ins, "search http status code error: %d", client->resp.status); - return -1; - } - - if (client->resp.payload_size <= 0) { - flb_plg_error(ctx->ins, "empty response"); - return -1; - } - - if (parse_fleet_search_json(ctx, client->resp.payload, client->resp.payload_size) == -1) { - flb_plg_error(ctx->ins, "unable to find fleet: %s", ctx->fleet_name); - return -1; - } - - if (ctx->fleet_id == NULL) { - return -1; - } - return 0; -} - -#ifdef FLB_SYSTEM_WINDOWS -#define link(a, b) CreateHardLinkA(b, a, 0) - -ssize_t readlink(const char *path, char *realpath, size_t srealpath) { - HANDLE hFile; - DWORD ret; - - hFile = CreateFile(path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, - FILE_ATTRIBUTE_NORMAL, NULL); - - if (hFile == INVALID_HANDLE_VALUE) { - return -1; - } - - ret = GetFinalPathNameByHandleA(hFile, realpath, srealpath, VOLUME_NAME_NT); - - if (ret < srealpath) { - CloseHandle(hFile); - return -1; - } - - CloseHandle(hFile); - return ret; -} - -#endif - -/* cb_collect callback */ -static int in_calyptia_fleet_collect(struct flb_input_instance *ins, - struct flb_config *config, - void *in_context) -{ - struct flb_in_calyptia_fleet_config *ctx = in_context; - struct flb_connection *u_conn; - struct flb_http_client *client; - flb_sds_t cfgname; - flb_sds_t cfgnewname; - flb_sds_t cfgoldname; - flb_sds_t cfgcurname; - flb_sds_t header; - flb_sds_t hdr; - FILE *cfgfp; - const char *fbit_last_modified; - int fbit_last_modified_len; - struct flb_tm tm_last_modified = { 0 }; - time_t time_last_modified; - char *data; - size_t b_sent; - int ret = -1; -#ifdef FLB_SYSTEM_WINDOWS - DWORD err; - LPSTR lpMsg; -#endif - - u_conn = flb_upstream_conn_get(ctx->u); - - if (!u_conn) { - flb_plg_error(ctx->ins, "could not get an upstream connection to %s:%u", - ctx->ins->host.name, ctx->ins->host.port); - goto conn_error; - } - - if (ctx->fleet_id == NULL) { - - if (get_calyptia_fleet_id_by_name(ctx, u_conn, config) == -1) { - flb_plg_error(ctx->ins, "unable to find fleet: %s", ctx->fleet_name); - goto conn_error; - } - } - - if (ctx->fleet_url == NULL) { - ctx->fleet_url = flb_sds_create_size(4096); - flb_sds_printf(&ctx->fleet_url, "/v1/fleets/%s/config?format=ini", ctx->fleet_id); - } - - client = flb_http_client(u_conn, FLB_HTTP_GET, ctx->fleet_url, - NULL, 0, - ctx->ins->host.name, ctx->ins->host.port, NULL, 0); - - if (!client) { - flb_plg_error(ins, "unable to create http client"); - goto client_error; - } - - flb_http_buffer_size(client, 8192); - - flb_http_add_header(client, - CALYPTIA_H_PROJECT, sizeof(CALYPTIA_H_PROJECT) - 1, - ctx->api_key, flb_sds_len(ctx->api_key)); - - ret = flb_http_do(client, &b_sent); - - if (ret != 0) { - flb_plg_error(ins, "http do error"); - goto http_error; - } - - if (client->resp.status != 200) { - flb_plg_error(ins, "http status code error: %d", client->resp.status); - goto http_error; - } - - if (client->resp.payload_size <= 0) { - flb_plg_error(ins, "empty response"); - goto http_error; - } - - /* copy and NULL terminate the payload */ - data = flb_sds_create_size(client->resp.payload_size + 1); - - if (!data) { - goto http_error; - } - memcpy(data, client->resp.payload, client->resp.payload_size); - data[client->resp.payload_size] = '\0'; - - ret = case_header_lookup(client, "Last-modified", strlen("Last-modified"), - &fbit_last_modified, &fbit_last_modified_len); - - if (ret == -1) { - flb_plg_error(ctx->ins, "unable to get last-modified header"); - goto http_error; - } - - flb_strptime(fbit_last_modified, "%a, %d %B %Y %H:%M:%S GMT", &tm_last_modified); - time_last_modified = mktime(&tm_last_modified.tm); - - cfgname = time_fleet_config_filename(ctx, time_last_modified); - - if (access(cfgname, F_OK) == -1 && errno == ENOENT) { - cfgfp = fopen(cfgname, "w+"); - - if (cfgfp == NULL) { - flb_plg_error(ctx->ins, "unable to open configuration file: %s", cfgname); - goto http_error; - } - - header = flb_sds_create_size(4096); - - if (ctx->fleet_name == NULL) { - hdr = flb_sds_printf(&header, - "[CUSTOM]\n" - " Name calyptia\n" - " api_key %s\n" - " fleet_id %s\n" - " add_label fleet_id %s\n" - " fleet.config_dir %s\n" - " calyptia_host %s\n" - " calyptia_port %d\n" - " calyptia_tls %s\n", - ctx->api_key, - ctx->fleet_id, - ctx->fleet_id, - ctx->config_dir, - ctx->ins->host.name, - ctx->ins->host.port, - tls_setting_string(ctx->ins->use_tls) - ); - } - else { - hdr = flb_sds_printf(&header, - "[CUSTOM]\n" - " Name calyptia\n" - " api_key %s\n" - " fleet_name %s\n" - " fleet_id %s\n" - " add_label fleet_id %s\n" - " fleet.config_dir %s\n" - " calyptia_host %s\n" - " calyptia_port %d\n" - " calyptia_tls %s\n", - ctx->api_key, - ctx->fleet_name, - ctx->fleet_id, - ctx->fleet_id, - ctx->config_dir, - ctx->ins->host.name, - ctx->ins->host.port, - tls_setting_string(ctx->ins->use_tls) - ); - } - if (hdr == NULL) { - fclose(cfgfp); - goto http_error; - } - if (ctx->machine_id) { - hdr = flb_sds_printf(&header, " machine_id %s\n", ctx->machine_id); - if (hdr == NULL) { - fclose(cfgfp); - goto http_error; - } - } - fwrite(header, strlen(header), 1, cfgfp); - flb_sds_destroy(header); - fwrite(data, client->resp.payload_size, 1, cfgfp); - fclose(cfgfp); - - cfgnewname = new_fleet_config_filename(ctx); - - if (exists_new_fleet_config(ctx) == FLB_TRUE) { - cfgoldname = old_fleet_config_filename(ctx); - rename(cfgnewname, cfgoldname); - unlink(cfgnewname); - flb_sds_destroy(cfgoldname); - } - - if (!link(cfgname, cfgnewname)) { -#ifdef FLB_SYSTEM_WINDOWS - err = GetLastError(); - FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_ALLOCATE_BUFFER, - NULL, err, 0, &lpMsg, 0, NULL); - flb_plg_error(ctx->ins, "unable to create hard link: %s", lpMsg); -#else - flb_errno(); -#endif - } - } - - if (ctx->config_timestamp < time_last_modified) { - flb_plg_debug(ctx->ins, "new configuration is newer than current: %ld < %ld", - ctx->config_timestamp, time_last_modified); - flb_plg_info(ctx->ins, "force the reloading of the configuration file=%d.", ctx->event_fd); - flb_sds_destroy(data); - - if (execute_reload(ctx, cfgname) == FLB_FALSE) { - cfgoldname = old_fleet_config_filename(ctx); - cfgcurname = cur_fleet_config_filename(ctx); - rename(cfgoldname, cfgcurname); - flb_sds_destroy(cfgcurname); - flb_sds_destroy(cfgoldname); - goto reload_error; - } - else { - FLB_INPUT_RETURN(0); - } - } - - ret = 0; - -reload_error: -http_error: - flb_http_client_destroy(client); -client_error: - flb_upstream_conn_release(u_conn); -conn_error: - FLB_INPUT_RETURN(ret); -} - -#ifdef FLB_SYSTEM_WINDOWS -#define _mkdir(a, b) mkdir(a) -#else -#define _mkdir(a, b) mkdir(a, b) -#endif - -/* recursively create directories, based on: - * https://stackoverflow.com/a/2336245 - * who found it at: - * http://nion.modprobe.de/blog/archives/357-Recursive-directory-creation.html - */ -static int __mkdir(const char *dir, int perms) { - char tmp[255]; - char *ptr = NULL; - size_t len; - int ret; - - ret = snprintf(tmp, sizeof(tmp),"%s",dir); - if (ret > sizeof(tmp)) { - return -1; - } - - len = strlen(tmp); - if (tmp[len - 1] == '/') { - tmp[len - 1] = 0; - } - - for (ptr = tmp + 1; *ptr; ptr++) { - if (*ptr == '/') { - *ptr = 0; - if (access(tmp, F_OK) != 0) { - ret = _mkdir(tmp, perms); - if (ret != 0) { - return ret; - } - } - *ptr = '/'; - } - } - return _mkdir(tmp, perms); -} - -static int create_fleet_directory(struct flb_in_calyptia_fleet_config *ctx) -{ - flb_sds_t myfleetdir; - - if (access(ctx->config_dir, F_OK) != 0) { - if (__mkdir(ctx->config_dir, 0700) != 0) { - return -1; - } - } - - myfleetdir = flb_sds_create_size(256); - - if (ctx->fleet_name != NULL) { - flb_sds_printf(&myfleetdir, "%s" PATH_SEPARATOR "%s" PATH_SEPARATOR "%s", - ctx->config_dir, ctx->machine_id, ctx->fleet_name); - } - else { - flb_sds_printf(&myfleetdir, "%s" PATH_SEPARATOR "%s" PATH_SEPARATOR "%s", - ctx->config_dir, ctx->machine_id, ctx->fleet_id); - } - - if (access(myfleetdir, F_OK) != 0) { - if (__mkdir(myfleetdir, 0700) !=0) { - return -1; - } - } - - flb_sds_destroy(myfleetdir); - return 0; -} - -static int load_fleet_config(struct flb_in_calyptia_fleet_config *ctx) -{ - flb_ctx_t *flb_ctx = flb_context_get(); - char *fname; - char *ext; - long timestamp; - char realname[4096]; - ssize_t len; - - if (create_fleet_directory(ctx) != 0) { - return -1; - } - - /* check if we are already using the fleet configuration file. */ - if (is_fleet_config(ctx, flb_ctx->config) == FLB_FALSE) { - /* check which one and load it */ - if (exists_cur_fleet_config(ctx) == FLB_TRUE) { - return execute_reload(ctx, cur_fleet_config_filename(ctx)); - } - else if (exists_new_fleet_config(ctx) == FLB_TRUE) { - return execute_reload(ctx, new_fleet_config_filename(ctx)); - } - } - else { - if (is_new_fleet_config(ctx, flb_ctx->config) || is_cur_fleet_config(ctx, flb_ctx->config)) { - len = readlink(flb_ctx->config->conf_path_file, realname, sizeof(realname)); - - if (len > sizeof(realname)) { - return FLB_FALSE; - } - - fname = basename(realname); - } - else { - fname = basename(flb_ctx->config->conf_path_file); - } - - if (fname == NULL) { - return FLB_FALSE; - } - - errno = 0; - timestamp = strtol(fname, &ext, 10); - - if ((errno == ERANGE && (timestamp == LONG_MAX || timestamp == LONG_MIN)) || - (errno != 0 && timestamp == 0)) { - flb_errno(); - return FLB_FALSE; - } - - /* unable to parse the timstamp */ - if (errno == ERANGE) { - return FLB_FALSE; - } - - ctx->config_timestamp = timestamp; - } - - return FLB_FALSE; -} - -static int in_calyptia_fleet_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - int ret; - int upstream_flags; - struct flb_in_calyptia_fleet_config *ctx; - (void) data; - -#ifdef _WIN32 - char *tmpdir; -#endif - - flb_plg_info(in, "initializing calyptia fleet input."); - - if (in->host.name == NULL) { - flb_plg_error(in, "no input 'Host' provided"); - return -1; - } - - /* Allocate space for the configuration */ - ctx = flb_calloc(1, sizeof(struct flb_in_calyptia_fleet_config)); - - if (!ctx) { - flb_errno(); - return -1; - } - ctx->ins = in; - ctx->collect_fd = -1; - - - /* Load the config map */ - ret = flb_input_config_map_set(in, (void *)ctx); - - if (ret == -1) { - flb_free(ctx); - flb_plg_error(in, "unable to load configuration"); - return -1; - } - -#ifdef _WIN32 - if (ctx->config_dir == NULL) { - tmpdir = getenv("TEMP"); - - if (tmpdir == NULL) { - flb_plg_error(in, "unable to find temporary directory (%%TEMP%%)."); - return -1; - } - - ctx->config_dir = flb_sds_create_size(4096); - - if (ctx->config_dir == NULL) { - flb_plg_error(in, "unable to allocate config-dir."); - return -1; - } - flb_sds_printf(&ctx->config_dir, "%s" PATH_SEPARATOR "%s", tmpdir, "calyptia-fleet"); - } -#endif - - upstream_flags = FLB_IO_TCP; - - if (in->use_tls) { - upstream_flags |= FLB_IO_TLS; - } - - ctx->u = flb_upstream_create(config, in->host.name, in->host.port, - upstream_flags, in->tls); - - if (!ctx->u) { - flb_plg_error(ctx->ins, "could not initialize upstream"); - flb_free(ctx); - return -1; - } - - if (ctx->interval_sec <= 0 && ctx->interval_nsec <= 0) { - /* Illegal settings. Override them. */ - ctx->interval_sec = atoi(DEFAULT_INTERVAL_SEC); - ctx->interval_nsec = atoi(DEFAULT_INTERVAL_NSEC); - } - - if (ctx->interval_sec < atoi(DEFAULT_INTERVAL_SEC)) { - ctx->interval_sec = atoi(DEFAULT_INTERVAL_SEC); - } - - /* Set the context */ - flb_input_set_context(in, ctx); - - /* if we load a new configuration then we will be reloaded anyways */ - if (load_fleet_config(ctx) == FLB_TRUE) { - return 0; - } - - /* Set our collector based on time */ - ret = flb_input_set_collector_time(in, - in_calyptia_fleet_collect, - ctx->interval_sec, - ctx->interval_nsec, - config); - - if (ret == -1) { - flb_plg_error(ctx->ins, "could not initialize collector for fleet input plugin"); - flb_free(ctx); - return -1; - } - - ctx->collect_fd = ret; - - return 0; -} - -static void cb_in_calyptia_fleet_pause(void *data, struct flb_config *config) -{ - struct flb_in_calyptia_fleet_config *ctx = data; - flb_input_collector_pause(ctx->collect_fd, ctx->ins); -} - -static void cb_in_calyptia_fleet_resume(void *data, struct flb_config *config) -{ - struct flb_in_calyptia_fleet_config *ctx = data; - flb_input_collector_resume(ctx->collect_fd, ctx->ins); -} - -static int in_calyptia_fleet_exit(void *data, struct flb_config *config) -{ - (void) *config; - struct flb_in_calyptia_fleet_config *ctx = (struct flb_in_calyptia_fleet_config *)data; - - flb_input_collector_delete(ctx->collect_fd, ctx->ins); - flb_upstream_destroy(ctx->u); - flb_free(ctx); - - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "api_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_in_calyptia_fleet_config, api_key), - "Calyptia Cloud API Key." - }, - { - FLB_CONFIG_MAP_STR, "config_dir", DEFAULT_CONFIG_DIR, - 0, FLB_TRUE, offsetof(struct flb_in_calyptia_fleet_config, config_dir), - "Base path for the configuration directory." - }, - { - FLB_CONFIG_MAP_STR, "fleet_id", NULL, - 0, FLB_TRUE, offsetof(struct flb_in_calyptia_fleet_config, fleet_id), - "Calyptia Fleet ID." - }, - { - FLB_CONFIG_MAP_STR, "fleet_name", NULL, - 0, FLB_TRUE, offsetof(struct flb_in_calyptia_fleet_config, fleet_name), - "Calyptia Fleet Name (used to lookup the fleet ID via the cloud API)." - }, - { - FLB_CONFIG_MAP_STR, "machine_id", NULL, - 0, FLB_TRUE, offsetof(struct flb_in_calyptia_fleet_config, machine_id), - "Agent Machine ID." - }, - { - FLB_CONFIG_MAP_INT, "event_fd", "-1", - 0, FLB_TRUE, offsetof(struct flb_in_calyptia_fleet_config, event_fd), - "Used internally to set the event fd." - }, - { - FLB_CONFIG_MAP_INT, "interval_sec", DEFAULT_INTERVAL_SEC, - 0, FLB_TRUE, offsetof(struct flb_in_calyptia_fleet_config, interval_sec), - "Set the collector interval" - }, - { - FLB_CONFIG_MAP_INT, "interval_nsec", DEFAULT_INTERVAL_NSEC, - 0, FLB_TRUE, offsetof(struct flb_in_calyptia_fleet_config, interval_nsec), - "Set the collector interval (nanoseconds)" - }, - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_input_plugin in_calyptia_fleet_plugin = { - .name = "calyptia_fleet", - .description = "Calyptia Fleet Input", - .cb_init = in_calyptia_fleet_init, - .cb_pre_run = NULL, - .cb_collect = in_calyptia_fleet_collect, - .cb_resume = cb_in_calyptia_fleet_resume, - .cb_pause = cb_in_calyptia_fleet_pause, - .cb_flush_buf = NULL, - .cb_exit = in_calyptia_fleet_exit, - .config_map = config_map, - .flags = FLB_INPUT_NET|FLB_INPUT_CORO|FLB_IO_OPT_TLS|FLB_INPUT_PRIVATE -}; diff --git a/fluent-bit/plugins/in_collectd/CMakeLists.txt b/fluent-bit/plugins/in_collectd/CMakeLists.txt deleted file mode 100644 index 1f8d64074..000000000 --- a/fluent-bit/plugins/in_collectd/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -set(src - typesdb.c - typesdb_parser.c - netprot.c - in_collectd.c) - -FLB_PLUGIN(in_collectd "${src}" "") diff --git a/fluent-bit/plugins/in_collectd/in_collectd.c b/fluent-bit/plugins/in_collectd/in_collectd.c deleted file mode 100644 index 06ef5ae8a..000000000 --- a/fluent-bit/plugins/in_collectd/in_collectd.c +++ /dev/null @@ -1,226 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "in_collectd.h" -#include "netprot.h" -#include "typesdb.h" - -/* - * Max payload size. By default, Collectd sends up to 1452 bytes - * per a UDP packet, but the limit can be increased up to 65535 - * bytes through a configuration parameter. - * - * See network_config_set_buffer_size() in collectd/src/network.c. - */ -#define BUFFER_SIZE 65535 - -#define DEFAULT_LISTEN "0.0.0.0" -#define DEFAULT_PORT 25826 - -/* This is where most Linux systems places a default TypesDB */ -#define DEFAULT_TYPESDB "/usr/share/collectd/types.db" - -static int in_collectd_callback(struct flb_input_instance *i_ins, - struct flb_config *config, void *in_context); - -static int in_collectd_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - int ret; - struct flb_in_collectd_config *ctx; - struct mk_list *tdb; - char *listen = DEFAULT_LISTEN; - int port = DEFAULT_PORT; - - /* Initialize context */ - ctx = flb_calloc(1, sizeof(struct flb_in_collectd_config)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->ins = in; - - ctx->bufsize = BUFFER_SIZE; - ctx->buf = flb_malloc(ctx->bufsize); - if (!ctx->buf) { - flb_errno(); - flb_free(ctx); - return -1; - } - - /* Load the config map */ - ret = flb_input_config_map_set(in, (void *)ctx); - if (ret == -1) { - flb_plg_error(in, "unable to load configuration"); - flb_free(ctx); - return -1; - } - - /* Listening address */ - if (in->host.listen) { - listen = in->host.listen; - } - - if (strlen(listen) > sizeof(ctx->listen) - 1) { - flb_plg_error(ctx->ins, "too long address '%s'", listen); - flb_free(ctx); - return -1; - } - strcpy(ctx->listen, listen); - - /* Listening port */ - if (in->host.port) { - port = in->host.port; - } - snprintf(ctx->port, sizeof(ctx->port), "%hu", (unsigned short) port); - - flb_plg_debug(ctx->ins, "Loading TypesDB from %s", ctx->types_db); - - tdb = typesdb_load_all(ctx, ctx->types_db); - if (!tdb) { - flb_plg_error(ctx->ins, "failed to load '%s'", ctx->types_db); - flb_free(ctx->buf); - flb_free(ctx); - return -1; - } - ctx->tdb = tdb; - - /* Set the context */ - flb_input_set_context(in, ctx); - - ctx->server_fd = flb_net_server_udp(ctx->port, ctx->listen); - if (ctx->server_fd < 0) { - flb_plg_error(ctx->ins, "failed to bind to %s:%s", ctx->listen, - ctx->port); - typesdb_destroy(ctx->tdb); - flb_free(ctx->buf); - flb_free(ctx); - return -1; - } - - /* Set the collector */ - ret = flb_input_set_collector_socket(in, - in_collectd_callback, - ctx->server_fd, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "failed set up a collector"); - flb_socket_close(ctx->server_fd); - typesdb_destroy(ctx->tdb); - flb_free(ctx->buf); - flb_free(ctx); - return -1; - } - ctx->coll_fd = ret; - - ret = flb_log_event_encoder_init(&ctx->log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, "error initializing event encoder : %d", ret); - - flb_socket_close(ctx->server_fd); - typesdb_destroy(ctx->tdb); - flb_free(ctx->buf); - flb_free(ctx); - - return -1; - } - - flb_plg_info(ctx->ins, "start listening to %s:%s", - ctx->listen, ctx->port); - - return 0; -} - -static int in_collectd_callback(struct flb_input_instance *i_ins, - struct flb_config *config, void *in_context) -{ - int len; - struct flb_in_collectd_config *ctx = in_context; - - len = recv(ctx->server_fd, ctx->buf, ctx->bufsize, 0); - if (len < 0) { - flb_errno(); - return -1; - } - if (len == 0) { - return 0; - } - - flb_log_event_encoder_reset(&ctx->log_encoder); - - if (netprot_to_msgpack(ctx->buf, len, ctx->tdb, &ctx->log_encoder)) { - flb_plg_error(ctx->ins, "netprot_to_msgpack fails"); - - return -1; - } - - if (ctx->log_encoder.output_length > 0) { - flb_input_log_append(i_ins, NULL, 0, - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - } - - return 0; -} - -static int in_collectd_exit(void *data, struct flb_config *config) -{ - struct flb_in_collectd_config *ctx = data; - - flb_log_event_encoder_destroy(&ctx->log_encoder); - flb_socket_close(ctx->server_fd); - flb_pipe_close(ctx->coll_fd); - typesdb_destroy(ctx->tdb); - flb_free(ctx->buf); - flb_free(ctx); - - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "typesdb", DEFAULT_TYPESDB, - 0, FLB_TRUE, offsetof(struct flb_in_collectd_config, types_db), - "Set the types database filename" - }, - /* EOF */ - {0} -}; - -struct flb_input_plugin in_collectd_plugin = { - .name = "collectd", - .description = "collectd input plugin", - .cb_init = in_collectd_init, - .cb_pre_run = NULL, - .cb_collect = NULL, - .cb_flush_buf = NULL, - .cb_pause = NULL, - .cb_resume = NULL, - .config_map = config_map, - .flags = FLB_INPUT_NET_SERVER, - .cb_exit = in_collectd_exit -}; diff --git a/fluent-bit/plugins/in_collectd/in_collectd.h b/fluent-bit/plugins/in_collectd/in_collectd.h deleted file mode 100644 index 8750c7bf5..000000000 --- a/fluent-bit/plugins/in_collectd/in_collectd.h +++ /dev/null @@ -1,46 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_COLLECTD_H -#define FLB_IN_COLLECTD_H - -#include -#include - -struct flb_in_collectd_config { - char *buf; - int bufsize; - - /* Server */ - char listen[256]; /* RFC-2181 */ - char port[6]; /* RFC-793 */ - - /* Sockets */ - flb_sockfd_t server_fd; - flb_pipefd_t coll_fd; - - flb_sds_t types_db; - struct mk_list *tdb; - struct flb_log_event_encoder log_encoder; - - /* Plugin input instance */ - struct flb_input_instance *ins; -}; - -#endif diff --git a/fluent-bit/plugins/in_collectd/netprot.c b/fluent-bit/plugins/in_collectd/netprot.c deleted file mode 100644 index 005db2edb..000000000 --- a/fluent-bit/plugins/in_collectd/netprot.c +++ /dev/null @@ -1,308 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * This module implements the binary network protocol of collectd. - * (https://collectd.org/wiki/index.php/Binary_protocol) - * - * The only interface you need to care is netprot_to_msgpack() that - * parses a UDP packet and converts it into MessagePack format. - */ - -#include -#include -#include -#include -#include -#include -#include "netprot.h" -#include "typesdb.h" - -#define be16read(x) (be16toh(*(uint16_t *) (x))) -#define be32read(x) (be32toh(*(uint32_t *) (x))) -#define be64read(x) (be64toh(*(uint64_t *) (x))) - -#define le16read(x) (le16toh(*(uint16_t *) (x))) -#define le32read(x) (le32toh(*(uint32_t *) (x))) -#define le64read(x) (le64toh(*(uint64_t *) (x))) - -/* Convert a high-resolution time into a normal UNIX time. */ -#define hr2time(x) ((double) (x) / 1073741824) - -/* Basic data field definitions for collectd */ -#define PART_HOST 0x0000 -#define PART_TIME 0x0001 -#define PART_PLUGIN 0x0002 -#define PART_PLUGIN_INSTANCE 0x0003 -#define PART_TYPE 0x0004 -#define PART_TYPE_INSTANCE 0x0005 -#define PART_VALUE 0x0006 -#define PART_INTERVAL 0x0007 - -#define PART_TIME_HR 0x0008 -#define PART_INTERVAL_HR 0x0009 - -/* - * The "DS_TYPE_*" are value types for PART_VALUE fields. - * - * Read https://collectd.org/wiki/index.php/Data_source for what - * these types mean. - */ -#define DS_TYPE_COUNTER 0 -#define DS_TYPE_GAUGE 1 -#define DS_TYPE_DERIVE 2 -#define DS_TYPE_ABSOLUTE 3 - -struct netprot_header -{ - double time; - double interval; - char *host; - char *plugin; - char *plugin_instance; - char *type; - char *type_instance; -}; - -static int netprot_pack_value(char *ptr, int size, struct netprot_header *hdr, - struct mk_list *tdb, - struct flb_log_event_encoder *encoder) -{ - int i; - char type; - char *pval; - uint16_t count; - struct typesdb_node *node; - int result; - - if (hdr->type == NULL) { - flb_error("[in_collectd] invalid data (type is NULL)"); - return -1; - } - - /* - * Since each value uses (1 + 8) bytes, the total buffer size must - * be 2-byte header plus bytes. - */ - count = be16read(ptr); - if (size != 2 + count * 9) { - flb_error("[in_collectd] data corrupted (size=%i, count=%i)", - size, count); - return -1; - } - - /* - * We need to query against TypesDB in order to get field names - * for the data set values. - */ - node = typesdb_find_node(tdb, hdr->type); - if (!node) { - flb_error("[in_collectd] no such type found '%s'", hdr->type); - return -1; - } - if (node->count != count) { - flb_error("[in_collectd] invalid value for '%s' (%i != %i)", - hdr->type, node->count, count); - return -1; - } - - result = flb_log_event_encoder_begin_record(encoder); - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_set_current_timestamp(encoder); - } - - if (hdr->type != NULL && - result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_append_body_values( - encoder, - FLB_LOG_EVENT_CSTRING_VALUE("type"), - FLB_LOG_EVENT_CSTRING_VALUE(hdr->type)); - } - - if (hdr->type_instance != NULL && - result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_append_body_values( - encoder, - FLB_LOG_EVENT_CSTRING_VALUE("type_instance"), - FLB_LOG_EVENT_CSTRING_VALUE(hdr->type_instance)); - } - - if (hdr->time > 0 && - result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_append_body_values( - encoder, - FLB_LOG_EVENT_CSTRING_VALUE("time"), - FLB_LOG_EVENT_DOUBLE_VALUE(hdr->time)); - } - - if (hdr->interval > 0 && - result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_append_body_values( - encoder, - FLB_LOG_EVENT_CSTRING_VALUE("interval"), - FLB_LOG_EVENT_DOUBLE_VALUE(hdr->interval)); - } - - if (hdr->plugin != NULL && - result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_append_body_values( - encoder, - FLB_LOG_EVENT_CSTRING_VALUE("plugin"), - FLB_LOG_EVENT_CSTRING_VALUE(hdr->plugin)); - } - - if (hdr->plugin_instance != NULL && - result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_append_body_values( - encoder, - FLB_LOG_EVENT_CSTRING_VALUE("plugin_instance"), - FLB_LOG_EVENT_CSTRING_VALUE(hdr->plugin_instance)); - } - - if (hdr->host != NULL && - result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_append_body_values( - encoder, - FLB_LOG_EVENT_CSTRING_VALUE("host"), - FLB_LOG_EVENT_CSTRING_VALUE(hdr->host)); - } - - for (i = 0; i < count && result == FLB_EVENT_ENCODER_SUCCESS ; i++) { - pval = ptr + 2 + count + 8 * i; - type = ptr[2 + i]; - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_append_body_cstring( - encoder, node->fields[i]); - } - - switch (type) { - case DS_TYPE_COUNTER: - result = flb_log_event_encoder_append_body_uint64( - encoder, be64read(pval)); - break; - case DS_TYPE_GAUGE: - result = flb_log_event_encoder_append_body_double( - encoder, *((double *) pval)); - break; - case DS_TYPE_DERIVE: - result = flb_log_event_encoder_append_body_int64( - encoder, (int64_t) be64read(pval)); - break; - case DS_TYPE_ABSOLUTE: - result = flb_log_event_encoder_append_body_uint64( - encoder, be64read(pval)); - break; - default: - flb_error("[in_collectd] unknown data type %i", type); - - result = FLB_EVENT_ENCODER_ERROR_INVALID_ARGUMENT; - } - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_commit_record(encoder); - } - else { - flb_log_event_encoder_rollback_record(encoder); - } - - if (result != FLB_EVENT_ENCODER_SUCCESS) { - return -1; - } - - return 0; -} - -/* - * Entry point function - */ -int netprot_to_msgpack(char *buf, int len, struct mk_list *tdb, - struct flb_log_event_encoder *encoder) -{ - uint16_t part_type; - uint16_t part_len; - int size; - char *ptr; - struct netprot_header hdr = {0}; - - while (len >= 4) { - part_type = be16read(buf); - part_len = be16read(buf + 2); - - if (len < part_len) { - flb_error("[in_collectd] data truncated (%i < %i)", len, part_len); - return -1; - } - ptr = buf + 4; - size = part_len - 4; - - switch (part_type) { - case PART_HOST: - if (ptr[size] == '\0') { - hdr.host = ptr; - } - break; - case PART_TIME: - hdr.time = (double) be64read(ptr); - break; - case PART_TIME_HR: - hdr.time = hr2time(be64read(ptr)); - break; - case PART_PLUGIN: - if (ptr[size] == '\0') { - hdr.plugin = ptr; - } - break; - case PART_PLUGIN_INSTANCE: - if (ptr[size] == '\0') { - hdr.plugin_instance = ptr; - } - break; - case PART_TYPE: - if (ptr[size] == '\0') { - hdr.type = ptr; - } - break; - case PART_TYPE_INSTANCE: - if (ptr[size] == '\0') { - hdr.type_instance = ptr; - } - break; - case PART_VALUE: - if (netprot_pack_value(ptr, size, &hdr, tdb, encoder)) { - return -1; - } - break; - case PART_INTERVAL: - hdr.interval = (double) be64read(ptr); - break; - case PART_INTERVAL_HR: - hdr.interval = hr2time(be64read(ptr)); - break; - default: - flb_debug("[in_collectd] skip unknown type %x", part_type); - break; - } - len -= part_len; - buf += part_len; - } - return 0; -} diff --git a/fluent-bit/plugins/in_collectd/netprot.h b/fluent-bit/plugins/in_collectd/netprot.h deleted file mode 100644 index c9292640c..000000000 --- a/fluent-bit/plugins/in_collectd/netprot.h +++ /dev/null @@ -1,22 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* Convert a binary buffer into MessagePack format */ -int netprot_to_msgpack(char *buf, int len, struct mk_list *tdb, - struct flb_log_event_encoder *encoder); diff --git a/fluent-bit/plugins/in_collectd/typesdb.c b/fluent-bit/plugins/in_collectd/typesdb.c deleted file mode 100644 index ef579583c..000000000 --- a/fluent-bit/plugins/in_collectd/typesdb.c +++ /dev/null @@ -1,223 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "in_collectd.h" -#include "typesdb.h" -#include "typesdb_parser.h" - -#include -#include -#include - - -/* Internal function to load from a single TypesDB */ -static int typesdb_load(struct flb_in_collectd_config *ctx, - struct mk_list *tdb, const char *path) -{ - int fd = open(path, O_RDONLY); - if (fd < 0) { - flb_errno(); - flb_plg_error(ctx->ins, "failed to open '%s'", path); - return -1; - } - - if (typesdb_parse(tdb, fd)) { - flb_plg_error(ctx->ins, "failed to parse '%s'", path); - close(fd); - return -1; - } - close(fd); - return 0; -} - -/* - * Load multiple TypesDB files at once. The return value is - * a linked list of typesdb_node objects. - * - * "paths" is a comma-separated list of file names. - */ -struct mk_list *typesdb_load_all(struct flb_in_collectd_config *ctx, - const char *paths) -{ - char *buf; - char *state; - char *path; - struct mk_list *tdb; - - buf = flb_strdup(paths); - if (!buf) { - flb_errno(); - return NULL; - } - - tdb = flb_malloc(sizeof(struct mk_list)); - if (!tdb) { - flb_errno(); - flb_free(buf); - return NULL; - } - mk_list_init(tdb); - - path = strtok_r(buf, ",", &state); - while (path) { - if (typesdb_load(ctx, tdb, path)) { - typesdb_destroy(tdb); - flb_free(buf); - return NULL; - } - path = strtok_r(NULL, ",", &state); - } - flb_free(buf); - return tdb; -} - -void typesdb_destroy(struct mk_list *tdb) -{ - struct typesdb_node *node; - struct mk_list *head; - struct mk_list *tmp; - - mk_list_foreach_safe(head, tmp, tdb) { - node = mk_list_entry(head, struct typesdb_node, _head); - typesdb_destroy_node(node); - } - flb_free(tdb); -} - -struct typesdb_node *typesdb_find_node(struct mk_list *tdb, const char *type) -{ - struct typesdb_node *node; - struct mk_list *head; - - if (type == NULL) { - return NULL; - } - - /* - * Search the linked list from the tail so that later entries - * take precedence over earlier ones. - */ - mk_list_foreach_r(head, tdb) { - node = mk_list_entry(head, struct typesdb_node, _head); - if (strcmp(node->type, type) == 0) { - return node; - } - } - return NULL; -} - -struct typesdb_node *typesdb_last_node(struct mk_list *tdb) -{ - return mk_list_entry_last(tdb, struct typesdb_node, _head); -} - -/* - * The folloings are API functions to modify a TypesDB instance. - */ -int typesdb_add_node(struct mk_list *tdb, const char *type) -{ - struct typesdb_node *node; - - node = flb_calloc(1, sizeof(struct typesdb_node)); - if (!node) { - flb_errno(); - return -1; - } - - node->type = flb_strdup(type); - if (!node->type) { - flb_errno(); - flb_free(node); - return -1; - } - - mk_list_add(&node->_head, tdb); - return 0; -} - -void typesdb_destroy_node(struct typesdb_node *node) -{ - int i; - - flb_free(node->type); - - if (node->fields) { - for (i = 0; i < node->count; i++) { - flb_free(node->fields[i]); - } - flb_free(node->fields); - } - mk_list_del(&node->_head); - flb_free(node); -} - -int typesdb_add_field(struct typesdb_node *node, const char *field) -{ - char *end; - int alloc; - char **fields; - - end = strchr(field, ':'); - if (!end) { - return -1; - } - - if (node->count >= node->alloc) { - alloc = node->alloc > 0 ? node->alloc * 2 : 1; - fields = flb_realloc(node->fields, alloc * sizeof(char *)); - if (!fields) { - flb_errno(); - return -1; - } - node->alloc = alloc; - node->fields = fields; - } - - node->fields[node->count] = flb_strndup(field, end - field); - if (!node->fields[node->count]) { - flb_errno(); - return -1; - } - node->count++; - return 0; -} - -/* A debug function to see the content of TypesDB */ -void typesdb_dump(struct mk_list *tdb) -{ - struct mk_list *head; - struct typesdb_node *node; - int i; - - mk_list_foreach(head, tdb) { - node = mk_list_entry(head, struct typesdb_node, _head); - - printf("%s", node->type); - for (i = 0; i < node->count; i++) { - printf("\t%s", node->fields[i]); - } - putchar('\n'); - } -} diff --git a/fluent-bit/plugins/in_collectd/typesdb.h b/fluent-bit/plugins/in_collectd/typesdb.h deleted file mode 100644 index 7da131be0..000000000 --- a/fluent-bit/plugins/in_collectd/typesdb.h +++ /dev/null @@ -1,45 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "in_collectd.h" - -struct typesdb_node { - char *type; - int alloc; - int count; - char **fields; - struct mk_list _head; -}; - -/* Load and destroy TypesDB */ -struct mk_list *typesdb_load_all(struct flb_in_collectd_config *ctx, - const char *paths); -void typesdb_destroy(struct mk_list *tdb); - -/* Find a node in TypesDB */ -struct typesdb_node *typesdb_find_node(struct mk_list *tdb, const char *type); -struct typesdb_node *typesdb_last_node(struct mk_list *tdb); - -/* Modify a TypesDB instance (used in typesdb_parser.c) */ -int typesdb_add_node(struct mk_list *tdb, const char *type); -void typesdb_destroy_node(struct typesdb_node *node); -int typesdb_add_field(struct typesdb_node *node, const char *field); - -/* For debugging */ -void typesdb_dump(struct mk_list *tdb); diff --git a/fluent-bit/plugins/in_collectd/typesdb_parser.c b/fluent-bit/plugins/in_collectd/typesdb_parser.c deleted file mode 100644 index 5e237ffaf..000000000 --- a/fluent-bit/plugins/in_collectd/typesdb_parser.c +++ /dev/null @@ -1,214 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * This file implements a collectd 5.x compatible parser for types.db(5). - * - * Note: it internally implements a finite state machine that consumes a - * single char at once, and pushes parsed tokens via typesdb_* methods. - */ - -#include -#include -#include -#include - -#include "typesdb.h" -#include "typesdb_parser.h" - -#define TDB_INVALID -1 -#define TDB_INIT 0 -#define TDB_LEFT 1 -#define TDB_SEP 2 -#define TDB_RIGHT 3 -#define TDB_RIGHT_SEP 4 -#define TDB_COMMENT 5 - -/* See collectd/src/daemon/types_list.c */ -#define MAX_LINE_SIZE 4096 - -/* - * tdb_* are state functions that take a single character as input. - * They do some action based on the input and return the next state. - */ -static int tdb_init(char c, struct mk_list *tdb, char *buf) -{ - switch (c) { - case '#': - return TDB_COMMENT; - case '\r': - case '\n': - return TDB_INIT; - default: - buf[0] = c; - buf[1] = '\0'; - return TDB_LEFT; - } -} - -static int tdb_left(char c, struct mk_list *tdb, char *buf) -{ - int len; - - switch (c) { - case ' ': - if (typesdb_add_node(tdb, buf)) { - return TDB_INVALID; - } - return TDB_SEP; - case '\r': - case '\n': - return TDB_INVALID; - default: - len = strlen(buf); - if (len >= MAX_LINE_SIZE - 1) { - return TDB_INVALID; - } - buf[len] = c; - buf[++len] = '\0'; - return TDB_LEFT; - } -} - -static int tdb_sep(char c, struct mk_list *tdb, char *buf) -{ - switch (c) { - case ' ': - return TDB_SEP; - case '\r': - case '\n': - return TDB_INVALID; - default: - buf[0] = c; - buf[1] = '\0'; - return TDB_RIGHT; - } -} - -static int tdb_right(char c, struct mk_list *tdb, char *buf) -{ - int len; - struct typesdb_node *node = typesdb_last_node(tdb); - - switch (c) { - case ' ': - case ',': - if (typesdb_add_field(node, buf)) { - flb_error("[in_collectd] cannot add value '%s'", buf); - return TDB_INVALID; - } - return TDB_RIGHT_SEP; - case '\r': - case '\n': - if (typesdb_add_field(node, buf)) { - flb_error("[in_collectd] cannot add value '%s'", buf); - return TDB_INVALID; - } - return TDB_INIT; - default: - len = strlen(buf); - if (len >= MAX_LINE_SIZE - 1) { - flb_error("[in_collectd] line too long > %i", MAX_LINE_SIZE); - return TDB_INVALID; - } - buf[len] = c; - buf[++len] = '\0'; - return TDB_RIGHT; - } -} - -static int tdb_right_sep(char c, struct mk_list *tdb, char *buf) -{ - switch (c) { - case ' ': - case ',': - return TDB_RIGHT_SEP; - case '\r': - case '\n': - return TDB_INIT; - default: - buf[0] = c; - buf[1] = '\0'; - return TDB_RIGHT; - } -} - -static int tdb_comment(char c, struct mk_list *tdb, char *buf) -{ - switch (c) { - case '\r': - case '\n': - return TDB_INIT; - default: - return TDB_COMMENT; - } -} - -/* - * Entry point function - */ -int typesdb_parse(struct mk_list *tdb, int fp) -{ - char tmp[1024]; - char buf[MAX_LINE_SIZE]; - char c; - int i; - int bytes; - int state = TDB_INIT; - - while (1) { - bytes = read(fp, tmp, 1024); - if (bytes < 0) { - flb_errno(); - return bytes; - } - if (bytes == 0) { - return 0; - } - for (i = 0; i < bytes; i++) { - c = tmp[i]; - switch (state) { - case TDB_INVALID: - return -1; - case TDB_INIT: - state = tdb_init(c, tdb, buf); - break; - case TDB_LEFT: - state = tdb_left(c, tdb, buf); - break; - case TDB_SEP: - state = tdb_sep(c, tdb, buf); - break; - case TDB_RIGHT: - state = tdb_right(c, tdb, buf); - break; - case TDB_RIGHT_SEP: - state = tdb_right_sep(c, tdb, buf); - break; - case TDB_COMMENT: - state = tdb_comment(c, tdb, buf); - break; - default: - flb_error("[in_collectd] unknown state %i", state); - return -1; - } - } - } - return 0; -} diff --git a/fluent-bit/plugins/in_collectd/typesdb_parser.h b/fluent-bit/plugins/in_collectd/typesdb_parser.h deleted file mode 100644 index 985570134..000000000 --- a/fluent-bit/plugins/in_collectd/typesdb_parser.h +++ /dev/null @@ -1,20 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -int typesdb_parse(struct mk_list *tdb, int fp); diff --git a/fluent-bit/plugins/in_cpu/CMakeLists.txt b/fluent-bit/plugins/in_cpu/CMakeLists.txt deleted file mode 100644 index 4ed82c5d7..000000000 --- a/fluent-bit/plugins/in_cpu/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - cpu.c) - -FLB_PLUGIN(in_cpu "${src}" "") diff --git a/fluent-bit/plugins/in_cpu/cpu.c b/fluent-bit/plugins/in_cpu/cpu.c deleted file mode 100644 index 5d049eb92..000000000 --- a/fluent-bit/plugins/in_cpu/cpu.c +++ /dev/null @@ -1,672 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "cpu.h" - -static inline void snapshot_key_format(int cpus, struct cpu_snapshot *snap_arr) -{ - int i; - struct cpu_snapshot *snap; - - snap = &snap_arr[0]; - memcpy(snap->k_cpu.name, "cpu", 3); - snap->k_cpu.name[3] = '\0'; - - for (i = 1; i <= cpus; i++) { - snap = (struct cpu_snapshot *) &snap_arr[i]; - CPU_KEY_FORMAT(snap, cpu, i); - CPU_KEY_FORMAT(snap, user, i); - CPU_KEY_FORMAT(snap, system, i); - } -} - -static int snapshots_init(int cpus, struct cpu_stats *cstats) -{ - cstats->snap_a = flb_calloc(1, sizeof(struct cpu_snapshot) * (cpus + 1)); - if (!cstats->snap_a) { - flb_errno(); - return -1; - } - - cstats->snap_b = flb_malloc(sizeof(struct cpu_snapshot) * (cpus + 1)); - if (!cstats->snap_b) { - flb_errno(); - return -1; - } - - /* Initialize each array */ - snapshot_key_format(cpus, cstats->snap_a); - snapshot_key_format(cpus, cstats->snap_b); - cstats->snap_active = CPU_SNAP_ACTIVE_A; - return 0; -} - -static inline void snapshots_switch(struct cpu_stats *cstats) -{ - if (cstats->snap_active == CPU_SNAP_ACTIVE_A) { - cstats->snap_active = CPU_SNAP_ACTIVE_B; - } - else { - cstats->snap_active = CPU_SNAP_ACTIVE_A; - } -} - -/* Retrieve CPU load from the system (through ProcFS) */ -static inline double proc_cpu_load(int cpus, struct cpu_stats *cstats) -{ - int i; - int ret; - char line[255]; - size_t len = 0; - char *fmt; - FILE *f; - struct cpu_snapshot *s; - struct cpu_snapshot *snap_arr; - - f = fopen("/proc/stat", "r"); - if (f == NULL) { - flb_errno(); - return -1; - } - - if (cstats->snap_active == CPU_SNAP_ACTIVE_A) { - snap_arr = cstats->snap_a; - } - else { - snap_arr = cstats->snap_b; - } - - /* Always read (n_cpus + 1) lines */ - for (i = 0; i <= cpus; i++) { - if (fgets(line, sizeof(line) - 1, f)) { - len = strlen(line); - if (line[len - 1] == '\n') { - line[--len] = 0; - if (len && line[len - 1] == '\r') { - line[--len] = 0; - } - } - - s = &snap_arr[i]; - if (i == 0) { - fmt = " cpu %lu %lu %lu %lu %lu"; - ret = sscanf(line, - fmt, - &s->v_user, - &s->v_nice, - &s->v_system, - &s->v_idle, - &s->v_iowait); - if (ret < 5) { - fclose(f); - return -1; - } - } - else { - fmt = " %s %lu %lu %lu %lu %lu"; - ret = sscanf(line, - fmt, - s->v_cpuid, - &s->v_user, - &s->v_nice, - &s->v_system, - &s->v_idle, - &s->v_iowait); - if (ret <= 5) { - fclose(f); - return -1; - } - } - } - else { - break; - } - } - - fclose(f); - return 0; -} - -/* Retrieve CPU stats for a given PID */ -static inline double proc_cpu_pid_load(struct flb_cpu *ctx, - pid_t pid, struct cpu_stats *cstats) -{ - int ret; - char *p; - char line[255]; - char *fmt = ") %c %d %d %d %d %d %u %lu %lu %lu %lu %lu %lu "; - FILE *f; - /* sscanf variables (ss_N) to perform scanning */ - unsigned char ss_state; - unsigned int ss_ppid; - unsigned int ss_pgrp; - unsigned int ss_session; - unsigned int ss_tty_nr; - unsigned int ss_tpgid; - unsigned int ss_flags; - unsigned long ss_minflt; - unsigned long ss_cmdinflt; - unsigned long ss_majflt; - unsigned long ss_cmajflt; - struct cpu_snapshot *s; - - /* Read the process stats */ - snprintf(line, sizeof(line) - 1, "/proc/%d/stat", pid); - f = fopen(line, "r"); - if (f == NULL) { - flb_errno(); - flb_plg_error(ctx->ins, "error opening stats file %s", line); - return -1; - } - - if (cstats->snap_active == CPU_SNAP_ACTIVE_A) { - s = cstats->snap_a; - } - else { - s = cstats->snap_b; - } - - if (fgets(line, sizeof(line) - 1, f) == NULL) { - flb_plg_error(ctx->ins, "cannot read process %ld stats", (long) pid); - fclose(f); - return -1; - } - - errno = 0; - - /* skip first two values (after process name) */ - p = line; - while (*p != ')') p++; - - errno = 0; - ret = sscanf(p, - fmt, - &ss_state, - &ss_ppid, - &ss_pgrp, - &ss_session, - &ss_tty_nr, - &ss_tpgid, - &ss_flags, - &ss_minflt, - &ss_cmdinflt, - &ss_majflt, - &ss_cmajflt, - &s->v_user, - &s->v_system); - if (errno != 0) { - flb_errno(); - flb_plg_error(ctx->ins, "pid sscanf failed ret=%i", ret); - } - - fclose(f); - return 0; -} - -/* - * Given the two snapshots, calculate the % used in user and kernel space, - * it returns the active snapshot. - */ -struct cpu_snapshot *snapshot_percent(struct cpu_stats *cstats, - struct flb_cpu *ctx) -{ - int i; - unsigned long sum_pre; - unsigned long sum_now; - struct cpu_snapshot *arr_pre = cstats->snap_b; - struct cpu_snapshot *arr_now = cstats->snap_a; - struct cpu_snapshot *snap_pre = NULL; - struct cpu_snapshot *snap_now = NULL; - - if (cstats->snap_active == CPU_SNAP_ACTIVE_A) { - arr_now = cstats->snap_a; - arr_pre = cstats->snap_b; - } - else if (cstats->snap_active == CPU_SNAP_ACTIVE_B) { - arr_now = cstats->snap_b; - arr_pre = cstats->snap_a; - } - - for (i = 0; i <= ctx->n_processors; i++) { - snap_pre = &arr_pre[i]; - snap_now = &arr_now[i]; - - /* Calculate overall CPU usage (user space + kernel space */ - sum_pre = (snap_pre->v_user + snap_pre->v_nice + snap_pre->v_system); - sum_now = (snap_now->v_user + snap_now->v_nice + snap_now->v_system); - - if (i == 0) { - snap_now->p_cpu = CPU_METRIC_SYS_AVERAGE(sum_pre, sum_now, ctx); - } - else { - snap_now->p_cpu = CPU_METRIC_USAGE(sum_pre, sum_now, ctx); - } - - /* User space CPU% */ - sum_pre = (snap_pre->v_user + snap_pre->v_nice); - sum_now = (snap_now->v_user + snap_now->v_nice); - if (i == 0) { - snap_now->p_user = CPU_METRIC_SYS_AVERAGE(sum_pre, sum_now, ctx); - } - else { - snap_now->p_user = CPU_METRIC_USAGE(sum_pre, sum_now, ctx); - } - - /* Kernel space CPU% */ - if (i == 0) { - snap_now->p_system = CPU_METRIC_SYS_AVERAGE(snap_pre->v_system, - snap_now->v_system, - ctx); - } - else { - snap_now->p_system = CPU_METRIC_USAGE(snap_pre->v_system, - snap_now->v_system, - ctx); - } - -#ifdef FLB_TRACE - if (i == 0) { - flb_trace("cpu[all] all=%s%f%s user=%s%f%s system=%s%f%s", - ANSI_BOLD, snap_now->p_cpu, ANSI_RESET, - ANSI_BOLD, snap_now->p_user, ANSI_RESET, - ANSI_BOLD, snap_now->p_system, ANSI_RESET); - } - else { - flb_trace("cpu[i=%i] all=%f user=%f system=%f", - i-1, snap_now->p_cpu, - snap_now->p_user, snap_now->p_system); - } -#endif - } - - return arr_now; -} - -struct cpu_snapshot *snapshot_pid_percent(struct cpu_stats *cstats, - struct flb_cpu *ctx) -{ - unsigned long sum_pre; - unsigned long sum_now; - struct cpu_snapshot *snap_pre = NULL; - struct cpu_snapshot *snap_now = NULL; - - if (cstats->snap_active == CPU_SNAP_ACTIVE_A) { - snap_now = cstats->snap_a; - snap_pre = cstats->snap_b; - } - else if (cstats->snap_active == CPU_SNAP_ACTIVE_B) { - snap_now = cstats->snap_b; - snap_pre = cstats->snap_a; - } - - /* Calculate overall CPU usage (user space + kernel space */ - sum_pre = (snap_pre->v_user + snap_pre->v_system); - sum_now = (snap_now->v_user + snap_now->v_system); - - snap_now->p_cpu = CPU_METRIC_SYS_AVERAGE(sum_pre, sum_now, ctx); - - /* User space CPU% */ - snap_now->p_user = CPU_METRIC_SYS_AVERAGE(snap_pre->v_user, - snap_now->v_user, - ctx); - - /* Kernel space CPU% */ - snap_now->p_system = CPU_METRIC_SYS_AVERAGE(snap_pre->v_system, - snap_now->v_system, - ctx); - -#ifdef FLB_TRACE - flb_trace("cpu[pid=%i] all=%s%f%s user=%s%f%s system=%s%f%s", - ctx->pid, - ANSI_BOLD, snap_now->p_cpu, ANSI_RESET, - ANSI_BOLD, snap_now->p_user, ANSI_RESET, - ANSI_BOLD, snap_now->p_system, ANSI_RESET); -#endif - - return snap_now; -} - -static int cpu_collect_system(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - int i; - int ret; - struct flb_cpu *ctx = in_context; - struct cpu_stats *cstats = &ctx->cstats; - struct cpu_snapshot *s; - - (void) config; - - /* Get overall system CPU usage */ - ret = proc_cpu_load(ctx->n_processors, cstats); - if (ret != 0) { - flb_plg_error(ins, "error retrieving overall system CPU stats"); - return -1; - } - - s = snapshot_percent(cstats, ctx); - - ret = flb_log_event_encoder_begin_record(&ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp( - &ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - &ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE("cpu_p"), - FLB_LOG_EVENT_DOUBLE_VALUE(s[0].p_cpu), - - FLB_LOG_EVENT_CSTRING_VALUE("user_p"), - FLB_LOG_EVENT_DOUBLE_VALUE(s[0].p_user), - - FLB_LOG_EVENT_CSTRING_VALUE("system_p"), - FLB_LOG_EVENT_DOUBLE_VALUE(s[0].p_system)); - } - - for (i = 1; - i < ctx->n_processors + 1 && - ret == FLB_EVENT_ENCODER_SUCCESS; - i++) { - struct cpu_snapshot *e = &s[i]; - - ret = flb_log_event_encoder_append_body_values( - &ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE(e->k_cpu.name), - FLB_LOG_EVENT_DOUBLE_VALUE(e->p_cpu), - - FLB_LOG_EVENT_CSTRING_VALUE(e->k_user.name), - FLB_LOG_EVENT_DOUBLE_VALUE(e->p_user), - - FLB_LOG_EVENT_CSTRING_VALUE(e->k_system.name), - FLB_LOG_EVENT_DOUBLE_VALUE(e->p_system)); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(&ctx->log_encoder); - } - - snapshots_switch(cstats); - - flb_plg_trace(ins, "CPU %0.2f%%", s->p_cpu); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(ins, NULL, 0, - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - ret = 0; - } - else { - flb_plg_error(ctx->ins, "Error encoding record : %d", ret); - - ret = -1; - } - - flb_log_event_encoder_reset(&ctx->log_encoder); - - return ret; -} - -static int cpu_collect_pid(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - int ret; - struct flb_cpu *ctx = in_context; - struct cpu_stats *cstats = &ctx->cstats; - struct cpu_snapshot *s; - - (void) config; - - /* Get overall system CPU usage */ - ret = proc_cpu_pid_load(ctx, ctx->pid, cstats); - if (ret != 0) { - flb_plg_error(ctx->ins, "error retrieving PID CPU stats"); - return -1; - } - - s = snapshot_pid_percent(cstats, ctx); - - ret = flb_log_event_encoder_begin_record(&ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp( - &ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - &ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE("cpu_p"), - FLB_LOG_EVENT_DOUBLE_VALUE(s->p_cpu), - - FLB_LOG_EVENT_CSTRING_VALUE("user_p"), - FLB_LOG_EVENT_DOUBLE_VALUE(s->p_user), - - FLB_LOG_EVENT_CSTRING_VALUE("system_p"), - FLB_LOG_EVENT_DOUBLE_VALUE(s->p_system)); - } - - snapshots_switch(cstats); - flb_plg_trace(ctx->ins, "PID %i CPU %0.2f%%", ctx->pid, s->p_cpu); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(&ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(ins, NULL, 0, - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - - ret = 0; - } - else { - flb_plg_error(ctx->ins, "Error encoding record : %d", ret); - - ret = -1; - } - - flb_log_event_encoder_reset(&ctx->log_encoder); - - return ret; -} - -/* Callback to gather CPU usage between now and previous snapshot */ -static int cb_cpu_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_cpu *ctx = in_context; - - /* if a PID is get, get CPU stats only for that process */ - if (ctx->pid >= 0) { - return cpu_collect_pid(ins, config, in_context); - } - else { - /* Get all system CPU stats */ - return cpu_collect_system(ins, config, in_context); - } -} - -/* Init CPU input */ -static int cb_cpu_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - int ret; - struct flb_cpu *ctx; - (void) data; - - /* Allocate space for the configuration */ - ctx = flb_calloc(1, sizeof(struct flb_cpu)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->ins = in; - - ret = flb_input_config_map_set(in, (void *)ctx); - if (ret == -1) { - flb_free(ctx); - return -1; - } - - /* Gather number of processors and CPU ticks */ - ctx->n_processors = sysconf(_SC_NPROCESSORS_ONLN); - ctx->cpu_ticks = sysconf(_SC_CLK_TCK); - - /* Collection time setting */ - if (ctx->interval_sec <= 0 && ctx->interval_nsec <= 0) { - /* Illegal settings. Override them. */ - ctx->interval_sec = atoi(DEFAULT_INTERVAL_SEC); - ctx->interval_nsec = atoi(DEFAULT_INTERVAL_NSEC); - } - - /* Initialize buffers for CPU stats */ - ret = snapshots_init(ctx->n_processors, &ctx->cstats); - if (ret != 0) { - flb_free(ctx); - return -1; - } - - /* Get CPU load, ready to be updated once fired the calc callback */ - if (ctx->pid > 0) { - ret = proc_cpu_pid_load(ctx, ctx->pid, &ctx->cstats); - } - else { - ret = proc_cpu_load(ctx->n_processors, &ctx->cstats); - } - if (ret != 0) { - flb_error("[cpu] Could not obtain CPU data"); - flb_free(ctx); - return -1; - } - - ctx->cstats.snap_active = CPU_SNAP_ACTIVE_B; - - /* Set the context */ - flb_input_set_context(in, ctx); - - /* Set our collector based on time, CPU usage every 1 second */ - ret = flb_input_set_collector_time(in, - cb_cpu_collect, - ctx->interval_sec, - ctx->interval_nsec, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not set collector for CPU input plugin"); - return -1; - } - ctx->coll_fd = ret; - - ret = flb_log_event_encoder_init(&ctx->log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, "error initializing event encoder : %d", ret); - - /* Release snapshots */ - flb_free(ctx->cstats.snap_a); - flb_free(ctx->cstats.snap_b); - - /* done */ - flb_free(ctx); - - return -1; - } - - return 0; -} - -static void cb_cpu_pause(void *data, struct flb_config *config) -{ - struct flb_cpu *ctx = data; - flb_input_collector_pause(ctx->coll_fd, ctx->ins); -} - -static void cb_cpu_resume(void *data, struct flb_config *config) -{ - struct flb_cpu *ctx = data; - flb_input_collector_resume(ctx->coll_fd, ctx->ins); -} - -static int cb_cpu_exit(void *data, struct flb_config *config) -{ - (void) *config; - struct flb_cpu *ctx = data; - struct cpu_stats *cs; - - flb_log_event_encoder_destroy(&ctx->log_encoder); - - /* Release snapshots */ - cs = &ctx->cstats; - flb_free(cs->snap_a); - flb_free(cs->snap_b); - - /* done */ - flb_free(ctx); - - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_INT, "pid", "-1", - 0, FLB_TRUE, offsetof(struct flb_cpu, pid), - "Configure a single process to measure usage via their PID" - }, - { - FLB_CONFIG_MAP_INT, "interval_sec", DEFAULT_INTERVAL_SEC, - 0, FLB_TRUE, offsetof(struct flb_cpu, interval_sec), - "Set the collector interval" - }, - { - FLB_CONFIG_MAP_INT, "interval_nsec", DEFAULT_INTERVAL_NSEC, - 0, FLB_TRUE, offsetof(struct flb_cpu, interval_nsec), - "Set the collector interval (sub seconds)" - }, - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_input_plugin in_cpu_plugin = { - .name = "cpu", - .description = "CPU Usage", - .cb_init = cb_cpu_init, - .cb_pre_run = NULL, - .cb_collect = cb_cpu_collect, - .cb_flush_buf = NULL, - .config_map = config_map, - .cb_pause = cb_cpu_pause, - .cb_resume = cb_cpu_resume, - .cb_exit = cb_cpu_exit -}; diff --git a/fluent-bit/plugins/in_cpu/cpu.h b/fluent-bit/plugins/in_cpu/cpu.h deleted file mode 100644 index 93cbd88c1..000000000 --- a/fluent-bit/plugins/in_cpu/cpu.h +++ /dev/null @@ -1,129 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_CPU_H -#define FLB_IN_CPU_H - -#include -#include -#include -#include - -/* Default collection time: every 1 second (0 nanoseconds) */ -#define DEFAULT_INTERVAL_SEC "1" -#define DEFAULT_INTERVAL_NSEC "0" -#define IN_CPU_KEY_LEN 16 - -struct cpu_key { - uint8_t length; - char name[IN_CPU_KEY_LEN]; -}; - -struct cpu_snapshot { - /* data snapshots */ - char v_cpuid[8]; - unsigned long v_user; - unsigned long v_nice; - unsigned long v_system; - unsigned long v_idle; - unsigned long v_iowait; - - /* percent values */ - double p_cpu; /* Overall CPU usage */ - double p_user; /* user space (user + nice) */ - double p_system; /* kernel space percent */ - - /* necessary... */ - struct cpu_key k_cpu; - struct cpu_key k_user; - struct cpu_key k_system; -}; - -#define CPU_SNAP_ACTIVE_A 0 -#define CPU_SNAP_ACTIVE_B 1 - -struct cpu_stats { - uint8_t snap_active; - - /* CPU snapshots, we always keep two snapshots */ - struct cpu_snapshot *snap_a; - struct cpu_snapshot *snap_b; -}; - -/* CPU Input configuration & context */ -struct flb_cpu { - /* setup */ - pid_t pid; /* optional PID */ - int n_processors; /* number of core processors */ - int cpu_ticks; /* CPU ticks (Kernel setting) */ - int coll_fd; /* collector id/fd */ - int interval_sec; /* interval collection time (Second) */ - int interval_nsec; /* interval collection time (Nanosecond) */ - struct cpu_stats cstats; - struct flb_input_instance *ins; - struct flb_log_event_encoder log_encoder; -}; - -#define CPU_KEY_FORMAT(s, key, i) \ - s->k_##key.length = snprintf(s->k_##key.name, \ - IN_CPU_KEY_LEN, \ - "cpu%i.p_%s", i - 1, #key) - -#define ULL_ABS(a, b) (a > b) ? a - b : b - a - -/* - * This routine calculate the average CPU utilization of the system, it - * takes in consideration the number CPU cores, so it return a value - * between 0 and 100 based on 'capacity'. - */ -static inline double CPU_METRIC_SYS_AVERAGE(unsigned long pre, - unsigned long now, - struct flb_cpu *ctx) -{ - double diff; - double total = 0; - - if (pre == now) { - return 0.0; - } - - diff = ULL_ABS(now, pre); - total = (((diff / ctx->cpu_ticks) * 100) / ctx->n_processors) / (ctx->interval_sec + 1e-9*ctx->interval_nsec); - - return total; -} - -/* Returns the CPU % utilization of a given CPU core */ -static inline double CPU_METRIC_USAGE(unsigned long pre, unsigned long now, - struct flb_cpu *ctx) -{ - double diff; - double total = 0; - - if (pre == now) { - return 0.0; - } - - diff = ULL_ABS(now, pre); - - total = ((diff * 100) / ctx->cpu_ticks) / (ctx->interval_sec + 1e-9*ctx->interval_nsec); - return total; -} - -#endif diff --git a/fluent-bit/plugins/in_disk/CMakeLists.txt b/fluent-bit/plugins/in_disk/CMakeLists.txt deleted file mode 100644 index a969ad47e..000000000 --- a/fluent-bit/plugins/in_disk/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - in_disk.c) - -FLB_PLUGIN(in_disk "${src}" "") diff --git a/fluent-bit/plugins/in_disk/in_disk.c b/fluent-bit/plugins/in_disk/in_disk.c deleted file mode 100644 index 607d030b1..000000000 --- a/fluent-bit/plugins/in_disk/in_disk.c +++ /dev/null @@ -1,387 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "in_disk.h" - -#define LINE_SIZE 256 -#define BUF_SIZE 32 - -static char *shift_line(const char *line, char separator, int *idx, - char *buf, int buf_size) -{ - char pack_mode = FLB_FALSE; - int idx_buf = 0; - - while (1) { - if (line[*idx] == '\0') { - /* end of line */ - return NULL; - } - else if (line[*idx] != separator) { - pack_mode = FLB_TRUE; - buf[idx_buf] = line[*idx]; - idx_buf++; - - if (idx_buf >= buf_size) { - buf[idx_buf-1] = '\0'; - return NULL; - } - } - else if (pack_mode == FLB_TRUE) { - buf[idx_buf] = '\0'; - return buf; - } - *idx += 1; - } -} - -static int update_disk_stats(struct flb_in_disk_config *ctx) -{ - char line[LINE_SIZE] = {0}; - char buf[BUF_SIZE] = {0}; - char skip_line = FLB_FALSE; - uint64_t temp_total = 0; - FILE *fp = NULL; - int i_line = 0; - int i_entry = 0; - int i_field = 0; - - fp = fopen("/proc/diskstats", "r"); - if (fp == NULL) { - flb_errno(); - return -1; - } - - while (fgets(line, LINE_SIZE-1, fp) != NULL) { - i_line = 0; - i_field = 0; - skip_line = FLB_FALSE; - while (skip_line != FLB_TRUE && - shift_line(line, ' ', &i_line, buf, BUF_SIZE-1) != NULL) { - i_field++; - switch(i_field) { - case 3: /* device name */ - if (ctx->dev_name != NULL && strstr(buf, ctx->dev_name) == NULL) { - skip_line = FLB_TRUE; - } - break; - case 6: /* sectors read */ - temp_total = strtoull(buf, NULL, 10); - ctx->prev_read_total[i_entry] = ctx->read_total[i_entry]; - ctx->read_total[i_entry] = temp_total; - break; - case 10: /* sectors written */ - temp_total = strtoull(buf, NULL, 10); - ctx->prev_write_total[i_entry] = ctx->write_total[i_entry]; - ctx->write_total[i_entry] = temp_total; - - skip_line = FLB_TRUE; - break; - default: - continue; - } - } - i_entry++; - } - - fclose(fp); - return 0; -} - - -/* cb_collect callback */ -static int in_disk_collect(struct flb_input_instance *i_ins, - struct flb_config *config, void *in_context) -{ - unsigned long write_total; - unsigned long read_total; - int entry; - struct flb_in_disk_config *ctx; - int ret; - int i; - - (void) *config; - - ret = 0; - ctx = (struct flb_in_disk_config *) in_context; - entry = ctx->entry; - - /* The type of sector size is unsigned long in kernel source */ - read_total = 0; - write_total = 0; - - update_disk_stats(ctx); - - if (ctx->first_snapshot == FLB_TRUE) { - ctx->first_snapshot = FLB_FALSE; /* assign first_snapshot with FLB_FALSE */ - } - else { - for (i = 0; i < entry; i++) { - if (ctx->read_total[i] >= ctx->prev_read_total[i]) { - read_total += ctx->read_total[i] - ctx->prev_read_total[i]; - } - else { - /* Overflow */ - read_total += ctx->read_total[i] + - (ULONG_MAX - ctx->prev_read_total[i]); - } - - if (ctx->write_total[i] >= ctx->prev_write_total[i]) { - write_total += ctx->write_total[i] - ctx->prev_write_total[i]; - } - else { - /* Overflow */ - write_total += ctx->write_total[i] + - (ULONG_MAX - ctx->prev_write_total[i]); - } - } - - read_total *= 512; - write_total *= 512; - - - ret = flb_log_event_encoder_begin_record(&ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp( - &ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - &ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE(STR_KEY_READ), - FLB_LOG_EVENT_UINT64_VALUE(read_total), - - FLB_LOG_EVENT_CSTRING_VALUE(STR_KEY_WRITE), - FLB_LOG_EVENT_UINT64_VALUE(write_total)); - } - - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(&ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(i_ins, NULL, 0, - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - - ret = 0; - } - else { - flb_plg_error(i_ins, "Error encoding record : %d", ret); - - ret = -1; - } - - flb_log_event_encoder_reset(&ctx->log_encoder); - } - - return 0; -} - -static int get_diskstats_entries(void) -{ - char line[LINE_SIZE] = {0}; - int ret = 0; - FILE *fp = NULL; - - fp = fopen("/proc/diskstats", "r"); - if (fp == NULL) { - perror("fopen"); - return 0; - } - while(fgets(line, LINE_SIZE-1, fp) != NULL) { - ret++; - } - - fclose(fp); - return ret; -} - -static int configure(struct flb_in_disk_config *disk_config, - struct flb_input_instance *in) -{ - (void) *in; - int entry = 0; - int i; - int ret; - - /* Load the config map */ - ret = flb_input_config_map_set(in, (void *)disk_config); - if (ret == -1) { - flb_plg_error(in, "unable to load configuration."); - return -1; - } - - /* interval settings */ - if (disk_config->interval_sec <= 0 && disk_config->interval_nsec <= 0) { - /* Illegal settings. Override them. */ - disk_config->interval_sec = atoi(DEFAULT_INTERVAL_SEC); - disk_config->interval_nsec = atoi(DEFAULT_INTERVAL_NSEC); - } - - entry = get_diskstats_entries(); - if (entry == 0) { - /* no entry to count */ - return -1; - } - - disk_config->read_total = (uint64_t*)flb_malloc(sizeof(uint64_t)*entry); - disk_config->write_total = (uint64_t*)flb_malloc(sizeof(uint64_t)*entry); - disk_config->prev_read_total = (uint64_t*)flb_malloc(sizeof(uint64_t)*entry); - disk_config->prev_write_total = (uint64_t*)flb_malloc(sizeof(uint64_t)*entry); - disk_config->entry = entry; - - if ( disk_config->read_total == NULL || - disk_config->write_total == NULL || - disk_config->prev_read_total == NULL || - disk_config->prev_write_total == NULL) { - flb_plg_error(in, "could not allocate memory"); - return -1; - } - - /* initialize */ - for (i=0; iread_total[i] = 0; - disk_config->write_total[i] = 0; - disk_config->prev_read_total[i] = 0; - disk_config->prev_write_total[i] = 0; - } - update_disk_stats(disk_config); - - disk_config->first_snapshot = FLB_TRUE; /* assign first_snapshot with FLB_TRUE */ - - return 0; -} - -/* Initialize plugin */ -static int in_disk_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - struct flb_in_disk_config *disk_config = NULL; - int ret = -1; - - /* Allocate space for the configuration */ - disk_config = flb_calloc(1, sizeof(struct flb_in_disk_config)); - if (disk_config == NULL) { - return -1; - } - - disk_config->read_total = NULL; - disk_config->write_total = NULL; - disk_config->prev_read_total = NULL; - disk_config->prev_write_total = NULL; - - /* Initialize head config */ - ret = configure(disk_config, in); - if (ret < 0) { - goto init_error; - } - - flb_input_set_context(in, disk_config); - - ret = flb_input_set_collector_time(in, - in_disk_collect, - disk_config->interval_sec, - disk_config->interval_nsec, config); - if (ret < 0) { - flb_plg_error(in, "could not set collector for disk input plugin"); - goto init_error; - } - - ret = flb_log_event_encoder_init(&disk_config->log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(in, "error initializing event encoder : %d", ret); - - goto init_error; - } - - return 0; - - init_error: - flb_free(disk_config->read_total); - flb_free(disk_config->write_total); - flb_free(disk_config->prev_read_total); - flb_free(disk_config->prev_write_total); - flb_free(disk_config); - return -1; -} - -static int in_disk_exit(void *data, struct flb_config *config) -{ - (void) *config; - struct flb_in_disk_config *disk_config = data; - - flb_log_event_encoder_destroy(&disk_config->log_encoder); - - flb_free(disk_config->read_total); - flb_free(disk_config->write_total); - flb_free(disk_config->prev_read_total); - flb_free(disk_config->prev_write_total); - flb_free(disk_config); - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_INT, "interval_sec", DEFAULT_INTERVAL_SEC, - 0, FLB_TRUE, offsetof(struct flb_in_disk_config, interval_sec), - "Set the collector interval" - }, - { - FLB_CONFIG_MAP_INT, "interval_nsec", DEFAULT_INTERVAL_NSEC, - 0, FLB_TRUE, offsetof(struct flb_in_disk_config, interval_nsec), - "Set the collector interval (nanoseconds)" - }, - { - FLB_CONFIG_MAP_STR, "dev_name", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_in_disk_config, dev_name), - "Set the device name" - }, - /* EOF */ - {0} -}; - -struct flb_input_plugin in_disk_plugin = { - .name = "disk", - .description = "Diskstats", - .cb_init = in_disk_init, - .cb_pre_run = NULL, - .cb_collect = in_disk_collect, - .cb_flush_buf = NULL, - .cb_exit = in_disk_exit, - .config_map = config_map -}; diff --git a/fluent-bit/plugins/in_disk/in_disk.h b/fluent-bit/plugins/in_disk/in_disk.h deleted file mode 100644 index 4f4506c7c..000000000 --- a/fluent-bit/plugins/in_disk/in_disk.h +++ /dev/null @@ -1,48 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef FLB_IN_DISK_H -#define FLB_IN_DISK_H - -#include -#include -#include -#include - -#define DEFAULT_INTERVAL_SEC "1" -#define DEFAULT_INTERVAL_NSEC "0" - -#define STR_KEY_WRITE "write_size" -#define STR_KEY_READ "read_size" - -struct flb_in_disk_config { - uint64_t *read_total; - uint64_t *write_total; - uint64_t *prev_read_total; - uint64_t *prev_write_total; - flb_sds_t dev_name; - int entry; - int interval_sec; - int interval_nsec; - int first_snapshot; /* a feild to indicate whethor or not this is the first collect*/ - struct flb_log_event_encoder log_encoder; -}; - -extern struct flb_input_plugin in_disk_plugin; - -#endif /* FLB_IN_DISK_H */ diff --git a/fluent-bit/plugins/in_docker/CMakeLists.txt b/fluent-bit/plugins/in_docker/CMakeLists.txt deleted file mode 100644 index 01cf3a848..000000000 --- a/fluent-bit/plugins/in_docker/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(src - docker.c - cgroup_v1.c - ) - -FLB_PLUGIN(in_docker "${src}" "") diff --git a/fluent-bit/plugins/in_docker/cgroup_v1.c b/fluent-bit/plugins/in_docker/cgroup_v1.c deleted file mode 100644 index a6fe355e2..000000000 --- a/fluent-bit/plugins/in_docker/cgroup_v1.c +++ /dev/null @@ -1,397 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include -#include -#include "docker.h" - -/* This method returns list of currently running docker ids. */ -static struct mk_list *get_active_dockers() -{ - DIR *dp; - struct dirent *ep; - struct mk_list *list; - - list = flb_malloc(sizeof(struct mk_list)); - if (!list) { - flb_errno(); - return NULL; - } - mk_list_init(list); - - dp = opendir(DOCKER_CGROUP_V1_CPU_DIR); - if (dp != NULL) { - ep = readdir(dp); - - while(ep != NULL) { - if (ep->d_type == OS_DIR_TYPE) { - if (strcmp(ep->d_name, CURRENT_DIR) != 0 - && strcmp(ep->d_name, PREV_DIR) != 0 - && strlen(ep->d_name) == DOCKER_LONG_ID_LEN) { /* precautionary check */ - - docker_info *docker = in_docker_init_docker_info(ep->d_name); - mk_list_add(&docker->_head, list); - } - } - ep = readdir(dp); - } - closedir(dp); - } - - return list; -} - -static char *read_line(FILE *fin) -{ - char *buffer; - char *tmp; - int read_chars = 0; - int bufsize = 1215; - char *line; - - line = (char *) flb_calloc(bufsize, sizeof(char)); - if (!line) { - flb_errno(); - return NULL; - } - - buffer = line; - - while (fgets(buffer, bufsize - read_chars, fin)) { - read_chars = strlen(line); - - if (line[read_chars - 1] == '\n') { - line[read_chars - 1] = '\0'; - return line; - } - else { - bufsize = 2 * bufsize; - tmp = flb_realloc(line, bufsize); - if (!tmp) { - flb_errno(); - return NULL; - } - else { - line = tmp; - buffer = line + read_chars; - } - } - } - - flb_free(line); - return NULL; -} - -/* This routine returns path to docker's cgroup CPU usage file. */ -static char *get_cpu_used_file(char *id) -{ - char *path; - - if (!id) { - return NULL; - } - - path = (char *) flb_calloc(105, sizeof(char)); - if (!path) { - flb_errno(); - return NULL; - } - - strcat(path, DOCKER_CGROUP_V1_CPU_DIR); - strcat(path, "/"); - strcat(path, id); - strcat(path, "/"); - strcat(path, DOCKER_CGROUP_V1_CPU_USAGE_FILE); - - return path; -} - -/* This routine returns path to docker's cgroup memory limit file. */ -static char *get_mem_limit_file(char *id) -{ - char *path; - - if (!id) { - return NULL; - } - - path = (char *) flb_calloc(116, sizeof(char)); - if (!path) { - flb_errno(); - return NULL; - } - strcat(path, DOCKER_CGROUP_V1_MEM_DIR); - strcat(path, "/"); - strcat(path, id); - strcat(path, "/"); - strcat(path, DOCKER_CGROUP_V1_MEM_LIMIT_FILE); - - return path; -} - -/* This routine returns path to docker's cgroup memory used file. */ -static char *get_mem_used_file(char *id) -{ - char *path; - - if (!id) { - return NULL; - } - - path = (char *) flb_calloc(116, sizeof(char)); - if (!path) { - flb_errno(); - return NULL; - } - strcat(path, DOCKER_CGROUP_V1_MEM_DIR); - strcat(path, "/"); - strcat(path, id); - strcat(path, "/"); - strcat(path, DOCKER_CGROUP_V1_MEM_USAGE_FILE); - - return path; -} - -static char *get_config_file(char *id) -{ - char *path; - - if (!id) { - return NULL; - } - - path = (char *) flb_calloc(107, sizeof(char)); - if (!path) { - flb_errno(); - return NULL; - } - strcat(path, DOCKER_LIB_ROOT); - strcat(path, "/"); - strcat(path, id); - strcat(path, "/"); - strcat(path, DOCKER_CONFIG_JSON); - - return path; -} - -static char *extract_name(char *line, char *start) -{ - int skip = 9; - int len = 0; - char *name; - char buff[256]; - char *curr; - - if (start != NULL) { - curr = start + skip; - while (*curr != '"') { - buff[len++] = *curr; - curr++; - } - - if (len > 0) { - name = (char *) flb_calloc(len + 1, sizeof(char)); - if (!name) { - flb_errno(); - return NULL; - } - memcpy(name, buff, len); - - return name; - } - } - - return NULL; -} - -static char *get_container_name(struct flb_docker *ctx, char *id) -{ - char *container_name = NULL; - char *config_file; - FILE *f = NULL; - char *line; - - config_file = get_config_file(id); - if (!config_file) { - return NULL; - } - - f = fopen(config_file, "r"); - if (!f) { - flb_errno(); - flb_plg_error(ctx->ins, "cannot open %s", config_file); - flb_free(config_file); - return NULL; - } - - while ((line = read_line(f))) { - char *index = strstr(line, DOCKER_NAME_ARG); - if (index != NULL) { - container_name = extract_name(line, index); - flb_free(line); - break; - } - flb_free(line); - } - - flb_free(config_file); - fclose(f); - - return container_name; -} - -/* Returns CPU metrics for docker id. */ -static cpu_snapshot *get_docker_cpu_snapshot(struct flb_docker *ctx, char *id) -{ - int c; - unsigned long cpu_used = 0; - char *usage_file; - cpu_snapshot *snapshot = NULL; - FILE *f; - - usage_file = get_cpu_used_file(id); - if (!usage_file) { - return NULL; - } - - f = fopen(usage_file, "r"); - if (!f) { - flb_errno(); - flb_plg_error(ctx->ins, "error gathering CPU data from %s", - usage_file); - flb_free(usage_file); - return NULL; - } - - c = fscanf(f, "%ld", &cpu_used); - if (c != 1) { - flb_plg_error(ctx->ins, "error scanning used CPU value from %s", - usage_file); - flb_free(usage_file); - fclose(f); - return NULL; - } - - snapshot = (cpu_snapshot *) flb_calloc(1, sizeof(cpu_snapshot)); - if (!snapshot) { - flb_errno(); - fclose(f); - flb_free(usage_file); - return NULL; - } - - snapshot->used = cpu_used; - - flb_free(usage_file); - fclose(f); - return snapshot; -} - -/* Returns memory used by a docker in bytes. */ -static uint64_t get_docker_mem_used(struct flb_docker *ctx, char *id) -{ - int c; - char *usage_file = NULL; - uint64_t mem_used = 0; - FILE *f; - - usage_file = get_mem_used_file(id); - if (!usage_file) { - return 0; - } - - f = fopen(usage_file, "r"); - if (!f) { - flb_errno(); - flb_plg_error(ctx->ins, "cannot retrieve memory used from %s", - usage_file); - flb_free(usage_file); - return 0; - } - - c = fscanf(f, "%ld", &mem_used); - if (c != 1) { - flb_plg_error(ctx->ins, "cannot scan memory usage value from %s", - usage_file); - flb_free(usage_file); - fclose(f); - return 0; - } - - flb_free(usage_file); - fclose(f); - - return mem_used; -} - -/* Returns memory limit for a docker in bytes. */ -static uint64_t get_docker_mem_limit(char *id) -{ - char *limit_file = get_mem_limit_file(id); - uint64_t mem_limit = 0; - FILE *f; - - if (!limit_file) { - return 0; - } - - f = fopen(limit_file, "r"); - if (!f) { - flb_errno(); - flb_free(limit_file); - return 0; - } - - fscanf(f, "%ld", &mem_limit); - flb_free(limit_file); - fclose(f); - - return mem_limit; -} - -/* Get memory snapshot for a docker id. */ -static mem_snapshot *get_docker_mem_snapshot(struct flb_docker *ctx, char *id) -{ - mem_snapshot *snapshot = NULL; - - snapshot = (mem_snapshot *) flb_calloc(1, sizeof(mem_snapshot)); - if (!snapshot) { - flb_errno(); - return NULL; - } - - snapshot->used = get_docker_mem_used(ctx, id); - snapshot->limit = get_docker_mem_limit(id); - - return snapshot; -} - -int in_docker_set_cgroup_api_v1(struct cgroup_api *api) -{ - api->cgroup_version = 1; - api->get_active_docker_ids = get_active_dockers; - api->get_container_name = get_container_name; - api->get_cpu_snapshot = get_docker_cpu_snapshot; - api->get_mem_snapshot = get_docker_mem_snapshot; - - return 0; -} diff --git a/fluent-bit/plugins/in_docker/docker.c b/fluent-bit/plugins/in_docker/docker.c deleted file mode 100644 index 135c9f6b4..000000000 --- a/fluent-bit/plugins/in_docker/docker.c +++ /dev/null @@ -1,560 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include "docker.h" - -static int cb_docker_collect(struct flb_input_instance *i_ins, - struct flb_config *config, void *in_context); - -docker_info *in_docker_init_docker_info(char *id) -{ - int len; - docker_info *docker; - - docker = flb_malloc(sizeof(docker_info)); - if (!docker) { - flb_errno(); - return NULL; - } - - len = strlen(id); - docker->id = flb_malloc(sizeof(char)*(len + 1)); - if (!docker->id) { - flb_errno(); - flb_free(docker); - return NULL; - } - strcpy(docker->id, id); - docker->id[len] = '\0'; - - return docker; -} - -static docker_snapshot *init_snapshot(char *id) -{ - int id_len; - docker_snapshot *snapshot; - - snapshot = (docker_snapshot *) flb_malloc(sizeof(docker_snapshot)); - if (!snapshot) { - flb_errno(); - return NULL; - } - - id_len = strlen(id) + 1; - snapshot->id = (char *) flb_malloc((id_len)*sizeof(char)); - if (!snapshot->id) { - flb_errno(); - flb_free(snapshot); - return NULL; - } - strcpy(snapshot->id, id); - - return snapshot; -} - -static bool is_exists(struct mk_list *list, char *id) -{ - int id_len; - char *cmp; - docker_info *item; - bool result = false; - struct mk_list *head; - - if (!list || !id) { - return result; - } - - mk_list_foreach(head, list) { - item = mk_list_entry(head, docker_info, _head); - - /* id could be of length 12 or 64 */ - id_len = strlen(item->id); - cmp = flb_calloc(id_len + 1, sizeof(char)); - if (!cmp) { - flb_errno(); - return NULL; - } - memcpy(cmp, id, id_len); - if (strcmp(item->id, cmp) == 0) { - result = true; - } - flb_free(cmp); - } - - return result; -} - -static void free_snapshots(struct mk_list *snaps); -/* Returns dockers CPU/Memory metrics. */ -static struct mk_list *get_docker_stats(struct flb_docker *ctx, struct mk_list *dockers) -{ - docker_snapshot *snapshot; - struct docker_info *docker; - struct mk_list *head; - struct mk_list *snapshots; - - if (!dockers) { - return NULL; - } - - snapshots = flb_malloc(sizeof(struct mk_list)); - if (!snapshots) { - flb_errno(); - return NULL; - } - - mk_list_init(snapshots); - mk_list_foreach(head, dockers) { - docker = mk_list_entry(head, docker_info, _head); - snapshot = init_snapshot(docker->id); - if (snapshot == NULL) { - free_snapshots(snapshots); - return NULL; - } - snapshot->name = ctx->cgroup_api.get_container_name(ctx, docker->id); - if (snapshot->name == NULL) { - free_snapshots(snapshots); - flb_free(snapshot->id); - flb_free(snapshot); - return NULL; - } - snapshot->cpu = ctx->cgroup_api.get_cpu_snapshot(ctx, docker->id); - if (snapshot->cpu == NULL) { - free_snapshots(snapshots); - flb_free(snapshot->name); - flb_free(snapshot->id); - flb_free(snapshot); - return NULL; - } - snapshot->mem = ctx->cgroup_api.get_mem_snapshot(ctx, docker->id); - if (snapshot->mem == NULL) { - free_snapshots(snapshots); - flb_free(snapshot->cpu); - flb_free(snapshot->name); - flb_free(snapshot->id); - flb_free(snapshot); - return NULL; - } - - mk_list_add(&snapshot->_head, snapshots); - } - - return snapshots; -} - -/* Returns a list of docker ids from space delimited string. */ -static struct mk_list *get_ids_from_str(char *space_delimited_str) -{ - struct mk_list *str_parts; - struct mk_list *parts_head; - struct mk_list *tmp; - struct flb_split_entry *part; - struct mk_list *dockers; - docker_info *docker; - - dockers = flb_malloc(sizeof(struct mk_list)); - if (!dockers) { - flb_errno(); - return NULL; - } - - mk_list_init(dockers); - str_parts = flb_utils_split(space_delimited_str, ' ', 256); - mk_list_foreach_safe(parts_head, tmp, str_parts) { - part = mk_list_entry(parts_head, struct flb_split_entry, _head); - if (part->len == DOCKER_LONG_ID_LEN - || part->len == DOCKER_SHORT_ID_LEN) { - docker = in_docker_init_docker_info(part->value); - mk_list_add(&docker->_head, dockers); - } - } - - flb_utils_split_free(str_parts); - return dockers; -} - -/* Initializes blacklist/whitelist. */ -static void init_filter_lists(struct flb_input_instance *f_ins, - struct flb_docker *ctx) -{ - struct mk_list *head; - struct flb_kv *kv; - - ctx->whitelist = NULL; - ctx->blacklist = NULL; - - /* Iterate all filter properties */ - mk_list_foreach(head, &f_ins->properties) { - kv = mk_list_entry(head, struct flb_kv, _head); - - if (strcasecmp(kv->key, "include") == 0) { - ctx->whitelist = get_ids_from_str(kv->val); - } - else if (strcasecmp(kv->key, "exclude") == 0) { - ctx->blacklist = get_ids_from_str(kv->val); - } - } -} - -/* Filters list of active dockers as per config. This returns a new list */ -static struct mk_list *apply_filters(struct flb_docker *ctx, - struct mk_list *dockers) -{ - struct mk_list *head; - struct mk_list *tmp; - docker_info *new; - docker_info *docker; - struct mk_list *filtered; - - if (ctx->whitelist == NULL && ctx->blacklist == NULL) { - return dockers; - } - - filtered = flb_malloc(sizeof(struct mk_list)); - if (!filtered) { - flb_errno(); - return NULL; - } - - mk_list_init(filtered); - - /* whitelist */ - mk_list_foreach_safe(head, tmp, dockers) { - docker = mk_list_entry(head, docker_info, _head); - if (ctx->whitelist == NULL) { - new = in_docker_init_docker_info(docker->id); - mk_list_add(&new->_head, filtered); - } - else { - if (is_exists(ctx->whitelist, docker->id)) { - new = in_docker_init_docker_info(docker->id); - mk_list_add(&new->_head, filtered); - } - } - } - - /* blacklist */ - if (ctx->blacklist != NULL) { - mk_list_foreach_safe(head, tmp, filtered) { - docker = mk_list_entry(head, docker_info, _head); - if (is_exists(ctx->blacklist, docker->id)) { - mk_list_del(&docker->_head); - flb_free(docker->id); - flb_free(docker); - } - } - } - - return filtered; -} - -/* Init Docker input */ -static int cb_docker_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - int ret; - struct flb_docker *ctx; - - /* Allocate space for the configuration */ - ctx = flb_calloc(1, sizeof(struct flb_docker)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->ins = in; - in_docker_set_cgroup_api_v1(&ctx->cgroup_api); /* TODO: support cgroup v2*/ - - init_filter_lists(in, ctx); - - /* Set the context */ - flb_input_set_context(in, ctx); - - /* Load the config map */ - ret = flb_input_config_map_set(in, (void *)ctx); - if (ret == -1) { - flb_free(ctx); - flb_plg_error(in, "unable to load configuration."); - return -1; - } - - if (ctx->interval_sec <= 0 && ctx->interval_nsec <= 0) { - ctx->interval_sec = atoi(DEFAULT_INTERVAL_SEC); - ctx->interval_nsec = atoi(DEFAULT_INTERVAL_NSEC); - } - - /* Set our collector based on time, CPU usage every 1 second */ - ret = flb_input_set_collector_time(in, - cb_docker_collect, ctx->interval_sec, - ctx->interval_nsec, config); - if (ret == -1) { - flb_plg_error(ctx->ins, "Could not set collector for Docker input plugin"); - flb_free(ctx); - return -1; - } - ctx->coll_fd = ret; - - ret = flb_log_event_encoder_init(&ctx->log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, "error initializing event encoder : %d", ret); - flb_free(ctx); - return -1; - } - - return ret; -} - -/* Flush snapshot as a message for output. */ -static void flush_snapshot(struct flb_docker *ctx, - struct flb_input_instance *i_ins, - docker_snapshot *snapshot) -{ - int result; - - if (!snapshot) { - return; - } - - result = flb_log_event_encoder_begin_record(&ctx->log_encoder); - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_set_current_timestamp( - &ctx->log_encoder); - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_append_body_values( - &ctx->log_encoder, - /* Docker ID [12 chars] */ - FLB_LOG_EVENT_CSTRING_VALUE("id"), - FLB_LOG_EVENT_STRING_VALUE(snapshot->id, DOCKER_SHORT_ID_LEN), - - /* Docker Name */ - FLB_LOG_EVENT_CSTRING_VALUE("name"), - FLB_LOG_EVENT_CSTRING_VALUE(snapshot->name), - - /* CPU used [nanoseconds] */ - FLB_LOG_EVENT_CSTRING_VALUE("cpu_used"), - FLB_LOG_EVENT_UINT32_VALUE(snapshot->cpu->used), - - /* Memory used [bytes] */ - FLB_LOG_EVENT_CSTRING_VALUE("mem_used"), - FLB_LOG_EVENT_UINT32_VALUE(snapshot->mem->used), - - /* Memory limit [bytes] */ - FLB_LOG_EVENT_CSTRING_VALUE("mem_limit"), - FLB_LOG_EVENT_UINT64_VALUE(snapshot->mem->limit)); - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_commit_record(&ctx->log_encoder); - } - - flb_trace("[in_docker] ID %s CPU %lu MEMORY %ld", snapshot->id, - snapshot->cpu->used, snapshot->mem->used); - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(i_ins, NULL, 0, - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - - } - else { - flb_plg_error(i_ins, "Error encoding record : %d", result); - } - - flb_log_event_encoder_reset(&ctx->log_encoder); -} - -static void flush_snapshots(struct flb_docker *ctx, - struct flb_input_instance *i_ins, - struct mk_list *snapshots) -{ - struct mk_list *head; - docker_snapshot *snapshot; - - mk_list_foreach(head, snapshots) { - snapshot = mk_list_entry(head, docker_snapshot, _head); - flush_snapshot(ctx, i_ins, snapshot); - } -} - -static void free_snapshots(struct mk_list *snaps) -{ - struct docker_snapshot *snap; - struct mk_list *tmp; - struct mk_list *head; - - if (snaps == NULL) { - return; - } - - mk_list_foreach_safe(head, tmp, snaps) { - snap = mk_list_entry(head, docker_snapshot, _head); - flb_free(snap->id); - flb_free(snap->name); - flb_free(snap->cpu); - flb_free(snap->mem); - flb_free(snap); - } - flb_free(snaps); -} - -static void free_docker_list(struct mk_list *dockers) -{ - struct mk_list *head; - struct mk_list *tmp; - struct docker_info *docker; - - if (dockers == NULL) { - return; - } - - mk_list_foreach_safe(head, tmp, dockers) { - docker = mk_list_entry(head, docker_info, _head); - flb_free(docker->id); - flb_free(docker); - } - flb_free(dockers); -} - -/* Callback to gather Docker CPU/Memory usage. */ -static int cb_docker_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct mk_list *active; - struct mk_list *filtered; - struct mk_list *snaps; - struct flb_docker *ctx = in_context; - (void) config; - - /* Get current active dockers. */ - active = ctx->cgroup_api.get_active_docker_ids(); - - filtered = apply_filters(ctx, active); - if (!filtered) { - free_docker_list(active); - return 0; - } - - /* Get Mem/CPU stats of dockers. */ - snaps = get_docker_stats(ctx, filtered); - if (!snaps) { - free_docker_list(active); - if (active != filtered) { - /* apply_filters can return the address of acive. - * In that case, filtered is already freed. - */ - free_docker_list(filtered); - } - return 0; - } - - flush_snapshots(ctx, ins, snaps); - - free_snapshots(snaps); - free_docker_list(active); - - if (ctx->whitelist != NULL || ctx->blacklist != NULL) { - free_docker_list(filtered); - } - - return 0; -} - -static void cb_docker_pause(void *data, struct flb_config *config) -{ - struct flb_docker *ctx = data; - flb_input_collector_pause(ctx->coll_fd, ctx->ins); -} - -static void cb_docker_resume(void *data, struct flb_config *config) -{ - struct flb_docker *ctx = data; - flb_input_collector_resume(ctx->coll_fd, ctx->ins); -} - -static int cb_docker_exit(void *data, struct flb_config *config) -{ - (void) *config; - struct flb_docker *ctx = data; - - /* done */ - flb_log_event_encoder_destroy(&ctx->log_encoder); - - free_docker_list(ctx->whitelist); - free_docker_list(ctx->blacklist); - flb_free(ctx); - - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_INT, "interval_sec", DEFAULT_INTERVAL_SEC, - 0, FLB_TRUE, offsetof(struct flb_docker, interval_sec), - "Set the collector interval" - }, - { - FLB_CONFIG_MAP_INT, "interval_nsec", DEFAULT_INTERVAL_NSEC, - 0, FLB_TRUE, offsetof(struct flb_docker, interval_nsec), - "Set the collector interval (nanoseconds)" - }, - { - FLB_CONFIG_MAP_STR, "include", NULL, - 0, FLB_FALSE, 0, - "A space-separated list of containers to include" - }, - { - FLB_CONFIG_MAP_STR, "exclude", NULL, - 0, FLB_FALSE, 0, - "A space-separated list of containers to exclude" - }, - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_input_plugin in_docker_plugin = { - .name = "docker", - .description = "Docker containers metrics", - .cb_init = cb_docker_init, - .cb_pre_run = NULL, - .cb_collect = cb_docker_collect, - .cb_flush_buf = NULL, - .cb_pause = cb_docker_pause, - .cb_resume = cb_docker_resume, - .cb_exit = cb_docker_exit, - .config_map = config_map -}; diff --git a/fluent-bit/plugins/in_docker/docker.h b/fluent-bit/plugins/in_docker/docker.h deleted file mode 100644 index d3814c390..000000000 --- a/fluent-bit/plugins/in_docker/docker.h +++ /dev/null @@ -1,94 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_DOCKER_H -#define FLB_IN_DOCKER_H - -#include -#include -#include -#include -#include - - -#define CURRENT_DIR "." -#define PREV_DIR ".." -#define OS_DIR_TYPE 4 -#define DOCKER_LONG_ID_LEN 64 -#define DOCKER_SHORT_ID_LEN 12 -#define DOCKER_CGROUP_V1_MEM_DIR "/sys/fs/cgroup/memory/docker" -#define DOCKER_CGROUP_V1_CPU_DIR "/sys/fs/cgroup/cpu/docker" -#define DOCKER_CGROUP_V1_MEM_LIMIT_FILE "memory.limit_in_bytes" -#define DOCKER_CGROUP_V1_MEM_USAGE_FILE "memory.usage_in_bytes" -#define DOCKER_CGROUP_V1_CPU_USAGE_FILE "cpuacct.usage" -#define DOCKER_LIB_ROOT "/var/lib/docker/containers" -#define DOCKER_CONFIG_JSON "config.v2.json" -#define DOCKER_NAME_ARG "\"Name\"" -#define DEFAULT_INTERVAL_SEC "1" -#define DEFAULT_INTERVAL_NSEC "0" - -typedef struct docker_info { - char *id; - struct mk_list _head; -} docker_info; - -typedef struct cpu_snapshot { - unsigned long used; -} cpu_snapshot; - -typedef struct mem_snapshot { - uint64_t limit; - uint64_t used; -} mem_snapshot; - -typedef struct docker_snapshot { - char *id; - char *name; - mem_snapshot *mem; - cpu_snapshot *cpu; - struct mk_list _head; -} docker_snapshot; - -struct flb_docker; - -struct cgroup_api { - int cgroup_version; - struct mk_list* (*get_active_docker_ids) (); - char* (*get_container_name) (struct flb_docker *, char *); - cpu_snapshot* (*get_cpu_snapshot) (struct flb_docker *, char *); - mem_snapshot* (*get_mem_snapshot) (struct flb_docker *, char *); -}; -int in_docker_set_cgroup_api_v1(struct cgroup_api *api); - -/* Docker Input configuration & context */ -struct flb_docker { - int coll_fd; /* collector id/fd */ - int interval_sec; /* interval collection time (Second) */ - int interval_nsec; /* interval collection time (Nanosecond) */ - struct mk_list *whitelist; /* dockers to monitor */ - struct mk_list *blacklist; /* dockers to exclude */ - struct cgroup_api cgroup_api; - struct flb_input_instance *ins; - struct flb_log_event_encoder log_encoder; -}; - -int in_docker_collect(struct flb_input_instance *i_ins, - struct flb_config *config, void *in_context); -docker_info *in_docker_init_docker_info(char *id); -#endif diff --git a/fluent-bit/plugins/in_docker_events/CMakeLists.txt b/fluent-bit/plugins/in_docker_events/CMakeLists.txt deleted file mode 100644 index dee7c0f27..000000000 --- a/fluent-bit/plugins/in_docker_events/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -set(src - docker_events.c - docker_events_config.c) - -FLB_PLUGIN(in_docker_events "${src}" "") diff --git a/fluent-bit/plugins/in_docker_events/docker_events.c b/fluent-bit/plugins/in_docker_events/docker_events.c deleted file mode 100644 index 7534eb1d6..000000000 --- a/fluent-bit/plugins/in_docker_events/docker_events.c +++ /dev/null @@ -1,476 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "docker_events.h" -#include "docker_events_config.h" - - -/** - * Creates the connection to docker's unix socket and sends the - * HTTP GET /events - * - * @param ctx Pointer to flb_in_de_config - * - * @return int 0 on success, -1 on failure - */ -static int de_unix_create(struct flb_in_de_config *ctx) -{ - ssize_t bytes; - unsigned long len; - size_t address_length; - struct sockaddr_un address; - char request[512]; - - ctx->fd = flb_net_socket_create(AF_UNIX, FLB_FALSE); - if (ctx->fd == -1) { - return -1; - } - - /* Prepare the unix socket path */ - len = strlen(ctx->unix_path); - address.sun_family = AF_UNIX; - sprintf(address.sun_path, "%s", ctx->unix_path); - address_length = sizeof(address.sun_family) + len + 1; - if (connect(ctx->fd, (struct sockaddr *)&address, address_length) == -1) { - flb_errno(); - close(ctx->fd); - return -1; - } - - strcpy(request, "GET /events HTTP/1.0\r\n\r\n"); - flb_plg_trace(ctx->ins, "writing to socket %s", request); - write(ctx->fd, request, strlen(request)); - - /* Read the initial http response */ - bytes = read(ctx->fd, ctx->buf, ctx->buf_size - 1); - if (bytes == -1) { - flb_errno(); - } - flb_plg_debug(ctx->ins, "read %zu bytes from socket", bytes); - - return 0; -} - -static int in_de_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context); - -static int reconnect_docker_sock(struct flb_input_instance *ins, - struct flb_config *config, - struct flb_in_de_config *ctx) -{ - int ret; - - /* remove old socket collector */ - if (ctx->coll_id >= 0) { - ret = flb_input_collector_delete(ctx->coll_id, ins); - if (ret < 0) { - flb_plg_error(ctx->ins, "failed to pause event"); - return -1; - } - ctx->coll_id = -1; - } - if (ctx->fd > 0) { - flb_plg_debug(ctx->ins, "close socket fd=%d", ctx->fd); - close(ctx->fd); - ctx->fd = -1; - } - - /* create socket again */ - if (de_unix_create(ctx) < 0) { - flb_plg_error(ctx->ins, "failed to re-initialize socket"); - if (ctx->fd > 0) { - flb_plg_debug(ctx->ins, "close socket fd=%d", ctx->fd); - close(ctx->fd); - ctx->fd = -1; - } - return -1; - } - /* set event */ - ctx->coll_id = flb_input_set_collector_event(ins, - in_de_collect, - ctx->fd, config); - if (ctx->coll_id < 0) { - flb_plg_error(ctx->ins, - "could not set collector for IN_DOCKER_EVENTS plugin"); - close(ctx->fd); - ctx->fd = -1; - return -1; - } - ret = flb_input_collector_start(ctx->coll_id, ins); - if (ret < 0) { - flb_plg_error(ctx->ins, - "could not start collector for IN_DOCKER_EVENTS plugin"); - flb_input_collector_delete(ctx->coll_id, ins); - close(ctx->fd); - ctx->coll_id = -1; - ctx->fd = -1; - return -1; - } - - flb_plg_info(ctx->ins, "Reconnect successful"); - return 0; -} - -static int cb_reconnect(struct flb_input_instance *ins, - struct flb_config *config, - void *in_context) -{ - struct flb_in_de_config *ctx = in_context; - int ret; - - flb_plg_info(ctx->ins, "Retry(%d/%d)", - ctx->current_retries, ctx->reconnect_retry_limits); - ret = reconnect_docker_sock(ins, config, ctx); - if (ret < 0) { - /* Failed to reconnect */ - ctx->current_retries++; - if (ctx->current_retries > ctx->reconnect_retry_limits) { - /* give up */ - flb_plg_error(ctx->ins, "Failed to retry. Giving up..."); - goto cb_reconnect_end; - } - flb_plg_info(ctx->ins, "Failed. Waiting for next retry.."); - return 0; - } - - cb_reconnect_end: - if(flb_input_collector_delete(ctx->retry_coll_id, ins) < 0) { - flb_plg_error(ctx->ins, "failed to delete timer event"); - } - ctx->current_retries = 0; - ctx->retry_coll_id = -1; - return ret; -} - -static int create_reconnect_event(struct flb_input_instance *ins, - struct flb_config *config, - struct flb_in_de_config *ctx) -{ - int ret; - - if (ctx->retry_coll_id >= 0) { - flb_plg_debug(ctx->ins, "already retring ?"); - return 0; - } - - /* try before creating event to stop incoming event */ - ret = reconnect_docker_sock(ins, config, ctx); - if (ret == 0) { - return 0; - } - - ctx->current_retries = 1; - ctx->retry_coll_id = flb_input_set_collector_time(ins, - cb_reconnect, - ctx->reconnect_retry_interval, - 0, - config); - if (ctx->retry_coll_id < 0) { - flb_plg_error(ctx->ins, "failed to create timer event"); - return -1; - } - ret = flb_input_collector_start(ctx->retry_coll_id, ins); - if (ret < 0) { - flb_plg_error(ctx->ins, "failed to start timer event"); - flb_input_collector_delete(ctx->retry_coll_id, ins); - ctx->retry_coll_id = -1; - return -1; - } - flb_plg_info(ctx->ins, "create reconnect event. interval=%d second", - ctx->reconnect_retry_interval); - - return 0; -} - -static int is_recoverable_error(int error) -{ - /* ENOTTY: - It reports on Docker in Docker mode. - https://github.com/fluent/fluent-bit/issues/3439#issuecomment-831424674 - */ - if (error == ENOTTY || error == EBADF) { - return FLB_TRUE; - } - return FLB_FALSE; -} - - -/** - * Callback function to process events recieved on the unix - * socket. - * - * @param ins Pointer to flb_input_instance - * @param config Pointer to flb_config - * @param in_context void Pointer used to cast to - * flb_in_de_config - * - * @return int Always returns success - */ -static int in_de_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - int ret = 0; - int error; - size_t str_len = 0; - struct flb_in_de_config *ctx = in_context; - - /* variables for parser */ - int parser_ret = -1; - void *out_buf = NULL; - size_t out_size = 0; - struct flb_time out_time; - - ret = read(ctx->fd, ctx->buf, ctx->buf_size - 1); - - if (ret > 0) { - str_len = ret; - ctx->buf[str_len] = '\0'; - - ret = flb_log_event_encoder_begin_record(&ctx->log_encoder); - - if (!ctx->parser) { - /* Initialize local msgpack buffer */ - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp( - &ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - &ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE(ctx->key), - FLB_LOG_EVENT_STRING_VALUE(ctx->buf, str_len)); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(&ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(ins, NULL, 0, - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - - } - else { - flb_plg_error(ctx->ins, "Error encoding record : %d", ret); - } - } - else { - flb_time_get(&out_time); - - parser_ret = flb_parser_do(ctx->parser, ctx->buf, str_len - 1, - &out_buf, &out_size, &out_time); - if (parser_ret >= 0) { - if (flb_time_to_nanosec(&out_time) == 0L) { - flb_time_get(&out_time); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_timestamp( - &ctx->log_encoder, - &out_time); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_body_from_raw_msgpack( - &ctx->log_encoder, - out_buf, - out_size); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(&ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(ins, NULL, 0, - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - - } - else { - flb_plg_error(ctx->ins, "Error encoding record : %d", ret); - } - - - flb_free(out_buf); - } - else { - flb_plg_trace(ctx->ins, "tried to parse: %s", ctx->buf); - flb_plg_trace(ctx->ins, "buf_size %zu", ctx->buf_size); - flb_plg_error(ctx->ins, "parser returned an error: %d", - parser_ret); - } - } - - flb_log_event_encoder_reset(&ctx->log_encoder); - } - else if (ret == 0) { - /* EOF */ - - /* docker service may be restarted */ - flb_plg_info(ctx->ins, "EOF detected. Re-initialize"); - if (ctx->reconnect_retry_limits > 0) { - ret = create_reconnect_event(ins, config, ctx); - if (ret < 0) { - return ret; - } - } - } - else { - error = errno; - flb_plg_error(ctx->ins, "read returned error: %d, %s", error, - strerror(error)); - if (is_recoverable_error(error)) { - if (ctx->reconnect_retry_limits > 0) { - ret = create_reconnect_event(ins, config, ctx); - if (ret < 0) { - return ret; - } - } - } - } - - return 0; -} - -/** - * Callback function to initialize docker events plugin - * - * @param ins Pointer to flb_input_instance - * @param config Pointer to flb_config - * @param data Unused - * - * @return int 0 on success, -1 on failure - */ -static int in_de_init(struct flb_input_instance *ins, - struct flb_config *config, void *data) -{ - struct flb_in_de_config *ctx = NULL; - (void) data; - - /* Allocate space for the configuration */ - ctx = de_config_init(ins, config); - if (!ctx) { - return -1; - } - ctx->ins = ins; - ctx->retry_coll_id = -1; - ctx->current_retries = 0; - - /* Set the context */ - flb_input_set_context(ins, ctx); - - if (de_unix_create(ctx) != 0) { - flb_plg_error(ctx->ins, "could not listen on unix://%s", - ctx->unix_path); - de_config_destroy(ctx); - return -1; - } - - ctx->coll_id = flb_input_set_collector_event(ins, in_de_collect, - ctx->fd, config); - if(ctx->coll_id < 0){ - flb_plg_error(ctx->ins, - "could not set collector for IN_DOCKER_EVENTS plugin"); - de_config_destroy(ctx); - return -1; - } - - flb_plg_info(ctx->ins, "listening for events on %s", ctx->unix_path); - return 0; -} - -/** - * Callback exit function to cleanup plugin - * - * @param data Pointer cast to flb_in_de_config - * @param config Unused - * - * @return int Always returns 0 - */ -static int in_de_exit(void *data, struct flb_config *config) -{ - (void) config; - struct flb_in_de_config *ctx = data; - - if (!ctx) { - return 0; - } - - de_config_destroy(ctx); - - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "unix_path", DEFAULT_UNIX_SOCKET_PATH, - 0, FLB_TRUE, offsetof(struct flb_in_de_config, unix_path), - "Define Docker unix socket path to read events" - }, - { - FLB_CONFIG_MAP_SIZE, "buffer_size", "8k", - 0, FLB_TRUE, offsetof(struct flb_in_de_config, buf_size), - "Set buffer size to read events" - }, - { - FLB_CONFIG_MAP_STR, "parser", NULL, - 0, FLB_FALSE, 0, - "Optional parser for records, if not set, records are packages under 'key'" - }, - { - FLB_CONFIG_MAP_STR, "key", DEFAULT_FIELD_NAME, - 0, FLB_TRUE, offsetof(struct flb_in_de_config, key), - "Set the key name to store unparsed Docker events" - }, - { - FLB_CONFIG_MAP_INT, "reconnect.retry_limits", "5", - 0, FLB_TRUE, offsetof(struct flb_in_de_config, reconnect_retry_limits), - "Maximum number to retry to connect docker socket" - }, - { - FLB_CONFIG_MAP_INT, "reconnect.retry_interval", "1", - 0, FLB_TRUE, offsetof(struct flb_in_de_config, reconnect_retry_interval), - "Retry interval to connect docker socket" - }, - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_input_plugin in_docker_events_plugin = { - .name = "docker_events", - .description = "Docker events", - .cb_init = in_de_init, - .cb_pre_run = NULL, - .cb_collect = in_de_collect, - .cb_flush_buf = NULL, - .cb_exit = in_de_exit, - .config_map = config_map, - .flags = FLB_INPUT_NET -}; diff --git a/fluent-bit/plugins/in_docker_events/docker_events.h b/fluent-bit/plugins/in_docker_events/docker_events.h deleted file mode 100644 index dc659d5ec..000000000 --- a/fluent-bit/plugins/in_docker_events/docker_events.h +++ /dev/null @@ -1,56 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_DE_H -#define FLB_IN_DE_H - -#include -#include -#include -#include - -#define DEFAULT_BUF_SIZE 8192 -#define MIN_BUF_SIZE 2048 -#define DEFAULT_FIELD_NAME "message" -#define DEFAULT_UNIX_SOCKET_PATH "/var/run/docker.sock" - -struct flb_in_de_config -{ - int fd; /* File descriptor */ - int coll_id; /* collector id */ - flb_sds_t unix_path; /* Unix path for socket */ - char *buf; - size_t buf_size; - flb_sds_t key; - - /* retries */ - int reconnect_retry_limits; - int reconnect_retry_interval; - - /* retries (internal) */ - int current_retries; - int retry_coll_id; - - struct flb_parser *parser; - struct flb_log_event_encoder log_encoder; - struct flb_input_instance *ins; /* Input plugin instace */ - -}; - -#endif diff --git a/fluent-bit/plugins/in_docker_events/docker_events_config.c b/fluent-bit/plugins/in_docker_events/docker_events_config.c deleted file mode 100644 index 8290686c1..000000000 --- a/fluent-bit/plugins/in_docker_events/docker_events_config.c +++ /dev/null @@ -1,106 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include "docker_events.h" -#include "docker_events_config.h" - -/** - * Function to initialize docker_events plugin. - * - * @param ins Pointer to flb_input_instance - * @param config Pointer to flb_config - * - * @return struct flb_in_de_config* Pointer to the plugin's - * structure on success, NULL on failure. - */ -struct flb_in_de_config *de_config_init(struct flb_input_instance *ins, - struct flb_config *config) -{ - int ret; - const char *tmp; - struct flb_in_de_config *ctx; - - ctx = flb_calloc(1, sizeof(struct flb_in_de_config)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - - /* Load the config map */ - ret = flb_input_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return NULL; - } - - /* Allocate buffer for events */ - ctx->buf = flb_malloc(ctx->buf_size); - if (!ctx->buf) { - flb_errno(); - flb_free(ctx); - return NULL; - } - - tmp = flb_input_get_property("parser", ins); - if (tmp) { - ctx->parser = flb_parser_get(tmp, config); - if (ctx->parser == NULL) { - flb_plg_error(ctx->ins, "requested parser '%s' not found", tmp); - flb_free(ctx->buf); - flb_free(ctx); - return NULL; - } - } - - ret = flb_log_event_encoder_init(&ctx->log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, "error initializing event encoder : %d", ret); - - de_config_destroy(ctx); - - ctx = NULL; - } - - return ctx; -} - -/** - * Function to destroy docker_events plugin. - * - * @param ctx Pointer to flb_in_de_config - * - * @return int 0 - */ -int de_config_destroy(struct flb_in_de_config *ctx) -{ - if (ctx->buf) { - flb_free(ctx->buf); - } - - flb_log_event_encoder_destroy(&ctx->log_encoder); - - flb_free(ctx); - return 0; -} diff --git a/fluent-bit/plugins/in_docker_events/docker_events_config.h b/fluent-bit/plugins/in_docker_events/docker_events_config.h deleted file mode 100644 index 94a6d87db..000000000 --- a/fluent-bit/plugins/in_docker_events/docker_events_config.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_DE_CONFIG_H -#define FLB_IN_DE_CONFIG_H - -#include "docker_events.h" - -struct flb_in_de_config *de_config_init(struct flb_input_instance *ins, - struct flb_config *config); -int de_config_destroy(struct flb_in_de_config *config); - -#endif diff --git a/fluent-bit/plugins/in_dummy/CMakeLists.txt b/fluent-bit/plugins/in_dummy/CMakeLists.txt deleted file mode 100644 index 52b03a2c8..000000000 --- a/fluent-bit/plugins/in_dummy/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - in_dummy.c) - -FLB_PLUGIN(in_dummy "${src}" "") diff --git a/fluent-bit/plugins/in_dummy/in_dummy.c b/fluent-bit/plugins/in_dummy/in_dummy.c deleted file mode 100644 index 75d1b7333..000000000 --- a/fluent-bit/plugins/in_dummy/in_dummy.c +++ /dev/null @@ -1,438 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "in_dummy.h" - -static void generate_timestamp(struct flb_dummy *ctx, - struct flb_time *result) -{ - struct flb_time current_timestamp; - struct flb_time delta; - - if (ctx->fixed_timestamp) { - if (ctx->dummy_timestamp_set) { - flb_time_copy(result, &ctx->dummy_timestamp); - } - else { - flb_time_copy(result, &ctx->base_timestamp); - } - } - else { - if (ctx->dummy_timestamp_set) { - flb_time_zero(&delta); - - flb_time_get(¤t_timestamp); - - flb_time_diff(¤t_timestamp, - &ctx->base_timestamp, - &delta); - - flb_time_add(&ctx->dummy_timestamp, - &delta, - result); - } - else { - flb_time_get(result); - } - } -} - -static int generate_event(struct flb_dummy *ctx) -{ - size_t chunk_offset; - size_t body_length; - char *body_buffer; - size_t body_start; - struct flb_time timestamp; - msgpack_unpacked object; - int result; - - result = FLB_EVENT_ENCODER_SUCCESS; - body_start = 0; - chunk_offset = 0; - - generate_timestamp(ctx, ×tamp); - - msgpack_unpacked_init(&object); - - while (result == FLB_EVENT_ENCODER_SUCCESS && - msgpack_unpack_next(&object, - ctx->ref_body_msgpack, - ctx->ref_body_msgpack_size, - &chunk_offset) == MSGPACK_UNPACK_SUCCESS) { - body_buffer = &ctx->ref_body_msgpack[body_start]; - body_length = chunk_offset - body_start; - - if (object.data.type == MSGPACK_OBJECT_MAP) { - flb_log_event_encoder_begin_record(ctx->encoder); - - flb_log_event_encoder_set_timestamp(ctx->encoder, ×tamp); - - result = flb_log_event_encoder_set_metadata_from_raw_msgpack( - ctx->encoder, - ctx->ref_metadata_msgpack, - ctx->ref_metadata_msgpack_size); - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_set_body_from_raw_msgpack( - ctx->encoder, - body_buffer, - body_length); - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_commit_record(ctx->encoder); - } - } - - body_start = chunk_offset; - } - - msgpack_unpacked_destroy(&object); - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = 0; - } - else { - result = -1; - } - - return result; -} - -/* cb_collect callback */ -static int in_dummy_collect(struct flb_input_instance *ins, - struct flb_config *config, - void *in_context) -{ - int result; - int index; - struct flb_dummy *ctx; - - ctx = (struct flb_dummy *) in_context; - - if (ctx->samples > 0 && (ctx->samples_count >= ctx->samples)) { - return -1; - } - - result = 0; - - if (ctx->samples_count == 0 || !ctx->fixed_timestamp) { - flb_log_event_encoder_reset(ctx->encoder); - - for (index = 0 ; index < ctx->copies && result == 0 ; index++) { - result = generate_event(ctx); - } - } - - if (result == 0) { - if (ctx->encoder->output_length > 0) { - flb_input_log_append(ins, NULL, 0, - ctx->encoder->output_buffer, - ctx->encoder->output_length); - } - else { - flb_plg_error(ins, "log chunk size == 0"); - } - } - else { - flb_plg_error(ins, "log chunk genartion error (%d)", result); - } - - if (ctx->samples > 0) { - ctx->samples_count++; - } - - return 0; -} - -static int config_destroy(struct flb_dummy *ctx) -{ - if (ctx->ref_body_msgpack != NULL) { - flb_free(ctx->ref_body_msgpack); - } - - if (ctx->ref_metadata_msgpack != NULL) { - flb_free(ctx->ref_metadata_msgpack); - } - - if (ctx->encoder != NULL) { - flb_log_event_encoder_destroy(ctx->encoder); - } - - flb_free(ctx); - - return 0; -} - -/* Set plugin configuration */ -static int configure(struct flb_dummy *ctx, - struct flb_input_instance *in, - struct timespec *tm) -{ - const char *msg; - int root_type; - int ret = -1; - - ctx->ref_metadata_msgpack = NULL; - ctx->ref_body_msgpack = NULL; - ctx->dummy_timestamp_set = FLB_FALSE; - - ret = flb_input_config_map_set(in, (void *) ctx); - if (ret == -1) { - return -1; - } - - /* interval settings */ - tm->tv_sec = 1; - tm->tv_nsec = 0; - - if (ctx->rate > 1) { - tm->tv_sec = 0; - tm->tv_nsec = 1000000000 / ctx->rate; - } - - /* dummy timestamp */ - flb_time_zero(&ctx->dummy_timestamp); - - if (ctx->start_time_sec >= 0 || ctx->start_time_nsec >= 0) { - ctx->dummy_timestamp_set = FLB_TRUE; - - if (ctx->start_time_sec >= 0) { - ctx->dummy_timestamp.tm.tv_sec = ctx->start_time_sec; - } - if (ctx->start_time_nsec >= 0) { - ctx->dummy_timestamp.tm.tv_nsec = ctx->start_time_nsec; - } - } - - flb_time_get(&ctx->base_timestamp); - - /* handle it explicitly since we need to validate it is valid JSON */ - msg = flb_input_get_property("dummy", in); - if (msg == NULL) { - msg = DEFAULT_DUMMY_MESSAGE; - } - - ret = flb_pack_json(msg, - strlen(msg), - &ctx->ref_body_msgpack, - &ctx->ref_body_msgpack_size, - &root_type, - NULL); - - if (ret != 0) { - flb_plg_warn(ctx->ins, "data is incomplete. Use default string."); - - ret = flb_pack_json(DEFAULT_DUMMY_MESSAGE, - strlen(DEFAULT_DUMMY_MESSAGE), - &ctx->ref_body_msgpack, - &ctx->ref_body_msgpack_size, - &root_type, - NULL); - if (ret != 0) { - flb_plg_error(ctx->ins, "unexpected error"); - return -1; - } - } - - /* handle it explicitly since we need to validate it is valid JSON */ - msg = flb_input_get_property("metadata", in); - - if (msg == NULL) { - msg = DEFAULT_DUMMY_METADATA; - } - - ret = flb_pack_json(msg, - strlen(msg), - &ctx->ref_metadata_msgpack, - &ctx->ref_metadata_msgpack_size, - &root_type, - NULL); - - if (ret != 0) { - flb_plg_warn(ctx->ins, "data is incomplete. Use default string."); - - ret = flb_pack_json(DEFAULT_DUMMY_METADATA, - strlen(DEFAULT_DUMMY_METADATA), - &ctx->ref_metadata_msgpack, - &ctx->ref_metadata_msgpack_size, - &root_type, - NULL); - - if (ret != 0) { - flb_plg_error(ctx->ins, "unexpected error"); - return -1; - } - } - - return 0; -} - - - - -/* Initialize plugin */ -static int in_dummy_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - int ret = -1; - struct flb_dummy *ctx = NULL; - struct timespec tm; - - /* Allocate space for the configuration */ - ctx = flb_malloc(sizeof(struct flb_dummy)); - if (ctx == NULL) { - return -1; - } - ctx->ins = in; - ctx->samples = 0; - ctx->samples_count = 0; - - /* Initialize head config */ - ret = configure(ctx, in, &tm); - if (ret < 0) { - config_destroy(ctx); - return -1; - } - - ctx->encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ctx->encoder == NULL) { - flb_plg_error(in, "could not initialize event encoder"); - config_destroy(ctx); - - return -1; - } - - flb_input_set_context(in, ctx); - - ret = flb_input_set_collector_time(in, - in_dummy_collect, - tm.tv_sec, - tm.tv_nsec, config); - if (ret < 0) { - flb_plg_error(ctx->ins, "could not set collector for dummy input plugin"); - config_destroy(ctx); - return -1; - } - - ctx->coll_fd = ret; - - flb_time_get(&ctx->base_timestamp); - - return 0; -} - -static void in_dummy_pause(void *data, struct flb_config *config) -{ - struct flb_dummy *ctx = data; - - flb_input_collector_pause(ctx->coll_fd, ctx->ins); -} - -static void in_dummy_resume(void *data, struct flb_config *config) -{ - struct flb_dummy *ctx = data; - - flb_input_collector_resume(ctx->coll_fd, ctx->ins); -} - -static int in_dummy_exit(void *data, struct flb_config *config) -{ - (void) *config; - struct flb_dummy *ctx = data; - - config_destroy(ctx); - - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_INT, "samples", "0", - 0, FLB_TRUE, offsetof(struct flb_dummy, samples), - "set a number of times to generate event." - }, - { - FLB_CONFIG_MAP_STR, "dummy", DEFAULT_DUMMY_MESSAGE, - 0, FLB_FALSE, 0, - "set the sample record to be generated. It should be a JSON object." - }, - { - FLB_CONFIG_MAP_STR, "metadata", DEFAULT_DUMMY_METADATA, - 0, FLB_FALSE, 0, - "set the sample metadata to be generated. It should be a JSON object." - }, - { - FLB_CONFIG_MAP_INT, "rate", "1", - 0, FLB_TRUE, offsetof(struct flb_dummy, rate), - "set a number of events per second." - }, - { - FLB_CONFIG_MAP_INT, "copies", "1", - 0, FLB_TRUE, offsetof(struct flb_dummy, copies), - "set the number of copies to generate per collectd." - }, - { - FLB_CONFIG_MAP_INT, "start_time_sec", "-1", - 0, FLB_TRUE, offsetof(struct flb_dummy, start_time_sec), - "set a dummy base timestamp in seconds." - }, - { - FLB_CONFIG_MAP_INT, "start_time_nsec", "-1", - 0, FLB_TRUE, offsetof(struct flb_dummy, start_time_nsec), - "set a dummy base timestamp in nanoseconds." - }, - { - FLB_CONFIG_MAP_BOOL, "fixed_timestamp", "off", - 0, FLB_TRUE, offsetof(struct flb_dummy, fixed_timestamp), - "used a fixed timestamp, allows the message to pre-generated once." - }, - {0} -}; - - -struct flb_input_plugin in_dummy_plugin = { - .name = "dummy", - .description = "Generate dummy data", - .cb_init = in_dummy_init, - .cb_pre_run = NULL, - .cb_collect = in_dummy_collect, - .cb_flush_buf = NULL, - .config_map = config_map, - .cb_pause = in_dummy_pause, - .cb_resume = in_dummy_resume, - .cb_exit = in_dummy_exit -}; diff --git a/fluent-bit/plugins/in_dummy/in_dummy.h b/fluent-bit/plugins/in_dummy/in_dummy.h deleted file mode 100644 index d351420cb..000000000 --- a/fluent-bit/plugins/in_dummy/in_dummy.h +++ /dev/null @@ -1,58 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_DUMMY_H -#define FLB_IN_DUMMY_H - -#include -#include -#include - -#define DEFAULT_DUMMY_MESSAGE "{\"message\":\"dummy\"}" -#define DEFAULT_DUMMY_METADATA "{}" - -struct flb_dummy { - int coll_fd; - - int rate; - int copies; - int samples; - int samples_count; - - int dummy_timestamp_set; - struct flb_time base_timestamp; - struct flb_time dummy_timestamp; - - int start_time_sec; - int start_time_nsec; - - bool fixed_timestamp; - - char *ref_metadata_msgpack; - size_t ref_metadata_msgpack_size; - - char *ref_body_msgpack; - size_t ref_body_msgpack_size; - - struct flb_log_event_encoder *encoder; - - struct flb_input_instance *ins; -}; - -#endif diff --git a/fluent-bit/plugins/in_elasticsearch/CMakeLists.txt b/fluent-bit/plugins/in_elasticsearch/CMakeLists.txt deleted file mode 100644 index 50a472f6a..000000000 --- a/fluent-bit/plugins/in_elasticsearch/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -if(NOT FLB_METRICS) - message(FATAL_ERROR "Elasticsearch input plugin requires FLB_HTTP_SERVER=On.") -endif() - -set(src - in_elasticsearch.c - in_elasticsearch_config.c - in_elasticsearch_bulk_conn.c - in_elasticsearch_bulk_prot.c - ) - -FLB_PLUGIN(in_elasticsearch "${src}" "") diff --git a/fluent-bit/plugins/in_elasticsearch/in_elasticsearch.c b/fluent-bit/plugins/in_elasticsearch/in_elasticsearch.c deleted file mode 100644 index af1a594c6..000000000 --- a/fluent-bit/plugins/in_elasticsearch/in_elasticsearch.c +++ /dev/null @@ -1,245 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#include -#include -#include -#include - -#include "in_elasticsearch.h" -#include "in_elasticsearch_config.h" -#include "in_elasticsearch_bulk_conn.h" - -/* - * For a server event, the collection event means a new client have arrived, we - * accept the connection and create a new TCP instance which will wait for - * JSON map messages. - */ -static int in_elasticsearch_bulk_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_connection *connection; - struct in_elasticsearch_bulk_conn *conn; - struct flb_in_elasticsearch *ctx; - - ctx = in_context; - - connection = flb_downstream_conn_get(ctx->downstream); - - if (connection == NULL) { - flb_plg_error(ctx->ins, "could not accept new connection"); - - return -1; - } - - flb_plg_trace(ctx->ins, "new TCP connection arrived FD=%i", - connection->fd); - - conn = in_elasticsearch_bulk_conn_add(connection, ctx); - - if (conn == NULL) { - flb_downstream_conn_release(connection); - - return -1; - } - - return 0; -} - -static void bytes_to_groupname(unsigned char *data, char *buf, size_t len) { - int index; - char charset[] = "0123456789" - "abcdefghijklmnopqrstuvwxyz" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; - - while (len-- > 0) { - index = (int) data[len]; - index = index % (sizeof(charset) - 1); - buf[len] = charset[index]; - } -} - -static void bytes_to_nodename(unsigned char *data, char *buf, size_t len) { - int index; - char charset[] = "0123456789" - "abcdefghijklmnopqrstuvwxyz"; - - while (len-- > 0) { - index = (int) data[len]; - index = index % (sizeof(charset) - 1); - buf[len] = charset[index]; - } -} - -static int in_elasticsearch_bulk_init(struct flb_input_instance *ins, - struct flb_config *config, void *data) -{ - unsigned short int port; - int ret; - struct flb_in_elasticsearch *ctx; - unsigned char rand[16]; - - (void) data; - - /* Create context and basic conf */ - ctx = in_elasticsearch_config_create(ins); - if (!ctx) { - return -1; - } - - ctx->collector_id = -1; - - /* Populate context with config map defaults and incoming properties */ - ret = flb_input_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_plg_error(ctx->ins, "configuration error"); - in_elasticsearch_config_destroy(ctx); - return -1; - } - - /* Set the context */ - flb_input_set_context(ins, ctx); - - port = (unsigned short int) strtoul(ctx->tcp_port, NULL, 10); - - if (flb_random_bytes(rand, 16)) { - flb_plg_error(ctx->ins, "cannot generate cluster name"); - in_elasticsearch_config_destroy(ctx); - return -1; - } - - bytes_to_groupname(rand, ctx->cluster_name, 16); - - if (flb_random_bytes(rand, 12)) { - flb_plg_error(ctx->ins, "cannot generate node name"); - in_elasticsearch_config_destroy(ctx); - return -1; - } - - bytes_to_nodename(rand, ctx->node_name, 12); - - ctx->downstream = flb_downstream_create(FLB_TRANSPORT_TCP, - ins->flags, - ctx->listen, - port, - ins->tls, - config, - &ins->net_setup); - - if (ctx->downstream == NULL) { - flb_plg_error(ctx->ins, - "could not initialize downstream on %s:%s. Aborting", - ctx->listen, ctx->tcp_port); - - in_elasticsearch_config_destroy(ctx); - - return -1; - } - - flb_input_downstream_set(ctx->downstream, ctx->ins); - - /* Collect upon data available on the standard input */ - ret = flb_input_set_collector_socket(ins, - in_elasticsearch_bulk_collect, - ctx->downstream->server_fd, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "Could not set collector for IN_ELASTICSEARCH input plugin"); - in_elasticsearch_config_destroy(ctx); - - return -1; - } - - ctx->collector_id = ret; - - return 0; -} - -static int in_elasticsearch_bulk_exit(void *data, struct flb_config *config) -{ - struct flb_in_elasticsearch *ctx; - - (void) config; - - ctx = data; - - if (ctx != NULL) { - in_elasticsearch_config_destroy(ctx); - } - - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_SIZE, "buffer_max_size", HTTP_BUFFER_MAX_SIZE, - 0, FLB_TRUE, offsetof(struct flb_in_elasticsearch, buffer_max_size), - "Set the maximum size of buffer" - }, - - { - FLB_CONFIG_MAP_SIZE, "buffer_chunk_size", HTTP_BUFFER_CHUNK_SIZE, - 0, FLB_TRUE, offsetof(struct flb_in_elasticsearch, buffer_chunk_size), - "Set the buffer chunk size" - }, - - { - FLB_CONFIG_MAP_STR, "tag_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_in_elasticsearch, tag_key), - "Specify a key name for extracting as a tag" - }, - - { - FLB_CONFIG_MAP_STR, "meta_key", "@meta", - 0, FLB_TRUE, offsetof(struct flb_in_elasticsearch, meta_key), - "Specify a key name for meta information" - }, - - { - FLB_CONFIG_MAP_STR, "hostname", "localhost", - 0, FLB_TRUE, offsetof(struct flb_in_elasticsearch, hostname), - "Specify hostname or FQDN. This parameter is effective for sniffering node information." - }, - - { - FLB_CONFIG_MAP_STR, "version", "8.0.0", - 0, FLB_TRUE, offsetof(struct flb_in_elasticsearch, es_version), - "Specify returning Elasticsearch server version." - }, - - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_input_plugin in_elasticsearch_plugin = { - .name = "elasticsearch", - .description = "HTTP Endpoints for Elasticsearch (Bulk API)", - .cb_init = in_elasticsearch_bulk_init, - .cb_pre_run = NULL, - .cb_collect = in_elasticsearch_bulk_collect, - .cb_flush_buf = NULL, - .cb_pause = NULL, - .cb_resume = NULL, - .cb_exit = in_elasticsearch_bulk_exit, - .config_map = config_map, - .flags = FLB_INPUT_NET_SERVER | FLB_IO_OPT_TLS -}; diff --git a/fluent-bit/plugins/in_elasticsearch/in_elasticsearch.h b/fluent-bit/plugins/in_elasticsearch/in_elasticsearch.h deleted file mode 100644 index 159dff88c..000000000 --- a/fluent-bit/plugins/in_elasticsearch/in_elasticsearch.h +++ /dev/null @@ -1,59 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_ELASTICSEARCH_H -#define FLB_IN_ELASTICSEARCH_H - -#include -#include -#include -#include -#include - -#include - -#define HTTP_BUFFER_MAX_SIZE "4M" -#define HTTP_BUFFER_CHUNK_SIZE "512K" - -struct flb_in_elasticsearch { - flb_sds_t listen; - flb_sds_t tcp_port; - const char *tag_key; - const char *meta_key; - flb_sds_t hostname; - flb_sds_t es_version; - char cluster_name[16]; - char node_name[12]; - - int collector_id; - - size_t buffer_max_size; /* Maximum buffer size */ - size_t buffer_chunk_size; /* Chunk allocation size */ - - struct flb_downstream *downstream; /* Client manager */ - struct mk_list connections; /* linked list of connections */ - - struct flb_log_event_encoder log_encoder; - - struct mk_server *server; - struct flb_input_instance *ins; -}; - - -#endif diff --git a/fluent-bit/plugins/in_elasticsearch/in_elasticsearch_bulk_conn.c b/fluent-bit/plugins/in_elasticsearch/in_elasticsearch_bulk_conn.c deleted file mode 100644 index f835af26a..000000000 --- a/fluent-bit/plugins/in_elasticsearch/in_elasticsearch_bulk_conn.c +++ /dev/null @@ -1,307 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include "in_elasticsearch.h" -#include "in_elasticsearch_bulk_conn.h" -#include "in_elasticsearch_bulk_prot.h" - -static void in_elasticsearch_bulk_conn_request_init(struct mk_http_session *session, - struct mk_http_request *request); - -static int in_elasticsearch_bulk_conn_event(void *data) -{ - int status; - size_t size; - ssize_t available; - ssize_t bytes; - char *tmp; - char *request_end; - size_t request_len; - struct flb_connection *connection; - struct in_elasticsearch_bulk_conn *conn; - struct mk_event *event; - struct flb_in_elasticsearch *ctx; - - connection = (struct flb_connection *) data; - - conn = connection->user_data; - - ctx = conn->ctx; - - event = &connection->event; - - if (event->mask & MK_EVENT_READ) { - available = (conn->buf_size - conn->buf_len) - 1; - if (available < 1) { - if (conn->buf_size + ctx->buffer_chunk_size > ctx->buffer_max_size) { - flb_plg_trace(ctx->ins, - "fd=%i incoming data exceed limit (%zu KB)", - event->fd, (ctx->buffer_max_size / 1024)); - in_elasticsearch_bulk_conn_del(conn); - return -1; - } - - size = conn->buf_size + ctx->buffer_chunk_size; - tmp = flb_realloc(conn->buf_data, size); - if (!tmp) { - flb_errno(); - in_elasticsearch_bulk_conn_del(conn); - return -1; - } - flb_plg_trace(ctx->ins, "fd=%i buffer realloc %i -> %zu", - event->fd, conn->buf_size, size); - - conn->buf_data = tmp; - conn->buf_size = size; - available = (conn->buf_size - conn->buf_len) - 1; - } - - /* Read data */ - bytes = flb_io_net_read(connection, - (void *) &conn->buf_data[conn->buf_len], - available); - - if (bytes <= 0) { - flb_plg_trace(ctx->ins, "fd=%i closed connection", event->fd); - in_elasticsearch_bulk_conn_del(conn); - return -1; - } - - flb_plg_trace(ctx->ins, "read()=%zi pre_len=%i now_len=%zi", - bytes, conn->buf_len, conn->buf_len + bytes); - conn->buf_len += bytes; - conn->buf_data[conn->buf_len] = '\0'; - - status = mk_http_parser(&conn->request, &conn->session.parser, - conn->buf_data, conn->buf_len, conn->session.server); - - if (status == MK_HTTP_PARSER_OK) { - /* Do more logic parsing and checks for this request */ - in_elasticsearch_bulk_prot_handle(ctx, conn, &conn->session, &conn->request); - - /* Evict the processed request from the connection buffer and reinitialize - * the HTTP parser. - */ - - request_end = NULL; - - if (NULL != conn->request.data.data) { - request_end = &conn->request.data.data[conn->request.data.len]; - } - else { - request_end = strstr(conn->buf_data, "\r\n\r\n"); - - if(NULL != request_end) { - request_end = &request_end[4]; - } - } - - if (NULL != request_end) { - request_len = (size_t)(request_end - conn->buf_data); - - if (0 < (conn->buf_len - request_len)) { - memmove(conn->buf_data, &conn->buf_data[request_len], - conn->buf_len - request_len); - - conn->buf_data[conn->buf_len - request_len] = '\0'; - conn->buf_len -= request_len; - } - else { - memset(conn->buf_data, 0, request_len); - - conn->buf_len = 0; - } - - /* Reinitialize the parser so the next request is properly - * handled, the additional memset intends to wipe any left over data - * from the headers parsed in the previous request. - */ - memset(&conn->session.parser, 0, sizeof(struct mk_http_parser)); - mk_http_parser_init(&conn->session.parser); - in_elasticsearch_bulk_conn_request_init(&conn->session, &conn->request); - } - } - else if (status == MK_HTTP_PARSER_ERROR) { - in_elasticsearch_bulk_prot_handle_error(ctx, conn, &conn->session, &conn->request); - - /* Reinitialize the parser so the next request is properly - * handled, the additional memset intends to wipe any left over data - * from the headers parsed in the previous request. - */ - memset(&conn->session.parser, 0, sizeof(struct mk_http_parser)); - mk_http_parser_init(&conn->session.parser); - in_elasticsearch_bulk_conn_request_init(&conn->session, &conn->request); - } - - /* FIXME: add Protocol handler here */ - return bytes; - } - - if (event->mask & MK_EVENT_CLOSE) { - flb_plg_trace(ctx->ins, "fd=%i hangup", event->fd); - in_elasticsearch_bulk_conn_del(conn); - return -1; - } - - return 0; - -} - -static void in_elasticsearch_bulk_conn_session_init(struct mk_http_session *session, - struct mk_server *server, - int client_fd) -{ - /* Alloc memory for node */ - session->_sched_init = MK_TRUE; - session->pipelined = MK_FALSE; - session->counter_connections = 0; - session->close_now = MK_FALSE; - session->status = MK_REQUEST_STATUS_INCOMPLETE; - session->server = server; - session->socket = client_fd; - - /* creation time in unix time */ - session->init_time = time(NULL); - - session->channel = mk_channel_new(MK_CHANNEL_SOCKET, session->socket); - session->channel->io = session->server->network; - - /* Init session request list */ - mk_list_init(&session->request_list); - - /* Initialize the parser */ - mk_http_parser_init(&session->parser); -} - -static void in_elasticsearch_bulk_conn_request_init(struct mk_http_session *session, - struct mk_http_request *request) -{ - memset(request, 0, sizeof(struct mk_http_request)); - - mk_http_request_init(session, request, session->server); - - request->in_headers.type = MK_STREAM_IOV; - request->in_headers.dynamic = MK_FALSE; - request->in_headers.cb_consumed = NULL; - request->in_headers.cb_finished = NULL; - request->in_headers.stream = &request->stream; - - mk_list_add(&request->in_headers._head, &request->stream.inputs); - - request->session = session; -} - -struct in_elasticsearch_bulk_conn *in_elasticsearch_bulk_conn_add(struct flb_connection *connection, - struct flb_in_elasticsearch *ctx) -{ - struct in_elasticsearch_bulk_conn *conn; - int ret; - - conn = flb_calloc(1, sizeof(struct in_elasticsearch_bulk_conn)); - if (!conn) { - flb_errno(); - return NULL; - } - - conn->connection = connection; - - /* Set data for the event-loop */ - MK_EVENT_NEW(&connection->event); - - connection->user_data = conn; - connection->event.type = FLB_ENGINE_EV_CUSTOM; - connection->event.handler = in_elasticsearch_bulk_conn_event; - - /* Connection info */ - conn->ctx = ctx; - conn->buf_len = 0; - - conn->buf_data = flb_malloc(ctx->buffer_chunk_size); - if (!conn->buf_data) { - flb_errno(); - - flb_plg_error(ctx->ins, "could not allocate new connection"); - flb_free(conn); - - return NULL; - } - conn->buf_size = ctx->buffer_chunk_size; - - /* Register instance into the event loop */ - ret = mk_event_add(flb_engine_evl_get(), - connection->fd, - FLB_ENGINE_EV_CUSTOM, - MK_EVENT_READ, - &connection->event); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not register new connection"); - - flb_free(conn->buf_data); - flb_free(conn); - - return NULL; - } - - /* Initialize HTTP Session: this is a custom context for Monkey HTTP */ - in_elasticsearch_bulk_conn_session_init(&conn->session, ctx->server, conn->connection->fd); - - /* Initialize HTTP Request: this is the initial request and it will be reinitialized - * automatically after the request is handled so it can be used for the next one. - */ - in_elasticsearch_bulk_conn_request_init(&conn->session, &conn->request); - - /* Link connection node to parent context list */ - mk_list_add(&conn->_head, &ctx->connections); - - return conn; -} - -int in_elasticsearch_bulk_conn_del(struct in_elasticsearch_bulk_conn *conn) -{ - if (conn->session.channel != NULL) { - mk_channel_release(conn->session.channel); - } - - /* The downstream unregisters the file descriptor from the event-loop - * so there's nothing to be done by the plugin - */ - flb_downstream_conn_release(conn->connection); - - mk_list_del(&conn->_head); - - flb_free(conn->buf_data); - flb_free(conn); - - return 0; -} - -void in_elasticsearch_bulk_conn_release_all(struct flb_in_elasticsearch *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - struct in_elasticsearch_bulk_conn *conn; - - mk_list_foreach_safe(head, tmp, &ctx->connections) { - conn = mk_list_entry(head, struct in_elasticsearch_bulk_conn, _head); - in_elasticsearch_bulk_conn_del(conn); - } -} diff --git a/fluent-bit/plugins/in_elasticsearch/in_elasticsearch_bulk_conn.h b/fluent-bit/plugins/in_elasticsearch/in_elasticsearch_bulk_conn.h deleted file mode 100644 index a5a7593ac..000000000 --- a/fluent-bit/plugins/in_elasticsearch/in_elasticsearch_bulk_conn.h +++ /dev/null @@ -1,55 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_ELASTICSEARCH_BULK_CONN -#define FLB_IN_ELASTICSEARCH_BULK_CONN - -#include -#include - -#include -#include -#include - -struct in_elasticsearch_bulk_conn { - /* Buffer */ - char *buf_data; /* Buffer data */ - int buf_len; /* Data length */ - int buf_size; /* Buffer size */ - - /* - * Parser context: we only held one parser per connection - * which is re-used everytime we have a new request. - */ - struct mk_http_parser parser; - struct mk_http_request request; - struct mk_http_session session; - struct flb_connection *connection; - - void *ctx; /* Plugin parent context */ - struct mk_list _head; /* link to flb_es_bulk->connections */ -}; - -struct in_elasticsearch_bulk_conn *in_elasticsearch_bulk_conn_add(struct flb_connection *connection, - struct flb_in_elasticsearch *ctx); -int in_elasticsearch_bulk_conn_del(struct in_elasticsearch_bulk_conn *conn); -void in_elasticsearch_bulk_conn_release_all(struct flb_in_elasticsearch *ctx); - - -#endif diff --git a/fluent-bit/plugins/in_elasticsearch/in_elasticsearch_bulk_prot.c b/fluent-bit/plugins/in_elasticsearch/in_elasticsearch_bulk_prot.c deleted file mode 100644 index c7acfd671..000000000 --- a/fluent-bit/plugins/in_elasticsearch/in_elasticsearch_bulk_prot.c +++ /dev/null @@ -1,922 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include -#include - -#include "in_elasticsearch.h" -#include "in_elasticsearch_bulk_conn.h" -#include "in_elasticsearch_bulk_prot.h" - -#define HTTP_CONTENT_JSON 0 -#define HTTP_CONTENT_NDJSON 1 - -static int send_empty_response(struct in_elasticsearch_bulk_conn *conn, int http_status) -{ - size_t sent; - flb_sds_t out; - - out = flb_sds_create_size(256); - if (!out) { - return -1; - } - - if (http_status == 200) { - flb_sds_printf(&out, - "HTTP/1.1 200 OK\r\n" - "Content-Type: application/json\r\n\r\n"); - } - - /* We should check this operations result */ - flb_io_net_write(conn->connection, - (void *) out, - flb_sds_len(out), - &sent); - - flb_sds_destroy(out); - - return 0; -} - -static int send_json_message_response(struct in_elasticsearch_bulk_conn *conn, int http_status, char *message) -{ - size_t sent; - int len; - flb_sds_t out; - - out = flb_sds_create_size(256); - if (!out) { - return -1; - } - - if (message) { - len = strlen(message); - } - else { - len = 0; - } - - if (http_status == 200) { - flb_sds_printf(&out, - "HTTP/1.1 200 OK\r\n" - "Content-Type: application/json\r\n" - "Content-Length: %i\r\n\r\n%s", - len, message); - } - - /* We should check this operations result */ - flb_io_net_write(conn->connection, - (void *) out, - flb_sds_len(out), - &sent); - - flb_sds_destroy(out); - - return 0; -} - -static int send_version_message_response(struct flb_in_elasticsearch *ctx, - struct in_elasticsearch_bulk_conn *conn, int http_status) -{ - size_t sent; - int len; - flb_sds_t out; - flb_sds_t resp; - - out = flb_sds_create_size(256); - if (!out) { - return -1; - } - resp = flb_sds_create_size(384); - if (!resp) { - flb_sds_destroy(out); - return -1; - } - - flb_sds_printf(&resp, - ES_VERSION_RESPONSE_TEMPLATE, - ctx->es_version); - - len = flb_sds_len(resp); - - if (http_status == 200) { - flb_sds_printf(&out, - "HTTP/1.1 200 OK\r\n" - "Content-Type: application/json\r\n" - "Content-Length: %i\r\n\r\n%s", - len, resp); - } - - /* We should check this operations result */ - flb_io_net_write(conn->connection, - (void *) out, - flb_sds_len(out), - &sent); - - flb_sds_destroy(resp); - flb_sds_destroy(out); - - return 0; -} - -static int send_dummy_sniffer_response(struct in_elasticsearch_bulk_conn *conn, int http_status, - struct flb_in_elasticsearch *ctx) -{ - size_t sent; - int len; - flb_sds_t out; - flb_sds_t resp; - flb_sds_t hostname; - - if (ctx->hostname != NULL) { - hostname = ctx->hostname; - } - else { - hostname = "localhost"; - } - - out = flb_sds_create_size(384); - if (!out) { - return -1; - } - - resp = flb_sds_create_size(384); - if (!resp) { - flb_sds_destroy(out); - return -1; - } - - flb_sds_printf(&resp, - ES_NODES_TEMPLATE, - ctx->cluster_name, ctx->node_name, - hostname, ctx->tcp_port, ctx->buffer_max_size); - - len = flb_sds_len(resp) ; - - if (http_status == 200) { - flb_sds_printf(&out, - "HTTP/1.1 200 OK\r\n" - "Content-Type: application/json\r\n" - "Content-Length: %i\r\n\r\n%s", - len, resp); - } - - /* We should check this operations result */ - flb_io_net_write(conn->connection, - (void *) out, - flb_sds_len(out), - &sent); - - flb_sds_destroy(resp); - flb_sds_destroy(out); - - return 0; -} - -static int send_response(struct in_elasticsearch_bulk_conn *conn, int http_status, char *message) -{ - size_t sent; - int len; - flb_sds_t out; - - out = flb_sds_create_size(256); - if (!out) { - return -1; - } - - if (message) { - len = strlen(message); - } - else { - len = 0; - } - - if (http_status == 200) { - flb_sds_printf(&out, - "HTTP/1.1 200 OK\r\n" - "Server: Fluent Bit v%s\r\n" - "Content-Type: application/json\r\n" - "Content-Length: %i\r\n\r\n%s", - FLB_VERSION_STR, - len, message); - } - else if (http_status == 400) { - flb_sds_printf(&out, - "HTTP/1.1 400 Forbidden\r\n" - "Server: Fluent Bit v%s\r\n" - "Content-Length: %i\r\n\r\n%s", - FLB_VERSION_STR, - len, message); - } - - /* We should check this operations result */ - flb_io_net_write(conn->connection, - (void *) out, - flb_sds_len(out), - &sent); - - flb_sds_destroy(out); - - return 0; -} - -/* implements functionality to get tag from key in record */ -static flb_sds_t tag_key(struct flb_in_elasticsearch *ctx, msgpack_object *map) -{ - size_t map_size = map->via.map.size; - msgpack_object_kv *kv; - msgpack_object key; - msgpack_object val; - char *key_str = NULL; - char *val_str = NULL; - size_t key_str_size = 0; - size_t val_str_size = 0; - int j; - int check = FLB_FALSE; - int found = FLB_FALSE; - flb_sds_t tag; - - kv = map->via.map.ptr; - - for(j=0; j < map_size; j++) { - check = FLB_FALSE; - found = FLB_FALSE; - key = (kv+j)->key; - if (key.type == MSGPACK_OBJECT_BIN) { - key_str = (char *) key.via.bin.ptr; - key_str_size = key.via.bin.size; - check = FLB_TRUE; - } - if (key.type == MSGPACK_OBJECT_STR) { - key_str = (char *) key.via.str.ptr; - key_str_size = key.via.str.size; - check = FLB_TRUE; - } - - if (check == FLB_TRUE) { - if (strncmp(ctx->tag_key, key_str, key_str_size) == 0) { - val = (kv+j)->val; - if (val.type == MSGPACK_OBJECT_BIN) { - val_str = (char *) val.via.bin.ptr; - val_str_size = val.via.str.size; - found = FLB_TRUE; - break; - } - if (val.type == MSGPACK_OBJECT_STR) { - val_str = (char *) val.via.str.ptr; - val_str_size = val.via.str.size; - found = FLB_TRUE; - break; - } - } - } - } - - if (found == FLB_TRUE) { - tag = flb_sds_create_len(val_str, val_str_size); - if (!tag) { - flb_errno(); - return NULL; - } - return tag; - } - - - flb_plg_error(ctx->ins, "Could not find tag_key %s in record", ctx->tag_key); - return NULL; -} - -static int get_write_op(struct flb_in_elasticsearch *ctx, msgpack_object *map, flb_sds_t *out_write_op, size_t *out_key_size) -{ - char *op_str = NULL; - size_t op_str_size = 0; - msgpack_object_kv *kv; - msgpack_object key; - int check = FLB_FALSE; - - kv = map->via.map.ptr; - key = kv[0].key; - if (key.type == MSGPACK_OBJECT_BIN) { - op_str = (char *) key.via.bin.ptr; - op_str_size = key.via.bin.size; - check = FLB_TRUE; - } - if (key.type == MSGPACK_OBJECT_STR) { - op_str = (char *) key.via.str.ptr; - op_str_size = key.via.str.size; - check = FLB_TRUE; - } - - if (check == FLB_TRUE) { - *out_write_op = flb_sds_create_len(op_str, op_str_size); - *out_key_size = op_str_size; - } - - return check; -} - -static int status_buffer_avail(struct flb_in_elasticsearch *ctx, flb_sds_t bulk_statuses, size_t threshold) -{ - if (flb_sds_avail(bulk_statuses) < threshold) { - flb_plg_warn(ctx->ins, "left buffer for bulk status(es) is too small"); - - return FLB_FALSE; - } - - return FLB_TRUE; -} - -static int process_ndpack(struct flb_in_elasticsearch *ctx, flb_sds_t tag, char *buf, size_t size, flb_sds_t bulk_statuses) -{ - int ret; - size_t off = 0; - size_t map_copy_index; - msgpack_object_kv *map_copy_entry; - msgpack_unpacked result; - struct flb_time tm; - msgpack_object *obj; - flb_sds_t tag_from_record = NULL; - int idx = 0; - flb_sds_t write_op; - size_t op_str_size = 0; - int op_ret = FLB_FALSE; - int error_op = FLB_FALSE; - - flb_time_get(&tm); - - msgpack_unpacked_init(&result); - while (msgpack_unpack_next(&result, buf, size, &off) == MSGPACK_UNPACK_SUCCESS) { - if (result.data.type == MSGPACK_OBJECT_MAP) { - if (idx > 0 && idx % 2 == 0) { - flb_sds_cat(bulk_statuses, ",", 1); - } - if (status_buffer_avail(ctx, bulk_statuses, 50) == FLB_FALSE) { - break; - } - if (idx % 2 == 0) { - op_ret = get_write_op(ctx, &result.data, &write_op, &op_str_size); - - if (op_ret) { - if (flb_sds_cmp(write_op, "index", op_str_size) == 0) { - flb_sds_cat(bulk_statuses, "{\"index\":", 9); - error_op = FLB_FALSE; - } - else if (flb_sds_cmp(write_op, "create", op_str_size) == 0) { - flb_sds_cat(bulk_statuses, "{\"create\":", 10); - error_op = FLB_FALSE; - } - else if (flb_sds_cmp(write_op, "update", op_str_size) == 0) { - flb_sds_cat(bulk_statuses, "{\"update\":", 10); - error_op = FLB_TRUE; - } - else if (flb_sds_cmp(write_op, "delete", op_str_size) == 0) { - flb_sds_cat(bulk_statuses, "{\"delete\":{\"status\":404,\"result\":\"not_found\"}}", 46); - error_op = FLB_TRUE; - idx += 1; /* Prepare to adjust to multiple of two - * in the end of the loop. - * Due to delete actions include only one line. */ - flb_sds_destroy(write_op); - - goto proceed; - } - else { - flb_sds_cat(bulk_statuses, "{\"unknown\":{\"status\":400,\"result\":\"bad_request\"}}", 49); - error_op = FLB_TRUE; - - flb_sds_destroy(write_op); - - break; - } - } else { - flb_sds_destroy(write_op); - flb_plg_error(ctx->ins, "meta information line is missing"); - error_op = FLB_TRUE; - - break; - } - - if (error_op == FLB_FALSE) { - flb_log_event_encoder_reset(&ctx->log_encoder); - - ret = flb_log_event_encoder_begin_record(&ctx->log_encoder); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_sds_destroy(write_op); - flb_plg_error(ctx->ins, "event encoder error : %d", ret); - error_op = FLB_TRUE; - - break; - } - - ret = flb_log_event_encoder_set_timestamp( - &ctx->log_encoder, - &tm); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_sds_destroy(write_op); - flb_plg_error(ctx->ins, "event encoder error : %d", ret); - error_op = FLB_TRUE; - - break; - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - &ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE((char *) ctx->meta_key), - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&result.data)); - } - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_sds_destroy(write_op); - flb_plg_error(ctx->ins, "event encoder error : %d", ret); - error_op = FLB_TRUE; - - break; - } - } - } - else if (idx % 2 == 1) { - if (error_op == FLB_FALSE) { - /* Pack body */ - - for (map_copy_index = 0 ; - map_copy_index < result.data.via.map.size && - ret == FLB_EVENT_ENCODER_SUCCESS ; - map_copy_index++) { - map_copy_entry = &result.data.via.map.ptr[map_copy_index]; - - ret = flb_log_event_encoder_append_body_values( - &ctx->log_encoder, - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&map_copy_entry->key), - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&map_copy_entry->val)); - } - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, "event encoder error : %d", ret); - error_op = FLB_TRUE; - - break; - } - - ret = flb_log_event_encoder_commit_record(&ctx->log_encoder); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, "event encoder error : %d", ret); - error_op = FLB_TRUE; - - break; - } - - tag_from_record = NULL; - - if (ctx->tag_key) { - obj = &result.data; - tag_from_record = tag_key(ctx, obj); - } - - if (tag_from_record) { - flb_input_log_append(ctx->ins, - tag_from_record, - flb_sds_len(tag_from_record), - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - - flb_sds_destroy(tag_from_record); - } - else if (tag) { - flb_input_log_append(ctx->ins, - tag, - flb_sds_len(tag), - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - } - else { - /* use default plugin Tag (it internal name, e.g: http.0 */ - flb_input_log_append(ctx->ins, NULL, 0, - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - } - - flb_log_event_encoder_reset(&ctx->log_encoder); - } - if (op_ret) { - if (flb_sds_cmp(write_op, "index", op_str_size) == 0) { - flb_sds_cat(bulk_statuses, "{\"status\":201,\"result\":\"created\"}}", 34); - } - else if (flb_sds_cmp(write_op, "create", op_str_size) == 0) { - flb_sds_cat(bulk_statuses, "{\"status\":201,\"result\":\"created\"}}", 34); - } - else if (flb_sds_cmp(write_op, "update", op_str_size) == 0) { - flb_sds_cat(bulk_statuses, "{\"status\":403,\"result\":\"forbidden\"}}", 36); - } - if (status_buffer_avail(ctx, bulk_statuses, 50) == FLB_FALSE) { - flb_sds_destroy(write_op); - - break; - } - } - flb_sds_destroy(write_op); - } - - proceed: - idx++; - } - else { - flb_plg_error(ctx->ins, "skip record from invalid type: %i", - result.data.type); - msgpack_unpacked_destroy(&result); - return -1; - } - } - - if (idx % 2 != 0) { - flb_plg_warn(ctx->ins, "decode payload of Bulk API is failed"); - msgpack_unpacked_destroy(&result); - if (error_op == FLB_FALSE) { - /* On lacking of body case in non-error case, there is no - * releasing memory code paths. We should proceed to do - * it here. */ - flb_sds_destroy(write_op); - } - - return -1; - } - - msgpack_unpacked_destroy(&result); - - return 0; -} - -static ssize_t parse_payload_ndjson(struct flb_in_elasticsearch *ctx, flb_sds_t tag, - char *payload, size_t size, flb_sds_t bulk_statuses) -{ - int ret; - int out_size; - char *pack; - struct flb_pack_state pack_state; - - /* Initialize packer */ - flb_pack_state_init(&pack_state); - - /* Pack JSON as msgpack */ - ret = flb_pack_json_state(payload, size, - &pack, &out_size, &pack_state); - flb_pack_state_reset(&pack_state); - - /* Handle exceptions */ - if (ret == FLB_ERR_JSON_PART) { - flb_plg_warn(ctx->ins, "JSON data is incomplete, skipping"); - return -1; - } - else if (ret == FLB_ERR_JSON_INVAL) { - flb_plg_warn(ctx->ins, "invalid JSON message, skipping"); - return -1; - } - else if (ret == -1) { - return -1; - } - - /* Process the packaged JSON and return the last byte used */ - process_ndpack(ctx, tag, pack, out_size, bulk_statuses); - flb_free(pack); - - return 0; -} - -static int process_payload(struct flb_in_elasticsearch *ctx, struct in_elasticsearch_bulk_conn *conn, - flb_sds_t tag, - struct mk_http_session *session, - struct mk_http_request *request, - flb_sds_t bulk_statuses) -{ - int type = -1; - int i = 0; - int ret = 0; - struct mk_http_header *header; - int extra_size = -1; - struct mk_http_header *headers_extra; - int gzip_compressed = FLB_FALSE; - void *gz_data = NULL; - size_t gz_size = -1; - - header = &session->parser.headers[MK_HEADER_CONTENT_TYPE]; - if (header->key.data == NULL) { - send_response(conn, 400, "error: header 'Content-Type' is not set\n"); - return -1; - } - - if (header->val.len >= 20 && - strncasecmp(header->val.data, "application/x-ndjson", 20) == 0) { - type = HTTP_CONTENT_NDJSON; - } - - if (header->val.len >= 16 && - strncasecmp(header->val.data, "application/json", 16) == 0) { - type = HTTP_CONTENT_JSON; - } - - if (type == -1) { - send_response(conn, 400, "error: invalid 'Content-Type'\n"); - return -1; - } - - if (request->data.len <= 0) { - send_response(conn, 400, "error: no payload found\n"); - return -1; - } - - extra_size = session->parser.headers_extra_count; - if (extra_size > 0) { - for (i = 0; i < extra_size; i++) { - headers_extra = &session->parser.headers_extra[i]; - if (headers_extra->key.len == 16 && - strncasecmp(headers_extra->key.data, "Content-Encoding", 16) == 0) { - if (headers_extra->val.len == 4 && - strncasecmp(headers_extra->val.data, "gzip", 4) == 0) { - flb_debug("[elasticsearch_bulk_prot] body is gzipped"); - gzip_compressed = FLB_TRUE; - } - } - } - } - - if (type == HTTP_CONTENT_NDJSON || type == HTTP_CONTENT_JSON) { - if (gzip_compressed == FLB_TRUE) { - ret = flb_gzip_uncompress((void *) request->data.data, request->data.len, - &gz_data, &gz_size); - if (ret == -1) { - flb_error("[elasticsearch_bulk_prot] gzip uncompress is failed"); - return -1; - } - parse_payload_ndjson(ctx, tag, gz_data, gz_size, bulk_statuses); - flb_free(gz_data); - } - else { - parse_payload_ndjson(ctx, tag, request->data.data, request->data.len, bulk_statuses); - } - } - - return 0; -} - -static inline int mk_http_point_header(mk_ptr_t *h, - struct mk_http_parser *parser, int key) -{ - struct mk_http_header *header; - - header = &parser->headers[key]; - if (header->type == key) { - h->data = header->val.data; - h->len = header->val.len; - return 0; - } - else { - h->data = NULL; - h->len = -1; - } - - return -1; -} - -/* - * Handle an incoming request. It perform extra checks over the request, if - * everything is OK, it enqueue the incoming payload. - */ -int in_elasticsearch_bulk_prot_handle(struct flb_in_elasticsearch *ctx, - struct in_elasticsearch_bulk_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request) -{ - int i; - int ret; - int len; - char *uri; - char *qs; - off_t diff; - flb_sds_t tag; - struct mk_http_header *header; - flb_sds_t bulk_statuses = NULL; - flb_sds_t bulk_response = NULL; - char *error_str = NULL; - - if (request->uri.data[0] != '/') { - send_response(conn, 400, "error: invalid request\n"); - return -1; - } - - /* Decode URI */ - uri = mk_utils_url_decode(request->uri); - if (!uri) { - uri = mk_mem_alloc_z(request->uri.len + 1); - if (!uri) { - return -1; - } - memcpy(uri, request->uri.data, request->uri.len); - uri[request->uri.len] = '\0'; - } - - /* Try to match a query string so we can remove it */ - qs = strchr(uri, '?'); - if (qs) { - /* remove the query string part */ - diff = qs - uri; - uri[diff] = '\0'; - } - - /* Refer the tag at first*/ - if (ctx->ins->tag && !ctx->ins->tag_default) { - tag = flb_sds_create(ctx->ins->tag); - if (tag == NULL) { - return -1; - } - } - else { - /* Compose the query string using the URI */ - len = strlen(uri); - - if (len == 1) { - tag = NULL; /* use default tag */ - } - else { - /* New tag skipping the URI '/' */ - tag = flb_sds_create_len(&uri[1], len - 1); - if (!tag) { - mk_mem_free(uri); - return -1; - } - - /* Sanitize, only allow alphanum chars */ - for (i = 0; i < flb_sds_len(tag); i++) { - if (!isalnum(tag[i]) && tag[i] != '_' && tag[i] != '.') { - tag[i] = '_'; - } - } - } - } - - /* Check if we have a Host header: Hostname ; port */ - mk_http_point_header(&request->host, &session->parser, MK_HEADER_HOST); - - /* Header: Connection */ - mk_http_point_header(&request->connection, &session->parser, - MK_HEADER_CONNECTION); - - /* HTTP/1.1 needs Host header */ - if (!request->host.data && request->protocol == MK_HTTP_PROTOCOL_11) { - flb_sds_destroy(tag); - mk_mem_free(uri); - return -1; - } - - /* Should we close the session after this request ? */ - mk_http_keepalive_check(session, request, ctx->server); - - /* Content Length */ - header = &session->parser.headers[MK_HEADER_CONTENT_LENGTH]; - if (header->type == MK_HEADER_CONTENT_LENGTH) { - request->_content_length.data = header->val.data; - request->_content_length.len = header->val.len; - } - else { - request->_content_length.data = NULL; - } - - if (request->method == MK_METHOD_HEAD) { - send_empty_response(conn, 200); - - flb_sds_destroy(tag); - mk_mem_free(uri); - - return 0; - } - - if (request->method == MK_METHOD_PUT) { - send_json_message_response(conn, 200, "{}"); - - flb_sds_destroy(tag); - mk_mem_free(uri); - - return 0; - } - - if (request->method == MK_METHOD_GET) { - if (strncmp(uri, "/_nodes/http", 12) == 0) { - send_dummy_sniffer_response(conn, 200, ctx); - } - else if (strlen(uri) == 1 && strncmp(uri, "/", 1) == 0) { - send_version_message_response(ctx, conn, 200); - } - else { - send_json_message_response(conn, 200, "{}"); - } - - flb_sds_destroy(tag); - mk_mem_free(uri); - - return 0; - } - - if (request->method == MK_METHOD_POST) { - if (strncmp(uri, "/_bulk", 6) == 0) { - bulk_statuses = flb_sds_create_size(ctx->buffer_max_size); - if (!bulk_statuses) { - flb_sds_destroy(tag); - mk_mem_free(uri); - return -1; - } - - bulk_response = flb_sds_create_size(ctx->buffer_max_size); - if (!bulk_response) { - flb_sds_destroy(bulk_statuses); - flb_sds_destroy(tag); - mk_mem_free(uri); - return -1; - } - } else { - flb_sds_destroy(tag); - mk_mem_free(uri); - - send_response(conn, 400, "error: invaild HTTP endpoint\n"); - - return -1; - } - } - - if (request->method != MK_METHOD_POST && - request->method != MK_METHOD_GET && - request->method != MK_METHOD_HEAD && - request->method != MK_METHOD_PUT) { - - if (bulk_statuses) { - flb_sds_destroy(bulk_statuses); - } - if (bulk_response) { - flb_sds_destroy(bulk_response); - } - - flb_sds_destroy(tag); - mk_mem_free(uri); - - send_response(conn, 400, "error: invalid HTTP method\n"); - return -1; - } - - ret = process_payload(ctx, conn, tag, session, request, bulk_statuses); - flb_sds_destroy(tag); - - len = flb_sds_len(bulk_statuses); - if (flb_sds_alloc(bulk_response) < len + 27) { - bulk_response = flb_sds_increase(bulk_response, len + 27 - flb_sds_alloc(bulk_response)); - } - error_str = strstr(bulk_statuses, "\"status\":40"); - if (error_str){ - flb_sds_cat(bulk_response, "{\"errors\":true,\"items\":[", 24); - } - else { - flb_sds_cat(bulk_response, "{\"errors\":false,\"items\":[", 25); - } - flb_sds_cat(bulk_response, bulk_statuses, flb_sds_len(bulk_statuses)); - flb_sds_cat(bulk_response, "]}", 2); - send_response(conn, 200, bulk_response); - - mk_mem_free(uri); - flb_sds_destroy(bulk_statuses); - flb_sds_destroy(bulk_response); - - return ret; -} - -/* - * Handle an incoming request which has resulted in an http parser error. - */ -int in_elasticsearch_bulk_prot_handle_error(struct flb_in_elasticsearch *ctx, - struct in_elasticsearch_bulk_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request) -{ - send_response(conn, 400, "error: invalid request\n"); - return -1; -} diff --git a/fluent-bit/plugins/in_elasticsearch/in_elasticsearch_bulk_prot.h b/fluent-bit/plugins/in_elasticsearch/in_elasticsearch_bulk_prot.h deleted file mode 100644 index be1aeceea..000000000 --- a/fluent-bit/plugins/in_elasticsearch/in_elasticsearch_bulk_prot.h +++ /dev/null @@ -1,40 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_ELASTICSEARCH_BULK_PROT -#define FLB_IN_ELASTICSEARCH_BULK_PROT - -#define ES_VERSION_RESPONSE_TEMPLATE \ - "{\"version\":{\"number\":\"%s\",\"build_flavor\":\"Fluent Bit OSS\"},\"tagline\":\"Fluent Bit's Bulk API compatible endpoint\"}" - -#define ES_NODES_TEMPLATE "{\"_nodes\":{\"total\":1,\"successful\":1,\"failed\":0}," \ - "\"nodes\":{\"%s\":{\"name\":\"%s\",\"version\":\"8.0.0\"," \ - "\"http\":{\"publish_address\":\"%s:%s\",\"max_content_length_in_bytes\":%ld}}}}" - -int in_elasticsearch_bulk_prot_handle(struct flb_in_elasticsearch *ctx, - struct in_elasticsearch_bulk_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request); - -int in_elasticsearch_bulk_prot_handle_error(struct flb_in_elasticsearch *ctx, - struct in_elasticsearch_bulk_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request); - -#endif diff --git a/fluent-bit/plugins/in_elasticsearch/in_elasticsearch_config.c b/fluent-bit/plugins/in_elasticsearch/in_elasticsearch_config.c deleted file mode 100644 index 4beb96320..000000000 --- a/fluent-bit/plugins/in_elasticsearch/in_elasticsearch_config.c +++ /dev/null @@ -1,105 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include "in_elasticsearch.h" -#include "in_elasticsearch_config.h" -#include "in_elasticsearch_bulk_conn.h" - -struct flb_in_elasticsearch *in_elasticsearch_config_create(struct flb_input_instance *ins) -{ - int ret; - char port[8]; - struct flb_in_elasticsearch *ctx; - - ctx = flb_calloc(1, sizeof(struct flb_in_elasticsearch)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - mk_list_init(&ctx->connections); - - /* Load the config map */ - ret = flb_input_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return NULL; - } - - /* Listen interface (if not set, defaults to 0.0.0.0:9200) */ - flb_input_net_default_listener("0.0.0.0", 9200, ins); - - ctx->listen = flb_sds_create(ins->host.listen); - snprintf(port, sizeof(port) - 1, "%d", ins->host.port); - ctx->tcp_port = flb_sds_create(port); - - /* HTTP Server specifics */ - ctx->server = flb_calloc(1, sizeof(struct mk_server)); - ctx->server->keep_alive = MK_TRUE; - - /* monkey detects server->workers == 0 as the server not being initialized at the - * moment so we want to make sure that it stays that way! - */ - - ret = flb_log_event_encoder_init(&ctx->log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, "error initializing event encoder : %d", ret); - - in_elasticsearch_config_destroy(ctx); - - return ctx = NULL; - } - - - return ctx; -} - -int in_elasticsearch_config_destroy(struct flb_in_elasticsearch *ctx) -{ - flb_log_event_encoder_destroy(&ctx->log_encoder); - - /* release all connections */ - in_elasticsearch_bulk_conn_release_all(ctx); - - - if (ctx->collector_id != -1) { - flb_input_collector_delete(ctx->collector_id, ctx->ins); - - ctx->collector_id = -1; - } - - if (ctx->downstream != NULL) { - flb_downstream_destroy(ctx->downstream); - } - - if (ctx->server) { - flb_free(ctx->server); - } - - flb_sds_destroy(ctx->listen); - flb_sds_destroy(ctx->tcp_port); - - flb_free(ctx); - - return 0; -} diff --git a/fluent-bit/plugins/in_elasticsearch/in_elasticsearch_config.h b/fluent-bit/plugins/in_elasticsearch/in_elasticsearch_config.h deleted file mode 100644 index 28108723d..000000000 --- a/fluent-bit/plugins/in_elasticsearch/in_elasticsearch_config.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_ELASTICSEARCH_CONFIG_H -#define FLB_IN_ELASTICSEARCH_CONFIG_H - -#include -#include "in_elasticsearch.h" - -struct flb_in_elasticsearch *in_elasticsearch_config_create(struct flb_input_instance *ins); -int in_elasticsearch_config_destroy(struct flb_in_elasticsearch *ctx); - -#endif diff --git a/fluent-bit/plugins/in_emitter/CMakeLists.txt b/fluent-bit/plugins/in_emitter/CMakeLists.txt deleted file mode 100644 index 596b53d08..000000000 --- a/fluent-bit/plugins/in_emitter/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -set(src - emitter.c - ) - -FLB_PLUGIN(in_emitter "${src}" "") diff --git a/fluent-bit/plugins/in_emitter/emitter.c b/fluent-bit/plugins/in_emitter/emitter.c deleted file mode 100644 index 821df9539..000000000 --- a/fluent-bit/plugins/in_emitter/emitter.c +++ /dev/null @@ -1,321 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#define DEFAULT_EMITTER_RING_BUFFER_FLUSH_FREQUENCY 2000 - -struct em_chunk { - flb_sds_t tag; - struct msgpack_sbuffer mp_sbuf; /* msgpack sbuffer */ - struct msgpack_packer mp_pck; /* msgpack packer */ - struct mk_list _head; -}; - -struct flb_emitter { - struct mk_list chunks; /* list of all pending chunks */ - struct flb_input_instance *ins; /* input instance */ - struct flb_ring_buffer *msgs; /* ring buffer for cross-thread messages */ - int ring_buffer_size; /* size of the ring buffer */ -}; - -struct em_chunk *em_chunk_create(const char *tag, int tag_len, - struct flb_emitter *ctx) -{ - struct em_chunk *ec; - - ec = flb_calloc(1, sizeof(struct em_chunk)); - if (!ec) { - flb_errno(); - return NULL; - } - - ec->tag = flb_sds_create_len(tag, tag_len); - if (!ec->tag) { - flb_errno(); - flb_free(ec); - return NULL; - } - - msgpack_sbuffer_init(&ec->mp_sbuf); - msgpack_packer_init(&ec->mp_pck, &ec->mp_sbuf, msgpack_sbuffer_write); - - mk_list_add(&ec->_head, &ctx->chunks); - - return ec; -} - -static void em_chunk_destroy(struct em_chunk *ec) -{ - mk_list_del(&ec->_head); - flb_sds_destroy(ec->tag); - msgpack_sbuffer_destroy(&ec->mp_sbuf); - flb_free(ec); -} - -int static do_in_emitter_add_record(struct em_chunk *ec, - struct flb_input_instance *in) -{ - struct flb_emitter *ctx = (struct flb_emitter *) in->context; - int ret; - - /* Associate this backlog chunk to this instance into the engine */ - ret = flb_input_log_append(in, - ec->tag, flb_sds_len(ec->tag), - ec->mp_sbuf.data, - ec->mp_sbuf.size); - if (ret == -1) { - flb_plg_error(ctx->ins, "error registering chunk with tag: %s", - ec->tag); - /* Release the echunk */ - em_chunk_destroy(ec); - return -1; - } - /* Release the echunk */ - em_chunk_destroy(ec); - return 0; -} - -/* - * Function used by filters to ingest custom records with custom tags, at the - * moment it's only used by rewrite_tag filter. - */ -int in_emitter_add_record(const char *tag, int tag_len, - const char *buf_data, size_t buf_size, - struct flb_input_instance *in) -{ - struct em_chunk temporary_chunk; - struct mk_list *head; - struct em_chunk *ec; - struct flb_emitter *ctx; - - ctx = (struct flb_emitter *) in->context; - ec = NULL; - - /* Use the ring buffer first if it exists */ - if (ctx->msgs) { - memset(&temporary_chunk, 0, sizeof(struct em_chunk)); - - temporary_chunk.tag = flb_sds_create_len(tag, tag_len); - - if (temporary_chunk.tag == NULL) { - flb_plg_error(ctx->ins, - "cannot allocate memory for tag: %s", - tag); - return -1; - } - - msgpack_sbuffer_init(&temporary_chunk.mp_sbuf); - msgpack_sbuffer_write(&temporary_chunk.mp_sbuf, buf_data, buf_size); - - return flb_ring_buffer_write(ctx->msgs, - (void *) &temporary_chunk, - sizeof(struct em_chunk)); - } - - /* Check if any target chunk already exists */ - mk_list_foreach(head, &ctx->chunks) { - ec = mk_list_entry(head, struct em_chunk, _head); - if (flb_sds_cmp(ec->tag, tag, tag_len) != 0) { - ec = NULL; - continue; - } - break; - } - - /* No candidate chunk found, so create a new one */ - if (!ec) { - ec = em_chunk_create(tag, tag_len, ctx); - if (!ec) { - flb_plg_error(ctx->ins, "cannot create new chunk for tag: %s", - tag); - return -1; - } - } - - /* Append raw msgpack data */ - msgpack_sbuffer_write(&ec->mp_sbuf, buf_data, buf_size); - - return do_in_emitter_add_record(ec, in); -} - -/* - * Triggered by refresh_interval, it re-scan the path looking for new files - * that match the original path pattern. - */ -static int in_emitter_ingest_ring_buffer(struct flb_input_instance *in, - struct flb_config *config, void *context) -{ - int ret; - struct flb_emitter *ctx = (struct flb_emitter *)context; - struct em_chunk ec; - (void) config; - (void) in; - - - while ((ret = flb_ring_buffer_read(ctx->msgs, (void *)&ec, - sizeof(struct em_chunk))) == 0) { - ret = flb_input_log_append(in, - ec.tag, flb_sds_len(ec.tag), - ec.mp_sbuf.data, - ec.mp_sbuf.size); - flb_sds_destroy(ec.tag); - msgpack_sbuffer_destroy(&ec.mp_sbuf); - } - return ret; -} - -static int in_emitter_start_ring_buffer(struct flb_input_instance *in, struct flb_emitter *ctx) -{ - if (ctx->ring_buffer_size <= 0) { - return 0; - } - - if (ctx->msgs != NULL) { - flb_warn("emitter %s already has a ring buffer", - flb_input_name(in)); - return 0; - } - - ctx->msgs = flb_ring_buffer_create(sizeof(void *) * ctx->ring_buffer_size); - if (!ctx->msgs) { - flb_error("emitter %s could not initialize ring buffer", - flb_input_name(in)); - return -1; - } - - return flb_input_set_collector_time(in, in_emitter_ingest_ring_buffer, - 1, 0, in->config); -} - -/* Initialize plugin */ -static int cb_emitter_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - struct flb_sched *scheduler; - struct flb_emitter *ctx; - int ret; - - scheduler = flb_sched_ctx_get(); - - ctx = flb_calloc(1, sizeof(struct flb_emitter)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->ins = in; - mk_list_init(&ctx->chunks); - - - ret = flb_input_config_map_set(in, (void *) ctx); - if (ret == -1) { - return -1; - } - - if (scheduler != config->sched && - scheduler != NULL && - ctx->ring_buffer_size == 0) { - - ctx->ring_buffer_size = DEFAULT_EMITTER_RING_BUFFER_FLUSH_FREQUENCY; - - flb_plg_debug(in, - "threaded emitter instances require ring_buffer_size" - " being set, using default value of %u", - ctx->ring_buffer_size); - } - - if (ctx->ring_buffer_size > 0) { - ret = in_emitter_start_ring_buffer(in, ctx); - if (ret == -1) { - flb_free(ctx); - return -1; - } - } - - /* export plugin context */ - flb_input_set_context(in, ctx); - - return 0; -} - -static int cb_emitter_exit(void *data, struct flb_config *config) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_emitter *ctx = data; - struct em_chunk *echunk; - struct em_chunk ec; - int ret; - - - mk_list_foreach_safe(head, tmp, &ctx->chunks) { - echunk = mk_list_entry(head, struct em_chunk, _head); - mk_list_del(&echunk->_head); - flb_free(echunk); - } - - if (ctx->msgs) { - while ((ret = flb_ring_buffer_read(ctx->msgs, (void *)&ec, - sizeof(struct em_chunk))) == 0) { - flb_sds_destroy(ec.tag); - msgpack_sbuffer_destroy(&ec.mp_sbuf); - } - flb_ring_buffer_destroy(ctx->msgs); - } - - flb_free(ctx); - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_INT, "ring_buffer_size", "0", - 0, FLB_TRUE, offsetof(struct flb_emitter, ring_buffer_size), - "use a ring buffer to ingest messages for the emitter (required across threads)." - }, - {0} -}; - -/* Plugin reference */ -struct flb_input_plugin in_emitter_plugin = { - .name = "emitter", - .description = "Record Emitter", - .cb_init = cb_emitter_init, - .cb_pre_run = NULL, - .cb_collect = NULL, - .cb_ingest = NULL, - .cb_flush_buf = NULL, - .config_map = config_map, - .cb_pause = NULL, - .cb_resume = NULL, - .cb_exit = cb_emitter_exit, - - /* This plugin can only be configured and invoked by the Engine only */ - .flags = FLB_INPUT_PRIVATE -}; diff --git a/fluent-bit/plugins/in_event_test/CMakeLists.txt b/fluent-bit/plugins/in_event_test/CMakeLists.txt deleted file mode 100644 index 9a9577a6c..000000000 --- a/fluent-bit/plugins/in_event_test/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - event_test.c) - -FLB_PLUGIN(in_event_test "${src}" "") diff --git a/fluent-bit/plugins/in_event_test/event_test.c b/fluent-bit/plugins/in_event_test/event_test.c deleted file mode 100644 index 557017fe2..000000000 --- a/fluent-bit/plugins/in_event_test/event_test.c +++ /dev/null @@ -1,407 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define STATUS_OK 1 -#define STATUS_ERROR 0 -#define STATUS_PENDING -1 -#define CALLBACK_TIME 2 /* 2 seconds */ - -#define SERVER_PORT "9092" -#define SERVER_IFACE "0.0.0.0" - -struct unit_test { - int id; - int coll_id; - int status; - char *desc; -}; - -struct unit_test tests[] = { - {0, 0, STATUS_PENDING, "collector time"}, - {1, 0, STATUS_PENDING, "collector fd_event"}, - {2, 0, STATUS_PENDING, "collector fd_server | socket"}, - {3, 0, STATUS_PENDING, "plugin paused from engine"}, - {4, 0, STATUS_PENDING, "plugin resumed from engine"}, -}; - -#define UNIT_TESTS_SIZE (sizeof(tests) / sizeof(struct unit_test)) - -struct event_test { - flb_pipefd_t pipe[2]; - int server_fd; - int client_coll_id; - struct flb_upstream *upstream; - struct unit_test *tests; - struct flb_input_instance *ins; -}; - -static void set_unit_test_status(struct event_test *ctx, int id, int status) -{ - struct unit_test *ut; - - ut = &ctx->tests[id]; - ut->status = status; -} - -static int config_destroy(struct event_test *ctx) -{ - if (!ctx) { - return 0; - } - - if (ctx->tests) { - flb_free(ctx->tests); - } - - if (ctx->pipe[0] > 0) { - flb_socket_close(ctx->pipe[0]); - } - if (ctx->pipe[1] > 0) { - flb_socket_close(ctx->pipe[1]); - } - if (ctx->server_fd > 0) { - flb_socket_close(ctx->server_fd); - } - - if (ctx->upstream) { - flb_upstream_destroy(ctx->upstream); - } - - flb_free(ctx); - return 0; -} - -static int cb_collector_time(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - int diff; - int ret; - uint64_t val; - time_t now; - struct unit_test *ut; - struct event_test *ctx = (struct event_test *) in_context; - - now = time(NULL); - diff = now - config->init_time; - /* For macOS, we sometimes get the +1 longer time elapse. - * To handle this, we simply add +1 as a delta for checking interval. */ - if (diff > (CALLBACK_TIME + 1)) { - flb_plg_error(ins, "cb_collector_time difference failed: %i seconds", diff); - set_unit_test_status(ctx, 0, STATUS_ERROR); - flb_engine_exit(config); - } - - /* disable the collector */ - ut = &ctx->tests[0]; - flb_input_collector_pause(ut->coll_id, ins); - - /* - * before to return, trigger test 1 (collector_fd_event) by writing a byte - * to our local pipe. - */ - val = 1; - ret = write(ctx->pipe[1], &val, sizeof(val)); - if (ret == -1) { - flb_errno(); - set_unit_test_status(ctx, 0, STATUS_ERROR); - flb_engine_exit(config); - } - - set_unit_test_status(ctx, 0, STATUS_OK); - flb_plg_info(ins, "[OK] collector_time"); - FLB_INPUT_RETURN(0); -} - -static int cb_collector_fd(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - uint64_t val = 0; - size_t bytes; - struct unit_test *ut; - struct event_test *ctx = (struct event_test *) in_context; - - bytes = read(ctx->pipe[0], &val, sizeof(val)); - if (bytes <= 0) { - flb_errno(); - set_unit_test_status(ctx, 1, STATUS_ERROR); - flb_engine_exit(config); - } - else { - flb_plg_info(ins, "[OK] collector_fd"); - } - - /* disable the collector */ - ut = &ctx->tests[1]; - flb_input_collector_pause(ut->coll_id, ins); - set_unit_test_status(ctx, 1, STATUS_OK); - - FLB_INPUT_RETURN(0); -} - -static int cb_collector_server_socket(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - int fd; - struct unit_test *ut; - struct event_test *ctx = in_context; - - /* Accept the new connection */ - fd = flb_net_accept(ctx->server_fd); - if (fd == -1) { - flb_plg_error(ins, "could not accept new connection"); - return -1; - } - - /* sleep co-routine for 500ms */ - flb_time_sleep(500); - flb_socket_close(fd); - - ut = &ctx->tests[2]; - flb_input_collector_pause(ut->coll_id, ins); - set_unit_test_status(ctx, 2, STATUS_OK); - - flb_plg_info(ins, "[OK] collector_server_socket"); - - /* tell the engine to deliver a pause request */ - flb_plg_info(ins, "test pause/resume in 5 seconds..."); - flb_input_test_pause_resume(ins, 5); - - /* return */ - FLB_INPUT_RETURN(0); -} - -static int cb_collector_server_client(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_connection *u_conn; - struct event_test *ctx = (struct event_test *) in_context; - - /* get the upstream connection (localhost) */ - u_conn = flb_upstream_conn_get(ctx->upstream); - if (!u_conn) { - flb_plg_error(ins, "could not connect to socket server"); - return -1; - } - - flb_time_sleep(200); - flb_upstream_conn_release(u_conn); - - /* disable this collector */ - flb_input_collector_pause(ctx->client_coll_id, ins); - FLB_INPUT_RETURN(0); -} - -static struct event_test *config_create(struct flb_input_instance *ins) -{ - size_t size; - struct event_test *ctx; - - /* Allocate space for the configuration */ - ctx = flb_calloc(1, sizeof(struct event_test)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - - size = sizeof(struct unit_test) * UNIT_TESTS_SIZE; - ctx->tests = flb_malloc(size); - if (!ctx->tests) { - flb_errno(); - flb_free(ctx); - return NULL; - } - memcpy(ctx->tests, &tests, size); - return ctx; -} - -/* Initialize plugin */ -static int cb_event_test_init(struct flb_input_instance *ins, - struct flb_config *config, void *data) -{ - int fd; - int ret; - struct unit_test *ut; - struct event_test *ctx = NULL; - struct flb_upstream *upstream; - - /* Allocate space for the configuration */ - ctx = config_create(ins); - if (!ctx) { - return -1; - } - flb_input_set_context(ins, ctx); - - /* unit test 0: collector_time */ - ret = flb_input_set_collector_time(ins, cb_collector_time, - CALLBACK_TIME, 0, config); - if (ret < 0) { - config_destroy(ctx); - return -1; - } - ut = &ctx->tests[0]; - ut->coll_id = ret; - - /* unit test 1: collector_fd_event */ - ret = flb_pipe_create(ctx->pipe); - if (ret == -1) { - flb_errno(); - config_destroy(ctx); - return -1; - } - ret = flb_input_set_collector_event(ins, - cb_collector_fd, - ctx->pipe[0], - config); - if (ret < 0) { - config_destroy(ctx); - return -1; - } - ut = &ctx->tests[1]; - ut->coll_id = ret; - - /* unit test 2: collector_socket */ - fd = flb_net_server(SERVER_PORT, SERVER_IFACE); - if (fd < 0) { - flb_errno(); - config_destroy(ctx); - return -1; - } - flb_net_socket_nonblocking(fd); - ctx->server_fd = fd; - - /* socket server */ - ret = flb_input_set_collector_socket(ins, - cb_collector_server_socket, - ctx->server_fd, - config); - if (ret == -1) { - config_destroy(ctx); - return -1; - } - ut = &ctx->tests[2]; - ut->coll_id = ret; - - /* socket client: connect to socket server to trigger the event */ - ret = flb_input_set_collector_time(ins, cb_collector_server_client, - CALLBACK_TIME * 2, 0, config); - if (ret < 0) { - config_destroy(ctx); - return -1; - } - ctx->client_coll_id = ret; - - /* upstream context for socket client */ - upstream = flb_upstream_create(config, "127.0.0.1", atoi(SERVER_PORT), - FLB_IO_TCP, NULL); - if (!upstream) { - config_destroy(ctx); - return -1; - } - ctx->upstream = upstream; - flb_input_upstream_set(ctx->upstream, ins); - - return 0; -} - -static int cb_event_test_pre_run(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - flb_plg_info(ins, "pre run OK"); - return -1; -} - -static void cb_event_test_pause(void *data, struct flb_config *config) -{ - struct event_test *ctx = data; - - set_unit_test_status(ctx, 3, STATUS_OK); - flb_plg_info(ctx->ins, "[OK] engine has paused the plugin"); -} - -static void cb_event_test_resume(void *data, struct flb_config *config) -{ - struct event_test *ctx = data; - - set_unit_test_status(ctx, 4, STATUS_OK); - flb_plg_info(ctx->ins, "[OK] engine has resumed the plugin"); - - flb_engine_exit(config); -} - -static int in_event_test_exit(void *data, struct flb_config *config) -{ - int i; - int failed = FLB_FALSE; - struct event_test *ctx = data; - struct unit_test *ut; - (void) *config; - - /* check tests */ - for (i = 0; i < UNIT_TESTS_SIZE; i++) { - ut = &ctx->tests[i]; - if (ut->status != STATUS_OK) { - flb_plg_error(ctx->ins, "unit test #%i '%s' failed", - i, ut->desc); - failed = FLB_TRUE; - } - else { - flb_plg_info(ctx->ins, "unit test #%i '%s' succeeded", - i, ut->desc); - } - } - - /* if one test failed, perform an abrupt exit with proper error */ - if (failed) { - exit(EXIT_FAILURE); - } - - config_destroy(ctx); - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - /* EOF */ - {0} -}; - -struct flb_input_plugin in_event_test_plugin = { - .name = "event_test", - .description = "Event tests for input plugins", - .cb_init = cb_event_test_init, - .cb_pre_run = cb_event_test_pre_run, - .cb_collect = NULL, - .cb_flush_buf = NULL, - .cb_pause = cb_event_test_pause, - .cb_resume = cb_event_test_resume, - .cb_exit = in_event_test_exit, - .config_map = config_map, - .flags = FLB_INPUT_CORO | FLB_INPUT_THREADED -}; diff --git a/fluent-bit/plugins/in_event_type/CMakeLists.txt b/fluent-bit/plugins/in_event_type/CMakeLists.txt deleted file mode 100644 index 596f5f94e..000000000 --- a/fluent-bit/plugins/in_event_type/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - event_type.c) - -FLB_PLUGIN(in_event_type "${src}" "") diff --git a/fluent-bit/plugins/in_event_type/event_type.c b/fluent-bit/plugins/in_event_type/event_type.c deleted file mode 100644 index bfad258c2..000000000 --- a/fluent-bit/plugins/in_event_type/event_type.c +++ /dev/null @@ -1,482 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include - -#define DEFAULT_INTERVAL_SEC "2" -#define DEFAULT_INTERVAL_NSEC "0" - -#define OTEL_SPAN_ID_LEN 8 - -struct event_type { - int coll_fd; - int type; - - int interval_sec; - int interval_nsec; -}; - -static struct ctrace_id *create_random_span_id() -{ - char *buf; - ssize_t ret; - struct ctrace_id *cid; - - buf = flb_malloc(OTEL_SPAN_ID_LEN); - if (!buf) { - ctr_errno(); - return NULL; - } - - ret = ctr_random_get(buf, OTEL_SPAN_ID_LEN); - if (ret < 0) { - flb_free(buf); - return NULL; - } - - cid = ctr_id_create(buf, OTEL_SPAN_ID_LEN); - flb_free(buf); - - return cid; - -} - -static int send_logs(struct flb_input_instance *ins) -{ - struct flb_log_event_encoder log_encoder; - int ret; - - ret = flb_log_event_encoder_init(&log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ins, "error initializing event encoder : %d", ret); - - return -1; - } - - ret = flb_log_event_encoder_begin_record(&log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp( - &log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_cstring( - &log_encoder, "event_type"); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_cstring( - &log_encoder, "some logs"); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(&log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(ins, NULL, 0, - log_encoder.output_buffer, - log_encoder.output_length); - - ret = 0; - } - else { - flb_plg_error(ins, "Error encoding record : %d", ret); - - ret = -1; - } - - flb_log_event_encoder_destroy(&log_encoder); - - return 0; -} - -static int send_metrics(struct flb_input_instance *ins) -{ - int ret; - double quantiles[5]; - struct cmt_histogram_buckets *buckets; - double val; - struct cmt *cmt; - uint64_t ts; - struct cmt_gauge *g1; - struct cmt_counter *c1; - struct cmt_summary *s1; - struct cmt_histogram *h1; - - ts = cfl_time_now(); - cmt = cmt_create(); - - c1 = cmt_counter_create(cmt, "kubernetes", "network", "load_counter", "Network load counter", - 2, (char *[]) {"hostname", "app"}); - - cmt_counter_get_val(c1, 0, NULL, &val); - cmt_counter_inc(c1, ts, 0, NULL); - cmt_counter_add(c1, ts, 2, 0, NULL); - cmt_counter_get_val(c1, 0, NULL, &val); - - cmt_counter_inc(c1, ts, 2, (char *[]) {"localhost", "cmetrics"}); - cmt_counter_get_val(c1, 2, (char *[]) {"localhost", "cmetrics"}, &val); - cmt_counter_add(c1, ts, 10.55, 2, (char *[]) {"localhost", "test"}); - cmt_counter_get_val(c1, 2, (char *[]) {"localhost", "test"}, &val); - cmt_counter_set(c1, ts, 12.15, 2, (char *[]) {"localhost", "test"}); - cmt_counter_set(c1, ts, 1, 2, (char *[]) {"localhost", "test"}); - - g1 = cmt_gauge_create(cmt, "kubernetes", "network", "load_gauge", "Network load gauge", 0, NULL); - - cmt_gauge_get_val(g1, 0, NULL, &val); - cmt_gauge_set(g1, ts, 2.0, 0, NULL); - cmt_gauge_get_val(g1, 0, NULL, &val); - cmt_gauge_inc(g1, ts, 0, NULL); - cmt_gauge_get_val(g1, 0, NULL, &val); - cmt_gauge_sub(g1, ts, 2, 0, NULL); - cmt_gauge_get_val(g1, 0, NULL, &val); - cmt_gauge_dec(g1, ts, 0, NULL); - cmt_gauge_get_val(g1, 0, NULL, &val); - cmt_gauge_inc(g1, ts, 0, NULL); - - buckets = cmt_histogram_buckets_create(3, 0.05, 5.0, 10.0); - - h1 = cmt_histogram_create(cmt, - "k8s", "network", "load_histogram", "Network load histogram", - buckets, - 1, (char *[]) {"my_label"}); - - cmt_histogram_observe(h1, ts, 0.001, 0, NULL); - cmt_histogram_observe(h1, ts, 0.020, 0, NULL); - cmt_histogram_observe(h1, ts, 5.0, 0, NULL); - cmt_histogram_observe(h1, ts, 8.0, 0, NULL); - cmt_histogram_observe(h1, ts, 1000, 0, NULL); - - cmt_histogram_observe(h1, ts, 0.001, 1, (char *[]) {"my_val"}); - cmt_histogram_observe(h1, ts, 0.020, 1, (char *[]) {"my_val"}); - cmt_histogram_observe(h1, ts, 5.0, 1, (char *[]) {"my_val"}); - cmt_histogram_observe(h1, ts, 8.0, 1, (char *[]) {"my_val"}); - cmt_histogram_observe(h1, ts, 1000, 1, (char *[]) {"my_val"});; - - quantiles[0] = 0.1; - quantiles[1] = 0.2; - quantiles[2] = 0.3; - quantiles[3] = 0.4; - quantiles[4] = 0.5; - - s1 = cmt_summary_create(cmt, - "k8s", "disk", "load_summary", "Disk load summary", - 5, quantiles, - 1, (char *[]) {"my_label"}); - - quantiles[0] = 1.1; - quantiles[1] = 2.2; - quantiles[2] = 3.3; - quantiles[3] = 4.4; - quantiles[4] = 5.5; - - cmt_summary_set_default(s1, ts, quantiles, 51.612894511314444, 10, 0, NULL); - - quantiles[0] = 11.11; - quantiles[1] = 0; - quantiles[2] = 33.33; - quantiles[3] = 44.44; - quantiles[4] = 55.55; - - cmt_summary_set_default(s1, ts, quantiles, 51.612894511314444, 10, 1, (char *[]) {"my_val"}); - - ret = flb_input_metrics_append(ins, NULL, 0, cmt); - - cmt_destroy(cmt); - return ret; -} - -static int send_traces(struct flb_input_instance *ins) -{ - int ret; - struct ctrace *ctx; - struct ctrace_opts opts; - struct ctrace_span *span_root; - struct ctrace_span *span_child; - struct ctrace_span_event *event; - struct ctrace_resource_span *resource_span; - struct ctrace_resource *resource; - struct ctrace_scope_span *scope_span; - struct ctrace_instrumentation_scope *instrumentation_scope; - struct ctrace_link *link; - struct ctrace_id *span_id; - struct ctrace_id *trace_id; - struct cfl_array *array; - struct cfl_array *sub_array; - struct cfl_kvlist *kv; - - ctr_opts_init(&opts); - - /* ctrace context */ - ctx = ctr_create(&opts); - if (!ctx) { - return -1; - } - - /* resource span */ - resource_span = ctr_resource_span_create(ctx); - ctr_resource_span_set_schema_url(resource_span, "https://ctraces/resource_span_schema_url"); - - /* create a 'resource' for the 'resource span' in question */ - resource = ctr_resource_span_get_resource(resource_span); - ctr_resource_set_dropped_attr_count(resource, 5); - - ctr_attributes_set_string(resource->attr, "service.name", "Fluent Bit Test Service"); - - /* scope span */ - scope_span = ctr_scope_span_create(resource_span); - ctr_scope_span_set_schema_url(scope_span, "https://ctraces/scope_span_schema_url"); - - /* create an optional instrumentation scope */ - instrumentation_scope = ctr_instrumentation_scope_create("ctrace", "a.b.c", 3, NULL); - ctr_scope_span_set_instrumentation_scope(scope_span, instrumentation_scope); - - /* generate a random trace_id */ - trace_id = ctr_id_create_random(CTR_ID_OTEL_TRACE_SIZE); - - /* generate a random ID for the new span */ - span_id = ctr_id_create_random(CTR_ID_OTEL_SPAN_SIZE); - - /* Create a root span */ - span_root = ctr_span_create(ctx, scope_span, "main", NULL); - if (!span_root) { - ctr_destroy(ctx); - ctr_opts_exit(&opts); - return -1; - } - - /* assign the random ID */ - ctr_span_set_span_id_with_cid(span_root, span_id); - - /* set random trace_id */ - ctr_span_set_trace_id_with_cid(span_root, trace_id); - - /* add some attributes to the span */ - ctr_span_set_attribute_string(span_root, "agent", "Fluent Bit"); - ctr_span_set_attribute_int64(span_root, "year", 2022); - ctr_span_set_attribute_bool(span_root, "open_source", CTR_TRUE); - ctr_span_set_attribute_double(span_root, "temperature", 25.5); - - /* pack an array: create an array context by using the CFL api */ - array = cfl_array_create(4); - cfl_array_append_string(array, "first"); - cfl_array_append_double(array, 2.0); - cfl_array_append_bool(array, CFL_FALSE); - - sub_array = cfl_array_create(3); - cfl_array_append_double(sub_array, 3.1); - cfl_array_append_double(sub_array, 5.2); - cfl_array_append_double(sub_array, 6.3); - cfl_array_append_array(array, sub_array); - - /* add array to the attribute list */ - ctr_span_set_attribute_array(span_root, "my_array", array); - - /* event: add one event and set attributes to it */ - event = ctr_span_event_add(span_root, "connect to remote server"); - - ctr_span_event_set_attribute_string(event, "syscall 1", "open()"); - ctr_span_event_set_attribute_string(event, "syscall 2", "connect()"); - ctr_span_event_set_attribute_string(event, "syscall 3", "write()"); - - /* add a key/value pair list */ - kv = cfl_kvlist_create(); - cfl_kvlist_insert_string(kv, "language", "c"); - - ctr_span_set_attribute_kvlist(span_root, "my-list", kv); - - /* create a child span */ - span_child = ctr_span_create(ctx, scope_span, "do-work", span_root); - if (!span_child) { - ctr_destroy(ctx); - ctr_opts_exit(&opts); - return -1; - } - - /* set trace_id */ - ctr_span_set_trace_id_with_cid(span_child, trace_id); - - /* use span_root ID as parent_span_id */ - ctr_span_set_parent_span_id_with_cid(span_child, span_id); - - /* delete old span id and generate a new one */ - ctr_id_destroy(span_id); - span_id = create_random_span_id(); - ctr_span_set_span_id_with_cid(span_child, span_id); - - /* destroy the IDs since is not longer needed */ - ctr_id_destroy(span_id); - ctr_id_destroy(trace_id); - - /* change span kind to client */ - ctr_span_kind_set(span_child, CTRACE_SPAN_CLIENT); - - /* create a Link (no valid IDs of course) */ - trace_id = ctr_id_create_random(CTR_ID_OTEL_TRACE_SIZE); - span_id = ctr_id_create_random(CTR_ID_OTEL_SPAN_SIZE); - - link = ctr_link_create_with_cid(span_child, trace_id, span_id); - ctr_link_set_trace_state(link, "aaabbbccc"); - ctr_link_set_dropped_attr_count(link, 2); - - /* delete IDs */ - ctr_id_destroy(span_id); - ctr_id_destroy(trace_id); - - ret = flb_input_trace_append(ins, NULL, 0, ctx); - - /* destroy the context */ - ctr_destroy(ctx); - - /* exit options (it release resources allocated) */ - ctr_opts_exit(&opts); - - return ret; -} - -static int cb_collector_time(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - int ret; - struct event_type *ctx = (struct event_type *) in_context; - - if (ctx->type == FLB_EVENT_TYPE_LOGS) { - ret = send_logs(ins); - flb_plg_debug(ins, "logs, ret=%i", ret); - } - else if (ctx->type == FLB_EVENT_TYPE_METRICS) { - ret = send_metrics(ins); - flb_plg_debug(ins, "metrics, ret=%i", ret); - } - else if (ctx->type == FLB_EVENT_TYPE_TRACES) { - ret = send_traces(ins); - flb_plg_debug(ins, "traces, ret=%i", ret); - } - - flb_plg_info(ins, "[OK] collector_time"); - FLB_INPUT_RETURN(0); -} - -/* Initialize plugin */ -static int cb_event_type_init(struct flb_input_instance *ins, - struct flb_config *config, void *data) -{ - int ret; - char *tmp; - struct event_type *ctx = NULL; - - ctx = flb_calloc(1, sizeof(struct event_type)); - if (!ctx) { - flb_errno(); - return -1; - } - - ret = flb_input_config_map_set(ins, (void *) ctx); - if (ret == -1) { - return -1; - } - - flb_input_set_context(ins, ctx); - - - ctx->type = FLB_EVENT_TYPE_LOGS; - tmp = (char *) flb_input_get_property("type", ins); - if (tmp) { - if (strcasecmp(tmp, "logs") == 0) { - ctx->type = FLB_EVENT_TYPE_LOGS; - } - else if (strcasecmp(tmp, "metrics") == 0) { - ctx->type = FLB_EVENT_TYPE_METRICS; - } - else if (strcasecmp(tmp, "traces") == 0) { - ctx->type = FLB_EVENT_TYPE_TRACES; - } - } - - /* unit test 0: collector_time */ - ret = flb_input_set_collector_time(ins, cb_collector_time, - ctx->interval_sec, ctx->interval_nsec, config); - if (ret < 0) { - return -1; - } - ctx->coll_fd = ret; - - return 0; -} - -static int cb_event_type_exit(void *data, struct flb_config *config) -{ - struct event_type *ctx = data; - - flb_free(ctx); - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "type", "logs", - 0, FLB_FALSE, 0, - "Set the type of event to deliver, optionsa are: logs, metrics or traces" - }, - { - FLB_CONFIG_MAP_INT, "interval_sec", DEFAULT_INTERVAL_SEC, - 0, FLB_TRUE, offsetof(struct event_type, interval_sec), - "Set the interval seconds between events generation" - }, - { - FLB_CONFIG_MAP_INT, "interval_nsec", DEFAULT_INTERVAL_NSEC, - 0, FLB_TRUE, offsetof(struct event_type, interval_nsec), - "Set the nanoseconds interval (sub seconds)" - }, - - /* EOF */ - {0} -}; - -struct flb_input_plugin in_event_type_plugin = { - .name = "event_type", - .description = "Event tests for input plugins", - .cb_init = cb_event_type_init, - .cb_pre_run = NULL, - .cb_collect = NULL, - .cb_flush_buf = NULL, - .cb_pause = NULL, - .cb_resume = NULL, - .cb_exit = cb_event_type_exit, - .config_map = config_map, - .flags = FLB_INPUT_CORO | FLB_INPUT_THREADED -}; diff --git a/fluent-bit/plugins/in_exec/CMakeLists.txt b/fluent-bit/plugins/in_exec/CMakeLists.txt deleted file mode 100644 index 73b601dcd..000000000 --- a/fluent-bit/plugins/in_exec/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - in_exec.c) - -FLB_PLUGIN(in_exec "${src}" "") diff --git a/fluent-bit/plugins/in_exec/in_exec.c b/fluent-bit/plugins/in_exec/in_exec.c deleted file mode 100644 index b5d66acdc..000000000 --- a/fluent-bit/plugins/in_exec/in_exec.c +++ /dev/null @@ -1,491 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include "in_exec_win32_compat.h" - -#include "in_exec.h" - -/* cb_collect callback */ -static int in_exec_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - int ret = -1; - int cmdret; - int flb_exit_code; - uint64_t val; - size_t str_len = 0; - FILE *cmdp = NULL; - struct flb_exec *ctx = in_context; - - /* variables for parser */ - int parser_ret = -1; - void *out_buf = NULL; - size_t out_size = 0; - struct flb_time out_time; - - if (ctx->oneshot == FLB_TRUE) { - ret = flb_pipe_r(ctx->ch_manager[0], &val, sizeof(val)); - if (ret == -1) { - flb_errno(); - return -1; - } - } - - cmdp = flb_popen(ctx->cmd, "r"); - if (cmdp == NULL) { - flb_plg_debug(ctx->ins, "command %s failed", ctx->cmd); - goto collect_end; - } - - if (ctx->parser) { - while (fgets(ctx->buf, ctx->buf_size, cmdp) != NULL) { - str_len = strnlen(ctx->buf, ctx->buf_size); - if (ctx->buf[str_len - 1] == '\n') { - ctx->buf[--str_len] = '\0'; /* chomp */ - } - - flb_time_get(&out_time); - parser_ret = flb_parser_do(ctx->parser, ctx->buf, str_len, - &out_buf, &out_size, &out_time); - if (parser_ret >= 0) { - if (flb_time_to_nanosec(&out_time) == 0L) { - flb_time_get(&out_time); - } - - ret = flb_log_event_encoder_begin_record(&ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_timestamp( - &ctx->log_encoder, - &out_time); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_body_from_raw_msgpack( - &ctx->log_encoder, - out_buf, - out_size); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(&ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(ctx->ins, NULL, 0, - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - - } - else { - flb_plg_error(ctx->ins, "Error encoding record : %d", ret); - } - - flb_log_event_encoder_reset(&ctx->log_encoder); - - flb_free(out_buf); - } - else { - flb_plg_trace(ctx->ins, "tried to parse '%s'", ctx->buf); - flb_plg_trace(ctx->ins, "buf_size %zu", ctx->buf_size); - flb_plg_error(ctx->ins, "parser returned an error"); - } - } - } - else { - while (fgets(ctx->buf, ctx->buf_size, cmdp) != NULL) { - str_len = strnlen(ctx->buf, ctx->buf_size); - if (ctx->buf[str_len - 1] == '\n') { - ctx->buf[--str_len] = '\0'; /* chomp */ - } - - ret = flb_log_event_encoder_begin_record(&ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp( - &ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_cstring( - &ctx->log_encoder, "exec"); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_string( - &ctx->log_encoder, - ctx->buf, - str_len); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(&ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(ctx->ins, NULL, 0, - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - - } - else { - flb_plg_error(ctx->ins, "Error encoding record : %d", ret); - } - - flb_log_event_encoder_reset(&ctx->log_encoder); - } - } - - ret = 0; /* success */ - - collect_end: - if(cmdp != NULL){ - /* - * If we're propagating the child exit code to the fluent-bit exit code - * in one-shot mode, popen() will have invoked our child command via - * its own shell, so unless the shell itself exited on a signal the - * translation is already done for us. - * For references on exit code handling in wrappers see - * https://www.gnu.org/software/bash/manual/html_node/Exit-Status.html - * and - * https://skarnet.org/software/execline/exitcodes.html - */ - cmdret = flb_pclose(cmdp); - if (cmdret == -1) { - flb_errno(); - flb_plg_debug(ctx->ins, - "unexpected error while waiting for exit of command %s ", - ctx->cmd); - /* - * The exit code of the shell run by popen() could not be - * determined; exit with 128, which is not a code that could be - * returned through a shell by a real child command. - */ - flb_exit_code = 128; - } else if (FLB_WIFEXITED(cmdret)) { - flb_plg_debug(ctx->ins, "command %s exited with code %d", - ctx->cmd, FLB_WEXITSTATUS(cmdret)); - /* - * Propagate shell exit code, which may encode a normal or signal - * exit for the real child process, directly to the caller. This - * could be greater than 127 if the shell encoded a signal exit - * status from the child process into its own return code. - */ - flb_exit_code = FLB_WEXITSTATUS(cmdret); - } else if (FLB_WIFSIGNALED(cmdret)) { - flb_plg_debug(ctx->ins, "command %s exited with signal %d", - ctx->cmd, FLB_WTERMSIG(cmdret)); - /* - * Follow the shell convention of returning 128+signo for signal - * exits. The consumer of fluent-bit's exit code will be unable to - * differentiate between the shell exiting on a signal and the - * process called by the shell exiting on a signal. - */ - flb_exit_code = 128 + FLB_WTERMSIG(cmdret); - } else { - flb_plg_debug(ctx->ins, "command %s exited with unknown status", - ctx->cmd); - flb_exit_code = 128; - } - - /* - * In one-shot mode, exit fluent-bit once the child process terminates. - */ - if (ctx->exit_after_oneshot == FLB_TRUE) { - /* - * propagate the child process exit code as the fluent-bit exit - * code so fluent-bit with the exec plugin can be used as a - * command wrapper. - */ - if (ctx->propagate_exit_code == FLB_TRUE) { - config->exit_status_code = flb_exit_code; - } - flb_plg_info(ctx->ins, - "one-shot command exited, terminating fluent-bit"); - flb_engine_exit(config); - } else { - flb_plg_debug(ctx->ins, - "one-shot command exited but exit_after_oneshot not set"); - } - } - - return ret; -} - -/* read config file and*/ -static int in_exec_config_read(struct flb_exec *ctx, - struct flb_input_instance *in, - struct flb_config *config -) -{ - int ret; - - ctx->ins = in; - - /* Load the config map */ - ret = flb_input_config_map_set(in, (void *)ctx); - if (ret == -1) { - flb_plg_error(in, "unable to load configuration"); - return -1; - } - - /* filepath setting */ - if (ctx->cmd == NULL) { - flb_plg_error(in, "no input 'command' was given"); - return -1; - } - - if (ctx->parser_name != NULL) { - ctx->parser = flb_parser_get(ctx->parser_name, config); - if (ctx->parser == NULL) { - flb_plg_error(in, "requested parser '%s' not found", ctx->parser_name); - } - } - - if (ctx->buf_size == -1) { - flb_plg_error(in, "buffer size is invalid"); - return -1; - } - - if (ctx->interval_sec <= 0 && ctx->interval_nsec <= 0) { - /* Illegal settings. Override them. */ - ctx->interval_sec = atoi(DEFAULT_INTERVAL_SEC); - ctx->interval_nsec = atoi(DEFAULT_INTERVAL_NSEC); - } - - /* - * propagate_exit_code is not being forced to imply exit_after_oneshot in - * case somebody in future wishes to make the exec plugin exit on nonzero - * exit codes for normal repeating commands. - */ - if (ctx->propagate_exit_code && !ctx->exit_after_oneshot) { - flb_plg_error(in, - "propagate_exit_code=True option makes no sense without " - "exit_after_oneshot=True"); - return -1; - } - - if (ctx->exit_after_oneshot && !ctx->oneshot) { - flb_plg_debug(in, "exit_after_oneshot implies oneshot mode, enabling"); - ctx->oneshot = FLB_TRUE; - } - - if (ctx->oneshot) { - ctx->interval_sec = -1; - ctx->interval_nsec = -1; - } - - ret = flb_log_event_encoder_init(&ctx->log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(in, "error initializing event encoder : %d", ret); - - return -1; - } - - flb_plg_debug(in, "interval_sec=%d interval_nsec=%d oneshot=%i buf_size=%zu", - ctx->interval_sec, ctx->interval_nsec, ctx->oneshot, ctx->buf_size); - - return 0; -} - -static void delete_exec_config(struct flb_exec *ctx) -{ - if (!ctx) { - return; - } - - flb_log_event_encoder_destroy(&ctx->log_encoder); - - /* release buffer */ - if (ctx->buf != NULL) { - flb_free(ctx->buf); - } - - if (ctx->ch_manager[0] > -1) { - flb_pipe_close(ctx->ch_manager[0]); - } - - if (ctx->ch_manager[1] > -1) { - flb_pipe_close(ctx->ch_manager[1]); - } - - flb_free(ctx); -} - -/* Initialize plugin */ -static int in_exec_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - struct flb_exec *ctx = NULL; - int ret = -1; - - /* Allocate space for the configuration */ - ctx = flb_calloc(1, sizeof(struct flb_exec)); - if (!ctx) { - return -1; - } - ctx->parser = NULL; - - /* Initialize exec config */ - ret = in_exec_config_read(ctx, in, config); - if (ret < 0) { - goto init_error; - } - - ctx->buf = flb_malloc(ctx->buf_size); - if (ctx->buf == NULL) { - flb_plg_error(in, "could not allocate exec buffer"); - goto init_error; - } - - flb_input_set_context(in, ctx); - - ctx->ch_manager[0] = -1; - ctx->ch_manager[1] = -1; - - if (ctx->oneshot == FLB_TRUE) { - if (flb_pipe_create(ctx->ch_manager)) { - flb_plg_error(in, "could not create pipe for oneshot command"); - goto init_error; - } - - ret = flb_input_set_collector_event(in, - in_exec_collect, - ctx->ch_manager[0], config); - } - else { - ret = flb_input_set_collector_time(in, - in_exec_collect, - ctx->interval_sec, - ctx->interval_nsec, config); - } - if (ret < 0) { - flb_plg_error(in, "could not set collector for exec input plugin"); - goto init_error; - } - - return 0; - - init_error: - delete_exec_config(ctx); - - return -1; -} - -static int in_exec_prerun(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - int ret; - uint64_t val = 0xc003; /* dummy constant */ - struct flb_exec *ctx = in_context; - (void) ins; - (void) config; - - if (ctx->oneshot == FLB_FALSE) { - return 0; - } - - /* Kick the oneshot execution */ - ret = flb_pipe_w(ctx->ch_manager[1], &val, sizeof(val)); - if (ret == -1) { - flb_errno(); - return -1; - } - return 0; -} - -static int in_exec_exit(void *data, struct flb_config *config) -{ - (void) *config; - struct flb_exec *ctx = data; - - delete_exec_config(ctx); - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "command", NULL, - 0, FLB_TRUE, offsetof(struct flb_exec, cmd), - "Set the command to execute" - }, - { - FLB_CONFIG_MAP_STR, "parser", NULL, - 0, FLB_TRUE, offsetof(struct flb_exec, parser_name), - "Set a parser" - }, - { - FLB_CONFIG_MAP_INT, "interval_sec", DEFAULT_INTERVAL_SEC, - 0, FLB_TRUE, offsetof(struct flb_exec, interval_sec), - "Set the collector interval" - }, - { - FLB_CONFIG_MAP_INT, "interval_nsec", DEFAULT_INTERVAL_NSEC, - 0, FLB_TRUE, offsetof(struct flb_exec, interval_nsec), - "Set the collector interval (nanoseconds)" - }, - { - FLB_CONFIG_MAP_SIZE, "buf_size", DEFAULT_BUF_SIZE, - 0, FLB_TRUE, offsetof(struct flb_exec, buf_size), - "Set the buffer size" - }, - { - FLB_CONFIG_MAP_BOOL, "oneshot", "false", - 0, FLB_TRUE, offsetof(struct flb_exec, oneshot), - "execute the command only once" - }, - { - FLB_CONFIG_MAP_BOOL, "exit_after_oneshot", "false", - 0, FLB_TRUE, offsetof(struct flb_exec, exit_after_oneshot), - "exit fluent-bit after the command terminates in one-shot mode" - }, - { - FLB_CONFIG_MAP_BOOL, "propagate_exit_code", "false", - 0, FLB_TRUE, offsetof(struct flb_exec, propagate_exit_code), - "propagate oneshot exit command fluent-bit exit code using " - "shell exit code translation conventions" - }, - /* EOF */ - {0} -}; - -struct flb_input_plugin in_exec_plugin = { - .name = "exec", - .description = "Exec Input", - .cb_init = in_exec_init, - .cb_pre_run = in_exec_prerun, - .cb_collect = in_exec_collect, - .cb_flush_buf = NULL, - .cb_exit = in_exec_exit, - .config_map = config_map -}; diff --git a/fluent-bit/plugins/in_exec/in_exec.h b/fluent-bit/plugins/in_exec/in_exec.h deleted file mode 100644 index efde8c8d9..000000000 --- a/fluent-bit/plugins/in_exec/in_exec.h +++ /dev/null @@ -1,52 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef FLB_IN_EXEC_H -#define FLB_IN_EXEC_H - -#include -#include -#include -#include -#include -#include -#include - -#include - -#define DEFAULT_BUF_SIZE "4096" -#define DEFAULT_INTERVAL_SEC "1" -#define DEFAULT_INTERVAL_NSEC "0" - -struct flb_exec { - flb_sds_t cmd; - flb_sds_t parser_name; - struct flb_parser *parser; - char *buf; - size_t buf_size; - struct flb_input_instance *ins; - int oneshot; - flb_pipefd_t ch_manager[2]; - int interval_sec; - int interval_nsec; - struct flb_log_event_encoder log_encoder; - int exit_after_oneshot; - int propagate_exit_code; -}; - -#endif /* FLB_IN_EXEC_H */ diff --git a/fluent-bit/plugins/in_exec/in_exec_win32_compat.h b/fluent-bit/plugins/in_exec/in_exec_win32_compat.h deleted file mode 100644 index 9f0dfe695..000000000 --- a/fluent-bit/plugins/in_exec/in_exec_win32_compat.h +++ /dev/null @@ -1,94 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef FLB_IN_EXEC_WIN32_COMPAT_H -#define FLB_IN_EXEC_WIN32_COMPAT_H - -#include -#include - -/* - * Work around lack of sys/wait.h and POSIX exit status macros from waitpid() - * in win32's _popen() and _pclose() implementation, since fluent-bit uses - * these in the in_exec plugin. - * - * On POSIX-like OSes this'll just use the standard macros with a name alias. - * - * On windows, where the concept of a signal exit does not exist, it defines - * dummy macros to indicate that the process exited normally and extract the - * exit code. - * - * These macros are for use with flb_pclose() only. Do not use them with - * other APIs that may differ in return value semantics. - */ -#ifdef FLB_HAVE_SYS_WAIT_H -#include -#define FLB_WIFEXITED(status) WIFEXITED((status)) -#define FLB_WEXITSTATUS(status) WEXITSTATUS((status)) -#define FLB_WIFSIGNALED(status) WIFSIGNALED((status)) -#define FLB_WTERMSIG(status) WTERMSIG((status)) -#else -#define FLB_WIFEXITED(status) (1) -#define FLB_WEXITSTATUS(status) ((status) & 0x00ff) -#define FLB_WIFSIGNALED(status) (0) -#define FLB_WTERMSIG(status) (-1) -#endif - -/* - * Because Windows has to do everything differently, call _popen() and - * _pclose() instead of the POSIX popen() and pclose() functions. - * - * flb_pclose() has different return value semantics on Windows vs non-windows - * targets because it propagates the pclose() or _pclose() return value - * directly. You MUST use the FLB_WIFEXITED(), FLB_WEXITSTATUS(), - * FLB_WIFSIGNALED() and FLB_WTERMSIG() macros to consume the return value, - * rather than the underlying POSIX macros or manual bit-shifts. - */ -#if !defined(FLB_SYSTEM_WINDOWS) -static inline FILE* flb_popen(const char *command, const char *type) { - return popen(command, type); -} -static inline int flb_pclose(FILE *stream) { - return pclose(stream); -} -#define FLB_PCLOSE pclose -#else -static inline FILE* flb_popen(const char *command, const char *type) { - return _popen(command, type); -} -/* - * flb_pclose() has the same return value on Windows as win32 _pclose(), rather - * than posix pclose(). The process exit code is not bit-shifted to the high - * byte. - * - * The MSVC docs for _pclose() at - * https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/pclose?view=msvc-170 - * are misleading; they say that "The format of the return value is the same as - * for _cwait, except the low-order and high-order bytes are swapped." But - * _cwait isn't documented as having any meaningful return on success, the - * process exit code is meant to be in its "termstat" out parameter per - * https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/cwait?view=msvc-170 - * The return code of _pclose() actually appears to be the process exit code - * without the bit-shift that waitpid() applies. - */ -static inline int flb_pclose(FILE *stream) { - return _pclose(stream); -} -#endif - -#endif /* FLB_IN_EXEC_WIN32_COMPAT_H */ diff --git a/fluent-bit/plugins/in_exec_wasi/CMakeLists.txt b/fluent-bit/plugins/in_exec_wasi/CMakeLists.txt deleted file mode 100644 index 7dcb817a1..000000000 --- a/fluent-bit/plugins/in_exec_wasi/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -set(WAMR_ROOT_DIR ../../${FLB_PATH_LIB_WASM_MICRO_RUNTIME}) -set(WASM_INCLUDE_DIRS - ${WAMR_ROOT_DIR}/core/iwasm/include - ) - -set(src - in_exec_wasi.c) - -FLB_PLUGIN(in_exec_wasi "${src}" "") -target_include_directories(flb-plugin-in_exec_wasi PRIVATE ${WASM_INCLUDE_DIRS}) -target_link_libraries(flb-plugin-in_exec_wasi flb-wasm-static vmlib-static) diff --git a/fluent-bit/plugins/in_exec_wasi/in_exec_wasi.c b/fluent-bit/plugins/in_exec_wasi/in_exec_wasi.c deleted file mode 100644 index bf765430d..000000000 --- a/fluent-bit/plugins/in_exec_wasi/in_exec_wasi.c +++ /dev/null @@ -1,451 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#ifdef FLB_SYSTEM_WINDOWS -#define STDIN_FILENO (_fileno( stdin )) -#define STDOUT_FILENO (_fileno( stdout )) -#define STDERR_FILENO (_fileno( stderr )) -#else -#include -#endif - -#include "in_exec_wasi.h" - -/* cb_collect callback */ -static int in_exec_wasi_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - int ret = -1; - uint64_t val; - size_t str_len = 0; - struct flb_exec_wasi *ctx = in_context; - struct flb_wasm *wasm = NULL; - FILE *stdoutp = tmpfile(); - - /* variables for parser */ - int parser_ret = -1; - void *out_buf = NULL; - size_t out_size = 0; - struct flb_time out_time; - - /* Validate the temporary file was created */ - if (stdoutp == NULL) { - flb_plg_error(ctx->ins, "failed to created temporary file"); - return -1; - } - - if (ctx->oneshot == FLB_TRUE) { - ret = flb_pipe_r(ctx->ch_manager[0], &val, sizeof(val)); - if (ret == -1) { - fclose(stdoutp); - flb_errno(); - return -1; - } - } - - wasm = flb_wasm_instantiate(config, ctx->wasi_path, ctx->accessible_dir_list, -1, fileno(stdoutp), -1); - if (wasm == NULL) { - flb_plg_debug(ctx->ins, "instantiate wasm [%s] failed", ctx->wasi_path); - goto collect_end; - } - ctx->wasm = wasm; - - ret = flb_wasm_call_wasi_main(ctx->wasm); - - if (!ret) { - flb_plg_error(ctx->ins, "WASI main function is not found"); - goto collect_end; - } - - if (ctx->parser) { - rewind(stdoutp); - - while (fgets(ctx->buf, ctx->buf_size, stdoutp) != NULL) { - str_len = strnlen(ctx->buf, ctx->buf_size); - if (ctx->buf[str_len - 1] == '\n') { - ctx->buf[--str_len] = '\0'; /* chomp */ - } - - flb_time_get(&out_time); - parser_ret = flb_parser_do(ctx->parser, ctx->buf, str_len, - &out_buf, &out_size, &out_time); - if (parser_ret >= 0) { - if (flb_time_to_nanosec(&out_time) == 0L) { - flb_time_get(&out_time); - } - - ret = flb_log_event_encoder_begin_record(&ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_timestamp( - &ctx->log_encoder, - &out_time); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_body_from_raw_msgpack( - &ctx->log_encoder, - out_buf, - out_size); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(&ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(ctx->ins, NULL, 0, - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - - } - else { - flb_plg_error(ctx->ins, "Error encoding record : %d", ret); - } - - flb_log_event_encoder_reset(&ctx->log_encoder); - - flb_free(out_buf); - } - else { - flb_plg_trace(ctx->ins, "tried to parse '%s'", ctx->buf); - flb_plg_trace(ctx->ins, "buf_size %zu", ctx->buf_size); - flb_plg_error(ctx->ins, "parser returned an error"); - } - } - } - else { - rewind(stdoutp); - - while (fgets(ctx->buf, ctx->buf_size, stdoutp) != NULL) { - str_len = strnlen(ctx->buf, ctx->buf_size); - if (ctx->buf[str_len - 1] == '\n') { - ctx->buf[--str_len] = '\0'; /* chomp */ - } - - ret = flb_log_event_encoder_begin_record(&ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp( - &ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_cstring( - &ctx->log_encoder, "wasi_stdout"); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_string( - &ctx->log_encoder, - ctx->buf, - str_len); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(&ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(ctx->ins, NULL, 0, - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - - } - else { - flb_plg_error(ctx->ins, "Error encoding record : %d", ret); - } - - flb_log_event_encoder_reset(&ctx->log_encoder); - } - } - - collect_end: - if (ctx->wasm != NULL) { - flb_wasm_destroy(ctx->wasm); - } - fclose(stdoutp); - - return ret; -} - -/* read config file and*/ -static int in_exec_wasi_config_read(struct flb_exec_wasi *ctx, - struct flb_input_instance *in, - struct flb_config *config) -{ - int ret; - - ctx->ins = in; - - /* Load the config map */ - ret = flb_input_config_map_set(in, (void *)ctx); - if (ret == -1) { - flb_plg_error(in, "unable to load configuration"); - return -1; - } - - /* filepath setting */ - if (ctx->wasi_path == NULL) { - flb_plg_error(in, "no input 'command' was given"); - return -1; - } - - if (ctx->parser_name != NULL) { - ctx->parser = flb_parser_get(ctx->parser_name, config); - if (ctx->parser == NULL) { - flb_plg_error(in, "requested parser '%s' not found", ctx->parser_name); - } - } - - if (ctx->buf_size == -1) { - flb_plg_error(in, "buffer size is invalid"); - return -1; - } - - if (ctx->interval_sec <= 0 && ctx->interval_nsec <= 0) { - /* Illegal settings. Override them. */ - ctx->interval_sec = atoi(DEFAULT_INTERVAL_SEC); - ctx->interval_nsec = atoi(DEFAULT_INTERVAL_NSEC); - } - - if (ctx->oneshot) { - ctx->interval_sec = -1; - ctx->interval_nsec = -1; - } - - ret = flb_log_event_encoder_init(&ctx->log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, "error initializing event encoder : %d", ret); - - return -1; - } - - flb_plg_debug(in, "interval_sec=%d interval_nsec=%d oneshot=%i buf_size=%zu", - ctx->interval_sec, ctx->interval_nsec, ctx->oneshot, ctx->buf_size); - - return 0; -} - -static void delete_exec_wasi_config(struct flb_exec_wasi *ctx) -{ - if (!ctx) { - return; - } - - flb_log_event_encoder_destroy(&ctx->log_encoder); - - /* release buffer */ - if (ctx->buf != NULL) { - flb_free(ctx->buf); - } - - if (ctx->ch_manager[0] > -1) { - flb_pipe_close(ctx->ch_manager[0]); - } - - if (ctx->ch_manager[1] > -1) { - flb_pipe_close(ctx->ch_manager[1]); - } - - flb_free(ctx); -} - -/* Initialize plugin */ -static int in_exec_wasi_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - struct flb_exec_wasi *ctx = NULL; - int ret = -1; - - /* Allocate space for the configuration */ - ctx = flb_malloc(sizeof(struct flb_exec_wasi)); - if (!ctx) { - return -1; - } - ctx->parser = NULL; - ctx->parser_name = NULL; - ctx->wasm = NULL; - ctx->wasi_path = NULL; - ctx->oneshot = FLB_FALSE; - - /* Initialize exec config */ - ret = in_exec_wasi_config_read(ctx, in, config); - if (ret < 0) { - goto init_error; - } - - flb_wasm_init(config); - - ctx->buf = flb_malloc(ctx->buf_size); - if (ctx->buf == NULL) { - flb_plg_error(in, "could not allocate exec buffer"); - goto init_error; - } - - flb_input_set_context(in, ctx); - - ctx->ch_manager[0] = -1; - ctx->ch_manager[1] = -1; - - if (ctx->oneshot == FLB_TRUE) { - if (flb_pipe_create(ctx->ch_manager)) { - flb_plg_error(in, "could not create pipe for oneshot command"); - goto init_error; - } - - ret = flb_input_set_collector_event(in, - in_exec_wasi_collect, - ctx->ch_manager[0], config); - } - else { - ret = flb_input_set_collector_time(in, - in_exec_wasi_collect, - ctx->interval_sec, - ctx->interval_nsec, config); - } - if (ret < 0) { - flb_plg_error(in, "could not set collector for exec input plugin"); - goto init_error; - } - ctx->coll_fd = ret; - - return 0; - - init_error: - delete_exec_wasi_config(ctx); - - return -1; -} - -static void in_exec_wasi_pause(void *data, struct flb_config *config) -{ - struct flb_exec_wasi *ctx = data; - - flb_input_collector_pause(ctx->coll_fd, ctx->ins); -} - -static void in_exec_wasi_resume(void *data, struct flb_config *config) -{ - struct flb_exec_wasi *ctx = data; - - flb_input_collector_resume(ctx->coll_fd, ctx->ins); -} - -static int in_exec_wasi_prerun(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - int ret; - uint64_t val = 0xc003; /* dummy constant */ - struct flb_exec_wasi *ctx = in_context; - (void) ins; - (void) config; - - if (ctx->oneshot == FLB_FALSE) { - return 0; - } - - /* Kick the oneshot execution */ - ret = flb_pipe_w(ctx->ch_manager[1], &val, sizeof(val)); - if (ret == -1) { - flb_errno(); - return -1; - } - return 0; -} - -static int in_exec_wasi_exit(void *data, struct flb_config *config) -{ - struct flb_exec_wasi *ctx = data; - - flb_wasm_destroy_all(config); - delete_exec_wasi_config(ctx); - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "wasi_path", NULL, - 0, FLB_TRUE, offsetof(struct flb_exec_wasi, wasi_path), - "Set the path of WASM program to execute" - }, - { - FLB_CONFIG_MAP_CLIST, "accessible_paths", ".", - 0, FLB_TRUE, offsetof(struct flb_exec_wasi, accessible_dir_list), - "Specifying paths to be accessible from a WASM program." - "Default value is current working directory" - }, - { - FLB_CONFIG_MAP_STR, "parser", NULL, - 0, FLB_TRUE, offsetof(struct flb_exec_wasi, parser_name), - "Set a parser" - }, - { - FLB_CONFIG_MAP_INT, "interval_sec", DEFAULT_INTERVAL_SEC, - 0, FLB_TRUE, offsetof(struct flb_exec_wasi, interval_sec), - "Set the collector interval" - }, - { - FLB_CONFIG_MAP_INT, "interval_nsec", DEFAULT_INTERVAL_NSEC, - 0, FLB_TRUE, offsetof(struct flb_exec_wasi, interval_nsec), - "Set the collector interval (nanoseconds)" - }, - { - FLB_CONFIG_MAP_SIZE, "buf_size", DEFAULT_BUF_SIZE, - 0, FLB_TRUE, offsetof(struct flb_exec_wasi, buf_size), - "Set the buffer size" - }, - { - FLB_CONFIG_MAP_BOOL, "bool", "false", - 0, FLB_TRUE, offsetof(struct flb_exec_wasi, oneshot), - "execute the command only once" - }, - /* EOF */ - {0} -}; - -struct flb_input_plugin in_exec_wasi_plugin = { - .name = "exec_wasi", - .description = "Exec WASI Input", - .cb_init = in_exec_wasi_init, - .cb_pre_run = in_exec_wasi_prerun, - .cb_pause = in_exec_wasi_pause, - .cb_resume = in_exec_wasi_resume, - .cb_collect = in_exec_wasi_collect, - .cb_flush_buf = NULL, - .cb_exit = in_exec_wasi_exit, - .config_map = config_map -}; diff --git a/fluent-bit/plugins/in_exec_wasi/in_exec_wasi.h b/fluent-bit/plugins/in_exec_wasi/in_exec_wasi.h deleted file mode 100644 index 132407bb9..000000000 --- a/fluent-bit/plugins/in_exec_wasi/in_exec_wasi.h +++ /dev/null @@ -1,55 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef FLB_IN_EXEC_WASI_H -#define FLB_IN_EXEC_WASI_H - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#define DEFAULT_BUF_SIZE "4096" -#define DEFAULT_INTERVAL_SEC "1" -#define DEFAULT_INTERVAL_NSEC "0" - -struct flb_exec_wasi { - flb_sds_t wasi_path; - struct mk_list *accessible_dir_list; /* list of directories to be - * accesible from WASM */ - flb_sds_t parser_name; - struct flb_parser *parser; - char *buf; - size_t buf_size; - struct flb_input_instance *ins; - struct flb_wasm *wasm; - int oneshot; - flb_pipefd_t ch_manager[2]; - int interval_sec; - int interval_nsec; - struct flb_log_event_encoder log_encoder; - int coll_fd; -}; - -#endif /* FLB_IN_EXEC_WASI_H */ diff --git a/fluent-bit/plugins/in_fluentbit_metrics/CMakeLists.txt b/fluent-bit/plugins/in_fluentbit_metrics/CMakeLists.txt deleted file mode 100644 index 299e05b58..000000000 --- a/fluent-bit/plugins/in_fluentbit_metrics/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -set(src - metrics.c - ) - -FLB_PLUGIN(in_fluentbit_metrics "${src}" "") diff --git a/fluent-bit/plugins/in_fluentbit_metrics/metrics.c b/fluent-bit/plugins/in_fluentbit_metrics/metrics.c deleted file mode 100644 index 779a03651..000000000 --- a/fluent-bit/plugins/in_fluentbit_metrics/metrics.c +++ /dev/null @@ -1,201 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -struct flb_in_metrics { - /* config map options */ - int scrape_on_start; - int scrape_interval; - - /* internal */ - int coll_fd_start; - int coll_fd_runtime; - struct cmt_counter *c; - struct flb_input_instance *ins; -}; - -static int scrape_metrics(struct flb_config *config, struct flb_in_metrics *ctx) -{ - int ret; - size_t ts; - char *name; - struct cmt *cmt; - - /* Update internal metric */ - ts = cfl_time_now(); - name = (char *) flb_input_name(ctx->ins); - cmt_counter_inc(ctx->c, ts, 1, (char *[]) {name}); - - - cmt = flb_me_get_cmetrics(config); - if (!cmt) { - flb_plg_error(ctx->ins, "could not scrape metrics"); - return 0; - } - - /* Append the updated metrics */ - ret = flb_input_metrics_append(ctx->ins, NULL, 0, cmt); - if (ret != 0) { - flb_plg_error(ctx->ins, "could not append metrics"); - } - cmt_destroy(cmt); - - return 0; -} - -/* - * Update the metrics, this function is invoked every time 'scrape_interval' - * expires. - */ -static int cb_metrics_collect_runtime(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - return scrape_metrics(config, in_context); -} - -static int cb_metrics_collect_start(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_in_metrics *ctx = in_context; - - /* pause collector */ - flb_input_collector_pause(ctx->coll_fd_start, ctx->ins); - - return scrape_metrics(config, ctx); -} - -static int in_metrics_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - int ret; - struct flb_in_metrics *ctx; - - /* Create plugin context */ - ctx = flb_calloc(1, sizeof(struct flb_in_metrics)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->ins = in; - - /* Load the config map */ - ret = flb_input_config_map_set(in, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return -1; - } - - /* Associate context with the instance */ - flb_input_set_context(in, ctx); - - /* Scrape metrics on start / collector */ - if (ctx->scrape_interval > 2 && ctx->scrape_on_start) { - ret = flb_input_set_collector_time(in, - cb_metrics_collect_start, - 5, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set collector on start for Fluent Bit " - "metrics plugin"); - return -1; - } - ctx->coll_fd_start = ret; - } - - /* Create the runtime collector */ - ret = flb_input_set_collector_time(in, - cb_metrics_collect_runtime, - ctx->scrape_interval, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set collector for Fluent Bit metrics plugin"); - return -1; - } - ctx->coll_fd_runtime = ret; - - /* Internal metrics */ - ctx->c = cmt_counter_create(ctx->ins->cmt, - "fluentbit", "input_metrics", "scrapes_total", - "Number of total metrics scrapes", - 1, (char *[]) {"name"}); - return 0; -} - -static int in_metrics_exit(void *data, struct flb_config *config) -{ - struct flb_in_metrics *ctx = data; - - if (!ctx) { - return 0; - } - - flb_free(ctx); - return 0; -} - -static void in_metrics_pause(void *data, struct flb_config *config) -{ - struct flb_in_metrics *ctx = data; - - flb_input_collector_pause(ctx->coll_fd_runtime, ctx->ins); -} - -static void in_metrics_resume(void *data, struct flb_config *config) -{ - struct flb_in_metrics *ctx = data; - - flb_input_collector_resume(ctx->coll_fd_runtime, ctx->ins); -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_TIME, "scrape_interval", "2", - 0, FLB_TRUE, offsetof(struct flb_in_metrics, scrape_interval), - "scrape interval to collect the internal metrics of Fluent Bit." - }, - - { - FLB_CONFIG_MAP_BOOL, "scrape_on_start", "false", - 0, FLB_TRUE, offsetof(struct flb_in_metrics, scrape_on_start), - "scrape metrics upon start, useful to avoid waiting for 'scrape_interval' " - "for the first round of metrics." - }, - - /* EOF */ - {0} -}; - -struct flb_input_plugin in_fluentbit_metrics_plugin = { - .name = "fluentbit_metrics", - .description = "Fluent Bit internal metrics", - .cb_init = in_metrics_init, - .cb_pre_run = NULL, - .cb_flush_buf = NULL, - .config_map = config_map, - .cb_pause = in_metrics_pause, - .cb_resume = in_metrics_resume, - .cb_exit = in_metrics_exit, -}; diff --git a/fluent-bit/plugins/in_forward/CMakeLists.txt b/fluent-bit/plugins/in_forward/CMakeLists.txt deleted file mode 100644 index ce4f62728..000000000 --- a/fluent-bit/plugins/in_forward/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -set(src - fw.c - fw_conn.c - fw_prot.c - fw_config.c) - -FLB_PLUGIN(in_forward "${src}" "") diff --git a/fluent-bit/plugins/in_forward/fw.c b/fluent-bit/plugins/in_forward/fw.c deleted file mode 100644 index b85b198c6..000000000 --- a/fluent-bit/plugins/in_forward/fw.c +++ /dev/null @@ -1,325 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include - -#ifdef FLB_HAVE_UNIX_SOCKET -#include -#include -#include -#endif - -#include "fw.h" -#include "fw_conn.h" -#include "fw_config.h" - -#ifdef FLB_HAVE_UNIX_SOCKET -static int remove_existing_socket_file(char *socket_path) -{ - struct stat file_data; - int result; - - result = stat(socket_path, &file_data); - - if (result == -1) { - if (errno == ENOENT) { - return 0; - } - - flb_errno(); - - return -1; - } - - if (S_ISSOCK(file_data.st_mode) == 0) { - return -2; - } - - result = unlink(socket_path); - - if (result != 0) { - return -3; - } - - return 0; -} - -static int fw_unix_create(struct flb_in_fw_config *ctx) -{ - int ret; - - ret = remove_existing_socket_file(ctx->unix_path); - - if (ret != 0) { - if (ret == -2) { - flb_plg_error(ctx->ins, - "%s exists and it is not a unix socket. Aborting", - ctx->unix_path); - } - else { - flb_plg_error(ctx->ins, - "could not remove existing unix socket %s. Aborting", - ctx->unix_path); - } - - return -1; - } - - ctx->downstream = flb_downstream_create(FLB_TRANSPORT_UNIX_STREAM, - ctx->ins->flags, - ctx->unix_path, - 0, - ctx->ins->tls, - ctx->ins->config, - &ctx->ins->net_setup); - - if (ctx->downstream == NULL) { - return -1; - } - - if (ctx->unix_perm_str) { - if (chmod(ctx->unix_path, ctx->unix_perm)) { - flb_errno(); - - flb_plg_error(ctx->ins, "cannot set permission on '%s' to %04o", - ctx->unix_path, ctx->unix_perm); - - return -1; - } - } - - return 0; -} -#endif - -/* - * For a server event, the collection event means a new client have arrived, we - * accept the connection and create a new FW instance which will wait for - * MessagePack records. - */ -static int in_fw_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_connection *connection; - struct fw_conn *conn; - struct flb_in_fw_config *ctx; - - ctx = in_context; - - connection = flb_downstream_conn_get(ctx->downstream); - - if (connection == NULL) { - flb_plg_error(ctx->ins, "could not accept new connection"); - - return -1; - } - - if (!config->is_ingestion_active) { - flb_downstream_conn_release(connection); - - return -1; - } - - flb_plg_trace(ins, "new TCP connection arrived FD=%i", connection->fd); - - conn = fw_conn_add(connection, ctx); - - if (!conn) { - return -1; - } - - return 0; -} - -/* Initialize plugin */ -static int in_fw_init(struct flb_input_instance *ins, - struct flb_config *config, void *data) -{ - unsigned short int port; - int ret; - struct flb_in_fw_config *ctx; - - (void) data; - - /* Allocate space for the configuration */ - ctx = fw_config_init(ins); - if (!ctx) { - return -1; - } - - ctx->coll_fd = -1; - ctx->ins = ins; - mk_list_init(&ctx->connections); - - /* Set the context */ - flb_input_set_context(ins, ctx); - - /* Unix Socket mode */ - if (ctx->unix_path) { -#ifndef FLB_HAVE_UNIX_SOCKET - flb_plg_error(ctx->ins, "unix address is not supported %s:%s. Aborting", - ctx->listen, ctx->tcp_port); - fw_config_destroy(ctx); - return -1; -#else - ret = fw_unix_create(ctx); - if (ret != 0) { - flb_plg_error(ctx->ins, "could not listen on unix://%s", - ctx->unix_path); - fw_config_destroy(ctx); - return -1; - } - flb_plg_info(ctx->ins, "listening on unix://%s", ctx->unix_path); -#endif - } - else { - port = (unsigned short int) strtoul(ctx->tcp_port, NULL, 10); - - ctx->downstream = flb_downstream_create(FLB_TRANSPORT_TCP, - ctx->ins->flags, - ctx->listen, - port, - ctx->ins->tls, - config, - &ctx->ins->net_setup); - - if (ctx->downstream == NULL) { - flb_plg_error(ctx->ins, - "could not initialize downstream on unix://%s. Aborting", - ctx->listen); - - fw_config_destroy(ctx); - - return -1; - } - - if (ctx->downstream != NULL) { - flb_plg_info(ctx->ins, "listening on %s:%s", - ctx->listen, ctx->tcp_port); - } - else { - flb_plg_error(ctx->ins, "could not bind address %s:%s. Aborting", - ctx->listen, ctx->tcp_port); - - fw_config_destroy(ctx); - - return -1; - } - } - - flb_input_downstream_set(ctx->downstream, ctx->ins); - - flb_net_socket_nonblocking(ctx->downstream->server_fd); - - /* Collect upon data available on the standard input */ - ret = flb_input_set_collector_socket(ins, - in_fw_collect, - ctx->downstream->server_fd, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not set server socket collector"); - fw_config_destroy(ctx); - return -1; - } - - ctx->coll_fd = ret; - - return 0; -} - -static void in_fw_pause(void *data, struct flb_config *config) -{ - struct flb_in_fw_config *ctx = data; - - /* - * If the plugin is paused AND the ingestion not longer active, - * it means we are in a shutdown phase. This plugin can safetly - * close the socket server collector. - * - * This socket stop is a workaround since the server API will be - * refactored shortly. - */ - if (config->is_ingestion_active == FLB_FALSE) { - fw_conn_del_all(ctx); - } -} - -static int in_fw_exit(void *data, struct flb_config *config) -{ - (void) *config; - struct flb_in_fw_config *ctx = data; - - if (!ctx) { - return 0; - } - - fw_conn_del_all(ctx); - fw_config_destroy(ctx); - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "tag_prefix", NULL, - 0, FLB_TRUE, offsetof(struct flb_in_fw_config, tag_prefix), - "Prefix incoming tag with the defined value." - }, - { - FLB_CONFIG_MAP_STR, "unix_path", NULL, - 0, FLB_TRUE, offsetof(struct flb_in_fw_config, unix_path), - "The path to unix socket to receive a Forward message." - }, - { - FLB_CONFIG_MAP_STR, "unix_perm", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_in_fw_config, unix_perm_str), - "Set the permissions for the UNIX socket" - }, - { - FLB_CONFIG_MAP_SIZE, "buffer_chunk_size", FLB_IN_FW_CHUNK_SIZE, - 0, FLB_TRUE, offsetof(struct flb_in_fw_config, buffer_chunk_size), - "The buffer memory size used to receive a Forward message." - }, - { - FLB_CONFIG_MAP_SIZE, "buffer_max_size", FLB_IN_FW_CHUNK_MAX_SIZE, - 0, FLB_TRUE, offsetof(struct flb_in_fw_config, buffer_max_size), - "The maximum buffer memory size used to receive a Forward message." - }, - {0} -}; - -/* Plugin reference */ -struct flb_input_plugin in_forward_plugin = { - .name = "forward", - .description = "Fluentd in-forward", - .cb_init = in_fw_init, - .cb_pre_run = NULL, - .cb_collect = in_fw_collect, - .cb_flush_buf = NULL, - .cb_pause = in_fw_pause, - .cb_exit = in_fw_exit, - .config_map = config_map, - .flags = FLB_INPUT_NET_SERVER | FLB_IO_OPT_TLS -}; diff --git a/fluent-bit/plugins/in_forward/fw.h b/fluent-bit/plugins/in_forward/fw.h deleted file mode 100644 index 454f255b9..000000000 --- a/fluent-bit/plugins/in_forward/fw.h +++ /dev/null @@ -1,52 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_FW_H -#define FLB_IN_FW_H - -#include -#include -#include -#include - -struct flb_in_fw_config { - size_t buffer_max_size; /* Max Buffer size */ - size_t buffer_chunk_size; /* Chunk allocation size */ - - /* Network */ - char *listen; /* Listen interface */ - char *tcp_port; /* TCP Port */ - - flb_sds_t tag_prefix; /* tag prefix */ - - /* Unix Socket */ - char *unix_path; /* Unix path for socket */ - unsigned int unix_perm; /* Permission for socket */ - flb_sds_t unix_perm_str; /* Permission (config map) */ - - int coll_fd; - struct flb_downstream *downstream; /* Client manager */ - struct mk_list connections; /* List of active connections */ - struct flb_input_instance *ins; /* Input plugin instace */ - - struct flb_log_event_decoder *log_decoder; - struct flb_log_event_encoder *log_encoder; -}; - -#endif diff --git a/fluent-bit/plugins/in_forward/fw_config.c b/fluent-bit/plugins/in_forward/fw_config.c deleted file mode 100644 index 7edbab0c7..000000000 --- a/fluent-bit/plugins/in_forward/fw_config.c +++ /dev/null @@ -1,120 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include "fw.h" -#include "fw_conn.h" -#include "fw_config.h" - -struct flb_in_fw_config *fw_config_init(struct flb_input_instance *i_ins) -{ - char tmp[16]; - int ret = -1; - const char *p; - struct flb_in_fw_config *config; - - config = flb_calloc(1, sizeof(struct flb_in_fw_config)); - if (!config) { - flb_errno(); - return NULL; - } - config->coll_fd = -1; - - config->log_encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (config->log_encoder == NULL) { - flb_plg_error(i_ins, "could not initialize event encoder"); - fw_config_destroy(config); - - return NULL; - } - - config->log_decoder = flb_log_event_decoder_create(NULL, 0); - - if (config->log_decoder == NULL) { - flb_plg_error(i_ins, "could not initialize event decoder"); - fw_config_destroy(config); - - return NULL; - } - - ret = flb_input_config_map_set(i_ins, (void *)config); - if (ret == -1) { - flb_plg_error(i_ins, "config map set error"); - flb_free(config); - return NULL; - } - - p = flb_input_get_property("unix_path", i_ins); - if (p == NULL) { - /* Listen interface (if not set, defaults to 0.0.0.0:24224) */ - flb_input_net_default_listener("0.0.0.0", 24224, i_ins); - config->listen = i_ins->host.listen; - snprintf(tmp, sizeof(tmp) - 1, "%d", i_ins->host.port); - config->tcp_port = flb_strdup(tmp); - } - else { - /* Unix socket mode */ - if (config->unix_perm_str) { - config->unix_perm = strtol(config->unix_perm_str, NULL, 8) & 07777; - } - } - - if (!config->unix_path) { - flb_debug("[in_fw] Listen='%s' TCP_Port=%s", - config->listen, config->tcp_port); - } - return config; -} - -int fw_config_destroy(struct flb_in_fw_config *config) -{ - if (config->log_encoder != NULL) { - flb_log_event_encoder_destroy(config->log_encoder); - } - - if (config->log_decoder != NULL) { - flb_log_event_decoder_destroy(config->log_decoder); - } - - if (config->coll_fd != -1) { - flb_input_collector_delete(config->coll_fd, config->ins); - - config->coll_fd = -1; - } - - if (config->downstream != NULL) { - flb_downstream_destroy(config->downstream); - } - - if (config->unix_path) { - unlink(config->unix_path); - } - else { - flb_free(config->tcp_port); - } - - flb_free(config); - - return 0; -} diff --git a/fluent-bit/plugins/in_forward/fw_config.h b/fluent-bit/plugins/in_forward/fw_config.h deleted file mode 100644 index bbad17610..000000000 --- a/fluent-bit/plugins/in_forward/fw_config.h +++ /dev/null @@ -1,28 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_FW_CONFIG_H -#define FLB_IN_FW_CONFIG_H - -#include "fw.h" - -struct flb_in_fw_config *fw_config_init(struct flb_input_instance *i_ins); -int fw_config_destroy(struct flb_in_fw_config *config); - -#endif diff --git a/fluent-bit/plugins/in_forward/fw_conn.c b/fluent-bit/plugins/in_forward/fw_conn.c deleted file mode 100644 index 3ccd98c24..000000000 --- a/fluent-bit/plugins/in_forward/fw_conn.c +++ /dev/null @@ -1,199 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include - -#include "fw.h" -#include "fw_prot.h" -#include "fw_conn.h" - -/* Callback invoked every time an event is triggered for a connection */ -int fw_conn_event(void *data) -{ - int ret; - int bytes; - int available; - int size; - char *tmp; - struct fw_conn *conn; - struct mk_event *event; - struct flb_in_fw_config *ctx; - struct flb_connection *connection; - - connection = (struct flb_connection *) data; - - conn = connection->user_data; - - ctx = conn->ctx; - - event = &connection->event; - - - if (event->mask & MK_EVENT_READ) { - available = (conn->buf_size - conn->buf_len); - if (available < 1) { - if (conn->buf_size >= ctx->buffer_max_size) { - flb_plg_warn(ctx->ins, "fd=%i incoming data exceed limit (%lu bytes)", - event->fd, (ctx->buffer_max_size)); - fw_conn_del(conn); - return -1; - } - else if (conn->buf_size + ctx->buffer_chunk_size > ctx->buffer_max_size) { - /* no space to add buffer_chunk_size */ - /* set maximum size */ - size = ctx->buffer_max_size; - } - else { - size = conn->buf_size + ctx->buffer_chunk_size; - } - tmp = flb_realloc(conn->buf, size); - if (!tmp) { - flb_errno(); - return -1; - } - flb_plg_trace(ctx->ins, "fd=%i buffer realloc %i -> %i", - event->fd, conn->buf_size, size); - - conn->buf = tmp; - conn->buf_size = size; - available = (conn->buf_size - conn->buf_len); - } - - bytes = flb_io_net_read(connection, - (void *) &conn->buf[conn->buf_len], - available); - - if (bytes > 0) { - flb_plg_trace(ctx->ins, "read()=%i pre_len=%i now_len=%i", - bytes, conn->buf_len, conn->buf_len + bytes); - conn->buf_len += bytes; - - ret = fw_prot_process(ctx->ins, conn); - if (ret == -1) { - fw_conn_del(conn); - return -1; - } - return bytes; - } - else { - flb_plg_trace(ctx->ins, "fd=%i closed connection", event->fd); - fw_conn_del(conn); - return -1; - } - } - - if (event->mask & MK_EVENT_CLOSE) { - flb_plg_trace(ctx->ins, "fd=%i hangup", event->fd); - fw_conn_del(conn); - return -1; - } - return 0; -} - -/* Create a new Forward request instance */ -struct fw_conn *fw_conn_add(struct flb_connection *connection, struct flb_in_fw_config *ctx) -{ - struct fw_conn *conn; - int ret; - - conn = flb_malloc(sizeof(struct fw_conn)); - if (!conn) { - flb_errno(); - - return NULL; - } - - conn->connection = connection; - - /* Set data for the event-loop */ - connection->user_data = conn; - connection->event.type = FLB_ENGINE_EV_CUSTOM; - connection->event.handler = fw_conn_event; - - /* Connection info */ - conn->ctx = ctx; - conn->buf_len = 0; - conn->rest = 0; - conn->status = FW_NEW; - - /* Allocate read buffer */ - conn->buf = flb_malloc(ctx->buffer_chunk_size); - if (!conn->buf) { - flb_errno(); - flb_free(conn); - - return NULL; - } - conn->buf_size = ctx->buffer_chunk_size; - conn->in = ctx->ins; - - /* Register instance into the event loop */ - ret = mk_event_add(flb_engine_evl_get(), - connection->fd, - FLB_ENGINE_EV_CUSTOM, - MK_EVENT_READ, - &connection->event); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not register new connection"); - - flb_free(conn->buf); - flb_free(conn); - - return NULL; - } - - mk_list_add(&conn->_head, &ctx->connections); - - return conn; -} - -int fw_conn_del(struct fw_conn *conn) -{ - /* The downstream unregisters the file descriptor from the event-loop - * so there's nothing to be done by the plugin - */ - flb_downstream_conn_release(conn->connection); - - /* Release resources */ - mk_list_del(&conn->_head); - - flb_free(conn->buf); - flb_free(conn); - - return 0; -} - -int fw_conn_del_all(struct flb_in_fw_config *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - struct fw_conn *conn; - - mk_list_foreach_safe(head, tmp, &ctx->connections) { - conn = mk_list_entry(head, struct fw_conn, _head); - fw_conn_del(conn); - } - - return 0; -} \ No newline at end of file diff --git a/fluent-bit/plugins/in_forward/fw_conn.h b/fluent-bit/plugins/in_forward/fw_conn.h deleted file mode 100644 index 4c04d9400..000000000 --- a/fluent-bit/plugins/in_forward/fw_conn.h +++ /dev/null @@ -1,57 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_FW_CONN_H -#define FLB_IN_FW_CONN_H - -#define FLB_IN_FW_CHUNK_SIZE "1024000" /* 1MB */ -#define FLB_IN_FW_CHUNK_MAX_SIZE "6144000" /* =FLB_IN_FW_CHUNK_SIZE * 6. 6MB */ - -enum { - FW_NEW = 1, /* it's a new connection */ - FW_CONNECTED = 2, /* MQTT connection per protocol spec OK */ -}; - -struct fw_conn_stream { - char *tag; - size_t tag_len; -}; - -/* Respresents a connection */ -struct fw_conn { - int status; /* Connection status */ - - /* Buffer */ - char *buf; /* Buffer data */ - int buf_len; /* Data length */ - int buf_size; /* Buffer size */ - size_t rest; /* Unpacking offset */ - - struct flb_input_instance *in; /* Parent plugin instance */ - struct flb_in_fw_config *ctx; /* Plugin configuration context */ - struct flb_connection *connection; - - struct mk_list _head; -}; - -struct fw_conn *fw_conn_add(struct flb_connection *connection, struct flb_in_fw_config *ctx); -int fw_conn_del(struct fw_conn *conn); -int fw_conn_del_all(struct flb_in_fw_config *ctx); - -#endif diff --git a/fluent-bit/plugins/in_forward/fw_prot.c b/fluent-bit/plugins/in_forward/fw_prot.c deleted file mode 100644 index 2a23b6254..000000000 --- a/fluent-bit/plugins/in_forward/fw_prot.c +++ /dev/null @@ -1,846 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include -#include - -#include -#include - -#include - -#include "fw.h" -#include "fw_prot.h" -#include "fw_conn.h" - -/* Try parsing rounds up-to 32 bytes */ -#define EACH_RECV_SIZE 32 - -static int get_chunk_event_type(struct flb_input_instance *ins, msgpack_object options) -{ - int i; - int type = FLB_EVENT_TYPE_LOGS; - msgpack_object k; - msgpack_object v; - - if (options.type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ins, "invalid options field in record"); - return -1; - } - - for (i = 0; i < options.via.map.size; i++) { - k = options.via.map.ptr[i].key; - v = options.via.map.ptr[i].val; - - if (k.type != MSGPACK_OBJECT_STR) { - return -1; - } - - if (k.via.str.size != 13) { - continue; - } - - if (strncmp(k.via.str.ptr, "fluent_signal", 13) == 0) { - if (v.type != MSGPACK_OBJECT_POSITIVE_INTEGER) { - flb_plg_error(ins, "invalid value type in options fluent_signal"); - return -1; - } - - if (v.via.i64 != FLB_EVENT_TYPE_LOGS && v.via.i64 != FLB_EVENT_TYPE_METRICS && v.via.i64 != FLB_EVENT_TYPE_TRACES) { - flb_plg_error(ins, "invalid value in options fluent_signal"); - return -1; - } - - /* cast should be fine */ - type = (int) v.via.i64; - break; - } - } - - return type; -} - -static int is_gzip_compressed(msgpack_object options) -{ - int i; - msgpack_object k; - msgpack_object v; - - if (options.type != MSGPACK_OBJECT_MAP) { - return -1; - } - - - for (i = 0; i < options.via.map.size; i++) { - k = options.via.map.ptr[i].key; - v = options.via.map.ptr[i].val; - - if (k.type != MSGPACK_OBJECT_STR) { - return -1; - } - - if (k.via.str.size != 10) { - continue; - } - - if (strncmp(k.via.str.ptr, "compressed", 10) == 0) { - if (v.type != MSGPACK_OBJECT_STR) { - return -1; - } - - if (v.via.str.size != 4) { - return -1; - } - - if (strncmp(v.via.str.ptr, "gzip", 4) == 0) { - return FLB_TRUE; - } - else if (strncmp(v.via.str.ptr, "text", 4) == 0) { - return FLB_FALSE; - } - - return -1; - } - } - - return FLB_FALSE; -} - -static int send_ack(struct flb_input_instance *in, struct fw_conn *conn, - msgpack_object chunk) -{ - int result; - size_t sent; - ssize_t bytes; - msgpack_packer mp_pck; - msgpack_sbuffer mp_sbuf; - - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - msgpack_pack_map(&mp_pck, 1); - msgpack_pack_str(&mp_pck, 3); - msgpack_pack_str_body(&mp_pck, "ack", 3); - msgpack_pack_object(&mp_pck, chunk); - - - bytes = flb_io_net_write(conn->connection, - (void *) mp_sbuf.data, - mp_sbuf.size, - &sent); - - msgpack_sbuffer_destroy(&mp_sbuf); - - if (bytes == -1) { - flb_plg_error(in, "cannot send ACK response: %.*s", - chunk.via.str.size, chunk.via.str.ptr); - - result = -1; - } - else { - result = 0; - } - - return result; - -} - -static size_t get_options_metadata(msgpack_object *arr, int expected, size_t *idx) -{ - size_t i; - msgpack_object *options; - msgpack_object k; - msgpack_object v; - - if (arr->type != MSGPACK_OBJECT_ARRAY) { - return -1; - } - - /* Make sure the 'expected' entry position is valid for the array size */ - if (expected >= arr->via.array.size) { - return 0; - } - - options = &arr->via.array.ptr[expected]; - if (options->type == MSGPACK_OBJECT_NIL) { - /* - * Old Docker 18.x sends a NULL options parameter, just be friendly and - * let it pass. - */ - return 0; - } - - if (options->type != MSGPACK_OBJECT_MAP) { - return -1; - } - - if (options->via.map.size <= 0) { - return 0; - } - - for (i = 0; i < options->via.map.size; i++) { - k = options->via.map.ptr[i].key; - v = options->via.map.ptr[i].val; - - if (k.type != MSGPACK_OBJECT_STR) { - continue; - } - - if (k.via.str.size != 8) { - continue; - } - - if (strncmp(k.via.str.ptr, "metadata", 8) != 0) { - continue; - } - - if (v.type != MSGPACK_OBJECT_MAP) { - return -1; - } - - *idx = i; - - return 0; - } - - return 0; -} - -static size_t get_options_chunk(msgpack_object *arr, int expected, size_t *idx) -{ - size_t i; - msgpack_object *options; - msgpack_object k; - msgpack_object v; - - if (arr->type != MSGPACK_OBJECT_ARRAY) { - return -1; - } - - /* Make sure the 'expected' entry position is valid for the array size */ - if (expected >= arr->via.array.size) { - return 0; - } - - options = &arr->via.array.ptr[expected]; - if (options->type == MSGPACK_OBJECT_NIL) { - /* - * Old Docker 18.x sends a NULL options parameter, just be friendly and - * let it pass. - */ - return 0; - } - - if (options->type != MSGPACK_OBJECT_MAP) { - return -1; - } - - if (options->via.map.size <= 0) { - return 0; - } - - for (i = 0; i < options->via.map.size; i++) { - k = options->via.map.ptr[i].key; - v = options->via.map.ptr[i].val; - - if (k.type != MSGPACK_OBJECT_STR) { - continue; - } - - if (k.via.str.size != 5) { - continue; - } - - if (strncmp(k.via.str.ptr, "chunk", 5) != 0) { - continue; - } - - if (v.type != MSGPACK_OBJECT_STR) { - return -1; - } - - *idx = i; - return 0; - } - - return 0; -} - -static int fw_process_forward_mode_entry( - struct fw_conn *conn, - const char *tag, int tag_len, - msgpack_object *entry, - int chunk_id) -{ - int result; - struct flb_log_event event; - - result = flb_event_decoder_decode_object(conn->ctx->log_decoder, - &event, entry); - - if (result == FLB_EVENT_DECODER_SUCCESS) { - result = flb_log_event_encoder_begin_record(conn->ctx->log_encoder); - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_set_timestamp(conn->ctx->log_encoder, - &event.timestamp); - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_set_metadata_from_msgpack_object( - conn->ctx->log_encoder, - event.metadata); - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_set_body_from_msgpack_object( - conn->ctx->log_encoder, - event.body); - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_commit_record(conn->ctx->log_encoder); - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(conn->ctx->ins, tag, tag_len, - conn->ctx->log_encoder->output_buffer, - conn->ctx->log_encoder->output_length); - } - - flb_log_event_encoder_reset(conn->ctx->log_encoder); - - if (result != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_warn(conn->ctx->ins, "Event decoder failure : %d", result); - - return -1; - } - - return 0; -} - -static int fw_process_message_mode_entry( - struct flb_input_instance *in, - struct fw_conn *conn, - const char *tag, int tag_len, - msgpack_object *root, - msgpack_object *ts, - msgpack_object *body, - int chunk_id, int metadata_id) -{ - struct flb_time timestamp; - msgpack_object *metadata; - msgpack_object options; - int result; - msgpack_object chunk; - - metadata = NULL; - - if (chunk_id != -1 || metadata_id != -1) { - options = root->via.array.ptr[3]; - - if (metadata_id != -1) { - metadata = &options.via.map.ptr[metadata_id].val; - } - } - - result = flb_log_event_decoder_decode_timestamp(ts, ×tamp); - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_begin_record(conn->ctx->log_encoder); - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_set_timestamp(conn->ctx->log_encoder, - ×tamp); - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - if (metadata != NULL) { - result = flb_log_event_encoder_set_metadata_from_msgpack_object( - conn->ctx->log_encoder, - metadata); - } - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_set_body_from_msgpack_object( - conn->ctx->log_encoder, - body); - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_commit_record(conn->ctx->log_encoder); - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(in, tag, tag_len, - conn->ctx->log_encoder->output_buffer, - conn->ctx->log_encoder->output_length); - } - - flb_log_event_encoder_reset(conn->ctx->log_encoder); - - if (chunk_id != -1) { - chunk = options.via.map.ptr[chunk_id].val; - send_ack(in, conn, chunk); - } - - return 0; -} - -static size_t receiver_recv(struct fw_conn *conn, char *buf, size_t try_size) { - size_t off; - size_t actual_size; - - off = conn->buf_len - conn->rest; - actual_size = try_size; - - if (actual_size > conn->rest) { - actual_size = conn->rest; - } - - memcpy(buf, conn->buf + off, actual_size); - conn->rest -= actual_size; - - return actual_size; -} - -static size_t receiver_to_unpacker(struct fw_conn *conn, size_t request_size, - msgpack_unpacker *unpacker) -{ - size_t recv_len; - - /* make sure there's enough room, or expand the unpacker accordingly */ - if (msgpack_unpacker_buffer_capacity(unpacker) < request_size) { - msgpack_unpacker_reserve_buffer(unpacker, request_size); - assert(msgpack_unpacker_buffer_capacity(unpacker) >= request_size); - } - recv_len = receiver_recv(conn, msgpack_unpacker_buffer(unpacker), - request_size); - msgpack_unpacker_buffer_consumed(unpacker, recv_len); - - return recv_len; -} - -int fw_prot_process(struct flb_input_instance *ins, struct fw_conn *conn) -{ - int ret; - int stag_len; - int event_type; - int contain_options = FLB_FALSE; - size_t index = 0; - size_t off = 0; - size_t chunk_id = -1; - size_t metadata_id = -1; - const char *stag; - flb_sds_t out_tag = NULL; - size_t bytes; - size_t recv_len; - size_t gz_size; - void *gz_data; - msgpack_object tag; - msgpack_object entry; - msgpack_object map; - msgpack_object root; - msgpack_object chunk; - msgpack_unpacked result; - msgpack_unpacker *unp; - size_t all_used = 0; - struct flb_in_fw_config *ctx = conn->ctx; - struct cmt *cmt; - struct ctrace *ctr; - - /* - * [tag, time, record] - * [tag, [[time,record], [time,record], ...]] - */ - - out_tag = flb_sds_create_size(1024); - if (!out_tag) { - return -1; - } - - unp = msgpack_unpacker_new(1024); - msgpack_unpacked_init(&result); - conn->rest = conn->buf_len; - - while (1) { - recv_len = receiver_to_unpacker(conn, EACH_RECV_SIZE, unp); - if (recv_len == 0) { - /* No more data */ - msgpack_unpacker_free(unp); - msgpack_unpacked_destroy(&result); - - /* Adjust buffer data */ - if (conn->buf_len >= all_used && all_used > 0) { - memmove(conn->buf, conn->buf + all_used, - conn->buf_len - all_used); - conn->buf_len -= all_used; - } - flb_sds_destroy(out_tag); - - return 0; - } - - /* Always summarize the total number of bytes requested to parse */ - ret = msgpack_unpacker_next_with_size(unp, &result, &bytes); - - /* - * Upon parsing or memory errors, break the loop, return the error - * and expect the connection to be closed. - */ - if (ret == MSGPACK_UNPACK_PARSE_ERROR || - ret == MSGPACK_UNPACK_NOMEM_ERROR) { - /* A bit redunant, print out the real error */ - if (ret == MSGPACK_UNPACK_PARSE_ERROR) { - flb_plg_debug(ctx->ins, "err=MSGPACK_UNPACK_PARSE_ERROR"); - } - else { - flb_plg_error(ctx->ins, "err=MSGPACK_UNPACK_NOMEM_ERROR"); - } - - /* Cleanup buffers */ - msgpack_unpacked_destroy(&result); - msgpack_unpacker_free(unp); - flb_sds_destroy(out_tag); - - return -1; - } - - while (ret == MSGPACK_UNPACK_SUCCESS) { - /* - * For buffering optimization we always want to know the total - * number of bytes involved on the new object returned. Despites - * buf_off always know the given bytes, it's likely we used a bit - * less. This 'all_used' field keep a reference per object so - * when returning to the caller we can adjust the source buffer - * and deprecated consumed data. - * - * The 'last_parsed' field is Fluent Bit specific and is documented - * in: - * - * lib/msgpack-c/include/msgpack/unpack.h - * - * Other references: - * - * https://github.com/msgpack/msgpack-c/issues/514 - */ - all_used += bytes; - - - /* Map the array */ - root = result.data; - - if (root.type != MSGPACK_OBJECT_ARRAY) { - flb_plg_debug(ctx->ins, - "parser: expecting an array (type=%i), skip.", - root.type); - msgpack_unpacked_destroy(&result); - msgpack_unpacker_free(unp); - flb_sds_destroy(out_tag); - - return -1; - } - - if (root.via.array.size < 2) { - flb_plg_debug(ctx->ins, - "parser: array of invalid size, skip."); - msgpack_unpacked_destroy(&result); - msgpack_unpacker_free(unp); - flb_sds_destroy(out_tag); - - return -1; - } - - if (root.via.array.size == 3) { - contain_options = FLB_TRUE; - } - - /* Get the tag */ - tag = root.via.array.ptr[0]; - if (tag.type != MSGPACK_OBJECT_STR) { - flb_plg_debug(ctx->ins, - "parser: invalid tag format, skip."); - msgpack_unpacked_destroy(&result); - msgpack_unpacker_free(unp); - flb_sds_destroy(out_tag); - return -1; - } - - /* reference the tag associated with the record */ - stag = tag.via.str.ptr; - stag_len = tag.via.str.size; - - /* clear out_tag before using */ - flb_sds_len_set(out_tag, 0); - - /* Prefix the incoming record tag with a custom prefix */ - if (ctx->tag_prefix) { - /* prefix */ - flb_sds_cat_safe(&out_tag, - ctx->tag_prefix, flb_sds_len(ctx->tag_prefix)); - /* record tag */ - flb_sds_cat_safe(&out_tag, stag, stag_len); - } - else if (ins->tag && !ins->tag_default) { - /* if the input plugin instance Tag has been manually set, use it */ - flb_sds_cat_safe(&out_tag, ins->tag, flb_sds_len(ins->tag)); - } - else { - /* use the tag from the record */ - flb_sds_cat_safe(&out_tag, stag, stag_len); - } - - entry = root.via.array.ptr[1]; - - if (entry.type == MSGPACK_OBJECT_ARRAY) { - /* - * Forward format 1 (forward mode: [tag, [[time, map], ...]] - */ - - /* Check for options */ - chunk_id = -1; - ret = get_options_chunk(&root, 2, &chunk_id); - if (ret == -1) { - flb_plg_debug(ctx->ins, "invalid options field"); - msgpack_unpacked_destroy(&result); - msgpack_unpacker_free(unp); - flb_sds_destroy(out_tag); - - return -1; - } - - /* Process array */ - ret = 0; - - for(index = 0 ; - index < entry.via.array.size && - ret == 0 ; - index++) { - ret = fw_process_forward_mode_entry( - conn, - out_tag, flb_sds_len(out_tag), - &entry.via.array.ptr[index], - chunk_id); - } - - if (chunk_id != -1) { - msgpack_object options; - msgpack_object chunk; - - options = root.via.array.ptr[2]; - chunk = options.via.map.ptr[chunk_id].val; - - send_ack(conn->in, conn, chunk); - } - } - else if (entry.type == MSGPACK_OBJECT_POSITIVE_INTEGER || - entry.type == MSGPACK_OBJECT_EXT) { - /* - * Forward format 2 (message mode) : [tag, time, map, ...] - */ - map = root.via.array.ptr[2]; - if (map.type != MSGPACK_OBJECT_MAP) { - flb_plg_warn(ctx->ins, "invalid data format, map expected"); - msgpack_unpacked_destroy(&result); - msgpack_unpacker_free(unp); - flb_sds_destroy(out_tag); - return -1; - } - - /* Check for options */ - chunk_id = -1; - ret = get_options_chunk(&root, 3, &chunk_id); - if (ret == -1) { - flb_plg_debug(ctx->ins, "invalid options field"); - msgpack_unpacked_destroy(&result); - msgpack_unpacker_free(unp); - flb_sds_destroy(out_tag); - return -1; - } - - metadata_id = -1; - ret = get_options_metadata(&root, 3, &metadata_id); - if (ret == -1) { - flb_plg_debug(ctx->ins, "invalid options field"); - msgpack_unpacked_destroy(&result); - msgpack_unpacker_free(unp); - flb_sds_destroy(out_tag); - return -1; - } - - /* Process map */ - fw_process_message_mode_entry( - conn->in, conn, - out_tag, flb_sds_len(out_tag), - &root, &entry, &map, chunk_id, - metadata_id); - } - else if (entry.type == MSGPACK_OBJECT_STR || - entry.type == MSGPACK_OBJECT_BIN) { - /* PackedForward Mode */ - const char *data = NULL; - size_t len = 0; - - /* Check for options */ - chunk_id = -1; - ret = get_options_chunk(&root, 2, &chunk_id); - if (ret == -1) { - flb_plg_debug(ctx->ins, "invalid options field"); - msgpack_unpacked_destroy(&result); - msgpack_unpacker_free(unp); - flb_sds_destroy(out_tag); - return -1; - } - - if (entry.type == MSGPACK_OBJECT_STR) { - data = entry.via.str.ptr; - len = entry.via.str.size; - } - else if (entry.type == MSGPACK_OBJECT_BIN) { - data = entry.via.bin.ptr; - len = entry.via.bin.size; - } - - if (data) { - ret = is_gzip_compressed(root.via.array.ptr[2]); - if (ret == -1) { - flb_plg_error(ctx->ins, "invalid 'compressed' option"); - msgpack_unpacked_destroy(&result); - msgpack_unpacker_free(unp); - flb_sds_destroy(out_tag); - return -1; - } - - if (ret == FLB_TRUE) { - ret = flb_gzip_uncompress((void *) data, len, - &gz_data, &gz_size); - if (ret == -1) { - flb_plg_error(ctx->ins, "gzip uncompress failure"); - msgpack_unpacked_destroy(&result); - msgpack_unpacker_free(unp); - flb_sds_destroy(out_tag); - return -1; - } - - /* Append uncompressed data */ - flb_input_log_append(conn->in, - out_tag, flb_sds_len(out_tag), - gz_data, gz_size); - flb_free(gz_data); - } - else { - event_type = FLB_EVENT_TYPE_LOGS; - if (contain_options) { - ret = get_chunk_event_type(ins, root.via.array.ptr[2]); - if (ret == -1) { - msgpack_unpacked_destroy(&result); - msgpack_unpacker_free(unp); - flb_sds_destroy(out_tag); - return -1; - } - event_type = ret; - } - - if (event_type == FLB_EVENT_TYPE_LOGS) { - flb_input_log_append(conn->in, - out_tag, flb_sds_len(out_tag), - data, len); - } - else if (event_type == FLB_EVENT_TYPE_METRICS) { - ret = cmt_decode_msgpack_create(&cmt, (char *) data, len, &off); - if (ret == -1) { - msgpack_unpacked_destroy(&result); - msgpack_unpacker_free(unp); - flb_sds_destroy(out_tag); - return -1; - } - flb_input_metrics_append(conn->in, - out_tag, flb_sds_len(out_tag), - cmt); - } - else if (event_type == FLB_EVENT_TYPE_TRACES) { - off = 0; - ret = ctr_decode_msgpack_create(&ctr, (char *) data, len, &off); - if (ret == -1) { - msgpack_unpacked_destroy(&result); - msgpack_unpacker_free(unp); - flb_sds_destroy(out_tag); - return -1; - } - - flb_input_trace_append(ins, - out_tag, flb_sds_len(out_tag), - ctr); - } - } - - /* Handle ACK response */ - if (chunk_id != -1) { - chunk = root.via.array.ptr[2].via.map.ptr[chunk_id].val; - send_ack(ctx->ins, conn, chunk); - } - } - } - else { - flb_plg_warn(ctx->ins, "invalid data format, type=%i", - entry.type); - msgpack_unpacked_destroy(&result); - msgpack_unpacker_free(unp); - return -1; - } - - ret = msgpack_unpacker_next(unp, &result); - } - } - - msgpack_unpacked_destroy(&result); - msgpack_unpacker_free(unp); - flb_sds_destroy(out_tag); - - switch (ret) { - case MSGPACK_UNPACK_EXTRA_BYTES: - flb_plg_error(ctx->ins, "MSGPACK_UNPACK_EXTRA_BYTES"); - return -1; - case MSGPACK_UNPACK_CONTINUE: - flb_plg_trace(ctx->ins, "MSGPACK_UNPACK_CONTINUE"); - return 1; - case MSGPACK_UNPACK_PARSE_ERROR: - flb_plg_debug(ctx->ins, "err=MSGPACK_UNPACK_PARSE_ERROR"); - return -1; - case MSGPACK_UNPACK_NOMEM_ERROR: - flb_plg_error(ctx->ins, "err=MSGPACK_UNPACK_NOMEM_ERROR"); - return -1; - }; - - return 0; -} diff --git a/fluent-bit/plugins/in_forward/fw_prot.h b/fluent-bit/plugins/in_forward/fw_prot.h deleted file mode 100644 index 67eae5507..000000000 --- a/fluent-bit/plugins/in_forward/fw_prot.h +++ /dev/null @@ -1,28 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_FW_PROT_H -#define FLB_IN_FW_PROT_H - -#include "fw_conn.h" - -int fw_prot_parser(struct fw_conn *conn); -int fw_prot_process(struct flb_input_instance *ins, struct fw_conn *conn); - -#endif diff --git a/fluent-bit/plugins/in_head/CMakeLists.txt b/fluent-bit/plugins/in_head/CMakeLists.txt deleted file mode 100644 index 2410c9ebe..000000000 --- a/fluent-bit/plugins/in_head/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - in_head.c) - -FLB_PLUGIN(in_head "${src}" "") diff --git a/fluent-bit/plugins/in_head/in_head.c b/fluent-bit/plugins/in_head/in_head.c deleted file mode 100644 index 2619d1c18..000000000 --- a/fluent-bit/plugins/in_head/in_head.c +++ /dev/null @@ -1,473 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include "in_head.h" -#define BUF_SIZE_MAX 512 - -static int read_lines(struct flb_in_head_config *ctx) -{ - FILE *fp = NULL; - int i; - int index = 0; - int str_len; - char buf[BUF_SIZE_MAX] = {0}; - int new_len = 0; - char *tmp; - char *ret_buf; - - fp = fopen(ctx->filepath, "r"); - if (fp == NULL) { - flb_errno(); - return -1; - } - - for (i = 0; ilines; i++){ - ret_buf = fgets(buf, BUF_SIZE_MAX-1, fp); - if (ret_buf == NULL) { - break; - } - str_len = strlen(buf); - if (ctx->buf_size < str_len + index + 1) { - /* buffer full. re-allocate new buffer */ - new_len = ctx->buf_size + str_len + 1; - tmp = flb_malloc(new_len); - if (tmp == NULL) { - flb_plg_error(ctx->ins, "failed to allocate buffer"); - /* try to output partial data */ - break; - } - /* copy and release old buffer */ - strcpy(tmp, ctx->buf); - flb_free(ctx->buf); - - ctx->buf_size = new_len; - ctx->buf = tmp; - } - strncat(&ctx->buf[index], buf, str_len); - ctx->buf_len += str_len; - index += str_len; - } - - fclose(fp); - return 0; -} - -static int read_bytes(struct flb_in_head_config *ctx) -{ - int fd = -1; - /* open at every collect callback */ - fd = open(ctx->filepath, O_RDONLY); - if (fd < 0) { - flb_errno(); - return -1; - } - ctx->buf_len = read(fd, ctx->buf, ctx->buf_size); - close(fd); - - if (ctx->buf_len < 0) { - flb_errno(); - return -1; - } - else { - return 0; - } -} - -static int single_value_per_record(struct flb_input_instance *i_ins, - struct flb_in_head_config *ctx) -{ - int ret = -1; - - ctx->buf[0] = '\0'; /* clear buf */ - ctx->buf_len = 0; - - if (ctx->lines > 0) { - read_lines(ctx); - } - else { - read_bytes(ctx); - } - - flb_plg_trace(ctx->ins, "%s read_len=%zd buf_size=%zu", __FUNCTION__, - ctx->buf_len, ctx->buf_size); - - ret = flb_log_event_encoder_begin_record(&ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp( - &ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - &ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE(ctx->key), - FLB_LOG_EVENT_STRING_VALUE(ctx->buf, ctx->buf_len)); - - } - - if (ctx->add_path) { - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - &ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE("path"), - FLB_LOG_EVENT_STRING_VALUE(ctx->filepath, ctx->path_len)); - } - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(&ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(i_ins, NULL, 0, - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - - ret = 0; - } - else { - flb_plg_error(i_ins, "Error encoding record : %d", ret); - - ret = -1; - } - - flb_log_event_encoder_reset(&ctx->log_encoder); - - return ret; -} - -#define KEY_LEN_MAX 32 -static int split_lines_per_record(struct flb_input_instance *i_ins, - struct flb_in_head_config *ctx) -{ - FILE *fp = NULL; - int i; - int ret; - size_t str_len; - size_t key_len; - char *ret_buf; - char key_str[KEY_LEN_MAX] = {0}; - - fp = fopen(ctx->filepath, "r"); - if (fp == NULL) { - flb_errno(); - return -1; - } - - ret = flb_log_event_encoder_begin_record(&ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp( - &ctx->log_encoder); - } - - if (ctx->add_path) { - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - &ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE("path"), - FLB_LOG_EVENT_STRING_VALUE(ctx->filepath, ctx->path_len)); - } - } - - for (i = 0; i < ctx->lines; i++) { - ret_buf = fgets(ctx->buf, ctx->buf_size, fp); - if (ret_buf == NULL) { - ctx->buf[0] = '\0'; - str_len = 0; - } - else { - str_len = strnlen(ctx->buf, ctx->buf_size-1); - ctx->buf[str_len-1] = '\0';/* chomp str */ - } - - key_len = snprintf(key_str, KEY_LEN_MAX, "line%d", i); - if (key_len > KEY_LEN_MAX) { - key_len = KEY_LEN_MAX; - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - &ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE(key_str), - FLB_LOG_EVENT_STRING_VALUE(ctx->buf, ctx->buf_len)); - } - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(&ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(i_ins, NULL, 0, - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - - ret = 0; - } - else { - flb_plg_error(i_ins, "Error encoding record : %d", ret); - - ret = -1; - } - - flb_log_event_encoder_reset(&ctx->log_encoder); - - fclose(fp); - - return ret; -} - - -/* cb_collect callback */ -static int in_head_collect(struct flb_input_instance *i_ins, - struct flb_config *config, void *in_context) -{ - int ret = -1; - struct flb_in_head_config *ctx = in_context; - - if (ctx->lines > 0 && ctx->split_line) { - ret = split_lines_per_record(i_ins, ctx); - } - else { - ret = single_value_per_record(i_ins, ctx); - } - - return ret; -} - -/* read config file and*/ -static int in_head_config_read(struct flb_in_head_config *ctx, - struct flb_input_instance *in) -{ - int ret; - /* Load the config map */ - ret = flb_input_config_map_set(in, (void *)ctx); - if (ret == -1) { - flb_plg_error(in, "unable to load configuration"); - return -1; - } - - - ctx->key_len = strlen(ctx->key); - - /* only set lines if not explicitly set */ - if (ctx->split_line && ctx->lines <= 0) { - ctx->lines = 10; - } - - if (ctx->interval_sec <= 0 && ctx->interval_nsec <= 0) { - /* Illegal settings. Override them. */ - ctx->interval_sec = atoi(DEFAULT_INTERVAL_SEC); - ctx->interval_nsec = atoi(DEFAULT_INTERVAL_NSEC); - } - - if (ctx->add_path) { - ctx->path_len = strlen(ctx->filepath); - } - - ret = flb_log_event_encoder_init(&ctx->log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, "error initializing event encoder : %d", ret); - - return -1; - } - - flb_plg_debug(ctx->ins, "buf_size=%zu path=%s", - ctx->buf_size, ctx->filepath); - flb_plg_debug(ctx->ins, "interval_sec=%d interval_nsec=%d", - ctx->interval_sec, ctx->interval_nsec); - - return 0; -} - -static void delete_head_config(struct flb_in_head_config *ctx) -{ - if (!ctx) { - return; - } - - flb_log_event_encoder_destroy(&ctx->log_encoder); - - /* release buffer */ - if (ctx->buf) { - flb_free(ctx->buf); - } - - flb_free(ctx); -} - -/* Initialize plugin */ -static int in_head_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - int ret = -1; - struct flb_in_head_config *ctx; - - /* Allocate space for the configuration */ - ctx = flb_calloc(1, sizeof(struct flb_in_head_config)); - if (!ctx) { - return -1; - } - - ctx->buf = NULL; - ctx->buf_len = 0; - ctx->add_path = FLB_FALSE; - ctx->lines = 0; - ctx->ins = in; - - /* Initialize head config */ - ret = in_head_config_read(ctx, in); - if (ret < 0) { - goto init_error; - } - - ctx->buf = flb_malloc(ctx->buf_size); - if (!ctx->buf) { - flb_errno(); - goto init_error; - } - - flb_plg_trace(ctx->ins, "%s read_len=%zd buf_size=%zu", __FUNCTION__, - ctx->buf_len, ctx->buf_size); - - flb_input_set_context(in, ctx); - - ret = flb_input_set_collector_time(in, - in_head_collect, - ctx->interval_sec, - ctx->interval_nsec, config); - if (ret < 0) { - flb_plg_error(ctx->ins, "could not set collector for head input plugin"); - goto init_error; - } - - ctx->coll_fd = ret; - return 0; - - init_error: - delete_head_config(ctx); - - return -1; -} - -static void in_head_pause(void *data, struct flb_config *config) -{ - struct flb_in_head_config *ctx = data; - (void) config; - - /* Pause collector */ - flb_input_collector_pause(ctx->coll_fd, ctx->ins); -} - -static void in_head_resume(void *data, struct flb_config *config) -{ - struct flb_in_head_config *ctx = data; - (void) config; - - /* Resume collector */ - flb_input_collector_resume(ctx->coll_fd, ctx->ins); -} - -static int in_head_exit(void *data, struct flb_config *config) -{ - (void) *config; - struct flb_in_head_config *head_config = data; - - delete_head_config(head_config); - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "file", NULL, - 0, FLB_TRUE, offsetof(struct flb_in_head_config, filepath), - "Set the file" - }, - { - FLB_CONFIG_MAP_STR, "key", "head", - 0, FLB_TRUE, offsetof(struct flb_in_head_config, key), - "Set the record key" - }, - { - FLB_CONFIG_MAP_INT, "buf_size", DEFAULT_BUF_SIZE, - 0, FLB_TRUE, offsetof(struct flb_in_head_config, buf_size), - "Set the read buffer size" - }, - { - FLB_CONFIG_MAP_BOOL, "split_line", "false", - 0, FLB_TRUE, offsetof(struct flb_in_head_config, split_line), - "generate key/value pair per line" - }, - { - FLB_CONFIG_MAP_INT, "lines", "0", - 0, FLB_TRUE, offsetof(struct flb_in_head_config, lines), - "Line number to read" - }, - { - FLB_CONFIG_MAP_BOOL, "add_path", "false", - 0, FLB_TRUE, offsetof(struct flb_in_head_config, add_path), - "append filepath to records" - }, - { - FLB_CONFIG_MAP_INT, "interval_sec", DEFAULT_INTERVAL_SEC, - 0, FLB_TRUE, offsetof(struct flb_in_head_config, interval_sec), - "Set the collector interval" - }, - { - FLB_CONFIG_MAP_INT, "interval_nsec", DEFAULT_INTERVAL_NSEC, - 0, FLB_TRUE, offsetof(struct flb_in_head_config, interval_nsec), - "Set the collector interval (nanoseconds)" - }, - /* EOF */ - {0} -}; - -struct flb_input_plugin in_head_plugin = { - .name = "head", - .description = "Head Input", - .cb_init = in_head_init, - .cb_pre_run = NULL, - .cb_collect = in_head_collect, - .cb_flush_buf = NULL, - .cb_pause = in_head_pause, - .cb_resume = in_head_resume, - .config_map = config_map, - .cb_exit = in_head_exit -}; diff --git a/fluent-bit/plugins/in_head/in_head.h b/fluent-bit/plugins/in_head/in_head.h deleted file mode 100644 index 08376a942..000000000 --- a/fluent-bit/plugins/in_head/in_head.h +++ /dev/null @@ -1,59 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef FLB_IN_HEAD_H -#define FLB_IN_HEAD_H - -#include -#include -#include -#include - -#include - -#define DEFAULT_BUF_SIZE "256" -#define DEFAULT_INTERVAL_SEC "1" -#define DEFAULT_INTERVAL_NSEC "0" - -struct flb_in_head_config { - int coll_fd; - size_t buf_size; /* size of buf */ - ssize_t buf_len; /* read size */ - char *buf; /* read buf */ - flb_sds_t key; - int key_len; - - flb_sds_t filepath; /* to read */ - - int add_path; /* add path mode */ - size_t path_len; - - int lines; /* line num to read */ - int split_line; - - int interval_sec; - int interval_nsec; - - struct flb_log_event_encoder log_encoder; - - struct flb_input_instance *ins; -}; - -extern struct flb_input_plugin in_head_plugin; - -#endif /* FLB_IN_HEAD_H */ diff --git a/fluent-bit/plugins/in_health/CMakeLists.txt b/fluent-bit/plugins/in_health/CMakeLists.txt deleted file mode 100644 index afb509ece..000000000 --- a/fluent-bit/plugins/in_health/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - health.c) - -FLB_PLUGIN(in_health "${src}" "") diff --git a/fluent-bit/plugins/in_health/health.c b/fluent-bit/plugins/in_health/health.c deleted file mode 100644 index 72f59e51c..000000000 --- a/fluent-bit/plugins/in_health/health.c +++ /dev/null @@ -1,293 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#define DEFAULT_INTERVAL_SEC "1" -#define DEFAULT_INTERVAL_NSEC "0" - -/* Input configuration & context */ -struct flb_in_health_config { - /* Alert mode */ - int alert; - - /* Append Hostname */ - int add_host; - int len_host; - char* hostname; - - /* Append Port Number */ - int add_port; - int port; - - /* Time interval check */ - int interval_sec; - int interval_nsec; - - /* Networking */ - struct flb_upstream *u; - - struct flb_log_event_encoder log_encoder; - - /* Plugin instance */ - struct flb_input_instance *ins; -}; - -/* Collection aims to try to connect to the specified TCP server */ -static int in_health_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - uint8_t alive; - struct flb_in_health_config *ctx = in_context; - struct flb_connection *u_conn; - int ret; - - u_conn = flb_upstream_conn_get(ctx->u); - if (!u_conn) { - alive = FLB_FALSE; - } - else { - alive = FLB_TRUE; - flb_upstream_conn_release(u_conn); - } - - if (alive == FLB_TRUE && ctx->alert == FLB_TRUE) { - FLB_INPUT_RETURN(0); - } - - ret = flb_log_event_encoder_begin_record(&ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp( - &ctx->log_encoder); - } - - /* Status */ - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - &ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE("alive"), - FLB_LOG_EVENT_BOOLEAN_VALUE(alive)); - } - - if (ctx->add_host) { - /* append hostname */ - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - &ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE("hostname"), - FLB_LOG_EVENT_CSTRING_VALUE(ctx->hostname)); - } - } - - if (ctx->add_port) { - /* append port number */ - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - &ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE("port"), - FLB_LOG_EVENT_INT32_VALUE(ctx->port)); - } - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(&ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(ins, NULL, 0, - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - - ret = 0; - } - else { - flb_plg_error(ins, "Error encoding record : %d", ret); - - ret = -1; - } - - flb_log_event_encoder_reset(&ctx->log_encoder); - - FLB_INPUT_RETURN(ret); -} - -static int in_health_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - int ret; - int upstream_flags; - struct flb_in_health_config *ctx; - (void) data; - - if (in->host.name == NULL) { - flb_plg_error(in, "no input 'Host' provided"); - return -1; - } - - /* Allocate space for the configuration */ - ctx = flb_calloc(1, sizeof(struct flb_in_health_config)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->alert = FLB_FALSE; - ctx->add_host = FLB_FALSE; - ctx->len_host = 0; - ctx->hostname = NULL; - ctx->add_port = FLB_FALSE; - ctx->port = -1; - ctx->ins = in; - - - /* Load the config map */ - ret = flb_input_config_map_set(in, (void *)ctx); - if (ret == -1) { - flb_free(ctx); - flb_plg_error(in, "unable to load configuration"); - return -1; - } - - upstream_flags = FLB_IO_TCP; - - if (in->use_tls) { - upstream_flags |= FLB_IO_TLS; - } - - ctx->u = flb_upstream_create(config, in->host.name, in->host.port, - upstream_flags, in->tls); - - if (!ctx->u) { - flb_plg_error(ctx->ins, "could not initialize upstream"); - flb_free(ctx); - return -1; - } - - if (ctx->interval_sec <= 0 && ctx->interval_nsec <= 0) { - /* Illegal settings. Override them. */ - ctx->interval_sec = atoi(DEFAULT_INTERVAL_SEC); - ctx->interval_nsec = atoi(DEFAULT_INTERVAL_NSEC); - } - - if (ctx->add_host) { - ctx->len_host = strlen(in->host.name); - ctx->hostname = flb_strdup(in->host.name); - } - - if (ctx->add_port) { - ctx->port = in->host.port; - } - - /* Set the context */ - flb_input_set_context(in, ctx); - - /* Set our collector based on time */ - ret = flb_input_set_collector_time(in, - in_health_collect, - ctx->interval_sec, - ctx->interval_nsec, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not set collector for Health input plugin"); - flb_free(ctx); - return -1; - } - - ret = flb_log_event_encoder_init(&ctx->log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(in, "error initializing event encoder : %d", ret); - - flb_free(ctx); - - return -1; - } - - return 0; -} - -static int in_health_exit(void *data, struct flb_config *config) -{ - (void) *config; - struct flb_in_health_config *ctx = data; - - flb_log_event_encoder_destroy(&ctx->log_encoder); - - /* Remove msgpack buffer and destroy context */ - flb_upstream_destroy(ctx->u); - flb_free(ctx->hostname); - flb_free(ctx); - - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_BOOL, "alert", "false", - 0, FLB_TRUE, offsetof(struct flb_in_health_config, alert), - "Only generate records when the port is down" - }, - { - FLB_CONFIG_MAP_BOOL, "add_host", "false", - 0, FLB_TRUE, offsetof(struct flb_in_health_config, add_host), - "Append hostname to each record" - }, - { - FLB_CONFIG_MAP_BOOL, "add_port", "false", - 0, FLB_TRUE, offsetof(struct flb_in_health_config, add_port), - "Append port to each record" - }, - { - FLB_CONFIG_MAP_INT, "interval_sec", DEFAULT_INTERVAL_SEC, - 0, FLB_TRUE, offsetof(struct flb_in_health_config, interval_sec), - "Set the collector interval" - }, - { - FLB_CONFIG_MAP_INT, "interval_nsec", DEFAULT_INTERVAL_NSEC, - 0, FLB_TRUE, offsetof(struct flb_in_health_config, interval_nsec), - "Set the collector interval (nanoseconds)" - }, - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_input_plugin in_health_plugin = { - .name = "health", - .description = "Check TCP server health", - .cb_init = in_health_init, - .cb_pre_run = NULL, - .cb_collect = in_health_collect, - .cb_flush_buf = NULL, - .cb_exit = in_health_exit, - .config_map = config_map, - .flags = FLB_INPUT_NET|FLB_INPUT_CORO -}; diff --git a/fluent-bit/plugins/in_http/CMakeLists.txt b/fluent-bit/plugins/in_http/CMakeLists.txt deleted file mode 100644 index 69ebeab71..000000000 --- a/fluent-bit/plugins/in_http/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -if(NOT FLB_METRICS) - message(FATAL_ERROR "HTTP input plugin requires FLB_HTTP_SERVER=On.") -endif() - -set(src - http.c - http_conn.c - http_prot.c - http_config.c - ) - -FLB_PLUGIN(in_http "${src}" "monkey-core-static") diff --git a/fluent-bit/plugins/in_http/http.c b/fluent-bit/plugins/in_http/http.c deleted file mode 100644 index 7bbe6e5f4..000000000 --- a/fluent-bit/plugins/in_http/http.c +++ /dev/null @@ -1,204 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#include -#include -#include - -#include "http.h" -#include "http_conn.h" -#include "http_config.h" - -/* - * For a server event, the collection event means a new client have arrived, we - * accept the connection and create a new TCP instance which will wait for - * JSON map messages. - */ -static int in_http_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_connection *connection; - struct http_conn *conn; - struct flb_http *ctx; - - ctx = in_context; - - connection = flb_downstream_conn_get(ctx->downstream); - - if (connection == NULL) { - flb_plg_error(ctx->ins, "could not accept new connection"); - - return -1; - } - - flb_plg_trace(ctx->ins, "new TCP connection arrived FD=%i", - connection->fd); - - conn = http_conn_add(connection, ctx); - - if (conn == NULL) { - flb_downstream_conn_release(connection); - - return -1; - } - - return 0; -} - -static int in_http_init(struct flb_input_instance *ins, - struct flb_config *config, void *data) -{ - unsigned short int port; - int ret; - struct flb_http *ctx; - - (void) data; - - /* Create context and basic conf */ - ctx = http_config_create(ins); - if (!ctx) { - return -1; - } - - ctx->collector_id = -1; - - /* Populate context with config map defaults and incoming properties */ - ret = flb_input_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_plg_error(ctx->ins, "configuration error"); - http_config_destroy(ctx); - return -1; - } - - /* Set the context */ - flb_input_set_context(ins, ctx); - - port = (unsigned short int) strtoul(ctx->tcp_port, NULL, 10); - - ctx->downstream = flb_downstream_create(FLB_TRANSPORT_TCP, - ins->flags, - ctx->listen, - port, - ins->tls, - config, - &ins->net_setup); - - if (ctx->downstream == NULL) { - flb_plg_error(ctx->ins, - "could not initialize downstream on %s:%s. Aborting", - ctx->listen, ctx->tcp_port); - - http_config_destroy(ctx); - - return -1; - } - - flb_input_downstream_set(ctx->downstream, ctx->ins); - - if (ctx->successful_response_code != 200 && - ctx->successful_response_code != 201 && - ctx->successful_response_code != 204) { - flb_plg_error(ctx->ins, "%d is not supported response code. Use default 201", - ctx->successful_response_code); - ctx->successful_response_code = 201; - } - - /* Collect upon data available on the standard input */ - ret = flb_input_set_collector_socket(ins, - in_http_collect, - ctx->downstream->server_fd, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "Could not set collector for IN_TCP input plugin"); - http_config_destroy(ctx); - - return -1; - } - - ctx->collector_id = ret; - - return 0; -} - -static int in_http_exit(void *data, struct flb_config *config) -{ - struct flb_http *ctx; - - (void) config; - - ctx = data; - - if (ctx != NULL) { - http_config_destroy(ctx); - } - - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_SIZE, "buffer_max_size", HTTP_BUFFER_MAX_SIZE, - 0, FLB_TRUE, offsetof(struct flb_http, buffer_max_size), - "" - }, - - { - FLB_CONFIG_MAP_SIZE, "buffer_chunk_size", HTTP_BUFFER_CHUNK_SIZE, - 0, FLB_TRUE, offsetof(struct flb_http, buffer_chunk_size), - "" - }, - - { - FLB_CONFIG_MAP_SLIST_1, "success_header", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct flb_http, success_headers), - "Add an HTTP header key/value pair on success. Multiple headers can be set" - }, - - { - FLB_CONFIG_MAP_STR, "tag_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_http, tag_key), - "" - }, - { - FLB_CONFIG_MAP_INT, "successful_response_code", "201", - 0, FLB_TRUE, offsetof(struct flb_http, successful_response_code), - "Set successful response code. 200, 201 and 204 are supported." - }, - - - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_input_plugin in_http_plugin = { - .name = "http", - .description = "HTTP", - .cb_init = in_http_init, - .cb_pre_run = NULL, - .cb_collect = in_http_collect, - .cb_flush_buf = NULL, - .cb_pause = NULL, - .cb_resume = NULL, - .cb_exit = in_http_exit, - .config_map = config_map, - .flags = FLB_INPUT_NET_SERVER | FLB_IO_OPT_TLS -}; diff --git a/fluent-bit/plugins/in_http/http.h b/fluent-bit/plugins/in_http/http.h deleted file mode 100644 index f9832e9b2..000000000 --- a/fluent-bit/plugins/in_http/http.h +++ /dev/null @@ -1,58 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_HTTP_H -#define FLB_IN_HTTP_H - -#include -#include -#include -#include -#include - -#include - -#define HTTP_BUFFER_MAX_SIZE "4M" -#define HTTP_BUFFER_CHUNK_SIZE "512K" - -struct flb_http { - int successful_response_code; - flb_sds_t listen; - flb_sds_t tcp_port; - const char *tag_key; - - int collector_id; - - /* Success HTTP headers */ - struct mk_list *success_headers; - flb_sds_t success_headers_str; - - size_t buffer_max_size; /* Maximum buffer size */ - size_t buffer_chunk_size; /* Chunk allocation size */ - - struct flb_log_event_encoder log_encoder; - struct flb_downstream *downstream; /* Client manager */ - struct mk_list connections; /* linked list of connections */ - - struct mk_server *server; - struct flb_input_instance *ins; -}; - - -#endif diff --git a/fluent-bit/plugins/in_http/http_config.c b/fluent-bit/plugins/in_http/http_config.c deleted file mode 100644 index f23759a66..000000000 --- a/fluent-bit/plugins/in_http/http_config.c +++ /dev/null @@ -1,157 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include "http.h" -#include "http_config.h" -#include "http_conn.h" -#include "http_config.h" - -struct flb_http *http_config_create(struct flb_input_instance *ins) -{ - struct mk_list *header_iterator; - struct flb_slist_entry *header_value; - struct flb_slist_entry *header_name; - struct flb_config_map_val *header_pair; - char port[8]; - int ret; - struct flb_http *ctx; - - ctx = flb_calloc(1, sizeof(struct flb_http)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - mk_list_init(&ctx->connections); - - /* Load the config map */ - ret = flb_input_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return NULL; - } - - /* Listen interface (if not set, defaults to 0.0.0.0:9880) */ - flb_input_net_default_listener("0.0.0.0", 9880, ins); - - ctx->listen = flb_strdup(ins->host.listen); - snprintf(port, sizeof(port) - 1, "%d", ins->host.port); - ctx->tcp_port = flb_strdup(port); - - /* HTTP Server specifics */ - ctx->server = flb_calloc(1, sizeof(struct mk_server)); - ctx->server->keep_alive = MK_TRUE; - - /* monkey detects server->workers == 0 as the server not being initialized at the - * moment so we want to make sure that it stays that way! - */ - - ret = flb_log_event_encoder_init(&ctx->log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, "error initializing event encoder : %d", ret); - - http_config_destroy(ctx); - - return NULL; - } - - ctx->success_headers_str = flb_sds_create_size(1); - - if (ctx->success_headers_str == NULL) { - http_config_destroy(ctx); - - return NULL; - } - - flb_config_map_foreach(header_iterator, header_pair, ctx->success_headers) { - header_name = mk_list_entry_first(header_pair->val.list, - struct flb_slist_entry, - _head); - - header_value = mk_list_entry_last(header_pair->val.list, - struct flb_slist_entry, - _head); - - ret = flb_sds_cat_safe(&ctx->success_headers_str, - header_name->str, - flb_sds_len(header_name->str)); - - if (ret == 0) { - ret = flb_sds_cat_safe(&ctx->success_headers_str, - ": ", - 2); - } - - if (ret == 0) { - ret = flb_sds_cat_safe(&ctx->success_headers_str, - header_value->str, - flb_sds_len(header_value->str)); - } - - if (ret == 0) { - ret = flb_sds_cat_safe(&ctx->success_headers_str, - "\r\n", - 2); - } - - if (ret != 0) { - http_config_destroy(ctx); - - return NULL; - } - } - - return ctx; -} - -int http_config_destroy(struct flb_http *ctx) -{ - /* release all connections */ - http_conn_release_all(ctx); - - flb_log_event_encoder_destroy(&ctx->log_encoder); - - if (ctx->collector_id != -1) { - flb_input_collector_delete(ctx->collector_id, ctx->ins); - - ctx->collector_id = -1; - } - - if (ctx->downstream != NULL) { - flb_downstream_destroy(ctx->downstream); - } - - if (ctx->server) { - flb_free(ctx->server); - } - - if (ctx->success_headers_str != NULL) { - flb_sds_destroy(ctx->success_headers_str); - } - - - flb_free(ctx->listen); - flb_free(ctx->tcp_port); - flb_free(ctx); - return 0; -} diff --git a/fluent-bit/plugins/in_http/http_config.h b/fluent-bit/plugins/in_http/http_config.h deleted file mode 100644 index 8a9611116..000000000 --- a/fluent-bit/plugins/in_http/http_config.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_HTTP_CONFIG_H -#define FLB_IN_HTTP_CONFIG_H - -#include -#include "http.h" - -struct flb_http *http_config_create(struct flb_input_instance *ins); -int http_config_destroy(struct flb_http *ctx); - -#endif diff --git a/fluent-bit/plugins/in_http/http_conn.c b/fluent-bit/plugins/in_http/http_conn.c deleted file mode 100644 index a5d9efa98..000000000 --- a/fluent-bit/plugins/in_http/http_conn.c +++ /dev/null @@ -1,306 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include "http.h" -#include "http_conn.h" -#include "http_prot.h" - -static void http_conn_request_init(struct mk_http_session *session, - struct mk_http_request *request); - -static int http_conn_event(void *data) -{ - int status; - size_t size; - ssize_t available; - ssize_t bytes; - char *tmp; - char *request_end; - size_t request_len; - struct flb_connection *connection; - struct http_conn *conn; - struct mk_event *event; - struct flb_http *ctx; - - connection = (struct flb_connection *) data; - - conn = connection->user_data; - - ctx = conn->ctx; - - event = &connection->event; - - if (event->mask & MK_EVENT_READ) { - available = (conn->buf_size - conn->buf_len) - 1; - if (available < 1) { - if (conn->buf_size + ctx->buffer_chunk_size > ctx->buffer_max_size) { - flb_plg_trace(ctx->ins, - "fd=%i incoming data exceed limit (%zu KB)", - event->fd, (ctx->buffer_max_size / 1024)); - http_conn_del(conn); - return -1; - } - - size = conn->buf_size + ctx->buffer_chunk_size; - tmp = flb_realloc(conn->buf_data, size); - if (!tmp) { - flb_errno(); - return -1; - } - flb_plg_trace(ctx->ins, "fd=%i buffer realloc %i -> %zu", - event->fd, conn->buf_size, size); - - conn->buf_data = tmp; - conn->buf_size = size; - available = (conn->buf_size - conn->buf_len) - 1; - } - - /* Read data */ - bytes = flb_io_net_read(connection, - (void *) &conn->buf_data[conn->buf_len], - available); - - if (bytes <= 0) { - flb_plg_trace(ctx->ins, "fd=%i closed connection", event->fd); - http_conn_del(conn); - return -1; - } - - flb_plg_trace(ctx->ins, "read()=%zi pre_len=%i now_len=%zi", - bytes, conn->buf_len, conn->buf_len + bytes); - conn->buf_len += bytes; - conn->buf_data[conn->buf_len] = '\0'; - - status = mk_http_parser(&conn->request, &conn->session.parser, - conn->buf_data, conn->buf_len, conn->session.server); - - if (status == MK_HTTP_PARSER_OK) { - /* Do more logic parsing and checks for this request */ - http_prot_handle(ctx, conn, &conn->session, &conn->request); - - /* Evict the processed request from the connection buffer and reinitialize - * the HTTP parser. - */ - - request_end = NULL; - - if (NULL != conn->request.data.data) { - request_end = &conn->request.data.data[conn->request.data.len]; - } - else { - request_end = strstr(conn->buf_data, "\r\n\r\n"); - - if(NULL != request_end) { - request_end = &request_end[4]; - } - } - - if (NULL != request_end) { - request_len = (size_t)(request_end - conn->buf_data); - - if (0 < (conn->buf_len - request_len)) { - memmove(conn->buf_data, &conn->buf_data[request_len], - conn->buf_len - request_len); - - conn->buf_data[conn->buf_len - request_len] = '\0'; - conn->buf_len -= request_len; - } - else { - memset(conn->buf_data, 0, request_len); - - conn->buf_len = 0; - } - - /* Reinitialize the parser so the next request is properly - * handled, the additional memset intends to wipe any left over data - * from the headers parsed in the previous request. - */ - memset(&conn->session.parser, 0, sizeof(struct mk_http_parser)); - mk_http_parser_init(&conn->session.parser); - http_conn_request_init(&conn->session, &conn->request); - } - } - else if (status == MK_HTTP_PARSER_ERROR) { - http_prot_handle_error(ctx, conn, &conn->session, &conn->request); - - /* Reinitialize the parser so the next request is properly - * handled, the additional memset intends to wipe any left over data - * from the headers parsed in the previous request. - */ - memset(&conn->session.parser, 0, sizeof(struct mk_http_parser)); - mk_http_parser_init(&conn->session.parser); - http_conn_request_init(&conn->session, &conn->request); - } - - /* FIXME: add Protocol handler here */ - return bytes; - } - - if (event->mask & MK_EVENT_CLOSE) { - flb_plg_trace(ctx->ins, "fd=%i hangup", event->fd); - http_conn_del(conn); - return -1; - } - - return 0; - -} - -static void http_conn_session_init(struct mk_http_session *session, - struct mk_server *server, - int client_fd) -{ - /* Alloc memory for node */ - session->_sched_init = MK_TRUE; - session->pipelined = MK_FALSE; - session->counter_connections = 0; - session->close_now = MK_FALSE; - session->status = MK_REQUEST_STATUS_INCOMPLETE; - session->server = server; - session->socket = client_fd; - - /* creation time in unix time */ - session->init_time = time(NULL); - - session->channel = mk_channel_new(MK_CHANNEL_SOCKET, session->socket); - session->channel->io = session->server->network; - - /* Init session request list */ - mk_list_init(&session->request_list); - - /* Initialize the parser */ - mk_http_parser_init(&session->parser); -} - -static void http_conn_request_init(struct mk_http_session *session, - struct mk_http_request *request) -{ - memset(request, 0, sizeof(struct mk_http_request)); - - mk_http_request_init(session, request, session->server); - - request->in_headers.type = MK_STREAM_IOV; - request->in_headers.dynamic = MK_FALSE; - request->in_headers.cb_consumed = NULL; - request->in_headers.cb_finished = NULL; - request->in_headers.stream = &request->stream; - - mk_list_add(&request->in_headers._head, &request->stream.inputs); - - request->session = session; -} - -struct http_conn *http_conn_add(struct flb_connection *connection, - struct flb_http *ctx) -{ - struct http_conn *conn; - int ret; - - conn = flb_calloc(1, sizeof(struct http_conn)); - if (!conn) { - flb_errno(); - return NULL; - } - - conn->connection = connection; - - /* Set data for the event-loop */ - MK_EVENT_NEW(&connection->event); - - connection->user_data = conn; - connection->event.type = FLB_ENGINE_EV_CUSTOM; - connection->event.handler = http_conn_event; - - /* Connection info */ - conn->ctx = ctx; - conn->buf_len = 0; - - conn->buf_data = flb_malloc(ctx->buffer_chunk_size); - if (!conn->buf_data) { - flb_errno(); - - flb_plg_error(ctx->ins, "could not allocate new connection"); - flb_free(conn); - - return NULL; - } - conn->buf_size = ctx->buffer_chunk_size; - - /* Register instance into the event loop */ - ret = mk_event_add(flb_engine_evl_get(), - connection->fd, - FLB_ENGINE_EV_CUSTOM, - MK_EVENT_READ, - &connection->event); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not register new connection"); - - flb_free(conn->buf_data); - flb_free(conn); - - return NULL; - } - - /* Initialize HTTP Session: this is a custom context for Monkey HTTP */ - http_conn_session_init(&conn->session, ctx->server, conn->connection->fd); - - /* Initialize HTTP Request: this is the initial request and it will be reinitialized - * automatically after the request is handled so it can be used for the next one. - */ - http_conn_request_init(&conn->session, &conn->request); - - /* Link connection node to parent context list */ - mk_list_add(&conn->_head, &ctx->connections); - - return conn; -} - -int http_conn_del(struct http_conn *conn) -{ - if (conn->session.channel != NULL) { - mk_channel_release(conn->session.channel); - } - - /* The downstream unregisters the file descriptor from the event-loop - * so there's nothing to be done by the plugin - */ - flb_downstream_conn_release(conn->connection); - - mk_list_del(&conn->_head); - - flb_free(conn->buf_data); - flb_free(conn); - - return 0; -} - -void http_conn_release_all(struct flb_http *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - struct http_conn *conn; - - mk_list_foreach_safe(head, tmp, &ctx->connections) { - conn = mk_list_entry(head, struct http_conn, _head); - http_conn_del(conn); - } -} diff --git a/fluent-bit/plugins/in_http/http_conn.h b/fluent-bit/plugins/in_http/http_conn.h deleted file mode 100644 index 8e1078982..000000000 --- a/fluent-bit/plugins/in_http/http_conn.h +++ /dev/null @@ -1,54 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_HTTP_CONN -#define FLB_IN_HTTP_CONN - -#include -#include - -#include -#include -#include - -struct http_conn { - /* Buffer */ - char *buf_data; /* Buffer data */ - int buf_len; /* Data length */ - int buf_size; /* Buffer size */ - - /* - * Parser context: we only held one parser per connection - * which is re-used everytime we have a new request. - */ - struct mk_http_parser parser; - struct mk_http_request request; - struct mk_http_session session; - struct flb_connection *connection; - - void *ctx; /* Plugin parent context */ - struct mk_list _head; /* link to flb_http->connections */ -}; - -struct http_conn *http_conn_add(struct flb_connection *connection, struct flb_http *ctx); -int http_conn_del(struct http_conn *conn); -void http_conn_release_all(struct flb_http *ctx); - - -#endif diff --git a/fluent-bit/plugins/in_http/http_prot.c b/fluent-bit/plugins/in_http/http_prot.c deleted file mode 100644 index ab16eb328..000000000 --- a/fluent-bit/plugins/in_http/http_prot.c +++ /dev/null @@ -1,665 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include -#include - -#include "http.h" -#include "http_conn.h" - -#define HTTP_CONTENT_JSON 0 -#define HTTP_CONTENT_URLENCODED 1 - -static inline char hex2nibble(char c) -{ - if ((c >= 0x30) && (c <= '9')) { - return c - 0x30; - } - // 0x30-0x39 are digits, 0x41-0x46 A-F, - // so there is a gap at 0x40 - if ((c >= 'A') && (c <= 'F')) { - return (c - 'A') + 10; - } - if ((c >= 'a') && (c <= 'f')) { - return (c - 'a') + 10; - } - return 0; -} - -static int sds_uri_decode(flb_sds_t s) -{ - char buf[1024]; - char *optr; - char *iptr; - - - for (optr = buf, iptr = s; iptr < s + flb_sds_len(s) && optr-buf < sizeof(buf); iptr++) { - if (*iptr == '%') { - if (iptr+2 > (s + flb_sds_len(s))) { - return -1; - } - *optr++ = hex2nibble(*(iptr+1)) << 4 | hex2nibble(*(iptr+2)); - iptr+=2; - } else if (*iptr == '+') { - *optr++ = ' '; - } else { - *optr++ = *iptr; - } - } - - memcpy(s, buf, optr-buf); - s[optr-buf] = '\0'; - flb_sds_len_set(s, (optr-buf)); - - return 0; -} - -static int send_response(struct http_conn *conn, int http_status, char *message) -{ - struct flb_http *context; - size_t sent; - int len; - flb_sds_t out; - - context = (struct flb_http *) conn->ctx; - - out = flb_sds_create_size(256); - if (!out) { - return -1; - } - - if (message) { - len = strlen(message); - } - else { - len = 0; - } - - if (http_status == 201) { - flb_sds_printf(&out, - "HTTP/1.1 201 Created \r\n" - "Server: Fluent Bit v%s\r\n" - "%s" - "Content-Length: 0\r\n\r\n", - FLB_VERSION_STR, - context->success_headers_str); - } - else if (http_status == 200) { - flb_sds_printf(&out, - "HTTP/1.1 200 OK\r\n" - "Server: Fluent Bit v%s\r\n" - "%s" - "Content-Length: 0\r\n\r\n", - FLB_VERSION_STR, - context->success_headers_str); - } - else if (http_status == 204) { - flb_sds_printf(&out, - "HTTP/1.1 204 No Content\r\n" - "Server: Fluent Bit v%s\r\n" - "%s" - "\r\n\r\n", - FLB_VERSION_STR, - context->success_headers_str); - } - else if (http_status == 400) { - flb_sds_printf(&out, - "HTTP/1.1 400 Forbidden\r\n" - "Server: Fluent Bit v%s\r\n" - "Content-Length: %i\r\n\r\n%s", - FLB_VERSION_STR, - len, message); - } - - /* We should check this operations result */ - flb_io_net_write(conn->connection, - (void *) out, - flb_sds_len(out), - &sent); - - flb_sds_destroy(out); - - return 0; -} - -/* implements functionality to get tag from key in record */ -static flb_sds_t tag_key(struct flb_http *ctx, msgpack_object *map) -{ - size_t map_size = map->via.map.size; - msgpack_object_kv *kv; - msgpack_object key; - msgpack_object val; - char *key_str = NULL; - char *val_str = NULL; - size_t key_str_size = 0; - size_t val_str_size = 0; - int j; - int check = FLB_FALSE; - int found = FLB_FALSE; - flb_sds_t tag; - - kv = map->via.map.ptr; - - for(j=0; j < map_size; j++) { - check = FLB_FALSE; - found = FLB_FALSE; - key = (kv+j)->key; - if (key.type == MSGPACK_OBJECT_BIN) { - key_str = (char *) key.via.bin.ptr; - key_str_size = key.via.bin.size; - check = FLB_TRUE; - } - if (key.type == MSGPACK_OBJECT_STR) { - key_str = (char *) key.via.str.ptr; - key_str_size = key.via.str.size; - check = FLB_TRUE; - } - - if (check == FLB_TRUE) { - if (strncmp(ctx->tag_key, key_str, key_str_size) == 0) { - val = (kv+j)->val; - if (val.type == MSGPACK_OBJECT_BIN) { - val_str = (char *) val.via.bin.ptr; - val_str_size = val.via.str.size; - found = FLB_TRUE; - break; - } - if (val.type == MSGPACK_OBJECT_STR) { - val_str = (char *) val.via.str.ptr; - val_str_size = val.via.str.size; - found = FLB_TRUE; - break; - } - } - } - } - - if (found == FLB_TRUE) { - tag = flb_sds_create_len(val_str, val_str_size); - if (!tag) { - flb_errno(); - return NULL; - } - return tag; - } - - - flb_plg_error(ctx->ins, "Could not find tag_key %s in record", ctx->tag_key); - return NULL; -} - -int process_pack(struct flb_http *ctx, flb_sds_t tag, char *buf, size_t size) -{ - int ret; - size_t off = 0; - msgpack_unpacked result; - struct flb_time tm; - int i = 0; - msgpack_object *obj; - msgpack_object record; - flb_sds_t tag_from_record = NULL; - - flb_time_get(&tm); - - msgpack_unpacked_init(&result); - while (msgpack_unpack_next(&result, buf, size, &off) == MSGPACK_UNPACK_SUCCESS) { - if (result.data.type == MSGPACK_OBJECT_MAP) { - tag_from_record = NULL; - if (ctx->tag_key) { - obj = &result.data; - tag_from_record = tag_key(ctx, obj); - } - - ret = flb_log_event_encoder_begin_record(&ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_timestamp( - &ctx->log_encoder, - &tm); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_body_from_msgpack_object( - &ctx->log_encoder, - &result.data); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(&ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - if (tag_from_record) { - flb_input_log_append(ctx->ins, - tag_from_record, - flb_sds_len(tag_from_record), - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - - flb_sds_destroy(tag_from_record); - } - else if (tag) { - flb_input_log_append(ctx->ins, tag, flb_sds_len(tag), - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - } - else { - /* use default plugin Tag (it internal name, e.g: http.0 */ - flb_input_log_append(ctx->ins, NULL, 0, - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - } - } - else { - flb_plg_error(ctx->ins, "Error encoding record : %d", ret); - } - - flb_log_event_encoder_reset(&ctx->log_encoder); - } - else if (result.data.type == MSGPACK_OBJECT_ARRAY) { - obj = &result.data; - for (i = 0; i < obj->via.array.size; i++) - { - record = obj->via.array.ptr[i]; - - tag_from_record = NULL; - if (ctx->tag_key) { - tag_from_record = tag_key(ctx, &record); - } - - ret = flb_log_event_encoder_begin_record(&ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_timestamp( - &ctx->log_encoder, - &tm); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_body_from_msgpack_object( - &ctx->log_encoder, - &record); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(&ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - if (tag_from_record) { - flb_input_log_append(ctx->ins, - tag_from_record, - flb_sds_len(tag_from_record), - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - - flb_sds_destroy(tag_from_record); - } - else if (tag) { - flb_input_log_append(ctx->ins, tag, flb_sds_len(tag), - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - } - else { - /* use default plugin Tag (it internal name, e.g: http.0 */ - flb_input_log_append(ctx->ins, NULL, 0, - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - } - } - else { - flb_plg_error(ctx->ins, "Error encoding record : %d", ret); - } - - /* TODO : Optimize this - * - * This is wasteful, considering that we are emitting a series - * of records we should start and commit each one and then - * emit them all at once after the loop. - */ - - flb_log_event_encoder_reset(&ctx->log_encoder); - } - - break; - } - else { - flb_plg_error(ctx->ins, "skip record from invalid type: %i", - result.data.type); - - msgpack_unpacked_destroy(&result); - - return -1; - } - } - - msgpack_unpacked_destroy(&result); - - return 0; -} - -static ssize_t parse_payload_json(struct flb_http *ctx, flb_sds_t tag, - char *payload, size_t size) -{ - int ret; - int out_size; - char *pack; - struct flb_pack_state pack_state; - - /* Initialize packer */ - flb_pack_state_init(&pack_state); - - /* Pack JSON as msgpack */ - ret = flb_pack_json_state(payload, size, - &pack, &out_size, &pack_state); - flb_pack_state_reset(&pack_state); - - /* Handle exceptions */ - if (ret == FLB_ERR_JSON_PART) { - flb_plg_warn(ctx->ins, "JSON data is incomplete, skipping"); - return -1; - } - else if (ret == FLB_ERR_JSON_INVAL) { - flb_plg_warn(ctx->ins, "invalid JSON message, skipping"); - return -1; - } - else if (ret == -1) { - return -1; - } - - /* Process the packaged JSON and return the last byte used */ - process_pack(ctx, tag, pack, out_size); - flb_free(pack); - - return 0; -} - -static ssize_t parse_payload_urlencoded(struct flb_http *ctx, flb_sds_t tag, - char *payload, size_t size) -{ - struct mk_list *kvs; - struct mk_list *head = NULL; - struct flb_split_entry *cur = NULL; - char **keys = NULL; - char **vals = NULL; - char *sep; - char *start; - int idx = 0; - int ret = -1; - msgpack_packer pck; - msgpack_sbuffer sbuf; - - - /* initialize buffers */ - msgpack_sbuffer_init(&sbuf); - msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); - - kvs = flb_utils_split(payload, '&', -1 ); - if (kvs == NULL) { - goto split_error; - } - - keys = flb_calloc(mk_list_size(kvs), sizeof(char *)); - if (keys == NULL) { - goto keys_calloc_error; - } - - vals = flb_calloc(mk_list_size(kvs), sizeof(char *)); - if (vals == NULL) { - goto vals_calloc_error; - } - - mk_list_foreach(head, kvs) { - cur = mk_list_entry(head, struct flb_split_entry, _head); - if (cur->value[0] == '\n') { - start = &cur->value[1]; - } else { - start = cur->value; - } - sep = strchr(start, '='); - if (sep == NULL) { - vals[idx] = NULL; - continue; - } - *sep++ = '\0'; - - keys[idx] = flb_sds_create_len(start, strlen(start)); - vals[idx] = flb_sds_create_len(sep, strlen(sep)); - - flb_sds_trim(keys[idx]); - flb_sds_trim(vals[idx]); - idx++; - } - - msgpack_pack_map(&pck, mk_list_size(kvs)); - for (idx = 0; idx < mk_list_size(kvs); idx++) { - msgpack_pack_str(&pck, flb_sds_len(keys[idx])); - msgpack_pack_str_body(&pck, keys[idx], flb_sds_len(keys[idx])); - - if (sds_uri_decode(vals[idx]) != 0) { - goto decode_error; - } else { - msgpack_pack_str(&pck, flb_sds_len(vals[idx])); - msgpack_pack_str_body(&pck, vals[idx], strlen(vals[idx])); - } - } - - ret = process_pack(ctx, tag, sbuf.data, sbuf.size); - -decode_error: - for (idx = 0; idx < mk_list_size(kvs); idx++) { - if (keys[idx]) { - flb_sds_destroy(keys[idx]); - } - if (vals[idx]) { - flb_sds_destroy(vals[idx]); - } - } - flb_free(vals); -vals_calloc_error: - flb_free(keys); -keys_calloc_error: - flb_utils_split_free(kvs); -split_error: - msgpack_sbuffer_destroy(&sbuf); - return ret; -} - -static int process_payload(struct flb_http *ctx, struct http_conn *conn, - flb_sds_t tag, - struct mk_http_session *session, - struct mk_http_request *request) -{ - int type = -1; - struct mk_http_header *header; - - header = &session->parser.headers[MK_HEADER_CONTENT_TYPE]; - if (header->key.data == NULL) { - send_response(conn, 400, "error: header 'Content-Type' is not set\n"); - return -1; - } - - if (header->val.len == 16 && - strncasecmp(header->val.data, "application/json", 16) == 0) { - type = HTTP_CONTENT_JSON; - } - - if (header->val.len == 33 && - strncasecmp(header->val.data, "application/x-www-form-urlencoded", 33) == 0) { - type = HTTP_CONTENT_URLENCODED; - } - - if (type == -1) { - send_response(conn, 400, "error: invalid 'Content-Type'\n"); - return -1; - } - - if (request->data.len <= 0) { - send_response(conn, 400, "error: no payload found\n"); - return -1; - } - - if (type == HTTP_CONTENT_JSON) { - parse_payload_json(ctx, tag, request->data.data, request->data.len); - } else if (type == HTTP_CONTENT_URLENCODED) { - parse_payload_urlencoded(ctx, tag, request->data.data, request->data.len); - } - - return 0; -} - -static inline int mk_http_point_header(mk_ptr_t *h, - struct mk_http_parser *parser, int key) -{ - struct mk_http_header *header; - - header = &parser->headers[key]; - if (header->type == key) { - h->data = header->val.data; - h->len = header->val.len; - return 0; - } - else { - h->data = NULL; - h->len = -1; - } - - return -1; -} - -/* - * Handle an incoming request. It perform extra checks over the request, if - * everything is OK, it enqueue the incoming payload. - */ -int http_prot_handle(struct flb_http *ctx, struct http_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request) -{ - int i; - int ret; - int len; - char *uri; - char *qs; - off_t diff; - flb_sds_t tag; - struct mk_http_header *header; - - if (request->uri.data[0] != '/') { - send_response(conn, 400, "error: invalid request\n"); - return -1; - } - - /* Decode URI */ - uri = mk_utils_url_decode(request->uri); - if (!uri) { - uri = mk_mem_alloc_z(request->uri.len + 1); - if (!uri) { - return -1; - } - memcpy(uri, request->uri.data, request->uri.len); - uri[request->uri.len] = '\0'; - } - - /* Try to match a query string so we can remove it */ - qs = strchr(uri, '?'); - if (qs) { - /* remove the query string part */ - diff = qs - uri; - uri[diff] = '\0'; - } - - /* Compose the query string using the URI */ - len = strlen(uri); - - if (len == 1) { - tag = NULL; /* use default tag */ - } - else { - tag = flb_sds_create_size(len); - if (!tag) { - mk_mem_free(uri); - return -1; - } - - /* New tag skipping the URI '/' */ - flb_sds_cat(tag, uri + 1, len - 1); - - /* Sanitize, only allow alphanum chars */ - for (i = 0; i < flb_sds_len(tag); i++) { - if (!isalnum(tag[i]) && tag[i] != '_' && tag[i] != '.') { - tag[i] = '_'; - } - } - } - - mk_mem_free(uri); - - /* Check if we have a Host header: Hostname ; port */ - mk_http_point_header(&request->host, &session->parser, MK_HEADER_HOST); - - /* Header: Connection */ - mk_http_point_header(&request->connection, &session->parser, - MK_HEADER_CONNECTION); - - /* HTTP/1.1 needs Host header */ - if (!request->host.data && request->protocol == MK_HTTP_PROTOCOL_11) { - flb_sds_destroy(tag); - return -1; - } - - /* Should we close the session after this request ? */ - mk_http_keepalive_check(session, request, ctx->server); - - /* Content Length */ - header = &session->parser.headers[MK_HEADER_CONTENT_LENGTH]; - if (header->type == MK_HEADER_CONTENT_LENGTH) { - request->_content_length.data = header->val.data; - request->_content_length.len = header->val.len; - } - else { - request->_content_length.data = NULL; - } - - if (request->method != MK_METHOD_POST) { - flb_sds_destroy(tag); - send_response(conn, 400, "error: invalid HTTP method\n"); - return -1; - } - - ret = process_payload(ctx, conn, tag, session, request); - flb_sds_destroy(tag); - send_response(conn, ctx->successful_response_code, NULL); - return ret; -} - -/* - * Handle an incoming request which has resulted in an http parser error. - */ -int http_prot_handle_error(struct flb_http *ctx, struct http_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request) -{ - send_response(conn, 400, "error: invalid request\n"); - return -1; -} diff --git a/fluent-bit/plugins/in_http/http_prot.h b/fluent-bit/plugins/in_http/http_prot.h deleted file mode 100644 index 1cb603089..000000000 --- a/fluent-bit/plugins/in_http/http_prot.h +++ /dev/null @@ -1,31 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_HTTP_PROT -#define FLB_IN_HTTP_PROT - -int http_prot_handle(struct flb_http *ctx, struct http_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request); - -int http_prot_handle_error(struct flb_http *ctx, struct http_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request); - -#endif diff --git a/fluent-bit/plugins/in_kafka/CMakeLists.txt b/fluent-bit/plugins/in_kafka/CMakeLists.txt deleted file mode 100644 index 696e263fa..000000000 --- a/fluent-bit/plugins/in_kafka/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(src - in_kafka.c - ) - -FLB_PLUGIN(in_kafka "${src}" "rdkafka") -target_link_libraries(flb-plugin-in_kafka -lpthread) diff --git a/fluent-bit/plugins/in_kafka/in_kafka.c b/fluent-bit/plugins/in_kafka/in_kafka.c deleted file mode 100644 index 972ae4170..000000000 --- a/fluent-bit/plugins/in_kafka/in_kafka.c +++ /dev/null @@ -1,382 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2021 The Fluent Bit Authors - * Copyright (C) 2015-2018 Treasure Data Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "fluent-bit/flb_input.h" -#include "fluent-bit/flb_kafka.h" -#include "fluent-bit/flb_mem.h" -#include "in_kafka.h" -#include "rdkafka.h" - -static int try_json(struct flb_log_event_encoder *log_encoder, - rd_kafka_message_t *rkm) -{ - int root_type; - char *buf = NULL; - size_t bufsize; - int ret; - - ret = flb_pack_json(rkm->payload, rkm->len, &buf, &bufsize, &root_type, NULL); - if (ret) { - if (buf) { - flb_free(buf); - } - return ret; - } - flb_log_event_encoder_append_body_raw_msgpack(log_encoder, buf, bufsize); - flb_free(buf); - return 0; -} - -static int process_message(struct flb_in_kafka_config *ctx, - rd_kafka_message_t *rkm) -{ - struct flb_log_event_encoder *log_encoder = ctx->log_encoder; - int ret; - - ret = flb_log_event_encoder_begin_record(log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp(log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_cstring(log_encoder, "topic"); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - if (rkm->rkt) { - ret = flb_log_event_encoder_append_body_cstring(log_encoder, - rd_kafka_topic_name(rkm->rkt)); - } - else { - ret = flb_log_event_encoder_append_body_null(log_encoder); - } - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values(log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE("partition"), - FLB_LOG_EVENT_INT32_VALUE(rkm->partition)); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values(log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE("offset"), - FLB_LOG_EVENT_INT64_VALUE(rkm->offset)); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_cstring(log_encoder, "error"); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - if (rkm->err) { - ret = flb_log_event_encoder_append_body_cstring(log_encoder, - rd_kafka_message_errstr(rkm)); - } - else { - ret = flb_log_event_encoder_append_body_null(log_encoder); - } - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_cstring(log_encoder, "key"); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - if (rkm->key) { - ret = flb_log_event_encoder_append_body_string(log_encoder, - rkm->key, - rkm->key_len); - } - else { - ret = flb_log_event_encoder_append_body_null(log_encoder); - } - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_cstring(log_encoder, "payload"); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - if (rkm->payload) { - if (ctx->format != FLB_IN_KAFKA_FORMAT_JSON || - try_json(log_encoder, rkm)) { - ret = flb_log_event_encoder_append_body_string(log_encoder, - rkm->payload, - rkm->len); - } - } - else { - ret = flb_log_event_encoder_append_body_null(log_encoder); - } - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(log_encoder); - } - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_log_event_encoder_rollback_record(log_encoder); - } - - return ret; -} - -static int in_kafka_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - int ret; - struct flb_in_kafka_config *ctx = in_context; - rd_kafka_message_t *rkm; - - ret = FLB_EVENT_ENCODER_SUCCESS; - - while (ret == FLB_EVENT_ENCODER_SUCCESS) { - rkm = rd_kafka_consumer_poll(ctx->kafka.rk, 1); - - if (!rkm) { - break; - } - - flb_plg_debug(ins, "kafka message received"); - - ret = process_message(ctx, rkm); - - rd_kafka_message_destroy(rkm); - - /* TO-DO: commit the record based on `ret` */ - rd_kafka_commit(ctx->kafka.rk, NULL, 0); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(ins, NULL, 0, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); - ret = 0; - } - else { - flb_plg_error(ins, "Error encoding record : %d", ret); - ret = -1; - } - - flb_log_event_encoder_reset(ctx->log_encoder); - - return ret; -} - -/* Initialize plugin */ -static int in_kafka_init(struct flb_input_instance *ins, - struct flb_config *config, void *data) -{ - int ret; - const char *conf; - struct flb_in_kafka_config *ctx; - rd_kafka_conf_t *kafka_conf = NULL; - rd_kafka_topic_partition_list_t *kafka_topics = NULL; - rd_kafka_resp_err_t err; - char errstr[512]; - (void) data; - - /* Allocate space for the configuration context */ - ctx = flb_malloc(sizeof(struct flb_in_kafka_config)); - if (!ctx) { - return -1; - } - ctx->ins = ins; - - ret = flb_input_config_map_set(ins, (void*) ctx); - if (ret == -1) { - flb_plg_error(ins, "unable to load configuration."); - flb_free(ctx); - return -1; - } - - kafka_conf = flb_kafka_conf_create(&ctx->kafka, &ins->properties, 1); - if (!kafka_conf) { - flb_plg_error(ins, "Could not initialize kafka config object"); - goto init_error; - } - - ctx->kafka.rk = rd_kafka_new(RD_KAFKA_CONSUMER, kafka_conf, errstr, - sizeof(errstr)); - - /* Create Kafka consumer handle */ - if (!ctx->kafka.rk) { - flb_plg_error(ins, "Failed to create new consumer: %s", errstr); - goto init_error; - } - - conf = flb_input_get_property("topics", ins); - if (!conf) { - flb_plg_error(ins, "config: no topics specified"); - goto init_error; - } - - kafka_topics = flb_kafka_parse_topics(conf); - if (!kafka_topics) { - flb_plg_error(ins, "Failed to parse topic list"); - goto init_error; - } - - if (strcasecmp(ctx->format_str, "none") == 0) { - ctx->format = FLB_IN_KAFKA_FORMAT_NONE; - } - else if (strcasecmp(ctx->format_str, "json") == 0) { - ctx->format = FLB_IN_KAFKA_FORMAT_JSON; - } - else { - flb_plg_error(ins, "config: invalid format \"%s\"", ctx->format_str); - goto init_error; - } - - if ((err = rd_kafka_subscribe(ctx->kafka.rk, kafka_topics))) { - flb_plg_error(ins, "Failed to start consuming topics: %s", rd_kafka_err2str(err)); - goto init_error; - } - rd_kafka_topic_partition_list_destroy(kafka_topics); - kafka_topics = NULL; - - /* Set the context */ - flb_input_set_context(ins, ctx); - /* Collect upon data available on the pipe read fd */ - - int poll_seconds = ctx->poll_ms / 1000; - int poll_milliseconds = ctx->poll_ms % 1000; - - ret = flb_input_set_collector_time(ins, - in_kafka_collect, - poll_seconds, poll_milliseconds * 1e6, - config); - if (ret) { - flb_plg_error(ctx->ins, "could not set collector for kafka input plugin"); - goto init_error; - } - - ctx->log_encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ctx->log_encoder == NULL) { - flb_plg_error(ins, "could not initialize log encoder"); - goto init_error; - } - - return 0; - -init_error: - if (kafka_topics) { - rd_kafka_topic_partition_list_destroy(kafka_topics); - } - if (ctx->kafka.rk) { - rd_kafka_destroy(ctx->kafka.rk); - } - else if (kafka_conf) { - /* conf is already destroyed when rd_kafka is initialized */ - rd_kafka_conf_destroy(kafka_conf); - } - flb_free(ctx); - - return -1; -} - -/* Cleanup serial input */ -static int in_kafka_exit(void *in_context, struct flb_config *config) -{ - struct flb_in_kafka_config *ctx; - - if (!in_context) { - return 0; - } - - ctx = in_context; - rd_kafka_destroy(ctx->kafka.rk); - flb_free(ctx->kafka.brokers); - - if (ctx->log_encoder){ - flb_log_event_encoder_destroy(ctx->log_encoder); - } - - flb_free(ctx); - - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_INT, "poll_ms", FLB_IN_KAFKA_DEFAULT_POLL_MS, - 0, FLB_TRUE, offsetof(struct flb_in_kafka_config, poll_ms), - "Interval in milliseconds to check for new messages." - }, - { - FLB_CONFIG_MAP_STR, "topics", (char *)NULL, - 0, FLB_FALSE, 0, - "Set the kafka topics, delimited by commas." - }, - { - FLB_CONFIG_MAP_STR, "format", FLB_IN_KAFKA_DEFAULT_FORMAT, - 0, FLB_TRUE, offsetof(struct flb_in_kafka_config, format_str), - "Set the data format which will be used for parsing records." - }, - { - FLB_CONFIG_MAP_STR, "brokers", (char *)NULL, - 0, FLB_FALSE, 0, - "Set the kafka brokers, delimited by commas." - }, - { - FLB_CONFIG_MAP_STR, "client_id", (char *)NULL, - 0, FLB_FALSE, 0, - "Set the kafka client_id." - }, - { - FLB_CONFIG_MAP_STR, "group_id", (char *)NULL, - 0, FLB_FALSE, 0, - "Set the kafka group_id." - }, - { - FLB_CONFIG_MAP_STR_PREFIX, "rdkafka.", NULL, - /* FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct flb_in_kafka_config, rdkafka_opts), */ - 0, FLB_FALSE, 0, - "Set the librdkafka options" - }, - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_input_plugin in_kafka_plugin = { - .name = "kafka", - .description = "Kafka consumer input plugin", - .cb_init = in_kafka_init, - .cb_pre_run = NULL, - .cb_collect = in_kafka_collect, - .cb_flush_buf = NULL, - .cb_exit = in_kafka_exit, - .config_map = config_map -}; diff --git a/fluent-bit/plugins/in_kafka/in_kafka.h b/fluent-bit/plugins/in_kafka/in_kafka.h deleted file mode 100644 index 2992efff1..000000000 --- a/fluent-bit/plugins/in_kafka/in_kafka.h +++ /dev/null @@ -1,48 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2021 The Fluent Bit Authors - * Copyright (C) 2015-2018 Treasure Data Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_KAFKA_H -#define FLB_IN_KAFKA_H - -#include -#include -#include -#include -#include - - -#define FLB_IN_KAFKA_DEFAULT_POLL_MS "500" -#define FLB_IN_KAFKA_DEFAULT_FORMAT "none" - -enum { - FLB_IN_KAFKA_FORMAT_NONE, - FLB_IN_KAFKA_FORMAT_JSON, -}; - -struct flb_in_kafka_config { - struct flb_kafka kafka; - struct flb_input_instance *ins; - struct flb_log_event_encoder *log_encoder; - int poll_ms; - int format; - char *format_str; -}; - -#endif diff --git a/fluent-bit/plugins/in_kmsg/CMakeLists.txt b/fluent-bit/plugins/in_kmsg/CMakeLists.txt deleted file mode 100644 index a4272294d..000000000 --- a/fluent-bit/plugins/in_kmsg/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - in_kmsg.c) - -FLB_PLUGIN(in_kmsg "${src}" "") diff --git a/fluent-bit/plugins/in_kmsg/in_kmsg.c b/fluent-bit/plugins/in_kmsg/in_kmsg.c deleted file mode 100644 index 0f27c67a1..000000000 --- a/fluent-bit/plugins/in_kmsg/in_kmsg.c +++ /dev/null @@ -1,390 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "in_kmsg.h" - -/* - * Note: Functions timeval_diff() and in_kmsg_boot_time() are based - * on syslog-ng-3.5 source code. - */ -static inline uint64_t timeval_diff(struct timeval *t1, struct timeval *t2) -{ - return ((uint64_t) t1->tv_sec - (uint64_t) t2->tv_sec) * KMSG_USEC_PER_SEC + - ((uint64_t) t1->tv_usec - (uint64_t) t2->tv_usec); -} - -static int boot_time(struct timeval *boot_time) -{ - int fd, pos = 0; - int bytes; - uint64_t tdiff; - char buf[256]; - struct timeval curr_time; - - fd = open("/proc/uptime", O_RDONLY); - if (fd == -1) { - return -1; - } - - bytes = read(fd, buf, sizeof(buf)); - if (bytes <= 0) { - close(fd); - return -1; - } - - close(fd); - gettimeofday(&curr_time, NULL); - - /* Read the seconds part */ - while (pos < bytes && buf[pos] != '.') { - if (isdigit(buf[pos])) { - boot_time->tv_sec = boot_time->tv_sec * 10 + ((buf[pos]) - '0'); - } - else { - boot_time->tv_sec = 0; - return 0; - } - pos++; - } - pos++; - - /* Then the microsecond part */ - while (pos < bytes && buf[pos] != ' ') { - if (isdigit(buf[pos])) { - boot_time->tv_usec = boot_time->tv_usec * 10 + ((buf[pos]) - '0'); - } - else { - boot_time->tv_sec = 0; - boot_time->tv_usec = 0; - return 0; - } - pos++; - } - - tdiff = timeval_diff(&curr_time, boot_time); - boot_time->tv_sec = tdiff / KMSG_USEC_PER_SEC; - boot_time->tv_usec = tdiff % KMSG_USEC_PER_SEC; - - return 0; -} - -static inline int process_line(const char *line, - struct flb_input_instance *i_ins, - struct flb_in_kmsg_config *ctx) -{ - char priority; /* log priority */ - uint64_t sequence; /* sequence number */ - struct timeval tv; /* time value */ - int line_len; - uint64_t val; - const char *p = line; - char *end = NULL; - struct flb_time ts; - int ret; - - /* Increase buffer position */ - ctx->buffer_id++; - - errno = 0; - val = strtol(p, &end, 10); - if ((errno == ERANGE && (val == INT_MAX || val == INT_MIN)) - || (errno != 0 && val == 0)) { - goto fail; - } - - /* Priority */ - priority = FLB_KLOG_PRI(val); - - if (priority > ctx->prio_level) { - /* Drop line */ - return 0; - } - - /* Sequence */ - p = strchr(p, ','); - if (!p) { - goto fail; - } - p++; - - val = strtol(p, &end, 10); - if ((errno == ERANGE && (val == INT_MAX || val == INT_MIN)) - || (errno != 0 && val == 0)) { - goto fail; - } - - sequence = val; - p = ++end; - - /* Timestamp */ - val = strtol(p, &end, 10); - if ((errno == ERANGE && (val == INT_MAX || val == INT_MIN)) - || (errno != 0 && val == 0)) { - goto fail; - } - - tv.tv_sec = val/1000000; - tv.tv_usec = val - (tv.tv_sec * 1000000); - - flb_time_set(&ts, ctx->boot_time.tv_sec + tv.tv_sec, tv.tv_usec * 1000); - - /* Now process the human readable message */ - p = strchr(p, ';'); - if (!p) { - goto fail; - } - p++; - - line_len = strlen(p); - - ret = flb_log_event_encoder_begin_record(&ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_timestamp( - &ctx->log_encoder, - &ts); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - &ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE("priority"), - FLB_LOG_EVENT_CHAR_VALUE(priority), - - FLB_LOG_EVENT_CSTRING_VALUE("sequence"), - FLB_LOG_EVENT_UINT64_VALUE(sequence), - - FLB_LOG_EVENT_CSTRING_VALUE("sec"), - FLB_LOG_EVENT_UINT64_VALUE(tv.tv_sec), - - FLB_LOG_EVENT_CSTRING_VALUE("usec"), - FLB_LOG_EVENT_UINT64_VALUE(tv.tv_usec), - - FLB_LOG_EVENT_CSTRING_VALUE("msg"), - FLB_LOG_EVENT_STRING_VALUE((char *) p, line_len - 1)); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(&ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(ctx->ins, NULL, 0, - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - - ret = 0; - } - else { - flb_plg_error(ctx->ins, "Error encoding record : %d", ret); - - ret = -1; - } - - flb_log_event_encoder_reset(&ctx->log_encoder); - - flb_plg_debug(ctx->ins, "pri=%i seq=%" PRIu64 " sec=%ld usec=%ld msg_length=%i", - priority, - sequence, - (long int) tv.tv_sec, - (long int) tv.tv_usec, - line_len - 1); - return ret; - - fail: - ctx->buffer_id--; - return -1; -} - -/* Callback triggered when some Kernel Log buffer msgs are available */ -static int in_kmsg_collect(struct flb_input_instance *i_ins, - struct flb_config *config, void *in_context) -{ - int ret; - int bytes; - struct flb_in_kmsg_config *ctx = in_context; - - bytes = read(ctx->fd, ctx->buf_data, ctx->buf_size - 1); - if (bytes == -1) { - if (errno == -EPIPE) { - return -1; - } - return 0; - } - else if (bytes == 0) { - flb_errno(); - return 0; - } - ctx->buf_len += bytes; - - /* Always set a delimiter to avoid buffer trash */ - ctx->buf_data[ctx->buf_len] = '\0'; - - /* Check if our buffer is full */ - if (ctx->buffer_id + 1 == KMSG_BUFFER_SIZE) { - ret = flb_engine_flush(config, &in_kmsg_plugin); - if (ret == -1) { - ctx->buffer_id = 0; - } - } - - /* Process and enqueue the received line */ - process_line(ctx->buf_data, i_ins, ctx); - ctx->buf_len = 0; - - return 0; -} - -/* Init kmsg input */ -static int in_kmsg_init(struct flb_input_instance *ins, - struct flb_config *config, void *data) -{ - int fd; - int ret; - struct flb_in_kmsg_config *ctx; - (void) data; - - ctx = flb_calloc(1, sizeof(struct flb_in_kmsg_config)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->ins = ins; - ctx->buf_data = flb_malloc(FLB_KMSG_BUF_SIZE); - if (!ctx->buf_data) { - flb_errno(); - flb_free(ctx); - return -1; - } - ctx->buf_len = 0; - ctx->buf_size = FLB_KMSG_BUF_SIZE; - - /* Load the config map */ - ret = flb_input_config_map_set(ins, (void *)ctx); - if (ret == -1) { - flb_free(ctx); - return -1; - } - - /* set context */ - flb_input_set_context(ins, ctx); - - /* open device */ - fd = open(FLB_KMSG_DEV, O_RDONLY); - if (fd == -1) { - flb_errno(); - flb_free(ctx); - return -1; - } - ctx->fd = fd; - - /* get the system boot time */ - ret = boot_time(&ctx->boot_time); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not get system boot time for kmsg input plugin"); - flb_free(ctx); - return -1; - } - flb_plg_debug(ctx->ins, "prio_level is %d", ctx->prio_level); - - /* Set our collector based on a file descriptor event */ - ret = flb_input_set_collector_event(ins, - in_kmsg_collect, - ctx->fd, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set collector for kmsg input plugin"); - flb_free(ctx); - return -1; - } - - ret = flb_log_event_encoder_init(&ctx->log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, "error initializing event encoder : %d", ret); - - flb_free(ctx); - - return -1; - } - - return 0; -} - -static int in_kmsg_exit(void *data, struct flb_config *config) -{ - (void)*config; - struct flb_in_kmsg_config *ctx = data; - - flb_log_event_encoder_destroy(&ctx->log_encoder); - - if (ctx->fd >= 0) { - close(ctx->fd); - } - - flb_free(ctx->buf_data); - flb_free(ctx); - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_INT, "prio_level", "8", - 0, FLB_TRUE, offsetof(struct flb_in_kmsg_config, prio_level), - "The log level to filter. The kernel log is dropped if its priority is more than prio_level. " - "Allowed values are 0-8. Default is 8." - }, - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_input_plugin in_kmsg_plugin = { - .name = "kmsg", - .description = "Kernel Log Buffer", - .cb_init = in_kmsg_init, - .cb_pre_run = NULL, - .cb_collect = in_kmsg_collect, - .cb_flush_buf = NULL, - .cb_exit = in_kmsg_exit, - .config_map = config_map -}; diff --git a/fluent-bit/plugins/in_kmsg/in_kmsg.h b/fluent-bit/plugins/in_kmsg/in_kmsg.h deleted file mode 100644 index 48a28f5ec..000000000 --- a/fluent-bit/plugins/in_kmsg/in_kmsg.h +++ /dev/null @@ -1,68 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_KMSG -#define FLB_IN_KMSG - -#include -#include -#include - -#include - -#define FLB_KMSG_DEV "/dev/kmsg" -#define FLB_KMSG_BUF_SIZE 4096 - -/* Alert levels, taken from util-linux sources */ -#define FLB_KLOG_EMERG 0 -#define FLB_KLOG_ALERT 1 -#define FLB_KLOG_CRIT 2 -#define FLB_KLOG_ERR 3 -#define FLB_KLOG_WARNING 4 -#define FLB_KLOG_NOTICE 5 -#define FLB_KLOG_INFO 6 -#define FLB_KLOG_DEBUG 7 - -#define FLB_KLOG_PRIMASK 0x07 -#define FLB_KLOG_PRI(p) ((p) & FLB_KLOG_PRIMASK) - -#define KMSG_BUFFER_SIZE 256 -#define KMSG_USEC_PER_SEC 1000000 - -struct flb_in_kmsg_config { - int fd; /* descriptor -> FLB_KMSG_DEV */ - struct timeval boot_time; /* System boot time */ - - int prio_level; - - /* Line processing */ - int buffer_id; - - /* Buffer */ - char *buf_data; - size_t buf_len; - size_t buf_size; - struct flb_log_event_encoder log_encoder; - struct flb_input_instance *ins; -}; - - -extern struct flb_input_plugin in_kmsg_plugin; - -#endif diff --git a/fluent-bit/plugins/in_kubernetes_events/CMakeLists.txt b/fluent-bit/plugins/in_kubernetes_events/CMakeLists.txt deleted file mode 100644 index b860a55e3..000000000 --- a/fluent-bit/plugins/in_kubernetes_events/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -set(src - kubernetes_events_conf.c - kubernetes_events.c) - -FLB_PLUGIN(in_kubernetes_events "${src}" "") diff --git a/fluent-bit/plugins/in_kubernetes_events/kubernetes_events.c b/fluent-bit/plugins/in_kubernetes_events/kubernetes_events.c deleted file mode 100644 index 97719fba6..000000000 --- a/fluent-bit/plugins/in_kubernetes_events/kubernetes_events.c +++ /dev/null @@ -1,921 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "kubernetes_events.h" -#include "kubernetes_events_conf.h" - -#ifdef FLB_HAVE_SQLDB -#include "kubernetes_events_sql.h" -static int k8s_events_sql_insert_event(struct k8s_events *ctx, msgpack_object *item); -#endif - -static int file_to_buffer(const char *path, - char **out_buf, size_t *out_size) -{ - int ret; - int len; - char *buf; - ssize_t bytes; - FILE *fp; - struct stat st; - - if (!(fp = fopen(path, "r"))) { - return -1; - } - - ret = stat(path, &st); - if (ret == -1) { - flb_errno(); - fclose(fp); - return -1; - } - - buf = flb_calloc(1, (st.st_size + 1)); - if (!buf) { - flb_errno(); - fclose(fp); - return -1; - } - - bytes = fread(buf, st.st_size, 1, fp); - if (bytes < 1) { - flb_free(buf); - fclose(fp); - return -1; - } - - fclose(fp); - - // trim new lines - for (len = st.st_size; len > 0; len--) { - if (buf[len-1] != '\n' && buf[len-1] != '\r') { - break; - } - } - buf[len] = '\0'; - - *out_buf = buf; - *out_size = len; - - return 0; -} - -/* Set K8s Authorization Token and get HTTP Auth Header */ -static int get_http_auth_header(struct k8s_events *ctx) -{ - int ret; - char *temp; - char *tk = NULL; - size_t tk_size = 0; - - if (!ctx->token_file || strlen(ctx->token_file) == 0) { - return 0; - } - - ret = file_to_buffer(ctx->token_file, &tk, &tk_size); - if (ret == -1) { - flb_plg_warn(ctx->ins, "cannot open %s", ctx->token_file); - return -1; - } - ctx->token_created = time(NULL); - - /* Token */ - if (ctx->token != NULL) { - flb_free(ctx->token); - } - ctx->token = tk; - ctx->token_len = tk_size; - - /* HTTP Auth Header */ - if (ctx->auth == NULL) { - ctx->auth = flb_malloc(tk_size + 32); - } - else if (ctx->auth_len < tk_size + 32) { - temp = flb_realloc(ctx->auth, tk_size + 32); - if (temp == NULL) { - flb_errno(); - flb_free(ctx->auth); - ctx->auth = NULL; - return -1; - } - ctx->auth = temp; - } - - if (!ctx->auth) { - return -1; - } - - ctx->auth_len = snprintf(ctx->auth, tk_size + 32, "Bearer %s", tk); - return 0; -} - -/* Refresh HTTP Auth Header if K8s Authorization Token is expired */ -static int refresh_token_if_needed(struct k8s_events *ctx) -{ - int expired = FLB_FALSE; - int ret; - - if (!ctx->token_file || strlen(ctx->token_file) == 0) { - return 0; - } - - if (ctx->token_created > 0) { - if (time(NULL) > ctx->token_created + ctx->token_ttl) { - expired = FLB_TRUE; - } - } - - if (expired || ctx->token_created == 0) { - ret = get_http_auth_header(ctx); - if (ret == -1) { - return -1; - } - } - - return 0; -} -static int timestamp_lookup(struct k8s_events *ctx, char *ts, struct flb_time *time) -{ - struct flb_tm tm = { 0 }; - - if (flb_strptime(ts, "%Y-%m-%dT%H:%M:%SZ", &tm) == NULL) { - return -1; - } - - time->tm.tv_sec = flb_parser_tm2time(&tm); - time->tm.tv_nsec = 0; - - return 0; -} - -static msgpack_object *record_get_field_ptr(msgpack_object *obj, const char *fieldname) -{ - int i; - msgpack_object *k; - msgpack_object *v; - - if (obj->type != MSGPACK_OBJECT_MAP) { - return NULL; - } - - for (i = 0; i < obj->via.map.size; i++) { - k = &obj->via.map.ptr[i].key; - if (k->type != MSGPACK_OBJECT_STR) { - continue; - } - - if (strncmp(k->via.str.ptr, fieldname, strlen(fieldname)) == 0) { - v = &obj->via.map.ptr[i].val; - return v; - } - } - return NULL; -} - -static int record_get_field_sds(msgpack_object *obj, const char *fieldname, flb_sds_t *val) -{ - msgpack_object *v; - - v = record_get_field_ptr(obj, fieldname); - if (v == NULL) { - return 0; - } - if (v->type != MSGPACK_OBJECT_STR) { - return -1; - } - - *val = flb_sds_create_len(v->via.str.ptr, v->via.str.size); - return 0; -} - -static int record_get_field_time(msgpack_object *obj, const char *fieldname, time_t *val) -{ - msgpack_object *v; - struct flb_tm tm = { 0 }; - - v = record_get_field_ptr(obj, fieldname); - if (v == NULL) { - return -1; - } - if (v->type != MSGPACK_OBJECT_STR) { - return -1; - } - - if (flb_strptime(v->via.str.ptr, "%Y-%m-%dT%H:%M:%SZ", &tm) == NULL) { - return -2; - } - - *val = mktime(&tm.tm); - return 0; -} - -static int record_get_field_uint64(msgpack_object *obj, const char *fieldname, uint64_t *val) -{ - msgpack_object *v; - char *end; - - v = record_get_field_ptr(obj, fieldname); - if (v == NULL) { - return -1; - } - - // attempt to parse string as number... - if (v->type == MSGPACK_OBJECT_STR) { - *val = strtoul(v->via.str.ptr, &end, 10); - if (end == NULL || (end < v->via.str.ptr + v->via.str.size)) { - return -1; - } - return 0; - } - if (v->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - *val = v->via.u64; - return 0; - } - if (v->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { - *val = (uint64_t)v->via.i64; - return 0; - } - return -1; -} - -static int item_get_timestamp(msgpack_object *obj, time_t *event_time) -{ - int ret; - msgpack_object *metadata; - - // some events can have lastTimestamp and firstTimestamp set to - // NULL while having metadata.creationTimestamp set. - ret = record_get_field_time(obj, "lastTimestamp", event_time); - if (ret != -1) { - return FLB_TRUE; - } - - ret = record_get_field_time(obj, "firstTimestamp", event_time); - if (ret != -1) { - return FLB_TRUE; - } - - metadata = record_get_field_ptr(obj, "metadata"); - if (metadata == NULL) { - return FLB_FALSE; - } - - ret = record_get_field_time(metadata, "creationTimestamp", event_time); - if (ret != -1) { - return FLB_TRUE; - } - - return FLB_FALSE; -} - -static bool check_event_is_filtered(struct k8s_events *ctx, msgpack_object *obj) -{ - int ret; - time_t event_time; - time_t now; - msgpack_object *metadata; - flb_sds_t uid; - uint64_t resource_version; - - ret = item_get_timestamp(obj, &event_time); - if (ret == -FLB_FALSE) { - flb_plg_error(ctx->ins, "Cannot get timestamp for item in response"); - return FLB_FALSE; - } - - now = (time_t)(cfl_time_now() / 1000000000); - if (event_time < (now - ctx->retention_time)) { - flb_plg_debug(ctx->ins, "Item is older than retention_time: %ld < %ld", - event_time, (now - ctx->retention_time)); - return FLB_TRUE; - } - - metadata = record_get_field_ptr(obj, "metadata"); - if (metadata == NULL) { - flb_plg_error(ctx->ins, "Cannot unpack item metadata in response"); - return FLB_FALSE; - } - - ret = record_get_field_uint64(metadata, "resourceVersion", &resource_version); - if (ret == -1) { - flb_plg_error(ctx->ins, "Cannot get resourceVersion for item in response"); - return FLB_FALSE; - } - - ret = record_get_field_sds(metadata, "uid", &uid); - if (ret == -1) { - flb_plg_error(ctx->ins, "Cannot get resourceVersion for item in response"); - return FLB_FALSE; - } - - -#ifdef FLB_HAVE_SQLDB - bool exists; - - - if (ctx->db) { - sqlite3_bind_text(ctx->stmt_get_kubernetes_event_exists_by_uid, - 1, uid, -1, NULL); - ret = sqlite3_step(ctx->stmt_get_kubernetes_event_exists_by_uid); - if (ret != SQLITE_ROW) { - if (ret != SQLITE_DONE) { - flb_plg_error(ctx->ins, "cannot execute kubernetes event exists"); - } - sqlite3_clear_bindings(ctx->stmt_get_kubernetes_event_exists_by_uid); - sqlite3_reset(ctx->stmt_get_kubernetes_event_exists_by_uid); - flb_sds_destroy(uid); - return FLB_FALSE; - } - - exists = sqlite3_column_int64(ctx->stmt_get_kubernetes_event_exists_by_uid, 0); - - flb_plg_debug(ctx->ins, "is_filtered: uid=%s exists=%d", uid, exists); - sqlite3_clear_bindings(ctx->stmt_get_kubernetes_event_exists_by_uid); - sqlite3_reset(ctx->stmt_get_kubernetes_event_exists_by_uid); - flb_sds_destroy(uid); - - return exists > 0 ? FLB_TRUE : FLB_FALSE; - } -#endif - - // check if this is an old event. - if (ctx->last_resource_version && resource_version <= ctx->last_resource_version) { - flb_plg_debug(ctx->ins, "skipping old object: %lu (< %lu)", resource_version, - ctx->last_resource_version); - flb_sds_destroy(uid); - return FLB_TRUE; - } - - flb_sds_destroy(uid); - return FLB_FALSE; -} - -static int process_events(struct k8s_events *ctx, char *in_data, size_t in_size, uint64_t *max_resource_version, flb_sds_t *continue_token) -{ - int i; - int ret = -1; - int root_type; - size_t consumed = 0; - char *buf_data; - size_t buf_size; - size_t off = 0; - struct flb_time ts; - struct flb_ra_value *rval; - uint64_t resource_version; - msgpack_unpacked result; - msgpack_object root; - msgpack_object k; - msgpack_object *items = NULL; - msgpack_object *item = NULL; - msgpack_object *item_metadata = NULL; - msgpack_object *metadata = NULL; - - - ret = flb_pack_json(in_data, in_size, &buf_data, &buf_size, &root_type, &consumed); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not process payload, incomplete or bad formed JSON"); - goto json_error; - } - - /* unpack */ - msgpack_unpacked_init(&result); - ret = msgpack_unpack_next(&result, buf_data, buf_size, &off); - if (ret != MSGPACK_UNPACK_SUCCESS) { - flb_plg_error(ctx->ins, "Cannot unpack response"); - goto unpack_error; - } - - /* lookup the items array */ - root = result.data; - if (root.type != MSGPACK_OBJECT_MAP) { - return -1; - } - - // Traverse the EventList for the metadata (for the continue token) and the items. - // https://kubernetes.io/docs/reference/kubernetes-api/cluster-resources/event-v1/#EventList - for (i = 0; i < root.via.map.size; i++) { - k = root.via.map.ptr[i].key; - if (k.type != MSGPACK_OBJECT_STR) { - continue; - } - - if (strncmp(k.via.str.ptr, "items", 5) == 0) { - items = &root.via.map.ptr[i].val; - if (items->type != MSGPACK_OBJECT_ARRAY) { - flb_plg_error(ctx->ins, "Cannot unpack items"); - goto msg_error; - } - } - - if (strncmp(k.via.str.ptr, "metadata", 8) == 0) { - metadata = &root.via.map.ptr[i].val; - if (metadata->type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "Cannot unpack metadata"); - goto msg_error; - } - } - } - - if (items == NULL) { - flb_plg_error(ctx->ins, "Cannot find items in response"); - goto msg_error; - } - - if (metadata == NULL) { - flb_plg_error(ctx->ins, "Cannot find metatada in response"); - goto msg_error; - } - - ret = record_get_field_sds(metadata, "continue", continue_token); - if (ret == -1) { - if (ret == -1) { - flb_plg_error(ctx->ins, "Cannot process continue token"); - goto msg_error; - } - } - - for (i = 0; i < items->via.array.size; i++) { - if (items->via.array.ptr[i].type != MSGPACK_OBJECT_MAP) { - flb_plg_warn(ctx->ins, "Event that is not a map"); - continue; - } - item_metadata = record_get_field_ptr(&items->via.array.ptr[i], "metadata"); - if (item_metadata == NULL) { - flb_plg_warn(ctx->ins, "Event without metadata"); - continue; - } - ret = record_get_field_uint64(item_metadata, - "resourceVersion", &resource_version); - if (ret == -1) { - continue; - } - if (resource_version > *max_resource_version) { - *max_resource_version = resource_version; - } - } - - /* reset the log encoder */ - flb_log_event_encoder_reset(ctx->encoder); - - /* print every item from the items array */ - for (i = 0; i < items->via.array.size; i++) { - item = &items->via.array.ptr[i]; - if (item->type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "Cannot unpack item in response"); - goto msg_error; - } - - if (check_event_is_filtered(ctx, item) == FLB_TRUE) { - continue; - } - -#ifdef FLB_HAVE_SQLDB - if (ctx->db) { - k8s_events_sql_insert_event(ctx, item); - } -#endif - - /* get event timestamp */ - rval = flb_ra_get_value_object(ctx->ra_timestamp, *item); - if (!rval || rval->type != FLB_RA_STRING) { - flb_plg_error(ctx->ins, "cannot retrieve event timestamp"); - goto msg_error; - } - - /* convert timestamp */ - ret = timestamp_lookup(ctx, rval->val.string, &ts); - if (ret == -1) { - flb_plg_error(ctx->ins, "cannot lookup event timestamp"); - flb_ra_key_value_destroy(rval); - goto msg_error; - } - - /* encode content as a log event */ - flb_log_event_encoder_begin_record(ctx->encoder); - flb_log_event_encoder_set_timestamp(ctx->encoder, &ts); - - ret = flb_log_event_encoder_set_body_from_msgpack_object(ctx->encoder, item); - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(ctx->encoder); - } else { - flb_plg_warn(ctx->ins, "unable to encode: %lu", resource_version); - } - flb_ra_key_value_destroy(rval); - } - - if (ctx->encoder->output_length > 0) { - flb_input_log_append(ctx->ins, NULL, 0, - ctx->encoder->output_buffer, - ctx->encoder->output_length); - } - -msg_error: - msgpack_unpacked_destroy(&result); -unpack_error: - flb_free(buf_data); -json_error: - return ret; -} - -static struct flb_http_client *make_event_api_request(struct k8s_events *ctx, - struct flb_connection *u_conn, - flb_sds_t continue_token) -{ - flb_sds_t url; - struct flb_http_client *c; - - - if (continue_token == NULL && ctx->limit_request == 0 && ctx->namespace == NULL) { - return flb_http_client(u_conn, FLB_HTTP_GET, K8S_EVENTS_KUBE_API_URI, - NULL, 0, ctx->api_host, ctx->api_port, NULL, 0); - } - - if (ctx->namespace == NULL) { - url = flb_sds_create(K8S_EVENTS_KUBE_API_URI); - } else { - url = flb_sds_create_size(strlen(K8S_EVENTS_KUBE_NAMESPACE_API_URI) + - strlen(ctx->namespace)); - flb_sds_printf(&url, K8S_EVENTS_KUBE_NAMESPACE_API_URI, ctx->namespace); - } - - flb_sds_cat_safe(&url, "?", 1); - if (ctx->limit_request) { - if (continue_token != NULL) { - flb_sds_printf(&url, "continue=%s&", continue_token); - } - flb_sds_printf(&url, "limit=%d", ctx->limit_request); - } - c = flb_http_client(u_conn, FLB_HTTP_GET, url, - NULL, 0, ctx->api_host, ctx->api_port, NULL, 0); - flb_sds_destroy(url); - return c; -} - -#ifdef FLB_HAVE_SQLDB - -static int k8s_events_cleanup_db(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - int ret; - struct k8s_events *ctx = (struct k8s_events *)in_context; - time_t retention_time_ago; - time_t now = (cfl_time_now() / 1000000000); - - if (ctx->db == NULL) { - FLB_INPUT_RETURN(0); - } - - retention_time_ago = now - (ctx->retention_time); - sqlite3_bind_int64(ctx->stmt_delete_old_kubernetes_events, - 1, (int64_t)retention_time_ago); - ret = sqlite3_step(ctx->stmt_delete_old_kubernetes_events); - if (ret != SQLITE_ROW && ret != SQLITE_DONE) { - flb_plg_error(ctx->ins, "cannot execute delete old kubernetes events"); - } - - sqlite3_clear_bindings(ctx->stmt_delete_old_kubernetes_events); - sqlite3_reset(ctx->stmt_delete_old_kubernetes_events); - - FLB_INPUT_RETURN(0); -} - -static int k8s_events_sql_insert_event(struct k8s_events *ctx, msgpack_object *item) -{ - int ret; - uint64_t resource_version; - time_t last; - msgpack_object *meta; - flb_sds_t uid; - - - meta = record_get_field_ptr(item, "meta"); - if (meta == NULL) { - flb_plg_error(ctx->ins, "unable to find metadata to save event"); - return -1; - } - - ret = record_get_field_uint64(meta, "resourceVersion", &resource_version); - if (ret == -1) { - flb_plg_error(ctx->ins, "unable to find resourceVersion in metadata to save event"); - return -1; - } - - ret = record_get_field_sds(meta, "uid", &uid); - if (ret == -1) { - flb_plg_error(ctx->ins, "unable to find uid in metadata to save event"); - return -1; - } - - ret = item_get_timestamp(item, &last); - if (ret == -FLB_FALSE) { - flb_plg_error(ctx->ins, "Cannot get timestamp for item to save it"); - return -1; - } - - if (ret == -2) { - flb_plg_error(ctx->ins, "unable to parse lastTimestamp in item to save event"); - flb_sds_destroy(uid); - return -1; - } - - /* Bind parameters */ - sqlite3_bind_text(ctx->stmt_insert_kubernetes_event, 1, uid, -1, 0); - sqlite3_bind_int64(ctx->stmt_insert_kubernetes_event, 2, resource_version); - sqlite3_bind_int64(ctx->stmt_insert_kubernetes_event, 3, (int64_t)last); - - /* Run the insert */ - ret = sqlite3_step(ctx->stmt_insert_kubernetes_event); - if (ret != SQLITE_DONE) { - sqlite3_clear_bindings(ctx->stmt_insert_kubernetes_event); - sqlite3_reset(ctx->stmt_insert_kubernetes_event); - flb_plg_error(ctx->ins, "cannot execute insert kubernetes event %s inode=%lu", - uid, resource_version); - flb_sds_destroy(uid); - return -1; - } - - flb_plg_debug(ctx->ins, - "inserted k8s event: uid=%s, resource_version=%lu, last=%ld", - uid, resource_version, last); - sqlite3_clear_bindings(ctx->stmt_insert_kubernetes_event); - sqlite3_reset(ctx->stmt_insert_kubernetes_event); - - flb_sds_destroy(uid); - return flb_sqldb_last_id(ctx->db); -} - -#endif - -static int k8s_events_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - int ret; - size_t b_sent; - struct flb_connection *u_conn = NULL; - struct flb_http_client *c = NULL; - struct k8s_events *ctx = in_context; - flb_sds_t continue_token = NULL; - uint64_t max_resource_version = 0; - - if (pthread_mutex_trylock(&ctx->lock) != 0) { - FLB_INPUT_RETURN(0); - } - - u_conn = flb_upstream_conn_get(ctx->upstream); - if (!u_conn) { - flb_plg_error(ins, "upstream connection initialization error"); - goto exit; - } - - ret = refresh_token_if_needed(ctx); - if (ret == -1) { - flb_plg_error(ctx->ins, "failed to refresh token"); - goto exit; - } - - do { - c = make_event_api_request(ctx, u_conn, continue_token); - if (continue_token != NULL) { - flb_sds_destroy(continue_token); - continue_token = NULL; - } - if (!c) { - flb_plg_error(ins, "unable to create http client"); - goto exit; - } - flb_http_buffer_size(c, 0); - - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - if (ctx->auth_len > 0) { - flb_http_add_header(c, "Authorization", 13, ctx->auth, ctx->auth_len); - } - - ret = flb_http_do(c, &b_sent); - if (ret != 0) { - flb_plg_error(ins, "http do error"); - goto exit; - } - - if (c->resp.status == 200) { - ret = process_events(ctx, c->resp.payload, c->resp.payload_size, &max_resource_version, &continue_token); - } - else { - if (c->resp.payload_size > 0) { - flb_plg_error(ctx->ins, "http_status=%i:\n%s", c->resp.status, c->resp.payload); - } - else { - flb_plg_error(ctx->ins, "http_status=%i", c->resp.status); - } - } - flb_http_client_destroy(c); - c = NULL; - } while(continue_token != NULL); - - if (max_resource_version > ctx->last_resource_version) { - flb_plg_debug(ctx->ins, "set last resourceVersion=%lu", max_resource_version); - ctx->last_resource_version = max_resource_version; - } - -exit: - pthread_mutex_unlock(&ctx->lock); - if (c) { - flb_http_client_destroy(c); - } - if (u_conn) { - flb_upstream_conn_release(u_conn); - } - FLB_INPUT_RETURN(0); -} - -static int k8s_events_init(struct flb_input_instance *ins, - struct flb_config *config, void *data) -{ - struct k8s_events *ctx = NULL; - - ctx = k8s_events_conf_create(ins); - if (!ctx) { - return -1; - } - - ctx->coll_id = flb_input_set_collector_time(ins, - k8s_events_collect, - ctx->interval_sec, - ctx->interval_nsec, - config); - -#ifdef FLB_HAVE_SQLDB - if (ctx->db) { - ctx->coll_cleanup_id = flb_input_set_collector_time(ins, - k8s_events_cleanup_db, - ctx->interval_sec, - ctx->interval_nsec, - config); - } -#endif - - return 0; -} - -static int k8s_events_exit(void *data, struct flb_config *config) -{ - struct k8s_events *ctx = data; - - if (!ctx) { - return 0; - } - - k8s_events_conf_destroy(ctx); - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - /* Full Kubernetes API server URL */ - { - FLB_CONFIG_MAP_STR, "kube_url", "https://kubernetes.default.svc", - 0, FLB_FALSE, 0, - "Kubernetes API server URL" - }, - - /* Refresh interval */ - { - FLB_CONFIG_MAP_INT, "interval_sec", DEFAULT_INTERVAL_SEC, - 0, FLB_TRUE, offsetof(struct k8s_events, interval_sec), - "Set the polling interval for each channel" - }, - { - FLB_CONFIG_MAP_INT, "interval_nsec", DEFAULT_INTERVAL_NSEC, - 0, FLB_TRUE, offsetof(struct k8s_events, interval_nsec), - "Set the polling interval for each channel (sub seconds)" - }, - - /* TLS: set debug 'level' */ - { - FLB_CONFIG_MAP_INT, "tls.debug", "0", - 0, FLB_TRUE, offsetof(struct k8s_events, tls_debug), - "set TLS debug level: 0 (no debug), 1 (error), " - "2 (state change), 3 (info) and 4 (verbose)" - }, - - /* TLS: enable verification */ - { - FLB_CONFIG_MAP_BOOL, "tls.verify", "true", - 0, FLB_TRUE, offsetof(struct k8s_events, tls_verify), - "enable or disable verification of TLS peer certificate" - }, - - /* TLS: set tls.vhost feature */ - { - FLB_CONFIG_MAP_STR, "tls.vhost", NULL, - 0, FLB_TRUE, offsetof(struct k8s_events, tls_vhost), - "set optional TLS virtual host" - }, - - /* Kubernetes TLS: CA file */ - { - FLB_CONFIG_MAP_STR, "kube_ca_file", K8S_EVENTS_KUBE_CA, - 0, FLB_TRUE, offsetof(struct k8s_events, tls_ca_file), - "Kubernetes TLS CA file" - }, - - /* Kubernetes TLS: CA certs path */ - { - FLB_CONFIG_MAP_STR, "kube_ca_path", NULL, - 0, FLB_TRUE, offsetof(struct k8s_events, tls_ca_path), - "Kubernetes TLS ca path" - }, - - /* Kubernetes Token file */ - { - FLB_CONFIG_MAP_STR, "kube_token_file", K8S_EVENTS_KUBE_TOKEN, - 0, FLB_TRUE, offsetof(struct k8s_events, token_file), - "Kubernetes authorization token file" - }, - - /* Kubernetes Token file TTL */ - { - FLB_CONFIG_MAP_TIME, "kube_token_ttl", "10m", - 0, FLB_TRUE, offsetof(struct k8s_events, token_ttl), - "kubernetes token ttl, until it is reread from the token file. Default: 10m" - }, - - { - FLB_CONFIG_MAP_INT, "kube_request_limit", "0", - 0, FLB_TRUE, offsetof(struct k8s_events, limit_request), - "kubernetes limit parameter for events query, no limit applied when set to 0" - }, - - { - FLB_CONFIG_MAP_TIME, "kube_retention_time", "1h", - 0, FLB_TRUE, offsetof(struct k8s_events, retention_time), - "kubernetes retention time for events. Default: 1h" - }, - - { - FLB_CONFIG_MAP_STR, "kube_namespace", NULL, - 0, FLB_TRUE, offsetof(struct k8s_events, namespace), - "kubernetes namespace to get events from, gets event from all namespaces by default." - }, - -#ifdef FLB_HAVE_SQLDB - { - FLB_CONFIG_MAP_STR, "db", NULL, - 0, FLB_FALSE, 0, - "set a database file to keep track of recorded kubernetes events." - }, - { - FLB_CONFIG_MAP_STR, "db.sync", "normal", - 0, FLB_FALSE, 0, - "set a database sync method. values: extra, full, normal and off." - }, -#endif - - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_input_plugin in_kubernetes_events_plugin = { - .name = "kubernetes_events", - .description = "Kubernetes Events", - .cb_init = k8s_events_init, - .cb_pre_run = NULL, - .cb_collect = k8s_events_collect, - .cb_flush_buf = NULL, - .cb_exit = k8s_events_exit, - .config_map = config_map, - .flags = FLB_INPUT_NET | FLB_INPUT_CORO | FLB_INPUT_THREADED -}; diff --git a/fluent-bit/plugins/in_kubernetes_events/kubernetes_events.h b/fluent-bit/plugins/in_kubernetes_events/kubernetes_events.h deleted file mode 100644 index 3afd48570..000000000 --- a/fluent-bit/plugins/in_kubernetes_events/kubernetes_events.h +++ /dev/null @@ -1,106 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_KUBERNETES_EVENTS_H -#define FLB_IN_KUBERNETES_EVENTS_H - -#include -#include -#include -#include -#include - -#define DEFAULT_INTERVAL_SEC "0" -#define DEFAULT_INTERVAL_NSEC "500000000" - -/* Filter context */ -struct k8s_events { - int coll_id; - int coll_cleanup_id; - int interval_sec; /* interval collection time (Second) */ - int interval_nsec; /* interval collection time (Nanosecond) */ - int retention_time; /* retention time limit, default 1 hour */ - - /* Configuration parameters */ - char *api_host; - int api_port; - int api_https; - int tls_debug; - int tls_verify; - int kube_token_ttl; - flb_sds_t namespace; - - /* API Server end point */ - char kube_url[1024]; - - /* TLS CA certificate file */ - char *tls_ca_path; - char *tls_ca_file; - - /* TLS virtual host (optional), set by configmap */ - flb_sds_t tls_vhost; - - /* Kubernetes Token from FLB_KUBE_TOKEN file */ - char *token_file; - char *token; - int token_ttl; - size_t token_len; - int token_created; - - /* Pre-formatted HTTP Authorization header value */ - char *auth; - size_t auth_len; - - int dns_retries; - int dns_wait_time; - - struct flb_tls *tls; - - struct flb_log_event_encoder *encoder; - - /* record accessor */ - struct flb_record_accessor *ra_timestamp; - struct flb_record_accessor *ra_resource_version; - - /* others */ - struct flb_config *config; - struct flb_upstream *upstream; - struct flb_input_instance *ins; - - /* limit for event queries */ - int limit_request; - /* last highest seen resource_version */ - uint64_t last_resource_version; - -#ifdef FLB_HAVE_SQLDB - /* State database */ - struct flb_sqldb *db; - int db_sync; - int db_locking; - flb_sds_t db_journal_mode; - sqlite3_stmt *stmt_get_kubernetes_event_exists_by_uid; - sqlite3_stmt *stmt_insert_kubernetes_event; - sqlite3_stmt *stmt_delete_old_kubernetes_events; -#endif - - /* concurrency lock */ - pthread_mutex_t lock; -}; - -#endif \ No newline at end of file diff --git a/fluent-bit/plugins/in_kubernetes_events/kubernetes_events_conf.c b/fluent-bit/plugins/in_kubernetes_events/kubernetes_events_conf.c deleted file mode 100644 index 4f67d8cfc..000000000 --- a/fluent-bit/plugins/in_kubernetes_events/kubernetes_events_conf.c +++ /dev/null @@ -1,326 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "kubernetes_events_conf.h" - -#ifdef FLB_HAVE_SQLDB -#include "kubernetes_events_sql.h" - - -/* Open or create database required by tail plugin */ -static struct flb_sqldb *flb_kubernetes_event_db_open(const char *path, - struct flb_input_instance *in, - struct k8s_events *ctx, - struct flb_config *config) -{ - int ret; - char tmp[64]; - struct flb_sqldb *db; - - /* Open/create the database */ - db = flb_sqldb_open(path, in->name, config); - if (!db) { - return NULL; - } - - /* Create table schema if it don't exists */ - ret = flb_sqldb_query(db, SQL_CREATE_KUBERNETES_EVENTS, NULL, NULL); - if (ret != FLB_OK) { - flb_plg_error(ctx->ins, "db: could not create 'in_kubernetes_events' table"); - flb_sqldb_close(db); - return NULL; - } - - if (ctx->db_sync >= 0) { - snprintf(tmp, sizeof(tmp) - 1, SQL_PRAGMA_SYNC, - ctx->db_sync); - ret = flb_sqldb_query(db, tmp, NULL, NULL); - if (ret != FLB_OK) { - flb_plg_error(ctx->ins, "db could not set pragma 'sync'"); - flb_sqldb_close(db); - return NULL; - } - } - - if (ctx->db_locking == FLB_TRUE) { - ret = flb_sqldb_query(db, SQL_PRAGMA_LOCKING_MODE, NULL, NULL); - if (ret != FLB_OK) { - flb_plg_error(ctx->ins, "db: could not set pragma 'locking_mode'"); - flb_sqldb_close(db); - return NULL; - } - } - - if (ctx->db_journal_mode) { - snprintf(tmp, sizeof(tmp) - 1, SQL_PRAGMA_JOURNAL_MODE, - ctx->db_journal_mode); - ret = flb_sqldb_query(db, tmp, NULL, NULL); - if (ret != FLB_OK) { - flb_plg_error(ctx->ins, "db could not set pragma 'journal_mode'"); - flb_sqldb_close(db); - return NULL; - } - } - - return db; -} - -static int flb_kubernetes_event_db_close(struct flb_sqldb *db) -{ - flb_sqldb_close(db); - return 0; -} - -#endif - -static int network_init(struct k8s_events *ctx, struct flb_config *config) -{ - int io_type = FLB_IO_TCP; - - ctx->upstream = NULL; - - if (ctx->api_https == FLB_TRUE) { - if (!ctx->tls_ca_path && !ctx->tls_ca_file) { - ctx->tls_ca_file = flb_strdup(K8S_EVENTS_KUBE_CA); - } - - /* create a custom TLS context since we use user-defined certs */ - ctx->tls = flb_tls_create(FLB_TLS_CLIENT_MODE, - ctx->tls_verify, - ctx->tls_debug, - ctx->tls_vhost, - ctx->tls_ca_path, - ctx->tls_ca_file, - NULL, NULL, NULL); - if (!ctx->tls) { - return -1; - } - - io_type = FLB_IO_TLS; - } - - /* Create an Upstream context */ - ctx->upstream = flb_upstream_create(config, - ctx->api_host, - ctx->api_port, - io_type, - ctx->tls); - if (!ctx->upstream) { - flb_plg_error(ctx->ins, "network initialization failed"); - return -1; - } - - return 0; -} - -struct k8s_events *k8s_events_conf_create(struct flb_input_instance *ins) -{ - int off; - int ret; - const char *p; - const char *url; - const char *tmp; - struct k8s_events *ctx = NULL; - pthread_mutexattr_t attr; - - ctx = flb_calloc(1, sizeof(struct k8s_events)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - - pthread_mutexattr_init(&attr); - pthread_mutex_init(&ctx->lock, &attr); - - /* Load the config map */ - ret = flb_input_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return NULL; - } - flb_input_set_context(ins, ctx); - - ctx->encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); - if (!ctx->encoder) { - flb_plg_error(ins, "could not initialize event encoder"); - k8s_events_conf_destroy(ctx); - return NULL; - } - - /* Record accessor pattern */ - ctx->ra_timestamp = flb_ra_create(K8S_EVENTS_RA_TIMESTAMP, FLB_TRUE); - if (!ctx->ra_timestamp) { - flb_plg_error(ctx->ins, - "could not create record accessor for metadata items"); - k8s_events_conf_destroy(ctx); - return NULL; - } - - ctx->ra_resource_version = flb_ra_create(K8S_EVENTS_RA_RESOURCE_VERSION, FLB_TRUE); - if (!ctx->ra_resource_version) { - flb_plg_error(ctx->ins, "could not create record accessor for resource version"); - k8s_events_conf_destroy(ctx); - return NULL; - } - - /* Get Kubernetes API server */ - url = flb_input_get_property("kube_url", ins); - if (!url) { - ctx->api_host = flb_strdup(K8S_EVENTS_KUBE_API_HOST); - ctx->api_port = K8S_EVENTS_KUBE_API_PORT; - ctx->api_https = FLB_TRUE; - } - else { - tmp = url; - - /* Check the protocol */ - if (strncmp(tmp, "http://", 7) == 0) { - off = 7; - ctx->api_https = FLB_FALSE; - } - else if (strncmp(tmp, "https://", 8) == 0) { - off = 8; - ctx->api_https = FLB_TRUE; - } - else { - k8s_events_conf_destroy(ctx); - return NULL; - } - - /* Get hostname and TCP port */ - p = url + off; - tmp = strchr(p, ':'); - if (tmp) { - ctx->api_host = flb_strndup(p, tmp - p); - tmp++; - ctx->api_port = atoi(tmp); - } - else { - ctx->api_host = flb_strdup(p); - ctx->api_port = K8S_EVENTS_KUBE_API_PORT; - } - } - snprintf(ctx->kube_url, sizeof(ctx->kube_url) - 1, - "%s://%s:%i", - ctx->api_https ? "https" : "http", - ctx->api_host, ctx->api_port); - - flb_plg_info(ctx->ins, "API server: %s", ctx->kube_url); - - /* network setup */ - ret = network_init(ctx, ins->config); - if (ret == -1) { - k8s_events_conf_destroy(ctx); - return NULL; - } - -#ifdef FLB_HAVE_SQLDB - /* Initialize database */ - tmp = flb_input_get_property("db", ins); - if (tmp) { - ctx->db = flb_kubernetes_event_db_open(tmp, ins, ctx, ins->config); - if (!ctx->db) { - flb_plg_error(ctx->ins, "could not open/create database"); - k8s_events_conf_destroy(ctx); - return NULL; - } - } - - if (ctx->db) { - ret = sqlite3_prepare_v2(ctx->db->handler, - SQL_KUBERNETES_EVENT_EXISTS_BY_UID, - -1, - &ctx->stmt_get_kubernetes_event_exists_by_uid, - 0); - if (ret != SQLITE_OK) { - flb_plg_error(ctx->ins, "error preparing database SQL statement: stmt_get_kubernetes_event_exists_by_uid"); - k8s_events_conf_destroy(ctx); - return NULL; - } - - ret = sqlite3_prepare_v2(ctx->db->handler, - SQL_INSERT_KUBERNETES_EVENTS, - -1, - &ctx->stmt_insert_kubernetes_event, - 0); - if (ret != SQLITE_OK) { - flb_plg_error(ctx->ins, "error preparing database SQL statement: stmt_insert_kubernetes_event"); - k8s_events_conf_destroy(ctx); - return NULL; - } - - ret = sqlite3_prepare_v2(ctx->db->handler, - SQL_DELETE_OLD_KUBERNETES_EVENTS, - -1, - &ctx->stmt_delete_old_kubernetes_events, - 0); - if (ret != SQLITE_OK) { - flb_plg_error(ctx->ins, "error preparing database SQL statement: stmt_delete_old_kubernetes_events"); - k8s_events_conf_destroy(ctx); - return NULL; - } - } -#endif - - return ctx; -} - -void k8s_events_conf_destroy(struct k8s_events *ctx) -{ - if (ctx->ra_timestamp) { - flb_ra_destroy(ctx->ra_timestamp); - } - - if (ctx->ra_resource_version) { - flb_ra_destroy(ctx->ra_resource_version); - } - - if (ctx->upstream) { - flb_upstream_destroy(ctx->upstream); - } - - if (ctx->encoder) { - flb_log_event_encoder_destroy(ctx->encoder); - } - - if (ctx->api_host) { - flb_free(ctx->api_host); - } - if (ctx->token) { - flb_free(ctx->token); - } - if (ctx->auth) { - flb_free(ctx->auth); - } - -#ifdef FLB_HAVE_TLS - if (ctx->tls) { - flb_tls_destroy(ctx->tls); - } -#endif - -#ifdef FLB_HAVE_SQLDB - if (ctx->db) { - flb_kubernetes_event_db_close(ctx->db); - } -#endif - - flb_free(ctx); -} diff --git a/fluent-bit/plugins/in_kubernetes_events/kubernetes_events_conf.h b/fluent-bit/plugins/in_kubernetes_events/kubernetes_events_conf.h deleted file mode 100644 index 9d6b54197..000000000 --- a/fluent-bit/plugins/in_kubernetes_events/kubernetes_events_conf.h +++ /dev/null @@ -1,47 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_KUBERNETES_EVENTS_CONF_H -#define FLB_IN_KUBERNETES_EVENTS_CONF_H - -#include -#include -#include - -#include "kubernetes_events.h" - -/* Kubernetes API server info */ -#define K8S_EVENTS_KUBE_API_HOST "kubernetes.default.svc" -#define K8S_EVENTS_KUBE_API_PORT 443 -// /apis/events.k8s.io/v1/events -// /apis/events.k8s.io/v1/namespaces/{namespace}/events -#define K8S_EVENTS_KUBE_API_URI "/api/v1/events" -#define K8S_EVENTS_KUBE_NAMESPACE_API_URI "/api/v1/namespaces/%s/events" - -/* secrets */ -#define K8S_EVENTS_KUBE_TOKEN "/var/run/secrets/kubernetes.io/serviceaccount/token" -#define K8S_EVENTS_KUBE_CA "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" - -#define K8S_EVENTS_RA_TIMESTAMP "$metadata['creationTimestamp']" -#define K8S_EVENTS_RA_RESOURCE_VERSION "$metadata['resourceVersion']" - -struct k8s_events *k8s_events_conf_create(struct flb_input_instance *ins); -void k8s_events_conf_destroy(struct k8s_events *ctx); - -#endif \ No newline at end of file diff --git a/fluent-bit/plugins/in_kubernetes_events/kubernetes_events_sql.h b/fluent-bit/plugins/in_kubernetes_events/kubernetes_events_sql.h deleted file mode 100644 index 3076791cc..000000000 --- a/fluent-bit/plugins/in_kubernetes_events/kubernetes_events_sql.h +++ /dev/null @@ -1,60 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_KUBERNETES_EVENTS_SQL_H -#define FLB_KUBERNETES_EVENTS_SQL_H - -/* - * In Fluent Bit we try to have a common convention for table names, - * if the table belongs to an input/output plugin, use the plugins name - * with the name of the object or type. - * - * in_kubernetes_events plugin table to track kubernetes events: - * in_kubernetes_events - */ -#define SQL_CREATE_KUBERNETES_EVENTS \ - "CREATE TABLE IF NOT EXISTS in_kubernetes_events (" \ - " id INTEGER PRIMARY KEY," \ - " uid TEXT NOT NULL," \ - " resourceVersion INTEGER NOT NULL," \ - " created INTEGER NOT NULL" \ - ");" - -#define SQL_KUBERNETES_EVENT_EXISTS_BY_UID \ - "SELECT COUNT(id) " \ - " FROM in_kubernetes_events " \ - " WHERE uid=@uid;" - -#define SQL_INSERT_KUBERNETES_EVENTS \ - "INSERT INTO in_kubernetes_events (uid, resourceVersion, created)" \ - " VALUES (@uid, @resourceVersion, @created);" - -#define SQL_DELETE_OLD_KUBERNETES_EVENTS \ - "DELETE FROM in_kubernetes_events WHERE created <= @createdBefore;" - -#define SQL_PRAGMA_SYNC \ - "PRAGMA synchronous=%i;" - -#define SQL_PRAGMA_JOURNAL_MODE \ - "PRAGMA journal_mode=%s;" - -#define SQL_PRAGMA_LOCKING_MODE \ - "PRAGMA locking_mode=EXCLUSIVE;" - -#endif diff --git a/fluent-bit/plugins/in_lib/CMakeLists.txt b/fluent-bit/plugins/in_lib/CMakeLists.txt deleted file mode 100644 index 87a19c5b2..000000000 --- a/fluent-bit/plugins/in_lib/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -# FIXME: there is something wrong when linking objects and this -# static plugin, I should not require to link to a specific symbol -# if the object was already linked from fluent-bit core on src/, also -# jsmn should not be required. - -set(src - in_lib.c - ../../src/flb_pack.c) - -FLB_PLUGIN(in_lib "${src}" "jsmn") diff --git a/fluent-bit/plugins/in_lib/in_lib.c b/fluent-bit/plugins/in_lib/in_lib.c deleted file mode 100644 index 466f1afe8..000000000 --- a/fluent-bit/plugins/in_lib/in_lib.c +++ /dev/null @@ -1,279 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include "in_lib.h" - -static int in_lib_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - int ret; - int dec_ret; - int enc_ret; - int bytes; - int out_size; - int capacity; - int size; - char *ptr; - char *pack; - struct flb_log_event record; - struct flb_log_event_decoder decoder; - struct flb_in_lib_config *ctx = in_context; - - capacity = (ctx->buf_size - ctx->buf_len); - - /* Allocate memory as required (FIXME: this will be limited in later) */ - if (capacity == 0) { - size = ctx->buf_size + LIB_BUF_CHUNK; - ptr = flb_realloc(ctx->buf_data, size); - if (!ptr) { - flb_errno(); - return -1; - } - ctx->buf_data = ptr; - ctx->buf_size = size; - capacity = LIB_BUF_CHUNK; - } - - bytes = flb_pipe_r(ctx->fd, - ctx->buf_data + ctx->buf_len, - capacity); - flb_plg_trace(ctx->ins, "in_lib read() = %i", bytes); - if (bytes == -1) { - perror("read"); - if (errno == -EPIPE) { - return -1; - } - return 0; - } - ctx->buf_len += bytes; - - /* initially we should support json input */ - ret = flb_pack_json_state(ctx->buf_data, ctx->buf_len, - &pack, &out_size, &ctx->state); - if (ret == FLB_ERR_JSON_PART) { - flb_plg_warn(ctx->ins, "lib data incomplete, waiting for more data..."); - return 0; - } - else if (ret == FLB_ERR_JSON_INVAL) { - flb_plg_warn(ctx->ins, "lib data invalid"); - flb_pack_state_reset(&ctx->state); - flb_pack_state_init(&ctx->state); - return -1; - } - ctx->buf_len = 0; - - dec_ret = flb_log_event_decoder_init(&decoder, pack, out_size); - if (dec_ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %s", - flb_log_event_decoder_get_error_description(dec_ret)); - flb_free(pack); - flb_pack_state_reset(&ctx->state); - flb_pack_state_init(&ctx->state); - return -1; - } - - while ((dec_ret = flb_log_event_decoder_next( - &decoder, - &record)) == FLB_EVENT_DECODER_SUCCESS) { - enc_ret = flb_log_event_encoder_begin_record(&ctx->log_encoder); - if (enc_ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, - "flb_log_event_encoder_begin_record error : %s", - flb_log_event_encoder_get_error_description(enc_ret)); - flb_log_event_encoder_rollback_record(&ctx->log_encoder); - continue; - } - - enc_ret = flb_log_event_encoder_set_timestamp( - &ctx->log_encoder, - &record.timestamp); - if (enc_ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, - "flb_log_event_encoder_set_timestamp error : %s", - flb_log_event_encoder_get_error_description(enc_ret)); - flb_log_event_encoder_rollback_record(&ctx->log_encoder); - continue; - } - - enc_ret = flb_log_event_encoder_set_metadata_from_msgpack_object( - &ctx->log_encoder, - record.metadata); - if (enc_ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, - "flb_log_event_encoder_set_metadata_from_msgpack_object error : %s", - flb_log_event_encoder_get_error_description(enc_ret)); - flb_log_event_encoder_rollback_record(&ctx->log_encoder); - continue; - } - - enc_ret = flb_log_event_encoder_set_body_from_msgpack_object( - &ctx->log_encoder, - record.body); - if (enc_ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, - "flb_log_event_encoder_set_body_from_msgpack_object error : %s", - flb_log_event_encoder_get_error_description(enc_ret)); - flb_log_event_encoder_rollback_record(&ctx->log_encoder); - continue; - } - - enc_ret = flb_log_event_encoder_commit_record(&ctx->log_encoder); - if (enc_ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, - "flb_log_event_encoder_commit_record error : %s", - flb_log_event_encoder_get_error_description(enc_ret)); - flb_log_event_encoder_rollback_record(&ctx->log_encoder); - continue; - } - } - - dec_ret = flb_log_event_decoder_get_last_result(&decoder); - if (dec_ret == FLB_EVENT_DECODER_SUCCESS) { - flb_input_log_append(ctx->ins, NULL, 0, - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - - ret = 0; - } - else { - flb_plg_error(ctx->ins, - "flb_log_event_decoder_get_last_result error : %s", - flb_log_event_decoder_get_error_description(dec_ret)); - ret = -1; - } - - flb_log_event_encoder_reset(&ctx->log_encoder); - flb_log_event_decoder_destroy(&decoder); - - /* Reset the state */ - flb_free(pack); - - flb_pack_state_reset(&ctx->state); - flb_pack_state_init(&ctx->state); - - return ret; -} - -/* Initialize plugin */ -static int in_lib_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - int ret; - struct flb_in_lib_config *ctx; - (void) data; - - /* Allocate space for the configuration */ - ctx = flb_malloc(sizeof(struct flb_in_lib_config)); - if (!ctx) { - return -1; - } - ctx->ins = in; - - /* Buffer for incoming data */ - ctx->buf_size = LIB_BUF_CHUNK; - ctx->buf_data = flb_calloc(1, LIB_BUF_CHUNK); - ctx->buf_len = 0; - - if (!ctx->buf_data) { - flb_errno(); - flb_plg_error(ctx->ins, "Could not allocate initial buf memory buffer"); - flb_free(ctx); - return -1; - } - - /* Init communication channel */ - flb_input_channel_init(in); - ctx->fd = in->channel[0]; - - /* Set the context */ - flb_input_set_context(in, ctx); - - /* Collect upon data available on the standard input */ - ret = flb_input_set_collector_event(in, - in_lib_collect, - ctx->fd, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "Could not set collector for LIB input plugin"); - flb_free(ctx->buf_data); - flb_free(ctx); - return -1; - } - - ret = flb_log_event_encoder_init(&ctx->log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, "error initializing event encoder : %d", ret); - - flb_free(ctx->buf_data); - flb_free(ctx); - - return -1; - } - - flb_pack_state_init(&ctx->state); - - return 0; -} - -static int in_lib_exit(void *data, struct flb_config *config) -{ - struct flb_in_lib_config *ctx = data; - struct flb_pack_state *s; - - (void) config; - - flb_log_event_encoder_destroy(&ctx->log_encoder); - - if (ctx->buf_data) { - flb_free(ctx->buf_data); - } - - s = &ctx->state; - flb_pack_state_reset(s); - flb_free(ctx); - return 0; -} - -/* Plugin reference */ -struct flb_input_plugin in_lib_plugin = { - .name = "lib", - .description = "Library mode Input", - .cb_init = in_lib_init, - .cb_pre_run = NULL, - .cb_collect = NULL, - .cb_ingest = NULL, - .cb_flush_buf = NULL, - .cb_exit = in_lib_exit -}; diff --git a/fluent-bit/plugins/in_lib/in_lib.h b/fluent-bit/plugins/in_lib/in_lib.h deleted file mode 100644 index a5fc8a9ea..000000000 --- a/fluent-bit/plugins/in_lib/in_lib.h +++ /dev/null @@ -1,45 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_LIB_H -#define FLB_IN_LIB_H - -#include -#include -#include -#include -#include - -#define LIB_BUF_CHUNK 65536 - -pthread_key_t flb_active_lib_context; - -/* Library input configuration & context */ -struct flb_in_lib_config { - int fd; /* instance input channel */ - int buf_size; /* buffer size / capacity */ - int buf_len; /* read buffer length */ - char *buf_data; /* the real buffer */ - - struct flb_log_event_encoder log_encoder; - struct flb_pack_state state; - struct flb_input_instance *ins; -}; - -#endif diff --git a/fluent-bit/plugins/in_mem/CMakeLists.txt b/fluent-bit/plugins/in_mem/CMakeLists.txt deleted file mode 100644 index 613abd69f..000000000 --- a/fluent-bit/plugins/in_mem/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -set(src - mem.c - proc.c) - -FLB_PLUGIN(in_mem "${src}" "") diff --git a/fluent-bit/plugins/in_mem/mem.c b/fluent-bit/plugins/in_mem/mem.c deleted file mode 100644 index 391ba6144..000000000 --- a/fluent-bit/plugins/in_mem/mem.c +++ /dev/null @@ -1,320 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "mem.h" -#include "proc.h" - -struct flb_input_plugin in_mem_plugin; - -static int in_mem_collect(struct flb_input_instance *i_ins, - struct flb_config *config, void *in_context); -#if 0 -/* Locate a specific key into the buffer */ -static char *field(char *data, char *field) -{ - char *p; - char *q; - char *sep; - char *value; - int len = strlen(field); - - p = strstr(data, field); - if (!p) { - return NULL; - } - - sep = strchr(p, ':'); - p = ++sep; - p++; - - while (*p == ' ') p++; - - q = strchr(p, ' '); - len = q - p; - value = flb_malloc(len + 1); - strncpy(value, p, len); - value[len] = '\0'; - - return value; -} -#endif - -static uint64_t calc_kb(unsigned long amount, unsigned int unit) -{ - unsigned long long bytes = amount; - - /* - * Recent Linux versions return memory/swap sizes as multiples - * of a certain size unit. See sysinfo(2) for details. - */ - if (unit > 1) { - bytes = bytes * unit; - } - - bytes = bytes / 1024; - - return (uint64_t) bytes; -} - -static int mem_calc(struct flb_in_mem_info *m_info) -{ - int ret; - struct sysinfo info; - - ret = sysinfo(&info); - if (ret == -1) { - flb_errno(); - return -1; - } - - /* set values in KBs */ - m_info->mem_total = calc_kb(info.totalram, info.mem_unit); - - /* - * This value seems to be MemAvailable if it is supported - * or MemFree on legacy Linux. - */ - m_info->mem_free = calc_kb(info.freeram, info.mem_unit); - - m_info->mem_used = m_info->mem_total - m_info->mem_free; - - m_info->swap_total = calc_kb(info.totalswap, info.mem_unit); - m_info->swap_free = calc_kb(info.freeswap, info.mem_unit); - m_info->swap_used = m_info->swap_total - m_info->swap_free; - - return 0; -} - -static int in_mem_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - int ret; - struct flb_in_mem_config *ctx; - (void) data; - - /* Initialize context */ - ctx = flb_malloc(sizeof(struct flb_in_mem_config)); - if (!ctx) { - return -1; - } - ctx->idx = 0; - ctx->pid = 0; - ctx->page_size = sysconf(_SC_PAGESIZE); - ctx->ins = in; - - /* Load the config map */ - ret = flb_input_config_map_set(in, (void *)ctx); - if (ret == -1) { - flb_free(ctx); - return -1; - } - - /* Collection time setting */ - if (ctx->interval_sec <= 0) { - ctx->interval_sec = atoi(DEFAULT_INTERVAL_SEC); - } - if (ctx->interval_nsec <= 0) { - ctx->interval_nsec = atoi(DEFAULT_INTERVAL_NSEC); - } - - /* Set the context */ - flb_input_set_context(in, ctx); - - /* Set the collector */ - ret = flb_input_set_collector_time(in, - in_mem_collect, - ctx->interval_sec, - ctx->interval_nsec, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not set collector for memory input plugin"); - return -1; - } - - ret = flb_log_event_encoder_init(&ctx->log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, "error initializing event encoder : %d", ret); - - return -1; - } - - return 0; -} - -static int in_mem_collect(struct flb_input_instance *i_ins, - struct flb_config *config, void *in_context) -{ - int ret; - struct proc_task *task = NULL; - struct flb_in_mem_config *ctx = in_context; - struct flb_in_mem_info info; - - if (ctx->pid) { - task = proc_stat(ctx->pid, ctx->page_size); - if (!task) { - flb_plg_warn(ctx->ins, "could not measure PID %i", ctx->pid); - ctx->pid = 0; - } - } - - ret = mem_calc(&info); - - if (ret == -1) { - if (task) { - proc_free(task); - } - return -1; - } - - ret = flb_log_event_encoder_begin_record(&ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp( - &ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - &ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE("Mem.total"), - FLB_LOG_EVENT_UINT64_VALUE(info.mem_total), - - FLB_LOG_EVENT_CSTRING_VALUE("Mem.used"), - FLB_LOG_EVENT_UINT64_VALUE(info.mem_used), - - FLB_LOG_EVENT_CSTRING_VALUE("Mem.free"), - FLB_LOG_EVENT_UINT64_VALUE(info.mem_free), - - FLB_LOG_EVENT_CSTRING_VALUE("Swap.total"), - FLB_LOG_EVENT_UINT64_VALUE(info.swap_total), - - FLB_LOG_EVENT_CSTRING_VALUE("Swap.used"), - FLB_LOG_EVENT_UINT64_VALUE(info.swap_used), - - FLB_LOG_EVENT_CSTRING_VALUE("Swap.free"), - FLB_LOG_EVENT_UINT64_VALUE(info.swap_free)); - } - - if (task != NULL && - ret == FLB_EVENT_ENCODER_SUCCESS) { - /* RSS bytes */ - - ret = flb_log_event_encoder_append_body_values( - &ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE("proc_bytes"), - FLB_LOG_EVENT_UINT64_VALUE(task->proc_rss), - - FLB_LOG_EVENT_CSTRING_VALUE("proc_hr"), - FLB_LOG_EVENT_UINT64_VALUE(task->proc_rss_hr)); - - proc_free(task); - } - - flb_plg_trace(ctx->ins, "memory total=%lu kb, used=%lu kb, free=%lu kb", - info.mem_total, info.mem_used, info.mem_free); - flb_plg_trace(ctx->ins, "swap total=%lu kb, used=%lu kb, free=%lu kb", - info.swap_total, info.swap_used, info.swap_free); - ++ctx->idx; - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(&ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(i_ins, NULL, 0, - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - - ret = 0; - } - else { - flb_plg_error(i_ins, "Error encoding record : %d", ret); - - ret = -1; - } - - flb_log_event_encoder_reset(&ctx->log_encoder); - - return 0; -} - -static int in_mem_exit(void *data, struct flb_config *config) -{ - (void) *config; - struct flb_in_mem_config *ctx = data; - - if (!ctx) { - return 0; - } - - flb_log_event_encoder_destroy(&ctx->log_encoder); - - /* done */ - flb_free(ctx); - - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_INT, "interval_sec", DEFAULT_INTERVAL_SEC, - 0, FLB_TRUE, offsetof(struct flb_in_mem_config, interval_sec), - "Set the collector interval" - }, - { - FLB_CONFIG_MAP_INT, "interval_nsec", DEFAULT_INTERVAL_NSEC, - 0, FLB_TRUE, offsetof(struct flb_in_mem_config, interval_nsec), - "Set the collector interval (subseconds)" - }, - { - FLB_CONFIG_MAP_INT, "pid", "0", - 0, FLB_TRUE, offsetof(struct flb_in_mem_config, pid), - "Set the PID of the process to measure" - }, - /* EOF */ - {0} -}; - -struct flb_input_plugin in_mem_plugin = { - .name = "mem", - .description = "Memory Usage", - .cb_init = in_mem_init, - .cb_pre_run = NULL, - .cb_collect = in_mem_collect, - .cb_flush_buf = NULL, - .cb_exit = in_mem_exit, - .config_map = config_map -}; diff --git a/fluent-bit/plugins/in_mem/mem.h b/fluent-bit/plugins/in_mem/mem.h deleted file mode 100644 index 3c28ff907..000000000 --- a/fluent-bit/plugins/in_mem/mem.h +++ /dev/null @@ -1,51 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_MEM_H -#define FLB_IN_MEM_H - -#include -#include -#include -#include -#include - -#define DEFAULT_INTERVAL_SEC "1" -#define DEFAULT_INTERVAL_NSEC "0" - -struct flb_in_mem_info { - uint64_t mem_total; - uint64_t mem_used; - uint64_t mem_free; - uint64_t swap_total; - uint64_t swap_used; - uint64_t swap_free; -}; - -struct flb_in_mem_config { - int idx; - int page_size; - int interval_sec; - int interval_nsec; - pid_t pid; - struct flb_input_instance *ins; - struct flb_log_event_encoder log_encoder; -}; - -#endif diff --git a/fluent-bit/plugins/in_mem/proc.c b/fluent-bit/plugins/in_mem/proc.c deleted file mode 100644 index 0e70b9de1..000000000 --- a/fluent-bit/plugins/in_mem/proc.c +++ /dev/null @@ -1,185 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include -#include "proc.h" - -static char *human_readable_size(long size) -{ - long u = 1024, i, len = 128; - char *buf; - static const char *__units[] = { "b", "K", "M", "G", - "T", "P", "E", "Z", "Y", NULL - }; - - buf = flb_malloc(len); - if (!buf) { - flb_errno(); - return NULL; - } - - for (i = 0; __units[i] != NULL; i++) { - if ((size / u) == 0) { - break; - } - u *= 1024; - } - if (!i) { - snprintf(buf, len, "%ld %s", size, __units[0]); - } - else { - float fsize = (float) ((double) size / (u / 1024)); - snprintf(buf, len, "%.2f%s", fsize, __units[i]); - } - - return buf; -} - -/* Read file content into a memory buffer */ -static char *file_to_buffer(const char *path) -{ - FILE *fp; - char *buffer; - - if (!(fp = fopen(path, "r"))) { - flb_errno(); - return NULL; - } - - buffer = flb_calloc(1, PROC_STAT_BUF_SIZE); - if (!buffer) { - fclose(fp); - flb_errno(); - return NULL; - } - - fread(buffer, PROC_STAT_BUF_SIZE, 1, fp); - if (ferror(fp) || !feof(fp)) { - flb_free(buffer); - fclose(fp); - return NULL; - } - - fclose(fp); - return buffer; -} - - -struct proc_task *proc_stat(pid_t pid, int page_size) -{ - int ret; - char *p, *q; - char *buf; - char pid_path[PROC_PID_SIZE]; - struct proc_task *t; - - t = flb_calloc(1, sizeof(struct proc_task)); - if (!t) { - flb_errno(); - return NULL; - } - - /* Compose path for /proc/PID/stat */ - ret = snprintf(pid_path, PROC_PID_SIZE, "/proc/%i/stat", pid); - if (ret < 0) { - flb_free(t); - flb_errno(); - return NULL; - } - - buf = file_to_buffer(pid_path); - if (!buf) { - flb_free(t); - return NULL; - } - - sscanf(buf, "%d", &t->pid); - - /* - * workaround for process with spaces in the name, so we dont screw up - * sscanf(3). - */ - p = buf; - while (*p != '(') { - p++; - } - p++; - - /* seek from tail of file. */ - q = buf + (PROC_STAT_BUF_SIZE - 1); - while (*q != ')' && p < q) { - q--; - } - if (p >= q) { - flb_free(buf); - flb_free(t); - return NULL; - } - - strncpy(t->comm, p, q - p); - q += 2; - - /* Read pending values */ - sscanf(q, PROC_STAT_FORMAT, - &t->state, - &t->ppid, - &t->pgrp, - &t->session, - &t->tty_nr, - &t->tpgid, - &t->flags, - &t->minflt, - &t->cminflt, - &t->majflt, - &t->cmajflt, - &t->utime, - &t->stime, - &t->cutime, - &t->cstime, - &t->priority, - &t->nice, - &t->num_threads, - &t->itrealvalue, - &t->starttime, - &t->vsize, - &t->rss); - - /* Internal conversion */ - t->proc_rss = (t->rss * page_size); - t->proc_rss_hr = human_readable_size(t->proc_rss); - if ( t->proc_rss_hr == NULL ) { - flb_free(buf); - flb_free(t); - return NULL; - } - - flb_free(buf); - return t; -} - -void proc_free(struct proc_task *t) -{ - flb_free(t->proc_rss_hr); - flb_free(t); -} diff --git a/fluent-bit/plugins/in_mem/proc.h b/fluent-bit/plugins/in_mem/proc.h deleted file mode 100644 index 79b4c3b81..000000000 --- a/fluent-bit/plugins/in_mem/proc.h +++ /dev/null @@ -1,68 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef IN_MEM_PROC_H -#define IN_MEM_PROC_H - -#define PROC_PID_SIZE 1024 -#define PROC_STAT_BUF_SIZE 1024 - -/* - * This 'stat' format omits the first two fields, due to the nature - * of sscanf(3) and whitespaces, programs with spaces in the name can - * screw up when scanning the information. - */ -#define PROC_STAT_FORMAT "%c %d %d %d %d %d %u %lu %lu %lu %lu %lu %lu %ld %ld %ld %ld %ld %ld %llu %lu %ld" - -/* Our tast struct to read the /proc/PID/stat values */ -struct proc_task { - int pid; /* %d */ - char comm[256]; /* %s */ - char state; /* %c */ - int ppid; /* %d */ - int pgrp; /* %d */ - int session; /* %d */ - int tty_nr; /* %d */ - int tpgid; /* %d */ - unsigned int flags; /* %u */ - unsigned long minflt; /* %lu */ - unsigned long cminflt; /* %lu */ - unsigned long majflt; /* %lu */ - unsigned long cmajflt; /* %lu */ - unsigned long utime; /* %lu */ - unsigned long stime; /* %lu */ - long cutime; /* %ld */ - long cstime; /* %ld */ - long priority; /* %ld */ - long nice; /* %ld */ - long num_threads; /* %ld */ - long itrealvalue; /* %ld */ - unsigned long long starttime; /* %llu */ - unsigned long vsize; /* %lu */ - long rss; /* %ld */ - - /* Internal conversion */ - long proc_rss; /* bytes = (rss * PAGESIZE) */ - char *proc_rss_hr; /* RSS in human readable format */ -}; - -struct proc_task *proc_stat(pid_t pid, int page_size); -void proc_free(struct proc_task *t); - -#endif diff --git a/fluent-bit/plugins/in_mqtt/CMakeLists.txt b/fluent-bit/plugins/in_mqtt/CMakeLists.txt deleted file mode 100644 index 53259d541..000000000 --- a/fluent-bit/plugins/in_mqtt/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -set(src - mqtt.c - mqtt_conn.c - mqtt_prot.c - mqtt_config.c) - -FLB_PLUGIN(in_mqtt "${src}" "") diff --git a/fluent-bit/plugins/in_mqtt/mqtt.c b/fluent-bit/plugins/in_mqtt/mqtt.c deleted file mode 100644 index d1ae74f1e..000000000 --- a/fluent-bit/plugins/in_mqtt/mqtt.c +++ /dev/null @@ -1,162 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include - -#include "mqtt.h" -#include "mqtt_conn.h" -#include "mqtt_config.h" - -/* Initialize plugin */ -static int in_mqtt_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - unsigned short int port; - int ret; - struct flb_in_mqtt_config *ctx; - - (void) data; - - /* Allocate space for the configuration */ - ctx = mqtt_config_init(in); - if (!ctx) { - return -1; - } - ctx->ins = in; - ctx->msgp_len = 0; - - /* Set the context */ - flb_input_set_context(in, ctx); - - /* Create downstream */ - port = (unsigned short int) strtoul(ctx->tcp_port, NULL, 10); - - ctx->downstream = flb_downstream_create(FLB_TRANSPORT_TCP, - in->flags, - ctx->listen, - port, - in->tls, - config, - &in->net_setup); - - if (ctx->downstream == NULL) { - flb_plg_error(ctx->ins, - "could not initialize downstream on %s:%s. Aborting", - ctx->listen, ctx->tcp_port); - - mqtt_config_free(ctx); - - return -1; - } - - flb_input_downstream_set(ctx->downstream, ctx->ins); - - /* Collect upon data available on the standard input */ - ret = flb_input_set_collector_event(in, - in_mqtt_collect, - ctx->downstream->server_fd, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not set collector for MQTT input plugin"); - mqtt_config_free(ctx); - return -1; - } - - return 0; -} - -/* - * For a server event, the collection event means a new client have arrived, we - * accept the connection and create a new MQTT instance which will wait for - * events/data (MQTT control packages) - */ -int in_mqtt_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_connection *connection; - struct mqtt_conn *conn; - struct flb_in_mqtt_config *ctx; - - ctx = in_context; - - connection = flb_downstream_conn_get(ctx->downstream); - - if (connection == NULL) { - flb_plg_error(ctx->ins, "could not accept new connection"); - - return -1; - } - - flb_plg_debug(ctx->ins, "[fd=%i] new TCP connection", connection->fd); - - conn = mqtt_conn_add(connection, ctx); - - if (!conn) { - flb_downstream_conn_release(connection); - - return -1; - } - - return 0; -} - -static int in_mqtt_exit(void *data, struct flb_config *config) -{ - (void) *config; - struct flb_in_mqtt_config *ctx = data; - - if (!ctx) { - return 0; - } - - mqtt_conn_destroy_all(ctx); - - mqtt_config_free(ctx); - - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "payload_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_in_mqtt_config, payload_key), - "Key where the payload will be preserved" - }, - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_input_plugin in_mqtt_plugin = { - .name = "mqtt", - .description = "MQTT, listen for Publish messages", - .cb_init = in_mqtt_init, - .cb_pre_run = NULL, - .cb_collect = in_mqtt_collect, - .cb_flush_buf = NULL, - .cb_exit = in_mqtt_exit, - .config_map = config_map, - .flags = FLB_INPUT_NET_SERVER | FLB_IO_OPT_TLS -}; diff --git a/fluent-bit/plugins/in_mqtt/mqtt.h b/fluent-bit/plugins/in_mqtt/mqtt.h deleted file mode 100644 index 01c3b7be9..000000000 --- a/fluent-bit/plugins/in_mqtt/mqtt.h +++ /dev/null @@ -1,45 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_MQTT_H -#define FLB_IN_MQTT_H - -#include -#include - -#define MQTT_MSGP_BUF_SIZE 8192 - -struct flb_in_mqtt_config { - char *listen; /* Listen interface */ - char *tcp_port; /* TCP Port */ - - flb_sds_t payload_key; /* payload key */ - - int msgp_len; /* msgpack data length */ - char msgp[MQTT_MSGP_BUF_SIZE]; /* msgpack static buffer */ - struct flb_input_instance *ins; /* plugin input instance */ - struct flb_downstream *downstream; /* Client manager */ - struct mk_list conns; /* Active connections */ - struct flb_log_event_encoder *log_encoder; -}; - -int in_mqtt_collect(struct flb_input_instance *i_ins, - struct flb_config *config, void *in_context); - -#endif diff --git a/fluent-bit/plugins/in_mqtt/mqtt_config.c b/fluent-bit/plugins/in_mqtt/mqtt_config.c deleted file mode 100644 index 800834c05..000000000 --- a/fluent-bit/plugins/in_mqtt/mqtt_config.c +++ /dev/null @@ -1,82 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include -#include - -#include "mqtt.h" -#include "mqtt_config.h" - -struct flb_in_mqtt_config *mqtt_config_init(struct flb_input_instance *ins) -{ - char tmp[16]; - struct flb_in_mqtt_config *config; - int ret; - - config = flb_calloc(1, sizeof(struct flb_in_mqtt_config)); - if (!config) { - flb_errno(); - return NULL; - } - - ret = flb_input_config_map_set(ins, (void*) config); - if (ret == -1) { - flb_plg_error(ins, "could not initialize config map"); - flb_free(config); - return NULL; - } - - config->log_encoder = flb_log_event_encoder_create( - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (config->log_encoder == NULL) { - flb_plg_error(ins, "could not initialize event encoder"); - mqtt_config_free(config); - - return NULL; - } - - /* Listen interface (if not set, defaults to 0.0.0.0) */ - flb_input_net_default_listener("0.0.0.0", 1883, ins); - - /* Map 'listen' and 'port' into the local context */ - config->listen = ins->host.listen; - snprintf(tmp, sizeof(tmp) - 1, "%d", ins->host.port); - config->tcp_port = flb_strdup(tmp); - - mk_list_init(&config->conns); - return config; -} - -void mqtt_config_free(struct flb_in_mqtt_config *config) -{ - if (config->downstream != NULL) { - flb_downstream_destroy(config->downstream); - } - - if (config->log_encoder != NULL) { - flb_log_event_encoder_destroy(config->log_encoder); - } - - flb_free(config->tcp_port); - flb_free(config); -} diff --git a/fluent-bit/plugins/in_mqtt/mqtt_config.h b/fluent-bit/plugins/in_mqtt/mqtt_config.h deleted file mode 100644 index 709c3dd95..000000000 --- a/fluent-bit/plugins/in_mqtt/mqtt_config.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_MQTT_CONFIG_H -#define FLB_MQTT_CONFIG_H - -#include "mqtt.h" -#include - -struct flb_in_mqtt_config *mqtt_config_init(struct flb_input_instance *in); -void mqtt_config_free(struct flb_in_mqtt_config *config); - -#endif diff --git a/fluent-bit/plugins/in_mqtt/mqtt_conn.c b/fluent-bit/plugins/in_mqtt/mqtt_conn.c deleted file mode 100644 index 32ade2f6e..000000000 --- a/fluent-bit/plugins/in_mqtt/mqtt_conn.c +++ /dev/null @@ -1,157 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include - -#include "mqtt.h" -#include "mqtt_prot.h" -#include "mqtt_conn.h" - -/* Callback invoked every time an event is triggered for a connection */ -int mqtt_conn_event(void *data) -{ - int ret; - int bytes; - int available; - struct mk_event *event; - struct mqtt_conn *conn; - struct flb_in_mqtt_config *ctx; - struct flb_connection *connection; - - connection = (struct flb_connection *) data; - - conn = connection->user_data; - - ctx = conn->ctx; - - event = &connection->event; - - if (event->mask & MK_EVENT_READ) { - available = sizeof(conn->buf) - conn->buf_len; - - bytes = flb_io_net_read(connection, - (void *) &conn->buf[conn->buf_len], - available); - - if (bytes > 0) { - conn->buf_len += bytes; - flb_plg_trace(ctx->ins, "[fd=%i] read()=%i bytes", - connection->fd, - bytes); - - ret = mqtt_prot_parser(conn); - if (ret < 0) { - mqtt_conn_del(conn); - return -1; - } - } - else { - flb_plg_debug(ctx->ins, "[fd=%i] connection closed", - connection->fd); - - mqtt_conn_del(conn); - } - } - else if (event->mask & MK_EVENT_CLOSE) { - flb_plg_debug(ctx->ins, "[fd=%i] hangup", event->fd); - } - - return 0; -} - -/* Create a new mqtt request instance */ -struct mqtt_conn *mqtt_conn_add(struct flb_connection *connection, - struct flb_in_mqtt_config *ctx) -{ - struct mqtt_conn *conn; - int ret; - - conn = flb_malloc(sizeof(struct mqtt_conn)); - if (!conn) { - flb_errno(); - return NULL; - } - - conn->connection = connection; - - /* Set data for the event-loop */ - MK_EVENT_NEW(&connection->event); - - connection->user_data = conn; - connection->event.type = FLB_ENGINE_EV_CUSTOM; - connection->event.handler = mqtt_conn_event; - - /* Connection info */ - conn->ctx = ctx; - conn->buf_pos = 0; - conn->buf_len = 0; - conn->buf_frame_end = 0; - conn->status = MQTT_NEW; - - /* Register instance into the event loop */ - ret = mk_event_add(flb_engine_evl_get(), - connection->fd, - FLB_ENGINE_EV_CUSTOM, - MK_EVENT_READ, - &connection->event); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not register new connection"); - flb_free(conn); - - return NULL; - } - - mk_list_add(&conn->_head, &ctx->conns); - - return conn; -} - -int mqtt_conn_del(struct mqtt_conn *conn) -{ - /* The downstream unregisters the file descriptor from the event-loop - * so there's nothing to be done by the plugin - */ - flb_downstream_conn_release(conn->connection); - - /* Release resources */ - mk_list_del(&conn->_head); - - flb_free(conn); - - return 0; -} - -int mqtt_conn_destroy_all(struct flb_in_mqtt_config *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - struct mqtt_conn *conn; - - mk_list_foreach_safe(head, tmp, &ctx->conns) { - conn = mk_list_entry(head, struct mqtt_conn, _head); - mqtt_conn_del(conn); - } - - return 0; -} diff --git a/fluent-bit/plugins/in_mqtt/mqtt_conn.h b/fluent-bit/plugins/in_mqtt/mqtt_conn.h deleted file mode 100644 index 43f98f09e..000000000 --- a/fluent-bit/plugins/in_mqtt/mqtt_conn.h +++ /dev/null @@ -1,49 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_MQTT_CONN_H -#define FLB_MQTT_CONN_H - -#include - -enum { - MQTT_NEW = 1, /* it's a new connection */ - MQTT_CONNECTED = 2, /* MQTT connection per protocol spec OK */ - MQTT_NEXT = 4 /* Waiting for Control packets */ -}; - -/* This structure respresents a MQTT connection */ -struct mqtt_conn { - int status; /* Connection status */ - int packet_type; /* MQTT packet type */ - int packet_length; - int buf_frame_end; /* Frame end position */ - int buf_pos; /* Index position */ - int buf_len; /* Buffer content length */ - unsigned char buf[1024]; /* Buffer data */ - struct flb_in_mqtt_config *ctx; /* Plugin configuration context */ - struct flb_connection *connection; - struct mk_list _head; /* Link to flb_in_mqtt_config->conns */ -}; - -struct mqtt_conn *mqtt_conn_add(struct flb_connection *connection, struct flb_in_mqtt_config *ctx); -int mqtt_conn_del(struct mqtt_conn *conn); -int mqtt_conn_destroy_all(struct flb_in_mqtt_config *ctx); - -#endif diff --git a/fluent-bit/plugins/in_mqtt/mqtt_prot.c b/fluent-bit/plugins/in_mqtt/mqtt_prot.c deleted file mode 100644 index e0267daf2..000000000 --- a/fluent-bit/plugins/in_mqtt/mqtt_prot.c +++ /dev/null @@ -1,465 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "mqtt.h" -#include "mqtt_prot.h" - -#define BUFC() conn->buf[conn->buf_pos] -#define BUF_AVAIL() conn->buf_len - conn->buf_pos -#define BIT_SET(a, b) ((a) |= (1 << (b))) -#define BIT_CHECK(a, b) ((a) & (1 << (b))) - -/* -static inline void print_hex(struct mqtt_conn *conn) -{ - int x; - - printf("\n--------HEX--------> "); - printf("buf_pos=%i buf_len=%i\n", conn->buf_pos, conn->buf_len); - for (x = conn->buf_pos; x < conn->buf_len; x++) { - printf("%x ", conn->buf[x]); - } - printf("\n--------------------\n\n"); -} - -static inline void print_str(struct mqtt_conn *conn) -{ - int x; - - printf("\n--------HEX--------> "); - printf("buf_pos=%i buf_len=%i\n", conn->buf_pos, conn->buf_len); - for (x = conn->buf_pos; x < conn->buf_len; x++) { - printf("%c", conn->buf[x]); - } - printf("\n--------------------\n\n"); -} -*/ - -/* - * It drop the current packet from the buffer, it move the remaining bytes - * from right-to-left and adjust the new length. - */ -static inline int mqtt_packet_drop(struct mqtt_conn *conn) -{ - int move_bytes; - - if (conn->buf_pos == conn->buf_len) { - conn->buf_frame_end = 0; - conn->buf_len = 0; - conn->buf_pos = 0; - return 0; - } - - /* Check boundaries */ - if (conn->buf_pos + 1 > conn->buf_len) { - conn->buf_frame_end = 0; - conn->buf_len = 0; - conn->buf_pos = 0; - return 0; - } - - move_bytes = conn->buf_pos + 1; - memmove(conn->buf, - conn->buf + move_bytes, - conn->buf_len - move_bytes); - - conn->buf_frame_end = 0; - conn->buf_len -= move_bytes; - conn->buf_pos = 0; - - return 0; -} - -/* - * It writes the packet control header which includes the packet type - * and the remaining length of the packet. The incoming buffer must have - * at least 6 bytes of space. - * - * The function returns the number of bytes used. - */ -static inline int mqtt_packet_header(int type, int length, char *buf) -{ - int i = 0; - uint8_t byte; - - buf[i] = (type << 4) | 0; - i++; - - do { - byte = length % 128; - length = (length / 128); - if (length > 0) { - byte = (byte | 128); - } - buf[i] = byte; - i++; - } while (length > 0); - - return i; -} - -/* Collect a buffer of JSON data and convert it to Fluent Bit format */ -static int mqtt_data_append(char *topic, size_t topic_len, - char *msg, int msg_len, - void *in_context) -{ - int i; - int ret; - int root_type; - size_t out; - size_t off = 0; - char *pack; - msgpack_object root; - msgpack_unpacked result; - struct flb_in_mqtt_config *ctx = in_context; - - /* Convert our incoming JSON to MsgPack */ - ret = flb_pack_json(msg, msg_len, &pack, &out, &root_type, NULL); - if (ret != 0) { - flb_plg_warn(ctx->ins, "MQTT Packet incomplete or is not JSON"); - return -1; - } - - off = 0; - msgpack_unpacked_init(&result); - if (msgpack_unpack_next(&result, pack, out, &off) != MSGPACK_UNPACK_SUCCESS) { - msgpack_unpacked_destroy(&result); - return -1; - } - - if (result.data.type != MSGPACK_OBJECT_MAP){ - msgpack_unpacked_destroy(&result); - return -1; - } - root = result.data; - - - ret = flb_log_event_encoder_begin_record(ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp(ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE("topic"), - FLB_LOG_EVENT_STRING_VALUE(topic, topic_len)); - } - - if (ctx->payload_key) { - flb_log_event_encoder_append_body_string_length(ctx->log_encoder, flb_sds_len(ctx->payload_key)); - flb_log_event_encoder_append_body_string_body(ctx->log_encoder, ctx->payload_key, - flb_sds_len(ctx->payload_key)); - flb_log_event_encoder_body_begin_map(ctx->log_encoder); - } - - /* Re-pack original KVs */ - for (i = 0; - i < root.via.map.size && - ret == FLB_EVENT_ENCODER_SUCCESS; - i++) { - ret = flb_log_event_encoder_append_body_values( - ctx->log_encoder, - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&root.via.map.ptr[i].key), - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&root.via.map.ptr[i].val)); - } - - if (ctx->payload_key) { - flb_log_event_encoder_body_commit_map(ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(ctx->ins, NULL, 0, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); - ret = 0; - } - else { - flb_plg_error(ctx->ins, "log event encoding error : %d", ret); - - ret = -1; - } - - flb_log_event_encoder_reset(ctx->log_encoder); - - msgpack_unpacked_destroy(&result); - flb_free(pack); - - return ret; -} - - -/* - * Handle a CONNECT request control packet: - * - * basically we need to acknoledge the sender so it can start - * publishing messages to our service. - */ -static int mqtt_handle_connect(struct mqtt_conn *conn) -{ - int i; - int ret; - size_t sent; - char buf[4] = {0, 0, 0, 0}; - struct flb_in_mqtt_config *ctx = conn->ctx; - - i = mqtt_packet_header(MQTT_CONNACK, 2 , (char *) &buf); - BIT_SET(buf[i], 0); - i++; - buf[i] = MQTT_CONN_ACCEPTED; - - /* write CONNACK message */ - ret = flb_io_net_write(conn->connection, - (void *) buf, - 4, - &sent); - - flb_plg_trace(ctx->ins, "[fd=%i] CMD CONNECT (connack=%i bytes)", - conn->connection->fd, ret); - - return ret; -} - -/* - * Handle a PUBLISH control packet - */ -static int mqtt_handle_publish(struct mqtt_conn *conn) -{ - int topic; - int topic_len; - uint8_t qos; - size_t sent; - uint16_t hlen; - uint16_t packet_id; - char buf[4]; - struct flb_in_mqtt_config *ctx = conn->ctx; - - /* - * DUP: we skip duplicated messages. - * QOS: We process this. - * Retain: skipped - */ - - qos = ((conn->buf[0] >> 1) & 0x03); - conn->buf_pos++; - - /* Topic */ - hlen = BUFC() << 8; - conn->buf_pos++; - hlen |= BUFC(); - - /* Validate topic length against current buffer capacity (overflow) */ - if (hlen > (conn->buf_len - conn->buf_pos)) { - flb_plg_debug(ctx->ins, "invalid topic length"); - return -1; - } - - conn->buf_pos++; - topic = conn->buf_pos; - topic_len = hlen; - conn->buf_pos += hlen; - - /* Check QOS flag and respond if required */ - if (qos > MQTT_QOS_LEV0) { - /* Packet Identifier */ - packet_id = BUFC() << 8; - conn->buf_pos++; - packet_id |= BUFC(); - conn->buf_pos++; - - if (qos == MQTT_QOS_LEV1) { - mqtt_packet_header(MQTT_PUBACK, 2 , (char *) &buf); - } - else if (qos == MQTT_QOS_LEV2) { - mqtt_packet_header(MQTT_PUBREC, 2 , (char *) &buf); - } - /* Set the identifier that we are replying to */ - buf[2] = (packet_id >> 8) & 0xff; - buf[3] = (packet_id & 0xff); - - /* This operation should be checked */ - flb_io_net_write(conn->connection, - (void *) buf, - 4, - &sent); - } - - /* Message */ - mqtt_data_append((char *) (conn->buf + topic), topic_len, - (char *) (conn->buf + conn->buf_pos), - conn->buf_frame_end - conn->buf_pos + 1, - conn->ctx); - - flb_plg_trace(ctx->ins, "[fd=%i] CMD PUBLISH", - conn->connection->fd); - return 0; -} - -/* Handle a PINGREQ control packet */ -static int mqtt_handle_ping(struct mqtt_conn *conn) -{ - int ret; - size_t sent; - char buf[2] = {0, 0}; - struct flb_in_mqtt_config *ctx = conn->ctx; - - mqtt_packet_header(MQTT_PINGRESP, 0 , (char *) &buf); - - /* write PINGRESP message */ - - ret = flb_io_net_write(conn->connection, - (void *) buf, - 2, - &sent); - - flb_plg_trace(ctx->ins, "[fd=%i] CMD PING (pong=%i bytes)", - conn->connection->fd, ret); - return ret; -} - -int mqtt_prot_parser(struct mqtt_conn *conn) -{ - int ret; - int length = 0; - int pos = conn->buf_pos; - int mult; - struct flb_in_mqtt_config *ctx = conn->ctx; - - for (; conn->buf_pos < conn->buf_len; conn->buf_pos++) { - if (conn->status & (MQTT_NEW | MQTT_NEXT)) { - /* - * Do we have at least the Control Packet fixed header - * and the remaining length byte field ? - */ - if (BUF_AVAIL() < 2) { - conn->buf_pos = pos; - flb_plg_trace(ctx->ins, "[fd=%i] Need more data", - conn->connection->fd); - return MQTT_MORE; - } - - /* As the connection is new we expect a MQTT_CONNECT request */ - conn->packet_type = BUFC() >> 4; - if (conn->status == MQTT_NEW && conn->packet_type != MQTT_CONNECT) { - flb_plg_trace(ctx->ins, "[fd=%i] error, expecting MQTT_CONNECT", - conn->connection->fd); - return MQTT_ERROR; - } - conn->packet_length = conn->buf_pos; - conn->buf_pos++; - - /* Get the remaining length */ - mult = 1; - length = 0; - - do { - if (conn->buf_pos + 1 > conn->buf_len) { - conn->buf_pos = pos; - flb_plg_trace(ctx->ins, "[fd=%i] Need more data", - conn->connection->fd); - return MQTT_MORE; - } - - length += (BUFC() & 127) * mult; - mult *= 128; - if (mult > 128*128*128) { - return MQTT_ERROR; - } - - if (length + 2 > (conn->buf_len - pos)) { - conn->buf_pos = pos; - flb_plg_trace(ctx->ins, "[fd=%i] Need more data", - conn->connection->fd); - return MQTT_MORE; - } - - if ((BUFC() & 128) == 0) { - if (conn->buf_len - 2 < length) { - conn->buf_pos = pos; - flb_plg_trace(ctx->ins, "[fd=%i] Need more data", - conn->connection->fd); - return MQTT_MORE; - } - else { - conn->buf_frame_end = conn->buf_pos + length; - break; - } - } - - if (conn->buf_pos + 1 < conn->buf_len) { - conn->buf_pos++; - } - else { - conn->buf_pos = pos; - flb_plg_trace(ctx->ins, "[fd=%i] Need more data", - conn->connection->fd); - return MQTT_MORE; - } - } while (1); - - conn->packet_length = length; - - /* At this point we have a full control packet in place */ - if (conn->packet_type == MQTT_CONNECT) { - mqtt_handle_connect(conn); - } - else if (conn->packet_type == MQTT_PUBLISH) { - ret = mqtt_handle_publish(conn); - if (ret == -1) { - return MQTT_ERROR; - } - } - else if (conn->packet_type == MQTT_PINGREQ) { - mqtt_handle_ping(conn); - } - else if (conn->packet_type == MQTT_DISCONNECT) { - flb_plg_trace(ctx->ins, "[fd=%i] CMD DISCONNECT", - conn->connection->fd); - return MQTT_HANGUP; - } - else { - } - - /* Prepare for next round */ - conn->status = MQTT_NEXT; - conn->buf_pos = conn->buf_frame_end; - - mqtt_packet_drop(conn); - - if (conn->buf_len > 0) { - conn->buf_pos = -1; - } - } - } - conn->buf_pos--; - return 0; -} diff --git a/fluent-bit/plugins/in_mqtt/mqtt_prot.h b/fluent-bit/plugins/in_mqtt/mqtt_prot.h deleted file mode 100644 index 74c4fe32e..000000000 --- a/fluent-bit/plugins/in_mqtt/mqtt_prot.h +++ /dev/null @@ -1,62 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_MQTT_PROT_H -#define FLB_MQTT_PROT_H - -#include "mqtt_conn.h" - -/* - * Specs definition from 2.2.1 MQTT Control Packet: - * - * http://docs.oasis-open.org/mqtt/mqtt/v3.1.1/os/mqtt-v3.1.1-os.html#_Toc398718021 - */ -#define MQTT_CONNECT 1 -#define MQTT_CONNACK 2 -#define MQTT_PUBLISH 3 -#define MQTT_PUBACK 4 -#define MQTT_PUBREC 5 -#define MQTT_PUBREL 6 -#define MQTT_PUBCOMP 7 -#define MQTT_PINGREQ 12 -#define MQTT_PINGRESP 13 -#define MQTT_DISCONNECT 14 - -/* CONNACK status codes */ -#define MQTT_CONN_ACCEPTED 0 -#define MQTT_CONN_REFUSED_PROTOCOL 1 -#define MQTT_CONN_REFUSED_IDENTIF 2 -#define MQTT_CONN_REFUSED_SERVER 3 -#define MQTT_CONN_REFUSED_BADCRED 4 -#define MQTT_CONN_REFUSED_NOAUTH 5 - -/* QOS Flag status */ -#define MQTT_QOS_LEV0 0 /* no reply */ -#define MQTT_QOS_LEV1 1 /* PUBACK packet */ -#define MQTT_QOS_LEV2 2 /* PUBREC packet */ - -/* Specific macros for Fluent Bit handling, not related to MQTT spec */ -#define MQTT_HANGUP -2 /* MQTT client is closing */ -#define MQTT_ERROR -1 /* MQTT protocol error, hangup */ -#define MQTT_OK 0 /* Everything is OK */ -#define MQTT_MORE 1 /* need to read more data */ - -int mqtt_prot_parser(struct mqtt_conn *conn); - -#endif diff --git a/fluent-bit/plugins/in_netif/CMakeLists.txt b/fluent-bit/plugins/in_netif/CMakeLists.txt deleted file mode 100644 index 952e7b3c8..000000000 --- a/fluent-bit/plugins/in_netif/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - in_netif.c) - -FLB_PLUGIN(in_netif "${src}" "") diff --git a/fluent-bit/plugins/in_netif/in_netif.c b/fluent-bit/plugins/in_netif/in_netif.c deleted file mode 100644 index f82c685aa..000000000 --- a/fluent-bit/plugins/in_netif/in_netif.c +++ /dev/null @@ -1,392 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include -#include "in_netif.h" - -struct entry_define entry_name_linux[] = { - {"rx.bytes", FLB_TRUE}, - {"rx.packets", FLB_TRUE}, - {"rx.errors", FLB_TRUE}, - {"rx.drop", FLB_FALSE}, - {"rx.fifo", FLB_FALSE}, - {"rx.frame", FLB_FALSE}, - {"rx.compressed", FLB_FALSE}, - {"rx.multicast", FLB_FALSE}, - {"tx.bytes", FLB_TRUE}, - {"tx.packets", FLB_TRUE}, - {"tx.errors", FLB_TRUE}, - {"tx.drop", FLB_FALSE}, - {"tx.fifo", FLB_FALSE}, - {"tx.collisions", FLB_FALSE}, - {"tx.carrier", FLB_FALSE}, - {"tx.compressepd", FLB_FALSE} -}; - -static int config_destroy(struct flb_in_netif_config *ctx) -{ - if (ctx->log_encoder != NULL) { - flb_log_event_encoder_destroy(ctx->log_encoder); - } - - flb_free(ctx->entry); - flb_free(ctx); - return 0; -} - - -static int in_netif_exit(void *data, struct flb_config *config) -{ - (void) *config; - struct flb_in_netif_config *ctx = data; - - /* Destroy context */ - config_destroy(ctx); - - return 0; -} - -static int init_entry_linux(struct flb_in_netif_config *ctx) -{ - int i; - - ctx->entry_len = sizeof(entry_name_linux) / sizeof(struct entry_define); - ctx->entry = flb_malloc(sizeof(struct netif_entry) * ctx->entry_len); - if (!ctx->entry) { - flb_errno(); - return -1; - } - - for(i = 0; i < ctx->entry_len; i++) { - ctx->entry[i].name = entry_name_linux[i].name; - ctx->entry[i].name_len = strlen(entry_name_linux[i].name); - ctx->entry[i].prev = 0; - ctx->entry[i].now = 0; - if (ctx->verbose){ - ctx->entry[i].checked = FLB_TRUE; - } - else { - ctx->entry[i].checked = entry_name_linux[i].checked; - } - if (ctx->entry[i].checked) { - ctx->map_num++; - } - } - return 0; -} - -static int configure(struct flb_in_netif_config *ctx, - struct flb_input_instance *in) -{ - int ret; - ctx->map_num = 0; - - /* Load the config map */ - ret = flb_input_config_map_set(in, (void *)ctx); - if (ret == -1) { - flb_plg_error(in, "unable to load configuration"); - return -1; - } - - if (ctx->interval_sec <= 0 && ctx->interval_nsec <= 0) { - /* Illegal settings. Override them. */ - ctx->interval_sec = atoi(DEFAULT_INTERVAL_SEC); - ctx->interval_nsec = atoi(DEFAULT_INTERVAL_NSEC); - } - - if (ctx->interface == NULL) { - flb_plg_error(ctx->ins, "'interface' is not set"); - return -1; - } - ctx->interface_len = strlen(ctx->interface); - - ctx->first_snapshot = FLB_TRUE; /* assign first_snapshot with FLB_TRUE */ - - return init_entry_linux(ctx); -} - -static inline int is_specific_interface(struct flb_in_netif_config *ctx, - char* interface) -{ - if (ctx->interface != NULL && - !strncmp(ctx->interface, interface, ctx->interface_len)) { - return FLB_TRUE; - } - return FLB_FALSE; -} - -static int parse_proc_line(char *line, - struct flb_in_netif_config *ctx) -{ - struct mk_list *head = NULL; - struct mk_list *split = NULL; - struct flb_split_entry *sentry = NULL; - - int i = 0; - int entry_num; - - split = flb_utils_split(line, ' ', 256); - entry_num = mk_list_size(split); - if (entry_num != ctx->entry_len + 1) { - flb_utils_split_free(split); - return -1; - } - - mk_list_foreach(head, split) { - sentry = mk_list_entry(head, struct flb_split_entry ,_head); - if (i==0) { - /* interface name */ - if( is_specific_interface(ctx, sentry->value)){ - i++; - continue; - } - else { - /* skip this line */ - flb_utils_split_free(split); - return -1; - } - } - ctx->entry[i-1].now = strtoul(sentry->value ,NULL ,10); - i++; - } - - flb_utils_split_free(split); - - return 0; -} - -static inline uint64_t calc_diff(struct netif_entry *entry) -{ - if (entry->prev <= entry->now) { - return entry->now - entry->prev; - } - else { - return entry->now + (UINT64_MAX - entry->prev); - } -} - -#define LINE_LEN 256 -static int read_proc_file_linux(struct flb_in_netif_config *ctx) -{ - FILE *fp = NULL; - char line[LINE_LEN] = {0}; - int interface_found = FLB_FALSE; - - fp = fopen("/proc/net/dev", "r"); - if (fp == NULL) { - flb_errno(); - flb_plg_error(ctx->ins, "cannot open /proc/net/dev"); - return -1; - } - while(fgets(line, LINE_LEN-1, fp) != NULL){ - if(parse_proc_line(line, ctx) == 0) { - interface_found = FLB_TRUE; - } - } - fclose(fp); - if (interface_found != FLB_TRUE) { - return -1; - } - return 0; -} - -static int in_netif_collect_linux(struct flb_input_instance *i_ins, - struct flb_config *config, void *in_context) -{ - struct flb_in_netif_config *ctx = in_context; - char key_name[LINE_LEN] = {0}; - int key_len; - int i; - int entry_len = ctx->entry_len; - int ret; - - ret = 0; - - read_proc_file_linux(ctx); - - if (ctx->first_snapshot == FLB_TRUE) { - /* if in_netif are called for the first time, assign prev with now */ - for (i = 0; i < entry_len; i++) { - ctx->entry[i].prev = ctx->entry[i].now; - } - - /* assign first_snapshot with FLB_FALSE */ - ctx->first_snapshot = FLB_FALSE; - } - else { - ret = flb_log_event_encoder_begin_record(ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp(ctx->log_encoder); - } - - for (i = 0 ; - i < entry_len && - ret == FLB_EVENT_ENCODER_SUCCESS ; - i++) { - if (ctx->entry[i].checked) { - key_len = ctx->interface_len + ctx->entry[i].name_len + 1/* '.' */; - - snprintf(key_name, key_len + 1 /* add null character */, - "%s.%s", ctx->interface, ctx->entry[i].name); - - ret = flb_log_event_encoder_append_body_values( - ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE(key_name), - FLB_LOG_EVENT_UINT64_VALUE(calc_diff(&ctx->entry[i]))); - - ctx->entry[i].prev = ctx->entry[i].now; - } - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(i_ins, NULL, 0, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); - ret = 0; - } - else { - flb_plg_error(i_ins, "log event encoding error : %d", ret); - - ret = -1; - } - - flb_log_event_encoder_reset(ctx->log_encoder); - } - - return ret; -} - -static int in_netif_collect(struct flb_input_instance *i_ins, - struct flb_config *config, void *in_context) -{ - return in_netif_collect_linux(i_ins, config, in_context); -} - -static int in_netif_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - int ret; - - struct flb_in_netif_config *ctx = NULL; - (void) data; - - /* Allocate space for the configuration */ - ctx = flb_calloc(1, sizeof(struct flb_in_netif_config)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->ins = in; - - ctx->log_encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ctx->log_encoder == NULL) { - flb_plg_error(in, "could not initialize event encoder"); - config_destroy(ctx); - - return -1; - } - - if (configure(ctx, in) < 0) { - config_destroy(ctx); - return -1; - } - - /* Testing interface */ - if (ctx->test_at_init == FLB_TRUE) { - /* Try to read procfs */ - ret = read_proc_file_linux(ctx); - if (ret < 0) { - flb_plg_error(in, "%s: init test failed", ctx->interface); - config_destroy(ctx); - return -1; - } - flb_plg_info(in, "%s: init test passed", ctx->interface); - } - - /* Set the context */ - flb_input_set_context(in, ctx); - - /* Set our collector based on time */ - ret = flb_input_set_collector_time(in, - in_netif_collect, - ctx->interval_sec, - ctx->interval_nsec, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "Could not set collector for Proc input plugin"); - config_destroy(ctx); - return -1; - } - - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "interface", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_in_netif_config, interface), - "Set the interface, eg: eth0 or enp1s0" - }, - { - FLB_CONFIG_MAP_INT, "interval_sec", DEFAULT_INTERVAL_SEC, - 0, FLB_TRUE, offsetof(struct flb_in_netif_config, interval_sec), - "Set the collector interval" - }, - { - FLB_CONFIG_MAP_INT, "interval_nsec", DEFAULT_INTERVAL_NSEC, - 0, FLB_TRUE, offsetof(struct flb_in_netif_config, interval_nsec), - "Set the collector interval (nanoseconds)" - }, - { - FLB_CONFIG_MAP_BOOL, "verbose", "false", - 0, FLB_TRUE, offsetof(struct flb_in_netif_config, verbose), - "Enable verbosity" - }, - { - FLB_CONFIG_MAP_BOOL, "test_at_init", "false", - 0, FLB_TRUE, offsetof(struct flb_in_netif_config, test_at_init), - "Testing interface at initialization" - }, - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_input_plugin in_netif_plugin = { - .name = "netif", - .description = "Network Interface Usage", - .cb_init = in_netif_init, - .cb_pre_run = NULL, - .cb_collect = in_netif_collect, - .cb_flush_buf = NULL, - .cb_exit = in_netif_exit, - .config_map = config_map, - .flags = 0, -}; diff --git a/fluent-bit/plugins/in_netif/in_netif.h b/fluent-bit/plugins/in_netif/in_netif.h deleted file mode 100644 index e571ec1fb..000000000 --- a/fluent-bit/plugins/in_netif/in_netif.h +++ /dev/null @@ -1,70 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_NETIF_H -#define FLB_IN_NETIF_H - -#include -#include - -#include -#include -#include - -#define DEFAULT_INTERVAL_SEC "1" -#define DEFAULT_INTERVAL_NSEC "0" - -#define FLB_IN_NETIF_NAME "in_netif" - -struct entry_define -{ - char *name; - int checked; -}; - -struct netif_entry { - int checked; - - char *name; - int name_len; - - uint64_t prev; - uint64_t now; -}; - -struct flb_in_netif_config { - int interval_sec; - int interval_nsec; - - flb_sds_t interface; - int interface_len; - int test_at_init; - - int verbose; - int first_snapshot; /* a feild to indicate whethor or not this is the first collect */ - - struct netif_entry *entry; - int entry_len; - - int map_num; - struct flb_input_instance *ins; - struct flb_log_event_encoder *log_encoder; -}; - -#endif /*FLB_IN_NETIF_H*/ diff --git a/fluent-bit/plugins/in_nginx_exporter_metrics/CMakeLists.txt b/fluent-bit/plugins/in_nginx_exporter_metrics/CMakeLists.txt deleted file mode 100644 index a74b01b2f..000000000 --- a/fluent-bit/plugins/in_nginx_exporter_metrics/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - nginx.c) - -FLB_PLUGIN(in_nginx_exporter_metrics "${src}" "") diff --git a/fluent-bit/plugins/in_nginx_exporter_metrics/nginx.c b/fluent-bit/plugins/in_nginx_exporter_metrics/nginx.c deleted file mode 100644 index bee495e23..000000000 --- a/fluent-bit/plugins/in_nginx_exporter_metrics/nginx.c +++ /dev/null @@ -1,2363 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "nginx.h" - -/** - * parse the output of the nginx stub_status module. - * - * An example: - * Active connections: 1 - * server accepts handled requests - * 10 10 10 - * Reading: 0 Writing: 1 Waiting: 0 - * - * Would result in: - * struct nginx_status = { - * active = 1, - * reading = 0, - * writing = 1, - * waiting = 0 - * accepts = 10, - * handled = 10, - * requests = 10 - *} - */ -static int nginx_parse_stub_status(flb_sds_t buf, struct nginx_status *status) -{ - struct mk_list *llines; - struct mk_list *head = NULL; - char *lines[4]; - int line = 0; - int rc; - struct flb_split_entry *cur = NULL; - - - llines = flb_utils_split(buf, '\n', 4); - if (llines == NULL) { - return -1; - } - - mk_list_foreach(head, llines) { - cur = mk_list_entry(head, struct flb_split_entry, _head); - lines[line] = cur->value; - line++; - } - if (line < 4) { - goto error; - } - - rc = sscanf(lines[0], "Active connections: %" PRIu64 " \n", &status->active); - if (rc != 1) { - goto error; - } - rc = sscanf(lines[2], " %" PRIu64 " %" PRIu64 " %" PRIu64 " \n", - &status->accepts, &status->handled, &status->requests); - if (rc != 3) { - goto error; - } - rc = sscanf(lines[3], "Reading: %" PRIu64 " Writing: %" PRIu64 " Waiting: %" PRIu64 " \n", - &status->reading, &status->writing, &status->waiting); - if (rc != 3) { - goto error; - } - - flb_utils_split_free(llines); - return 0; -error: - flb_utils_split_free(llines); - return -1; -} - -/** - * Callback function to gather statistics from the nginx - * status module. - * - * @param ins Pointer to flb_input_instance - * @param config Pointer to flb_config - * @param in_context void Pointer used to cast to nginx_ctx - * - * @return int Always returns success - */ -static int nginx_collect_stub_status(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct nginx_ctx *ctx = (struct nginx_ctx *)in_context; - struct flb_connection *u_conn; - struct flb_http_client *client; - struct nginx_status status; - flb_sds_t data; - - size_t b_sent; - int ret = -1; - int rc = -1; - uint64_t ts = cfl_time_now(); - - - u_conn = flb_upstream_conn_get(ctx->upstream); - if (!u_conn) { - flb_plg_error(ins, "upstream connection initialization error"); - goto conn_error; - } - - client = flb_http_client(u_conn, FLB_HTTP_GET, ctx->status_url, - NULL, 0, ctx->ins->host.name, ctx->ins->host.port, NULL, 0); - if (!client) { - flb_plg_error(ins, "unable to create http client"); - goto client_error; - } - - ret = flb_http_do(client, &b_sent); - if (ret != 0) { - flb_plg_error(ins, "http do error"); - goto http_error; - } - - if (client->resp.status != 200) { - flb_plg_error(ins, "http status code error: %d", client->resp.status); - goto http_error; - } - - if (client->resp.payload_size <= 0) { - flb_plg_error(ins, "empty response"); - goto http_error; - } - - /* copy and NULL terminate the payload */ - data = flb_sds_create_size(client->resp.payload_size + 1); - if (!data) { - goto http_error; - } - memcpy(data, client->resp.payload, client->resp.payload_size); - data[client->resp.payload_size] = '\0'; - - /* work directly on the data here ... */ - if (nginx_parse_stub_status(data, &status) == -1) { - flb_plg_error(ins, "unable to parse stub status response"); - goto status_error; - } - - rc = 0; - - cmt_counter_set(ctx->connections_accepted, ts, (double)status.accepts, 0, NULL); - cmt_gauge_set(ctx->connections_active, ts, (double)status.active, 0, NULL); - cmt_counter_set(ctx->connections_handled, ts, (double)status.handled, 0, NULL); - - cmt_gauge_set(ctx->connections_reading, ts, (double)status.reading, 0, NULL); - cmt_gauge_set(ctx->connections_writing, ts, (double)status.writing, 0, NULL); - cmt_gauge_set(ctx->connections_waiting, ts, (double)status.waiting, 0, NULL); - - cmt_counter_set(ctx->connections_total, ts, (double)status.requests, 0, NULL); - -status_error: - flb_sds_destroy(data); -http_error: - flb_http_client_destroy(client); -client_error: - flb_upstream_conn_release(u_conn); -conn_error: - if (rc == 0 && ctx->is_up == FLB_FALSE) { - cmt_gauge_set(ctx->connection_up, ts, 1.0, 0, NULL); - ctx->is_up = FLB_TRUE; - } - else if (rc != 0 && ctx->is_up == FLB_TRUE) { - cmt_gauge_set(ctx->connection_up, ts, 0.0, 0, NULL); - ctx->is_up = FLB_FALSE; - } - ret = flb_input_metrics_append(ins, NULL, 0, ctx->cmt); - if (ret != 0) { - flb_plg_error(ins, "could not append metrics"); - } - - return rc; -} - - -int process_connections(void *ctx, uint64_t ts, char *buf, size_t size) -{ - struct nginx_plus_connections *plus = (struct nginx_plus_connections *)ctx; - size_t off = 0; - msgpack_unpacked result; - msgpack_object_kv *cur; - msgpack_object_str *key; - int i = 0; - - - msgpack_unpacked_init(&result); - while (msgpack_unpack_next(&result, buf, size, &off) == MSGPACK_UNPACK_SUCCESS) { - if (result.data.type == MSGPACK_OBJECT_MAP) { - for (i = 0; i < result.data.via.map.size; i++) { - - cur = &result.data.via.map.ptr[i]; - key = &cur->key.via.str; - - if (strncmp(key->ptr, "accepted", key->size) == 0) { - cmt_counter_set(plus->connections_accepted, ts, - (double)cur->val.via.i64, 0, NULL); - } - else if (strncmp(key->ptr, "dropped", key->size) == 0) { - cmt_counter_set(plus->connections_dropped, ts, - (double)cur->val.via.i64, 0, NULL); - } - else if (strncmp(key->ptr, "active", key->size) == 0) { - cmt_counter_set(plus->connections_active, ts, - (double)cur->val.via.i64, 0, NULL); - } - else if (strncmp(key->ptr, "idle", key->size) == 0) { - cmt_counter_set(plus->connections_idle, ts, - (double)cur->val.via.i64, 0, NULL); - } - } - break; - } - } - msgpack_unpacked_destroy(&result); - return 0; -} - -int process_ssl(void *ctx, uint64_t ts, char *buf, size_t size) -{ - struct nginx_plus_ssl *plus = (struct nginx_plus_ssl *)ctx; - size_t off = 0; - msgpack_unpacked result; - msgpack_object_kv *cur; - msgpack_object_str *key; - int i = 0; - - - msgpack_unpacked_init(&result); - while (msgpack_unpack_next(&result, buf, size, &off) == MSGPACK_UNPACK_SUCCESS) { - if (result.data.type == MSGPACK_OBJECT_MAP) { - for (i = 0; i < result.data.via.map.size; i++) { - cur = &result.data.via.map.ptr[i]; - key = &cur->key.via.str; - if (strncmp(key->ptr, "handshakes", key->size) == 0) { - cmt_counter_set(plus->handshakes, ts, - (double)cur->val.via.i64, 0, NULL); - } - else if (strncmp(key->ptr, "handshakes_failed", key->size) == 0) { - cmt_counter_set(plus->handshakes_failed, ts, - (double)cur->val.via.i64, 0, NULL); - } - else if (strncmp(key->ptr, "session_reuses", key->size) == 0) { - cmt_counter_set(plus->session_reuses, ts, - (double)cur->val.via.i64, 0, NULL); - } - } - break; - } - } - msgpack_unpacked_destroy(&result); - return 0; -} - -int process_http_requests(void *ctx, uint64_t ts, char *buf, size_t size) -{ - struct nginx_plus_http_requests *plus = (struct nginx_plus_http_requests *)ctx; - size_t off = 0; - msgpack_unpacked result; - msgpack_object_kv *cur; - msgpack_object_str *key; - int i = 0; - - - msgpack_unpacked_init(&result); - while (msgpack_unpack_next(&result, buf, size, &off) == MSGPACK_UNPACK_SUCCESS) { - if (result.data.type == MSGPACK_OBJECT_MAP) { - for (i = 0; i < result.data.via.map.size; i++) { - cur = &result.data.via.map.ptr[i]; - key = &cur->key.via.str; - if (strncmp(key->ptr, "total", key->size) == 0) { - cmt_counter_set(plus->total, ts, - (double)cur->val.via.i64, 0, NULL); - } - else if (strncmp(key->ptr, "current", key->size) == 0) { - cmt_counter_set(plus->current, ts, - (double)cur->val.via.i64, 0, NULL); - } - } - break; - } - } - msgpack_unpacked_destroy(&result); - return 0; -} - -static ssize_t parse_payload_json(struct nginx_ctx *nginx, void *ctx, uint64_t ts, - int (*process)(void *, uint64_t, char *, size_t), - char *payload, size_t size) -{ - int ret; - int out_size; - char *pack; - struct flb_pack_state pack_state; - - /* Initialize packer */ - flb_pack_state_init(&pack_state); - - /* Pack JSON as msgpack */ - ret = flb_pack_json_state(payload, size, - &pack, &out_size, &pack_state); - flb_pack_state_reset(&pack_state); - - /* Handle exceptions */ - if (ret == FLB_ERR_JSON_PART) { - flb_plg_warn(nginx->ins, "JSON data is incomplete, skipping"); - return -1; - } - else if (ret == FLB_ERR_JSON_INVAL) { - flb_plg_warn(nginx->ins, "invalid JSON message, skipping"); - return -1; - } - else if (ret == -1) { - return -1; - } - - /* Process the packaged JSON and return the last byte used */ - process(ctx, ts, pack, out_size); - flb_free(pack); - - return 0; -} - -/** - * Callback function to gather statistics from the nginx - * plus ngx_http module. - * - * @param ins Pointer to flb_input_instance - * @param config Pointer to flb_config - * @param in_context void Pointer used to cast to nginx_ctx - * - * @return int Always returns success - */ -static int nginx_collect_plus_connections(struct flb_input_instance *ins, - struct flb_config *config, struct nginx_ctx *ctx, uint64_t ts) -{ - struct flb_connection *u_conn; - struct flb_http_client *client; - char url[1024]; - size_t b_sent; - int ret = -1; - int rc = -1; - - - u_conn = flb_upstream_conn_get(ctx->upstream); - if (!u_conn) { - flb_plg_error(ins, "upstream connection initialization error"); - goto conn_error; - } - - snprintf(url, sizeof(url)-1, "%s/%d/connections", ctx->status_url, - ctx->nginx_plus_version); - client = flb_http_client(u_conn, FLB_HTTP_GET, url, - NULL, 0, ctx->ins->host.name, ctx->ins->host.port, NULL, 0); - if (!client) { - flb_plg_error(ins, "unable to create http client"); - goto client_error; - } - - ret = flb_http_do(client, &b_sent); - if (ret != 0) { - flb_plg_error(ins, "http do error"); - goto http_error; - } - - if (client->resp.status != 200) { - flb_plg_error(ins, "http status code error: %d", client->resp.status); - goto http_error; - } - - if (client->resp.payload_size <= 0) { - flb_plg_error(ins, "empty response"); - goto http_error; - } - - parse_payload_json(ctx, ctx->plus_connections, ts, process_connections, - client->resp.payload, client->resp.payload_size); - - rc = 0; -http_error: - flb_http_client_destroy(client); -client_error: - flb_upstream_conn_release(u_conn); -conn_error: - return rc; -} - -/** - * Callback function to gather statistics from the nginx - * plus ngx_http module. - * - * @param ins Pointer to flb_input_instance - * @param config Pointer to flb_config - * @param in_context void Pointer used to cast to nginx_ctx - * - * @return int Always returns success - */ -static int nginx_collect_plus_ssl(struct flb_input_instance *ins, - struct flb_config *config, struct nginx_ctx *ctx, uint64_t ts) -{ - struct flb_connection *u_conn; - struct flb_http_client *client; - char url[1024]; - size_t b_sent; - int ret = -1; - int rc = -1; - - - u_conn = flb_upstream_conn_get(ctx->upstream); - if (!u_conn) { - flb_plg_error(ins, "upstream connection initialization error"); - goto conn_error; - } - - snprintf(url, sizeof(url)-1, "%s/%d/ssl", ctx->status_url, ctx->nginx_plus_version); - client = flb_http_client(u_conn, FLB_HTTP_GET, url, - NULL, 0, ctx->ins->host.name, ctx->ins->host.port, NULL, 0); - if (!client) { - flb_plg_error(ins, "unable to create http client"); - goto client_error; - } - - ret = flb_http_do(client, &b_sent); - if (ret != 0) { - flb_plg_error(ins, "http do error"); - goto http_error; - } - - if (client->resp.status != 200) { - flb_plg_error(ins, "http status code error: %d", client->resp.status); - goto http_error; - } - - if (client->resp.payload_size <= 0) { - flb_plg_error(ins, "empty response"); - goto http_error; - } - - parse_payload_json(ctx, ctx->plus_ssl, ts, process_ssl, - client->resp.payload, client->resp.payload_size); - - rc = 0; -http_error: - flb_http_client_destroy(client); -client_error: - flb_upstream_conn_release(u_conn); -conn_error: - return rc; -} - -/** - * Callback function to gather statistics from the nginx - * plus ngx_http module. - * - * @param ins Pointer to flb_input_instance - * @param config Pointer to flb_config - * @param in_context void Pointer used to cast to nginx_ctx - * - * @return int Always returns success - */ -static int nginx_collect_plus_http_requests(struct flb_input_instance *ins, - struct flb_config *config, struct nginx_ctx *ctx, uint64_t ts) -{ - struct flb_connection *u_conn; - struct flb_http_client *client; - char url[1024]; - size_t b_sent; - int ret = -1; - int rc = -1; - - - u_conn = flb_upstream_conn_get(ctx->upstream); - if (!u_conn) { - flb_plg_error(ins, "upstream connection initialization error"); - goto conn_error; - } - - snprintf(url, sizeof(url)-1, "%s/%d/http/requests", ctx->status_url, - ctx->nginx_plus_version); - client = flb_http_client(u_conn, FLB_HTTP_GET, url, - NULL, 0, ctx->ins->host.name, ctx->ins->host.port, NULL, 0); - if (!client) { - flb_plg_error(ins, "unable to create http client"); - goto client_error; - } - - ret = flb_http_do(client, &b_sent); - if (ret != 0) { - flb_plg_error(ins, "http do error"); - goto http_error; - } - - if (client->resp.status != 200) { - flb_plg_error(ins, "http status code error: %d", client->resp.status); - goto http_error; - } - - if (client->resp.payload_size <= 0) { - flb_plg_error(ins, "empty response"); - goto http_error; - } - - parse_payload_json(ctx, ctx->plus_http_requests, ts, process_http_requests, - client->resp.payload, client->resp.payload_size); - - rc = 0; -http_error: - flb_http_client_destroy(client); -client_error: - flb_upstream_conn_release(u_conn); -conn_error: - return rc; -} - -void *process_server_zone(struct nginx_ctx *ctx, char *zone, uint64_t ts, - msgpack_object_map *map) -{ - msgpack_object_kv *responses; - msgpack_object_kv *cur; - msgpack_object_str *key; - int i = 0; - int x = 0; - char code[4] = { '0', 'x', 'x', 0}; - - - for (i = 0; i < map->size; i++) { - cur = &map->ptr[i]; - key = &cur->key.via.str; - if (strncmp(key->ptr, "processing", key->size) == 0) { - cmt_counter_set(ctx->server_zones->processing, ts, - (double)map->ptr[i].val.via.i64, 1, (char *[]){zone}); - } - else if (strncmp(key->ptr, "requests", key->size) == 0) { - cmt_counter_set(ctx->server_zones->requests, ts, - (double)map->ptr[i].val.via.i64, 1, (char *[]){zone}); - } - else if (strncmp(key->ptr, "discarded", key->size) == 0) { - cmt_counter_set(ctx->server_zones->discarded, ts, - (double)map->ptr[i].val.via.i64, 1, (char *[]){zone}); - } - else if (strncmp(key->ptr, "received", key->size) == 0) { - cmt_counter_set(ctx->server_zones->received, ts, - (double)map->ptr[i].val.via.i64, 1, (char *[]){zone}); - } - else if (strncmp(key->ptr, "sent", key->size) == 0) { - cmt_counter_set(ctx->server_zones->sent, ts, - (double)map->ptr[i].val.via.i64, 1, (char *[]){zone}); - } - else if (strncmp(key->ptr, "responses", key->size) == 0) { - for (x = 0; x < map->ptr[i].val.via.map.size; x++) { - responses = &map->ptr[i].val.via.map.ptr[x]; - if (responses->key.via.str.size == 3 && - responses->key.via.str.ptr[1] == 'x' && - responses->key.via.str.ptr[2] == 'x') { - code[0] = responses->key.via.str.ptr[0]; - cmt_counter_set(ctx->server_zones->responses, ts, - (double)responses->val.via.i64, - 2, (char *[]){zone, code}); - } - } - } - } - return ctx; -} - -void *process_location_zone(struct nginx_ctx *ctx, char *zone, uint64_t ts, - msgpack_object_map *map) -{ - msgpack_object_kv *responses; - msgpack_object_str *str; - int i = 0; - int x = 0; - char code[4] = { '0', 'x', 'x', 0}; - - for (i = 0; i < map->size; i++) { - - str = &map->ptr[i].key.via.str; - - if (strncmp(str->ptr, "requests", str->size) == 0) { - cmt_counter_set(ctx->location_zones->requests, ts, - (double)map->ptr[i].val.via.i64, 1, (char *[]){zone}); - } - else if (strncmp(str->ptr, "discarded", str->size) == 0) { - cmt_counter_set(ctx->location_zones->discarded, ts, - (double)map->ptr[i].val.via.i64, 1, (char *[]){zone}); - } - else if (strncmp(str->ptr, "received", str->size) == 0) { - cmt_counter_set(ctx->location_zones->received, ts, - (double)map->ptr[i].val.via.i64, 1, (char *[]){zone}); - } - else if (strncmp(str->ptr, "sent", str->size) == 0) { - cmt_counter_set(ctx->location_zones->sent, ts, - (double)map->ptr[i].val.via.i64, 1, (char *[]){zone}); - } - else if (strncmp(str->ptr, "responses", str->size) == 0) { - for (x = 0; x < map->ptr[i].val.via.map.size; x++) { - responses = &map->ptr[i].val.via.map.ptr[x]; - if (responses->key.via.str.size == 3 && - responses->key.via.str.ptr[1] == 'x' && - responses->key.via.str.ptr[2] == 'x') { - code[0] = responses->key.via.str.ptr[0]; - cmt_counter_set(ctx->location_zones->responses, ts, - (double)responses->val.via.i64, - 2, (char *[]){zone, code}); - } - } - } - } - //msgpack_unpacked_destroy(&result); - return ctx; -} - -void *process_stream_server_zone(struct nginx_ctx *ctx, char *zone, uint64_t ts, - msgpack_object_map *map) -{ - msgpack_object_kv *sessions; - msgpack_object_str *str; - int i = 0; - int x = 0; - char code[4] = { '0', 'x', 'x', 0}; - - - for (i = 0; i < map->size; i++) { - - str = &map->ptr[i].key.via.str; - - if (strncmp(str->ptr, "connections", str->size) == 0) { - cmt_counter_set(ctx->streams->connections, ts, - (double)map->ptr[i].val.via.i64, 1, (char *[]){zone}); - } - if (strncmp(str->ptr, "processing", str->size) == 0) { - cmt_counter_set(ctx->streams->processing, ts, - (double)map->ptr[i].val.via.i64, 1, (char *[]){zone}); - } - else if (strncmp(str->ptr, "discarded", str->size) == 0) { - cmt_counter_set(ctx->streams->discarded, ts, - (double)map->ptr[i].val.via.i64, 1, (char *[]){zone}); - } - else if (strncmp(str->ptr, "received", str->size) == 0) { - cmt_counter_set(ctx->streams->received, ts, - (double)map->ptr[i].val.via.i64, 1, (char *[]){zone}); - } - else if (strncmp(str->ptr, "sent", str->size) == 0) { - cmt_counter_set(ctx->streams->sent, ts, - (double)map->ptr[i].val.via.i64, 1, (char *[]){zone}); - } - else if (strncmp(str->ptr, "sessions", str->size) == 0) { - for (x = 0; x < map->ptr[i].val.via.map.size; x++) { - sessions = &map->ptr[i].val.via.map.ptr[x]; - if (sessions->key.via.str.size == 3 && - sessions->key.via.str.ptr[1] == 'x' && - sessions->key.via.str.ptr[2] == 'x') { - code[0] = sessions->key.via.str.ptr[0]; - cmt_counter_set(ctx->streams->sessions, ts, - (double)sessions->val.via.i64, - 2, (char *[]){zone, code}); - } - } - } - } - //msgpack_unpacked_destroy(&result); - return ctx; -} - -static int process_upstream_peers(struct nginx_ctx *ctx, char *backend, uint64_t ts, - msgpack_object_array *peers) -{ - int i = 0; - int p = 0; - int x = 0; - msgpack_object_map *map; - msgpack_object_kv *responses; - msgpack_object_str *key; - msgpack_object *kv; - char *server; - char code[4] = {'0', 'x', 'x', 0}; - - - for (i = 0; i < peers->size; i++) { - map = &peers->ptr[i].via.map; - for (p = 0, server = NULL; p < map->size; p++) { - key = &map->ptr[p].key.via.str; - kv = &map->ptr[p].val; - if (strncmp(key->ptr, "server", key->size) == 0) { - server = flb_calloc(1, kv->via.str.size+1); - memcpy(server, kv->via.str.ptr, kv->via.str.size); - break; - } - } - if (server == NULL) { - flb_plg_warn(ctx->ins, "no server for upstream"); - continue; - } - for (p = 0; p < map->size; p++) { - key = &map->ptr[p].key.via.str; - // initialize to zero for now to respond - // how the official exporter does... - cmt_gauge_set(ctx->upstreams->limit, ts, (double)0.0, 2, - (char *[]){backend, server}); - cmt_gauge_set(ctx->upstreams->header_time, ts, (double)0.0, 2, - (char *[]){backend, server}); - cmt_gauge_set(ctx->upstreams->response_time, ts, (double)0.0, 2, - (char *[]){backend, server}); - - if (strncmp(key->ptr, "active", key->size) == 0) { - cmt_gauge_set(ctx->upstreams->active, ts, - (double)map->ptr[p].val.via.i64, 2, - (char *[]){backend, server}); - } - else if (strncmp(key->ptr, "fails", key->size) == 0) { - cmt_counter_set(ctx->upstreams->fails, ts, - (double)map->ptr[p].val.via.i64, 2, - (char *[]){backend, server}); - } - else if (strncmp(key->ptr, "header_time", key->size) == 0) { - cmt_gauge_set(ctx->upstreams->header_time, ts, - (double)map->ptr[p].val.via.i64, 2, - (char *[]){backend, server}); - } - else if (strncmp(key->ptr, "limit", key->size) == 0) { - cmt_gauge_set(ctx->upstreams->limit, ts, - (double)map->ptr[p].val.via.i64, 2, - (char *[]){backend, server}); - } - else if (strncmp(key->ptr, "received", key->size) == 0) { - cmt_counter_set(ctx->upstreams->received, ts, - (double)map->ptr[p].val.via.i64, 2, - (char *[]){backend, server}); - } - else if (strncmp(key->ptr, "requests", key->size) == 0) { - cmt_counter_set(ctx->upstreams->requests, ts, - (double)map->ptr[p].val.via.i64, 2, - (char *[]){backend, server}); - } - else if (strncmp(key->ptr, "responses", key->size) == 0) { - for (x = 0; x < map->ptr[p].val.via.map.size; x++) { - responses = &map->ptr[p].val.via.map.ptr[x]; - if (responses->key.via.str.size == 3 && - responses->key.via.str.ptr[1] == 'x' && - responses->key.via.str.ptr[2] == 'x') { - code[0] = responses->key.via.str.ptr[0]; - cmt_counter_set(ctx->upstreams->responses, ts, - (double)responses->val.via.i64, - 3, (char *[]){backend, server, code}); - } - } - } - else if (strncmp(key->ptr, "response_time", key->size) == 0) { - cmt_gauge_set(ctx->upstreams->response_time, ts, - (double)map->ptr[p].val.via.i64, 2, - (char *[]){backend, server}); - } - else if (strncmp(key->ptr, "sent", key->size) == 0) { - cmt_counter_set(ctx->upstreams->sent, ts, - (double)map->ptr[p].val.via.i64, 2, - (char *[]){backend, server}); - } - else if (strncmp(key->ptr, "state", key->size) == 0) { - cmt_gauge_set(ctx->upstreams->state, ts, - (double)map->ptr[p].val.via.i64, 2, - (char *[]){backend, server}); - } - else if (strncmp(key->ptr, "unavail", key->size) == 0) { - cmt_counter_set(ctx->upstreams->unavail, ts, - (double)map->ptr[p].val.via.i64, 2, - (char *[]){backend, server}); - } - } - flb_free(server); - } - return 0; -} - -void *process_upstreams(struct nginx_ctx *ctx, char *backend, uint64_t ts, - msgpack_object_map *map) -{ - int i = 0; - msgpack_object_str *key; - - for (i = 0; i < map->size; i++) { - key = &map->ptr[i].key.via.str; - if (strncmp(key->ptr, "keepalives", key->size) == 0) { - cmt_gauge_set(ctx->upstreams->keepalives, ts, - (double)map->ptr[i].val.via.i64, 1, (char *[]){backend}); - } - else if (strncmp(key->ptr, "zombies", key->size) == 0) { - cmt_gauge_set(ctx->upstreams->zombies, ts, - (double)map->ptr[i].val.via.i64, 1, (char *[]){backend}); - } - // go into the peer... - else if (strncmp(key->ptr, "peers", key->size) == 0) { - process_upstream_peers(ctx, backend, ts, &map->ptr[i].val.via.array); - } - } - //msgpack_unpacked_destroy(&result); - return ctx; -} - -static int process_stream_upstream_peers(struct nginx_ctx *ctx, char *backend, - uint64_t ts, msgpack_object_array *peers) -{ - int i = 0; - int p = 0; - msgpack_object_map *map; - msgpack_object_str *key; - char *server; - - - for (i = 0; i < peers->size; i++) { - map = &peers->ptr[i].via.map; - for (p = 0, server = NULL; p < map->size; p++) { - key = &map->ptr[p].key.via.str; - if (strncmp(key->ptr, "server", key->size) == 0) { - server = flb_calloc(1, map->ptr[p].val.via.str.size+1); - memcpy(server, map->ptr[p].val.via.str.ptr, map->ptr[p].val.via.str.size); - break; - } - } - if (server == NULL) { - flb_plg_warn(ctx->ins, "no server for stream upstream"); - continue; - } - for (p = 0; p < map->size; p++) { - // initialize to zero for now to respond - // how the official exporter does... - cmt_gauge_set(ctx->stream_upstreams->limit, ts, (double)0.0, 2, - (char *[]){backend, server}); - cmt_gauge_set(ctx->stream_upstreams->response_time, ts, (double)0.0, 2, - (char *[]){backend, server}); - cmt_gauge_set(ctx->stream_upstreams->connect_time, ts, (double)0.0, 2, - (char *[]){backend, server}); - cmt_gauge_set(ctx->stream_upstreams->first_byte_time, ts, (double)0.0, 2, - (char *[]){backend, server}); - - key = &map->ptr[p].key.via.str; - if (strncmp(key->ptr, "active", key->size) == 0) { - cmt_gauge_set(ctx->stream_upstreams->active, ts, - (double)map->ptr[p].val.via.i64, 2, - (char *[]){backend, server}); - } - else if (strncmp(key->ptr, "fails", key->size) == 0) { - cmt_counter_set(ctx->stream_upstreams->fails, ts, - (double)map->ptr[p].val.via.i64, 2, - (char *[]){backend, server}); - } - else if (strncmp(key->ptr, "limit", key->size) == 0) { - cmt_gauge_set(ctx->stream_upstreams->limit, ts, - (double)map->ptr[p].val.via.i64, 2, - (char *[]){backend, server}); - } - else if (strncmp(key->ptr, "received", key->size) == 0) { - cmt_counter_set(ctx->stream_upstreams->received, ts, - (double)map->ptr[p].val.via.i64, 2, - (char *[]){backend, server}); - } - else if (strncmp(key->ptr, "connect_time", key->size) == 0) { - cmt_gauge_set(ctx->stream_upstreams->connect_time, ts, - (double)map->ptr[p].val.via.i64, 2, - (char *[]){backend, server}); - } - else if (strncmp(key->ptr, "first_byte_time", key->size) == 0) { - cmt_gauge_set(ctx->stream_upstreams->first_byte_time, ts, - (double)map->ptr[p].val.via.i64, 2, - (char *[]){backend, server}); - } - else if (strncmp(key->ptr, "connections", key->size) == 0) { - cmt_counter_set(ctx->stream_upstreams->connections, ts, - (double)map->ptr[p].val.via.i64, 2, - (char *[]){backend, server}); - } - else if (strncmp(key->ptr, "response_time", key->size) == 0) { - cmt_gauge_set(ctx->stream_upstreams->response_time, ts, - (double)map->ptr[p].val.via.i64, 2, - (char *[]){backend, server}); - } - else if (strncmp(key->ptr, "sent", key->size) == 0) { - cmt_counter_set(ctx->stream_upstreams->sent, ts, - (double)map->ptr[p].val.via.i64, 2, - (char *[]){backend, server}); - } - else if (strncmp(key->ptr, "state", key->size) == 0) { - cmt_gauge_set(ctx->stream_upstreams->state, ts, - (double)map->ptr[p].val.via.i64, 2, - (char *[]){backend, server}); - } - else if (strncmp(key->ptr, "unavail", key->size) == 0) { - cmt_counter_set(ctx->stream_upstreams->unavail, ts, - (double)map->ptr[p].val.via.i64, 2, - (char *[]){backend, server}); - } - } - flb_free(server); - } - return 0; -} - -void *process_stream_upstreams(struct nginx_ctx *ctx, char *backend, uint64_t ts, - msgpack_object_map *map) -{ - int i = 0; - msgpack_object_str *key; - - for (i = 0; i < map->size; i++) { - key = &map->ptr[i].key.via.str; - if (strncmp(key->ptr, "zombies", key->size) == 0) { - cmt_gauge_set(ctx->stream_upstreams->zombies, ts, - (double)map->ptr[i].val.via.i64, 1, (char *[]){backend}); - } - // go into the peer... - else if (strncmp(key->ptr, "peers", key->size) == 0) { - process_stream_upstream_peers(ctx, backend, ts, &map->ptr[i].val.via.array); - } - } - //msgpack_unpacked_destroy(&result); - return ctx; -} - -static ssize_t parse_payload_json_table(struct nginx_ctx *ctx, int64_t ts, - void *(*process)(struct nginx_ctx *, char *, - uint64_t, msgpack_object_map *), - char *payload, size_t size) -{ - size_t off = 0; - msgpack_unpacked result; - msgpack_object_str *name; - int i = 0; - int ret; - int out_size; - char *pack; - struct flb_pack_state pack_state; - char *zone; - - /* Initialize packer */ - flb_pack_state_init(&pack_state); - - /* Pack JSON as msgpack */ - ret = flb_pack_json_state(payload, size, &pack, &out_size, &pack_state); - flb_pack_state_reset(&pack_state); - - /* Handle exceptions */ - if (ret == FLB_ERR_JSON_PART) { - flb_plg_warn(ctx->ins, "JSON data is incomplete, skipping"); - return -1; - } - else if (ret == FLB_ERR_JSON_INVAL) { - flb_plg_warn(ctx->ins, "invalid JSON message, skipping"); - return -1; - } - else if (ret == -1) { - return -1; - } - - msgpack_unpacked_init(&result); - while (msgpack_unpack_next(&result, pack, out_size, &off) == MSGPACK_UNPACK_SUCCESS) { - if (result.data.type == MSGPACK_OBJECT_MAP) { - for (i = 0; i < result.data.via.map.size; i++) { - name = &result.data.via.map.ptr[i].key.via.str; - zone = flb_calloc(1, name->size+1); - memcpy(zone, name->ptr, name->size); - process(ctx, zone, ts, &result.data.via.map.ptr[i].val.via.map); - flb_free(zone); - } - } else { - msgpack_object_print(stdout, result.data); - } - } - - flb_free(pack); - return 0; -} - -/** - * Callback function to gather statistics from the nginx - * plus ngx_http module. - * - * @param ins Pointer to flb_input_instance - * @param config Pointer to flb_config - * @param in_context void Pointer used to cast to nginx_ctx - * - * @return int Always returns success - */ -static int nginx_collect_plus_server_zones(struct flb_input_instance *ins, - struct flb_config *config, struct nginx_ctx *ctx, uint64_t ts) -{ - struct flb_connection *u_conn; - struct flb_http_client *client; - char url[1024]; - size_t b_sent; - int ret = -1; - int rc = -1; - - - u_conn = flb_upstream_conn_get(ctx->upstream); - if (!u_conn) { - flb_plg_error(ins, "upstream connection initialization error"); - goto conn_error; - } - - snprintf(url, sizeof(url)-1, "%s/%d/http/server_zones", ctx->status_url, - ctx->nginx_plus_version); - client = flb_http_client(u_conn, FLB_HTTP_GET, url, - NULL, 0, ctx->ins->host.name, ctx->ins->host.port, NULL, 0); - if (!client) { - flb_plg_error(ins, "unable to create http client"); - goto client_error; - } - - ret = flb_http_do(client, &b_sent); - if (ret != 0) { - flb_plg_error(ins, "http do error"); - goto http_error; - } - - if (client->resp.status != 200) { - flb_plg_error(ins, "http status code error: %d", client->resp.status); - goto http_error; - } - - if (client->resp.payload_size <= 0) { - flb_plg_error(ins, "empty response"); - goto http_error; - } - - parse_payload_json_table(ctx, ts, process_server_zone, - client->resp.payload, client->resp.payload_size); - rc = 0; -http_error: - flb_http_client_destroy(client); -client_error: - flb_upstream_conn_release(u_conn); -conn_error: - return rc; -} - -/** - * Callback function to gather statistics from the nginx - * plus ngx_http module. - * - * @param ins Pointer to flb_input_instance - * @param config Pointer to flb_config - * @param in_context void Pointer used to cast to nginx_ctx - * - * @return int Always returns success - */ -static int nginx_collect_plus_location_zones(struct flb_input_instance *ins, - struct flb_config *config, struct nginx_ctx *ctx, uint64_t ts) -{ - struct flb_connection *u_conn; - struct flb_http_client *client; - char url[1024]; - size_t b_sent; - int ret = -1; - int rc = -1; - - - u_conn = flb_upstream_conn_get(ctx->upstream); - if (!u_conn) { - flb_plg_error(ins, "upstream connection initialization error"); - goto conn_error; - } - - snprintf(url, sizeof(url)-1, "%s/%d/http/location_zones", ctx->status_url, - ctx->nginx_plus_version); - client = flb_http_client(u_conn, FLB_HTTP_GET, url, - NULL, 0, ctx->ins->host.name, ctx->ins->host.port, NULL, 0); - if (!client) { - flb_plg_error(ins, "unable to create http client"); - goto client_error; - } - - ret = flb_http_do(client, &b_sent); - if (ret != 0) { - flb_plg_error(ins, "http do error"); - goto http_error; - } - - if (client->resp.status != 200) { - flb_plg_error(ins, "http status code error: [%s] %d", url, client->resp.status); - goto http_error; - } - - if (client->resp.payload_size <= 0) { - flb_plg_error(ins, "empty response"); - goto http_error; - } - - parse_payload_json_table(ctx, ts, process_location_zone, - client->resp.payload, client->resp.payload_size); - rc = 0; -http_error: - flb_http_client_destroy(client); -client_error: - flb_upstream_conn_release(u_conn); -conn_error: - return rc; -} - -/** - * Callback function to gather statistics from the nginx - * plus ngx_http module. - * - * @param ins Pointer to flb_input_instance - * @param config Pointer to flb_config - * @param in_context void Pointer used to cast to nginx_ctx - * - * @return int Always returns success - */ -static int nginx_collect_plus_upstreams(struct flb_input_instance *ins, - struct flb_config *config, struct nginx_ctx *ctx, uint64_t ts) -{ - struct flb_connection *u_conn; - struct flb_http_client *client; - char url[1024]; - size_t b_sent; - int ret = -1; - int rc = -1; - - - u_conn = flb_upstream_conn_get(ctx->upstream); - if (!u_conn) { - flb_plg_error(ins, "upstream connection initialization error"); - goto conn_error; - } - - snprintf(url, sizeof(url)-1, "%s/%d/http/upstreams", ctx->status_url, - ctx->nginx_plus_version); - client = flb_http_client(u_conn, FLB_HTTP_GET, url, - NULL, 0, ctx->ins->host.name, ctx->ins->host.port, NULL, 0); - if (!client) { - flb_plg_error(ins, "unable to create http client"); - goto client_error; - } - - ret = flb_http_do(client, &b_sent); - if (ret != 0) { - flb_plg_error(ins, "http do error"); - goto http_error; - } - - if (client->resp.status != 200) { - flb_plg_error(ins, "http status code error: [%s] %d", url, client->resp.status); - goto http_error; - } - - if (client->resp.payload_size <= 0) { - flb_plg_error(ins, "empty response"); - goto http_error; - } - - parse_payload_json_table(ctx, ts, process_upstreams, - client->resp.payload, client->resp.payload_size); - rc = 0; -http_error: - flb_http_client_destroy(client); -client_error: - flb_upstream_conn_release(u_conn); -conn_error: - return rc; -} - -/** - * Callback function to gather statistics from the nginx - * plus ngx_http module. - * - * @param ins Pointer to flb_input_instance - * @param config Pointer to flb_config - * @param in_context void Pointer used to cast to nginx_ctx - * - * @return int Always returns success - */ -static int nginx_collect_plus_stream_server_zones(struct flb_input_instance *ins, - struct flb_config *config, struct nginx_ctx *ctx, uint64_t ts) -{ - struct flb_connection *u_conn; - struct flb_http_client *client; - char url[1024]; - size_t b_sent; - int ret = -1; - int rc = -1; - - - u_conn = flb_upstream_conn_get(ctx->upstream); - if (!u_conn) { - flb_plg_error(ins, "upstream connection initialization error"); - goto conn_error; - } - - snprintf(url, sizeof(url)-1, "%s/%d/stream/server_zones", ctx->status_url, - ctx->nginx_plus_version); - client = flb_http_client(u_conn, FLB_HTTP_GET, url, - NULL, 0, ctx->ins->host.name, ctx->ins->host.port, NULL, 0); - if (!client) { - flb_plg_error(ins, "unable to create http client"); - goto client_error; - } - - ret = flb_http_do(client, &b_sent); - if (ret != 0) { - flb_plg_error(ins, "http do error"); - goto http_error; - } - - if (client->resp.status != 200) { - flb_plg_error(ins, "http status code error: [%s] %d", url, client->resp.status); - goto http_error; - } - - if (client->resp.payload_size <= 0) { - flb_plg_error(ins, "empty response"); - goto http_error; - } - - parse_payload_json_table(ctx, ts, process_stream_server_zone, - client->resp.payload, client->resp.payload_size); - rc = 0; -http_error: - flb_http_client_destroy(client); -client_error: - flb_upstream_conn_release(u_conn); -conn_error: - return rc; -} - -/** - * Callback function to gather statistics from the nginx - * plus ngx_http module. - * - * @param ins Pointer to flb_input_instance - * @param config Pointer to flb_config - * @param in_context void Pointer used to cast to nginx_ctx - * - * @return int Always returns success - */ -static int nginx_collect_plus_stream_upstreams(struct flb_input_instance *ins, - struct flb_config *config, - struct nginx_ctx *ctx, uint64_t ts) -{ - struct flb_connection *u_conn; - struct flb_http_client *client; - char url[1024]; - size_t b_sent; - int ret = -1; - int rc = -1; - - - u_conn = flb_upstream_conn_get(ctx->upstream); - if (!u_conn) { - flb_plg_error(ins, "upstream connection initialization error"); - goto conn_error; - } - - snprintf(url, sizeof(url)-1, "%s/%d/stream/upstreams", ctx->status_url, - ctx->nginx_plus_version); - client = flb_http_client(u_conn, FLB_HTTP_GET, url, - NULL, 0, ctx->ins->host.name, ctx->ins->host.port, NULL, 0); - if (!client) { - flb_plg_error(ins, "unable to create http client"); - goto client_error; - } - - ret = flb_http_do(client, &b_sent); - if (ret != 0) { - flb_plg_error(ins, "http do error"); - goto http_error; - } - - if (client->resp.status != 200) { - flb_plg_error(ins, "http status code error: [%s] %d", url, client->resp.status); - goto http_error; - } - - if (client->resp.payload_size <= 0) { - flb_plg_error(ins, "empty response"); - goto http_error; - } - - parse_payload_json_table(ctx, ts, process_stream_upstreams, - client->resp.payload, client->resp.payload_size); - rc = 0; -http_error: - flb_http_client_destroy(client); -client_error: - flb_upstream_conn_release(u_conn); -conn_error: - return rc; -} - -/** - * Get the current highest REST API version - * - * @param ins Pointer to flb_input_instance - * @param config Pointer to flb_config - * @param in_context void Pointer used to cast to nginx_ctx - * - * @return int highest version if > 0, error otherwise. - */ -static int nginx_plus_get_version(struct flb_input_instance *ins, - struct flb_config *config, - struct nginx_ctx *ctx) -{ - struct flb_connection *u_conn; - struct flb_http_client *client; - char url[1024]; - size_t b_sent; - int rc = -1; - int out_size; - char *pack; - struct flb_pack_state pack_state; - size_t off = 0; - msgpack_unpacked result; - int maxversion = 1; - int i = 0; - - - u_conn = flb_upstream_conn_get(ctx->upstream); - if (!u_conn) { - flb_plg_error(ins, "upstream connection initialization error"); - goto conn_error; - } - - snprintf(url, sizeof(url)-1, "%s/", ctx->status_url); - client = flb_http_client(u_conn, FLB_HTTP_GET, url, - NULL, 0, ctx->ins->host.name, ctx->ins->host.port, NULL, 0); - if (!client) { - flb_plg_error(ins, "unable to create http client"); - goto client_error; - } - - rc = flb_http_do(client, &b_sent); - if (rc != 0) { - flb_plg_error(ins, "http do error"); - goto http_error; - } - - if (client->resp.status != 200) { - flb_plg_error(ins, "http status code error: [%s] %d", url, client->resp.status); - goto http_error; - } - - if (client->resp.payload_size <= 0) { - flb_plg_error(ins, "empty response"); - goto http_error; - } - - /* Initialize packer */ - flb_pack_state_init(&pack_state); - - /* Pack JSON as msgpack */ - rc = flb_pack_json_state(client->resp.payload, client->resp.payload_size, - &pack, &out_size, &pack_state); - flb_pack_state_reset(&pack_state); - - /* Handle exceptions */ - if (rc == FLB_ERR_JSON_PART) { - flb_plg_warn(ins, "JSON data is incomplete, skipping"); - goto json_error; - } - else if (rc == FLB_ERR_JSON_INVAL) { - flb_plg_warn(ins, "invalid JSON message, skipping"); - goto json_error; - } - else if (rc == -1) { - flb_plg_error(ins, "unable to parse JSON response"); - goto json_error; - } - - msgpack_unpacked_init(&result); - while (msgpack_unpack_next(&result, pack, out_size, &off) == MSGPACK_UNPACK_SUCCESS) { - if (result.data.type == MSGPACK_OBJECT_ARRAY) { - for (i = 0; i < result.data.via.array.size; i++) { - if (result.data.via.array.ptr[i].via.i64 > maxversion) { - maxversion = result.data.via.array.ptr[i].via.i64; - } - } - } else { - flb_plg_error(ins, "NOT AN ARRAY"); - goto rest_error; - } - } - -rest_error: - msgpack_unpacked_destroy(&result); -json_error: - flb_free(pack); -http_error: - flb_http_client_destroy(client); -client_error: - flb_upstream_conn_release(u_conn); -conn_error: - return maxversion; -} - - -/** - * Callback function to gather statistics from the nginx - * plus ngx_http module. - * - * @param ins Pointer to flb_input_instance - * @param config Pointer to flb_config - * @param in_context void Pointer used to cast to nginx_ctx - * - * @return int Always returns success - */ -static int nginx_collect_plus(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - int version = -1; - struct nginx_ctx *ctx = (struct nginx_ctx *)in_context; - int rc = -1; - int ret = -1; - uint64_t ts = cfl_time_now(); - - - version = nginx_plus_get_version(ins, config, in_context); - if (version <= 0) { - flb_plg_error(ins, "bad NGINX plus REST API version = %d", version); - goto error; - } - ctx->nginx_plus_version = version; - - rc = nginx_collect_plus_connections(ins, config, ctx, ts); - if (rc != 0) { - goto error; - } - rc = nginx_collect_plus_ssl(ins, config, ctx, ts); - if (rc != 0) { - goto error; - } - rc = nginx_collect_plus_http_requests(ins, config, ctx, ts); - if (rc != 0) { - goto error; - } - rc = nginx_collect_plus_server_zones(ins, config, ctx, ts); - if (rc != 0) { - goto error; - } - - if (ctx->nginx_plus_version >= 5) { - rc = nginx_collect_plus_location_zones(ins, config, ctx, ts); - if (rc != 0) { - goto error; - } - } - - rc = nginx_collect_plus_upstreams(ins, config, ctx, ts); - if (rc != 0) { - goto error; - } - rc = nginx_collect_plus_stream_server_zones(ins, config, ctx, ts); - if (rc != 0) { - goto error; - } - rc = nginx_collect_plus_stream_upstreams(ins, config, ctx, ts); - if (rc != 0) { - goto error; - } -error: - if (rc == 0) { - cmt_gauge_set(ctx->connection_up, ts, (double)1.0, 0, NULL); - } else { - cmt_gauge_set(ctx->connection_up, ts, (double)0.0, 0, NULL); - } - ret = flb_input_metrics_append(ins, NULL, 0, ctx->cmt); - if (ret != 0) { - flb_plg_error(ins, "could not append metrics"); - } - return rc; -} - -/** - * Function to initialize nginx metrics plugin. - * - * @param ins Pointer to flb_input_instance - * @param config Pointer to flb_config - * - * @return struct nginx_ctx_init* Pointer to the plugin's - * structure on success, NULL on failure. - */ -struct nginx_ctx *nginx_ctx_init(struct flb_input_instance *ins, - struct flb_config *config) -{ - int ret; - int upstream_flags; - struct nginx_ctx *ctx; - struct flb_upstream *upstream; - - if (ins->host.name == NULL) { - ins->host.name = flb_sds_create("localhost"); - } - if (ins->host.port == 0) { - ins->host.port = 80; - } - - ctx = flb_calloc(1, sizeof(struct nginx_ctx)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->is_up = FLB_FALSE; - - ctx->ins = ins; - - /* Load the config map */ - ret = flb_input_config_map_set(ins, (void *)ctx); - if (ret == -1) { - flb_free(ctx); - return NULL; - } - - ctx->cmt = cmt_create(); - if (!ctx->cmt) { - flb_plg_error(ins, "could not initialize CMetrics"); - flb_free(ctx); - return NULL; - } - - upstream_flags = FLB_IO_TCP; - - if (ins->use_tls) { - upstream_flags |= FLB_IO_TLS; - } - - upstream = flb_upstream_create(config, ins->host.name, ins->host.port, - upstream_flags, ins->tls); - - if (!upstream) { - flb_plg_error(ins, "upstream initialization error"); - cmt_destroy(ctx->cmt); - flb_free(ctx); - return NULL; - } - ctx->upstream = upstream; - - return ctx; -} - -static int nginx_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - int rc; - struct nginx_ctx *ctx = (struct nginx_ctx *)in_context; - if (ctx->is_nginx_plus == FLB_TRUE) { - rc = nginx_collect_plus(ins, config, in_context); - } else { - rc = nginx_collect_stub_status(ins, config, in_context); - } - FLB_INPUT_RETURN(rc); -} - -static int nginx_ctx_destroy(struct nginx_ctx *ctx); -/** - * Callback function to initialize nginx metrics plugin - * - * @param ins Pointer to flb_input_instance - * @param config Pointer to flb_config - * @param data Unused - * - * @return int 0 on success, -1 on failure - */ -static int nginx_init(struct flb_input_instance *ins, - struct flb_config *config, void *data) -{ - struct nginx_ctx *ctx = NULL; - struct cmt_counter *c; - struct cmt_gauge *g; - int ret = -1; - - /* Allocate space for the configuration */ - ctx = nginx_ctx_init(ins, config); - if (!ctx) { - return -1; - } - - - flb_input_set_context(ins, ctx); - - if (ctx->is_nginx_plus == FLB_FALSE) { - /* These metrics follow the same format as those define here: - * https://github.com/nginxinc/nginx-prometheus-exporter#metrics-for-nginx-oss - */ - ctx->connections_accepted = cmt_counter_create(ctx->cmt, "nginx", "connections", - "accepted", - "Accepted client connections", 0, - NULL); - if (ctx->connections_accepted == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(ctx->connections_accepted); - - ctx->connections_active = cmt_gauge_create(ctx->cmt, "nginx", "connections", - "active", "active client connections", - 0, NULL); - if (ctx->connections_active == NULL) { - goto nginx_init_end; - } - - ctx->connections_handled = cmt_counter_create(ctx->cmt, "nginx", "connections", - "handled", - "Handled client connections", 0, - NULL); - if (ctx->connections_handled == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(ctx->connections_handled); - - ctx->connections_reading = cmt_gauge_create(ctx->cmt, "nginx", "connections", - "reading", - "reading client connections", - 0, NULL); - if (ctx->connections_reading == NULL) { - goto nginx_init_end; - } - - ctx->connections_writing = cmt_gauge_create(ctx->cmt, "nginx", "connections", - "writing", - "writing client connections", - 0, NULL); - if (ctx->connections_writing == NULL) { - goto nginx_init_end; - } - - ctx->connections_waiting = cmt_gauge_create(ctx->cmt, "nginx", "connections", - "waiting", - "waiting client connections", - 0, NULL); - if (ctx->connections_waiting == NULL) { - goto nginx_init_end; - } - - ctx->connections_total = cmt_counter_create(ctx->cmt, "nginx", "http", - "requests_total", - "Total http requests", 0, NULL); - if (ctx->connections_total == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(ctx->connections_total); - - ctx->connection_up = cmt_gauge_create(ctx->cmt, "nginx", "", "up", - "Shows the status of the last metric " - "scrape: 1 for a successful scrape and " - "0 for a failed one", - 0, NULL); - } else { - flb_plg_info(ins, "nginx-plus mode on"); - - ctx->plus_connections = flb_calloc(1, sizeof(struct nginx_plus_connections)); - ctx->plus_ssl = flb_calloc(1, sizeof(struct nginx_plus_ssl)); - ctx->plus_http_requests = flb_calloc(1, sizeof(struct nginx_plus_http_requests)); - ctx->server_zones = flb_calloc(1, sizeof(struct nginx_plus_server_zones)); - ctx->location_zones = flb_calloc(1, sizeof(struct nginx_plus_location_zones)); - ctx->upstreams = flb_calloc(1, sizeof(struct nginx_plus_upstreams)); - ctx->streams = flb_calloc(1, sizeof(struct nginx_plus_streams)); - ctx->stream_upstreams = flb_calloc(1, sizeof(struct nginx_plus_stream_upstreams)); - - g = cmt_gauge_create(ctx->cmt, "nginxplus", "", "up", - "Shows the status of the last metric scrape: " - "1 for a successful scrape and 0 for a failed " - "one", 0, NULL); - if (g == NULL) { - goto nginx_init_end; - } - ctx->connection_up = g; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", "connections", "accepted", - "NGINX Plus Total Connections", - 0, NULL); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->plus_connections->connections_accepted = c; - - - c = cmt_counter_create(ctx->cmt, - "nginxplus", "connections", "dropped", - "NGINX Plus Total Connections", - 0, NULL); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->plus_connections->connections_dropped = c; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", "connections", "active", - "NGINX Plus Total Connections", - 0, NULL); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->plus_connections->connections_active = c; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", "connections", "idle", - "NGINX Plus Total Connections", - 0, NULL); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->plus_connections->connections_idle = c; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", "ssl", "handshakes", - "NGINX Plus Total Connections", - 0, NULL); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->plus_ssl->handshakes = c; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", "ssl", "handshakes_failed", - "NGINX Plus Total Connections", - 0, NULL); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->plus_ssl->handshakes_failed = c; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", "ssl", "session_reuses", - "NGINX Plus Total Connections", - 0, NULL); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->plus_ssl->session_reuses = c; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", "http_requests", "total", - "NGINX Plus Total Connections", - 0, NULL); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->plus_http_requests->total = c; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", "http_requests", "current", - "NGINX Plus Total Connections", - 0, NULL); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->plus_http_requests->current = c; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", - "server_zone", - "discarded", - "NGINX Server Zone discarded", - 1, (char *[]){"server_zone"}); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->server_zones->discarded = c; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", - "server_zone", - "processing", - "NGINX Server Zone processing", - 1, (char *[]){"server_zone"}); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->server_zones->processing = c; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", - "server_zone", - "received", - "NGINX Server Zone received", - 1, (char *[]){"server_zone"}); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->server_zones->received = c; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", - "server_zone", - "requests", - "NGINX Server Zone requests", - 1, (char *[]){"server_zone"}); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->server_zones->requests = c; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", - "server_zone", - "responses", - "NGINX Server Zone responses", - 2, (char *[]){"server_zone", "code"}); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->server_zones->responses = c; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", - "server_zone", - "sent", - "NGINX Server Zone sent", - 1, (char *[]){"server_zone"}); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->server_zones->sent = c; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", - "server_zone", - "discarded", - "NGINX Server Zone discarded", - 1, (char *[]){"location_zone"}); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->location_zones->discarded = c; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", - "location_zone", - "received", - "NGINX Server Zone received", - 1, (char *[]){"location_zone"}); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->location_zones->received = c; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", - "location_zone", - "requests", - "NGINX Server Zone requests", - 1, (char *[]){"location_zone"}); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->location_zones->requests = c; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", - "location_zone", - "responses", - "NGINX Server Zone responses", - 2, (char *[]){"location_zone", "code"}); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->location_zones->responses = c; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", - "location_zone", - "sent", - "NGINX Server Zone sent", - 1, (char *[]){"location_zone"}); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->location_zones->sent = c; - - g = cmt_gauge_create(ctx->cmt, - "nginxplus", - "upstream", - "keepalives", - "NGINX Upstream Keepalives", - 1, (char *[]){"upstream"}); - if (g == NULL) { - goto nginx_init_end; - } - ctx->upstreams->keepalives = g; - - g = cmt_gauge_create(ctx->cmt, - "nginxplus", - "upstream", - "zombies", - "NGINX Upstream Zombies", - 1, (char *[]){"upstream"}); - if (g == NULL) { - goto nginx_init_end; - } - ctx->upstreams->zombies = g; - - g = cmt_gauge_create(ctx->cmt, - "nginxplus", - "upstream_server", - "active", - "NGINX Upstream Active", - 2, (char *[]){"upstream","server"}); - if (g == NULL) { - goto nginx_init_end; - } - ctx->upstreams->active = g; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", - "upstream_server", - "fails", - "NGINX Upstream Fails", - 2, (char *[]){"upstream","server"}); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->upstreams->fails = c; - - g = cmt_gauge_create(ctx->cmt, - "nginxplus", - "upstream_server", - "header_time", - "NGINX Upstream Header Time", - 2, (char *[]){"upstream","server"}); - if (g == NULL) { - goto nginx_init_end; - } - ctx->upstreams->header_time = g; - - g = cmt_gauge_create(ctx->cmt, - "nginxplus", - "upstream_server", - "limit", - "NGINX Upstream Limit", - 2, (char *[]){"upstream","server"}); - if (g == NULL) { - goto nginx_init_end; - } - ctx->upstreams->limit = g; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", - "upstream_server", - "received", - "NGINX Upstream Received", - 2, (char *[]){"upstream","server"}); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->upstreams->received = c; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", - "upstream_server", - "requests", - "NGINX Upstream Requests", - 2, (char *[]){"upstream","server"}); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->upstreams->requests = c; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", - "upstream_server", - "responses", - "NGINX Upstream Responses", - 3, (char *[]){"code", "upstream","server"}); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->upstreams->responses = c; - - g = cmt_gauge_create(ctx->cmt, - "nginxplus", - "upstream_server", - "response_time", - "NGINX Upstream Response Time", - 2, (char *[]){"upstream","server"}); - if (g == NULL) { - goto nginx_init_end; - } - ctx->upstreams->response_time = g; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", - "upstream_server", - "sent", - "NGINX Upstream Sent", - 2, (char *[]){"upstream","server"}); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->upstreams->sent = c; - - g = cmt_gauge_create(ctx->cmt, - "nginxplus", - "upstream_server", - "state", - "NGINX Upstream State", - 2, (char *[]){"upstream","server"}); - if (g == NULL) { - goto nginx_init_end; - } - ctx->upstreams->state = g; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", - "upstream_server", - "unavail", - "NGINX Upstream Unavailable", - 2, (char *[]){"upstream","server"}); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->upstreams->unavail = c; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", - "stream_server_zone", - "connections", - "NGINX Stream Server Zone connections", - 1, (char *[]){"server_zone"}); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->streams->connections = c; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", - "stream_server_zone", - "discarded", - "NGINX Stream Server Zone discarded", - 1, (char *[]){"server_zone"}); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->streams->discarded = c; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", - "stream_server_zone", - "processing", - "NGINX Stream Server Zone " - "processing", - 1, (char *[]){"server_zone"}); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->streams->processing = c; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", - "stream_server_zone", - "received", - "NGINX Stream Server Zone received", - 1, (char *[]){"server_zone"}); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->streams->received = c; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", - "server_zone", - "sent", - "NGINX Stream Server Zone sent", - 1, (char *[]){"server_zone"}); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->streams->sent = c; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", - "stream_server_zone", - "sessions", - "NGINX Stream Server Zone Sessions", - 2, (char *[]){"server_zone", "code"}); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->streams->sessions = c; - - g = cmt_gauge_create(ctx->cmt, - "nginxplus", - "stream_upstream", - "zombies", - "NGINX Upstream Zombies", - 1, (char *[]){"upstream"}); - if (g == NULL) { - goto nginx_init_end; - } - ctx->stream_upstreams->zombies = g; - - g = cmt_gauge_create(ctx->cmt, - "nginxplus", - "stream_upstream_server", - "active", - "NGINX Upstream Active", - 2, (char *[]){"upstream","server"}); - if (g == NULL) { - goto nginx_init_end; - } - ctx->stream_upstreams->active = g; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", - "stream_upstream_server", - "fails", - "NGINX Upstream Fails", - 2, (char *[]){"upstream","server"}); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->stream_upstreams->fails = c; - - g = cmt_gauge_create(ctx->cmt, - "nginxplus", - "stream_upstream_server", - "limit", - "NGINX Upstream Limit", - 2, (char *[]){"upstream","server"}); - if (g == NULL) { - goto nginx_init_end; - } - ctx->stream_upstreams->limit = g; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", - "stream_upstream_server", - "received", - "NGINX Upstream Received", - 2, (char *[]){"upstream","server"}); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->stream_upstreams->received = c; - - g = cmt_gauge_create(ctx->cmt, - "nginxplus", - "stream_upstream_server", - "connect_time", - "NGINX Upstream Header Time", - 2, (char *[]){"upstream", "server"}); - if (g == NULL) { - goto nginx_init_end; - } - ctx->stream_upstreams->connect_time = g; - - g = cmt_gauge_create(ctx->cmt, - "nginxplus", - "stream_upstream_server", - "first_byte_time", - "NGINX Upstream Header Time", - 2, (char *[]){"upstream", "server"}); - if (g == NULL) { - goto nginx_init_end; - } - ctx->stream_upstreams->first_byte_time = g; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", - "stream_upstream_server", - "connections", - "NGINX Upstream Requests", - 2, (char *[]){"upstream","server"}); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->stream_upstreams->connections = c; - - g = cmt_gauge_create(ctx->cmt, - "nginxplus", - "stream_upstream_server", - "response_time", - "NGINX Upstream Response Time", - 2, (char *[]){"upstream","server"}); - if (g == NULL) { - goto nginx_init_end; - } - ctx->stream_upstreams->response_time = g; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", - "stream_upstream_server", - "sent", - "NGINX Upstream Sent", - 2, (char *[]){"upstream","server"}); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->stream_upstreams->sent = c; - - g = cmt_gauge_create(ctx->cmt, - "nginxplus", - "stream_upstream_server", - "state", - "NGINX Upstream State", - 2, (char *[]){"upstream","server"}); - if (g == NULL) { - goto nginx_init_end; - } - ctx->stream_upstreams->state = g; - - c = cmt_counter_create(ctx->cmt, - "nginxplus", - "stream_upstream_server", - "unavail", - "NGINX Upstream Unavailable", - 2, (char *[]){"upstream","server"}); - if (c == NULL) { - goto nginx_init_end; - } - cmt_counter_allow_reset(c); - ctx->stream_upstreams->unavail = c; - - } - ctx->coll_id = flb_input_set_collector_time(ins, - nginx_collect, - 1, - 0, config); - ret = 0; - nginx_init_end: - if (ret < 0) { - nginx_ctx_destroy(ctx); - } - - return ret; -} - - -/** - * Function to destroy nginx metrics plugin. - * - * @param ctx Pointer to nginx_ctx - * - * @return int 0 - */ -static int nginx_ctx_destroy(struct nginx_ctx *ctx) -{ - if (ctx->upstream) { - flb_upstream_destroy(ctx->upstream); - } - if (ctx->cmt) { - cmt_destroy(ctx->cmt); - } - if (ctx->is_nginx_plus) { - if (ctx->plus_connections) flb_free(ctx->plus_connections); - if (ctx->plus_ssl) flb_free(ctx->plus_ssl); - if (ctx->plus_http_requests) flb_free(ctx->plus_http_requests); - if (ctx->server_zones) flb_free(ctx->server_zones); - if (ctx->location_zones) flb_free(ctx->location_zones); - if (ctx->upstreams) flb_free(ctx->upstreams); - if (ctx->streams) flb_free(ctx->streams); - if (ctx->stream_upstreams) flb_free(ctx->stream_upstreams); - } - flb_free(ctx); - return 0; -} - -/** - * Callback exit function to cleanup plugin - * - * @param data Pointer cast to flb_in_de_config - * @param config Unused - * - * @return int Always returns 0 - */ -static int nginx_exit(void *data, struct flb_config *config) -{ - struct nginx_ctx *ctx = (struct nginx_ctx *)data; - - if (!ctx) { - return 0; - } - - nginx_ctx_destroy(ctx); - - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "status_url", DEFAULT_STATUS_URL, - 0, FLB_TRUE, offsetof(struct nginx_ctx, status_url), - "Define URL of stub status handler" - }, - { - FLB_CONFIG_MAP_BOOL, "nginx_plus", "true", - 0, FLB_TRUE, offsetof(struct nginx_ctx, is_nginx_plus), - "Turn on NGINX plus mode" - }, - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_input_plugin in_nginx_exporter_metrics_plugin = { - .name = "nginx_metrics", - .description = "Nginx status metrics", - .cb_init = nginx_init, - .cb_pre_run = NULL, - .cb_collect = nginx_collect, - .cb_flush_buf = NULL, - .cb_exit = nginx_exit, - .config_map = config_map, - .flags = FLB_INPUT_NET|FLB_INPUT_CORO, -}; diff --git a/fluent-bit/plugins/in_nginx_exporter_metrics/nginx.h b/fluent-bit/plugins/in_nginx_exporter_metrics/nginx.h deleted file mode 100644 index 97538587b..000000000 --- a/fluent-bit/plugins/in_nginx_exporter_metrics/nginx.h +++ /dev/null @@ -1,150 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_NGINX_H -#define FLB_IN_NGINX_H - -#include -#include -#include -#include - -#define DEFAULT_STATUS_URL "/status" - -struct nginx_ctx -{ - int coll_id; /* collector id */ - flb_sds_t status_url; - struct flb_parser *parser; - struct flb_input_instance *ins; /* Input plugin instace */ - struct flb_upstream *upstream; - struct cmt *cmt; - struct cmt_counter *connections_accepted; - struct cmt_counter *connections_handled; - struct cmt_counter *connections_total; - struct cmt_gauge *connection_active; - struct cmt_gauge *connections_active; - struct cmt_gauge *connections_reading; - struct cmt_gauge *connections_writing; - struct cmt_gauge *connections_waiting; - struct cmt_gauge *connection_up; - bool is_up; - bool is_nginx_plus; - int nginx_plus_version; - - struct nginx_plus_connections *plus_connections; - struct nginx_plus_http_requests *plus_http_requests; - struct nginx_plus_ssl *plus_ssl; - struct nginx_plus_server_zones *server_zones; - struct nginx_plus_location_zones *location_zones; - struct nginx_plus_upstreams *upstreams; - struct nginx_plus_streams *streams; - struct nginx_plus_stream_upstreams *stream_upstreams; -}; - -struct nginx_status -{ - uint64_t active; - uint64_t reading; - uint64_t writing; - uint64_t waiting; - uint64_t accepts; - uint64_t handled; - uint64_t requests; -}; - -struct nginx_plus_connections { - struct cmt_counter *connections_accepted; - struct cmt_counter *connections_dropped; - struct cmt_counter *connections_active; - struct cmt_counter *connections_idle; -}; - -struct nginx_plus_ssl { - struct cmt_counter *handshakes; - struct cmt_counter *handshakes_failed; - struct cmt_counter *session_reuses; -}; - -struct nginx_plus_http_requests { - struct cmt_counter *total; - struct cmt_counter *current; -}; - -struct nginx_plus_server_zones { - struct cmt_counter *discarded; - struct cmt_counter *processing; - struct cmt_counter *received; - struct cmt_counter *requests; - struct cmt_counter *responses; - struct cmt_counter *sent; -}; - -struct nginx_plus_upstreams { - //struct nginx_plux_upstream_peer **peers; - struct cmt_gauge *keepalives; - struct cmt_gauge *zombies; - // per peer - struct cmt_gauge *active; - struct cmt_counter *fails; - struct cmt_gauge *header_time; - struct cmt_gauge *limit; - struct cmt_counter *received; - struct cmt_counter *requests; - struct cmt_counter *responses; - struct cmt_gauge *response_time; - struct cmt_counter *sent; - struct cmt_gauge *state; - struct cmt_counter *unavail; -}; - -struct nginx_plus_location_zones { - struct cmt_counter *discarded; - struct cmt_counter *received; - struct cmt_counter *requests; - struct cmt_counter *responses; - struct cmt_counter *sent; -}; - -struct nginx_plus_streams { - struct cmt_counter *connections; - struct cmt_counter *discarded; - struct cmt_counter *processing; - struct cmt_counter *received; - struct cmt_counter *sent; - struct cmt_counter *sessions; -}; - -struct nginx_plus_stream_upstreams { - struct cmt_gauge *zombies; - // per peer - struct cmt_gauge *active; - struct cmt_counter *fails; - struct cmt_gauge *limit; - struct cmt_counter *received; - struct cmt_gauge *connect_time; - struct cmt_gauge *first_byte_time; - struct cmt_counter *connections; - struct cmt_gauge *response_time; - struct cmt_counter *sent; - struct cmt_gauge *state; - struct cmt_counter *unavail; -}; - -#endif \ No newline at end of file diff --git a/fluent-bit/plugins/in_node_exporter_metrics/CMakeLists.txt b/fluent-bit/plugins/in_node_exporter_metrics/CMakeLists.txt deleted file mode 100644 index 16584dc06..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/CMakeLists.txt +++ /dev/null @@ -1,26 +0,0 @@ -set(src - ne_cpu.c - ne_meminfo.c - ne_diskstats.c - ne_filesystem.c - ne_uname.c - ne_stat_linux.c - ne_vmstat_linux.c - ne_netdev.c - ne_time.c - ne_loadavg.c - ne_filefd_linux.c - ne_textfile.c - ne_utils.c - ne_config.c - ne.c - ) - -if(FLB_HAVE_SYSTEMD_SDBUS) -set(src - ${src} - ne_systemd.c - ) -endif() - -FLB_PLUGIN(in_node_exporter_metrics "${src}" "") diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne.c b/fluent-bit/plugins/in_node_exporter_metrics/ne.c deleted file mode 100644 index d77817957..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne.c +++ /dev/null @@ -1,1107 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "ne.h" -#include "ne_config.h" -#include "ne_filefd_linux.h" - -/* collectors */ -#include "ne_cpu.h" -#include "ne_cpufreq.h" -#include "ne_meminfo.h" -#include "ne_diskstats.h" -#include "ne_filesystem.h" -#include "ne_uname.h" -#include "ne_stat_linux.h" -#include "ne_time.h" -#include "ne_loadavg.h" -#include "ne_vmstat_linux.h" -#include "ne_netdev.h" -#include "ne_textfile.h" -#include "ne_systemd.h" - -static int ne_timer_cpu_metrics_cb(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_ne *ctx = in_context; - - ne_cpu_update(ctx); - - return 0; -} - -static int ne_timer_cpufreq_metrics_cb(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_ne *ctx = in_context; - - ne_cpufreq_update(ctx); - - return 0; -} - -static int ne_timer_meminfo_metrics_cb(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_ne *ctx = in_context; - - ne_meminfo_update(ctx); - - return 0; -} - -static int ne_timer_diskstats_metrics_cb(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_ne *ctx = in_context; - - ne_diskstats_update(ctx); - - return 0; -} - -static int ne_timer_filesystem_metrics_cb(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_ne *ctx = in_context; - - ne_filesystem_update(ctx); - - return 0; -} - -static int ne_timer_uname_metrics_cb(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_ne *ctx = in_context; - - ne_uname_update(ctx); - - return 0; -} - -static int ne_timer_stat_metrics_cb(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_ne *ctx = in_context; - - ne_stat_update(ctx); - - return 0; -} - -static int ne_timer_time_metrics_cb(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_ne *ctx = in_context; - - ne_time_update(ctx); - - return 0; -} - -static int ne_timer_loadavg_metrics_cb(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_ne *ctx = in_context; - - ne_loadavg_update(ctx); - - return 0; -} - -static int ne_timer_vmstat_metrics_cb(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_ne *ctx = in_context; - - ne_vmstat_update(ctx); - - return 0; -} - -static int ne_timer_netdev_metrics_cb(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_ne *ctx = in_context; - - ne_netdev_update(ctx); - - return 0; -} - -static int ne_timer_filefd_metrics_cb(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_ne *ctx = in_context; - - ne_filefd_update(ctx); - - return 0; -} - -static int ne_timer_textfile_metrics_cb(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_ne *ctx = in_context; - - ne_textfile_update(ctx); - - return 0; -} - -static int ne_timer_systemd_metrics_cb(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_ne *ctx = in_context; - - ne_systemd_update(ctx); - - return 0; -} - -struct flb_ne_callback { - char *name; - void (*func)(char *, void *, void *); -}; - -static int ne_update_cb(struct flb_ne *ctx, char *name); - -static void update_metrics(struct flb_input_instance *ins, struct flb_ne *ctx) -{ - int ret; - struct mk_list *head; - struct flb_slist_entry *entry; - - /* Update our metrics */ - if (ctx->metrics) { - mk_list_foreach(head, ctx->metrics) { - entry = mk_list_entry(head, struct flb_slist_entry, _head); - ret = flb_callback_exists(ctx->callback, entry->str); - if (ret == FLB_TRUE) { - ne_update_cb(ctx, entry->str); - } - else { - flb_plg_debug(ctx->ins, "Callback for metrics '%s' is not registered", entry->str); - } - } - } -} - -/* - * Update the metrics, this function is invoked every time 'scrape_interval' - * expires. - */ -static int cb_ne_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - int ret; - struct flb_ne *ctx = in_context; - - update_metrics(ins, ctx); - - /* Append the updated metrics */ - ret = flb_input_metrics_append(ins, NULL, 0, ctx->cmt); - if (ret != 0) { - flb_plg_error(ins, "could not append metrics"); - } - - return 0; -} - -static void ne_cpu_update_cb(char *name, void *p1, void *p2) -{ - struct flb_ne *ctx = p1; - - ne_cpu_update(ctx); -} - -static void ne_cpufreq_update_cb(char *name, void *p1, void *p2) -{ - struct flb_ne *ctx = p1; - - ne_cpufreq_update(ctx); -} - -static void ne_meminfo_update_cb(char *name, void *p1, void *p2) -{ - struct flb_ne *ctx = p1; - - ne_meminfo_update(ctx); -} - -static void ne_diskstats_update_cb(char *name, void *p1, void *p2) -{ - struct flb_ne *ctx = p1; - - ne_diskstats_update(ctx); -} - -static void ne_filesystem_update_cb(char *name, void *p1, void *p2) -{ - struct flb_ne *ctx = p1; - - ne_filesystem_update(ctx); -} - -static void ne_uname_update_cb(char *name, void *p1, void *p2) -{ - struct flb_ne *ctx = p1; - - ne_uname_update(ctx); -} - -static void ne_stat_update_cb(char *name, void *p1, void *p2) -{ - struct flb_ne *ctx = p1; - - ne_stat_update(ctx); -} - -static void ne_time_update_cb(char *name, void *p1, void *p2) -{ - struct flb_ne *ctx = p1; - - ne_time_update(ctx); -} - -static void ne_loadavg_update_cb(char *name, void *p1, void *p2) -{ - struct flb_ne *ctx = p1; - - ne_loadavg_update(ctx); -} - -static void ne_vmstat_update_cb(char *name, void *p1, void *p2) -{ - struct flb_ne *ctx = p1; - - ne_vmstat_update(ctx); -} - -static void ne_netdev_update_cb(char *name, void *p1, void *p2) -{ - struct flb_ne *ctx = p1; - - ne_netdev_update(ctx); -} - -static void ne_filefd_update_cb(char *name, void *p1, void *p2) -{ - struct flb_ne *ctx = p1; - - ne_filefd_update(ctx); -} - -static void ne_textfile_update_cb(char *name, void *p1, void *p2) -{ - struct flb_ne *ctx = p1; - - ne_textfile_update(ctx); -} - -static void ne_systemd_update_cb(char *name, void *p1, void *p2) -{ - struct flb_ne *ctx = p1; - - ne_systemd_update(ctx); -} - -static int ne_update_cb(struct flb_ne *ctx, char *name) -{ - int ret; - - ret = flb_callback_do(ctx->callback, name, ctx, NULL); - return ret; -} - -/* - * Callbacks Table - */ -struct flb_ne_callback ne_callbacks[] = { - /* metrics */ - { "cpufreq", ne_cpufreq_update_cb }, - { "cpu", ne_cpu_update_cb }, - { "meminfo", ne_meminfo_update_cb }, - { "diskstats", ne_diskstats_update_cb }, - { "filesystem", ne_filesystem_update_cb }, - { "uname", ne_uname_update_cb }, - { "stat", ne_stat_update_cb }, - { "time", ne_time_update_cb }, - { "loadavg", ne_loadavg_update_cb }, - { "vmstat", ne_vmstat_update_cb }, - { "netdev", ne_netdev_update_cb }, - { "filefd", ne_filefd_update_cb }, - { "textfile", ne_textfile_update_cb }, - { "systemd", ne_systemd_update_cb }, - { 0 } -}; - -static int in_ne_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - int ret; - int metric_idx = -1; - struct flb_ne *ctx; - struct mk_list *head; - struct flb_slist_entry *entry; - struct flb_ne_callback *cb; - - /* Create plugin context */ - ctx = flb_ne_config_create(in, config); - if (!ctx) { - flb_errno(); - return -1; - } - - /* Initialize fds */ - ctx->coll_fd = -1; - ctx->coll_cpu_fd = -1; - ctx->coll_cpufreq_fd = -1; - ctx->coll_meminfo_fd = -1; - ctx->coll_diskstats_fd = -1; - ctx->coll_filesystem_fd = -1; - ctx->coll_uname_fd = -1; - ctx->coll_stat_fd = -1; - ctx->coll_time_fd = -1; - ctx->coll_loadavg_fd = -1; - ctx->coll_vmstat_fd = -1; - ctx->coll_netdev_fd = -1; - ctx->coll_filefd_fd = -1; - ctx->coll_textfile_fd = -1; - ctx->coll_systemd_fd = -1; - - ctx->callback = flb_callback_create(in->name); - if (!ctx->callback) { - flb_plg_error(ctx->ins, "Create callback failed"); - return -1; - } - - /* Associate context with the instance */ - flb_input_set_context(in, ctx); - - /* Create the collector */ - ret = flb_input_set_collector_time(in, - cb_ne_collect, - ctx->scrape_interval, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set collector for Node Exporter Metrics plugin"); - return -1; - } - ctx->coll_fd = ret; - - /* Check and initialize enabled metrics */ - if (ctx->metrics) { - mk_list_foreach(head, ctx->metrics) { - entry = mk_list_entry(head, struct flb_slist_entry, _head); - ret = flb_callback_exists(ctx->callback, entry->str); - - if (ret == FLB_FALSE) { - if (strncmp(entry->str, "cpufreq", 7) == 0) { - if (ctx->cpu_scrape_interval == 0) { - flb_plg_debug(ctx->ins, "enabled metrics %s", entry->str); - metric_idx = 0; - } - else if (ctx->cpufreq_scrape_interval > 0) { - /* Create the cpufreq collector */ - ret = flb_input_set_collector_time(in, - ne_timer_cpufreq_metrics_cb, - ctx->cpufreq_scrape_interval, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set cpufreq collector for Node Exporter Metrics plugin"); - return -1; - } - ctx->coll_cpufreq_fd = ret; - } - ne_cpufreq_init(ctx); - } - else if (strncmp(entry->str, "cpu", 3) == 0) { - if (ctx->cpufreq_scrape_interval == 0) { - flb_plg_debug(ctx->ins, "enabled metrics %s", entry->str); - metric_idx = 1; - } - else if (ctx->cpu_scrape_interval > 0) { - /* Create the cpu collector */ - ret = flb_input_set_collector_time(in, - ne_timer_cpu_metrics_cb, - ctx->cpu_scrape_interval, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set cpu collector for Node Exporter Metrics plugin"); - return -1; - } - ctx->coll_cpu_fd = ret; - } - ne_cpu_init(ctx); - } - else if (strncmp(entry->str, "meminfo", 7) == 0) { - if (ctx->meminfo_scrape_interval == 0) { - flb_plg_debug(ctx->ins, "enabled metrics %s", entry->str); - metric_idx = 2; - } - else if (ctx->meminfo_scrape_interval > 0) { - /* Create the meminfo collector */ - ret = flb_input_set_collector_time(in, - ne_timer_meminfo_metrics_cb, - ctx->meminfo_scrape_interval, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set meminfo collector for Node Exporter Metrics plugin"); - return -1; - } - ctx->coll_meminfo_fd = ret; - } - ne_meminfo_init(ctx); - } - else if (strncmp(entry->str, "diskstats", 9) == 0) { - if (ctx->diskstats_scrape_interval == 0) { - flb_plg_debug(ctx->ins, "enabled metrics %s", entry->str); - metric_idx = 3; - } - else if (ctx->diskstats_scrape_interval > 0) { - /* Create the diskstats collector */ - ret = flb_input_set_collector_time(in, - ne_timer_diskstats_metrics_cb, - ctx->diskstats_scrape_interval, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set diskstats collector for Node Exporter Metrics plugin"); - return -1; - } - ctx->coll_diskstats_fd = ret; - } - ne_diskstats_init(ctx); - } - else if (strncmp(entry->str, "filesystem", 10) == 0) { - if (ctx->diskstats_scrape_interval == 0) { - flb_plg_debug(ctx->ins, "enabled metrics %s", entry->str); - metric_idx = 4; - } - else if (ctx->filesystem_scrape_interval > 0) { - /* Create the diskstats collector */ - ret = flb_input_set_collector_time(in, - ne_timer_filesystem_metrics_cb, - ctx->filesystem_scrape_interval, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set filesystem collector for Node Exporter Metrics plugin"); - return -1; - } - ctx->coll_filesystem_fd = ret; - } - ne_filesystem_init(ctx); - } - else if (strncmp(entry->str, "uname", 5) == 0) { - if (ctx->uname_scrape_interval == 0) { - flb_plg_debug(ctx->ins, "enabled metrics %s", entry->str); - metric_idx = 5; - } - else if (ctx->uname_scrape_interval > 0) { - /* Create the uname collector */ - ret = flb_input_set_collector_time(in, - ne_timer_uname_metrics_cb, - ctx->uname_scrape_interval, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set uname collector for Node Exporter Metrics plugin"); - return -1; - } - ctx->coll_uname_fd = ret; - } - ne_uname_init(ctx); - } - else if (strncmp(entry->str, "stat", 4) == 0) { - if (ctx->stat_scrape_interval == 0) { - flb_plg_debug(ctx->ins, "enabled metrics %s", entry->str); - metric_idx = 6; - } - else if (ctx->stat_scrape_interval > 0) { - /* Create the meminfo collector */ - ret = flb_input_set_collector_time(in, - ne_timer_stat_metrics_cb, - ctx->stat_scrape_interval, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set meminfo collector for Node Exporter Metrics plugin"); - return -1; - } - ctx->coll_stat_fd = ret; - } - ne_stat_init(ctx); - } - else if (strncmp(entry->str, "time", 4) == 0) { - if (ctx->time_scrape_interval == 0) { - flb_plg_debug(ctx->ins, "enabled metrics %s", entry->str); - metric_idx = 7; - } - else if (ctx->time_scrape_interval > 0) { - /* Create the time collector */ - ret = flb_input_set_collector_time(in, - ne_timer_time_metrics_cb, - ctx->time_scrape_interval, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set time collector for Node Exporter Metrics plugin"); - return -1; - } - ctx->coll_time_fd = ret; - } - ne_time_init(ctx); - } - else if (strncmp(entry->str, "loadavg", 7) == 0) { - if (ctx->loadavg_scrape_interval == 0) { - flb_plg_debug(ctx->ins, "enabled metrics %s", entry->str); - metric_idx = 8; - } - else if (ctx->loadavg_scrape_interval > 0) { - /* Create the loadavg collector */ - ret = flb_input_set_collector_time(in, - ne_timer_loadavg_metrics_cb, - ctx->loadavg_scrape_interval, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set loadavg collector for Node Exporter Metrics plugin"); - return -1; - } - ctx->coll_loadavg_fd = ret; - } - ne_loadavg_init(ctx); - } - else if (strncmp(entry->str, "vmstat", 6) == 0) { - if (ctx->vmstat_scrape_interval == 0) { - flb_plg_debug(ctx->ins, "enabled metrics %s", entry->str); - metric_idx = 9; - } - else if (ctx->vmstat_scrape_interval > 0) { - /* Create the vmstat collector */ - ret = flb_input_set_collector_time(in, - ne_timer_vmstat_metrics_cb, - ctx->vmstat_scrape_interval, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set vmstat collector for Node Exporter Metrics plugin"); - return -1; - } - ctx->coll_vmstat_fd = ret; - } - ne_vmstat_init(ctx); - } - else if (strncmp(entry->str, "netdev", 6) == 0) { - if (ctx->netdev_scrape_interval == 0) { - flb_plg_debug(ctx->ins, "enabled metrics %s", entry->str); - metric_idx = 10; - } - else if (ctx->netdev_scrape_interval > 0) { - /* Create the netdev collector */ - ret = flb_input_set_collector_time(in, - ne_timer_netdev_metrics_cb, - ctx->netdev_scrape_interval, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set netdev collector for Node Exporter Metrics plugin"); - return -1; - } - ctx->coll_netdev_fd = ret; - } - ne_netdev_init(ctx); - } - else if (strncmp(entry->str, "filefd", 6) == 0) { - if (ctx->filefd_scrape_interval == 0) { - flb_plg_debug(ctx->ins, "enabled metrics %s", entry->str); - metric_idx = 11; - } - else if (ctx->filefd_scrape_interval > 0) { - /* Create the filefd collector */ - ret = flb_input_set_collector_time(in, - ne_timer_filefd_metrics_cb, - ctx->filefd_scrape_interval, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set filefd collector for Node Exporter Metrics plugin"); - return -1; - } - ctx->coll_filefd_fd = ret; - } - ne_filefd_init(ctx); - } - else if (strncmp(entry->str, "textfile", 8) == 0) { - if (ctx->textfile_scrape_interval == 0) { - flb_plg_debug(ctx->ins, "enabled metrics %s", entry->str); - metric_idx = 12; - } - else if (ctx->textfile_scrape_interval > 0) { - /* Create the filefd collector */ - ret = flb_input_set_collector_time(in, - ne_timer_textfile_metrics_cb, - ctx->textfile_scrape_interval, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set textfile collector for Node Exporter Metrics plugin"); - return -1; - } - ctx->coll_textfile_fd = ret; - } - ne_textfile_init(ctx); - } - else if (strncmp(entry->str, "systemd", 7) == 0) { - if (ctx->systemd_scrape_interval == 0) { - flb_plg_debug(ctx->ins, "enabled metrics %s", entry->str); - metric_idx = 13; - } - else if (ctx->systemd_scrape_interval > 0) { - /* Create the filefd collector */ - ret = flb_input_set_collector_time(in, - ne_timer_systemd_metrics_cb, - ctx->systemd_scrape_interval, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set systemd collector for Node Exporter Metrics plugin"); - return -1; - } - ctx->coll_systemd_fd = ret; - } - ne_systemd_init(ctx); - } - else { - flb_plg_warn(ctx->ins, "Unknown metrics: %s", entry->str); - metric_idx = -1; - } - - if (metric_idx >= 0) { - cb = &ne_callbacks[metric_idx]; - ret = flb_callback_set(ctx->callback, cb->name, cb->func); - if (ret == -1) { - flb_plg_error(ctx->ins, "error setting up default " - "callback '%s'", cb->name); - } - } - } - } - } - else { - flb_plg_error(ctx->ins, "No metrics is specified"); - - return -1; - } - - return 0; -} - -static int in_ne_exit(void *data, struct flb_config *config) -{ - int ret; - struct flb_ne *ctx = data; - struct mk_list *head; - struct flb_slist_entry *entry; - - if (!ctx) { - return 0; - } - - /* Teardown for callback tied up resources */ - if (ctx->metrics) { - mk_list_foreach(head, ctx->metrics) { - entry = mk_list_entry(head, struct flb_slist_entry, _head); - ret = flb_callback_exists(ctx->callback, entry->str); - - if (ret == FLB_TRUE) { - if (strncmp(entry->str, "cpufreq", 7) == 0) { - /* nop */ - } - else if (strncmp(entry->str, "cpu", 3) == 0) { - /* nop */ - } - else if (strncmp(entry->str, "meminfo", 7) == 0) { - ne_meminfo_exit(ctx); - } - else if (strncmp(entry->str, "diskstats", 9) == 0) { - ne_diskstats_exit(ctx); - } - else if (strncmp(entry->str, "filesystem", 10) == 0) { - ne_filesystem_exit(ctx); - } - else if (strncmp(entry->str, "uname", 5) == 0) { - /* nop */ - } - else if (strncmp(entry->str, "stat", 4) == 0) { - /* nop */ - } - else if (strncmp(entry->str, "time", 4) == 0) { - /* nop */ - } - else if (strncmp(entry->str, "loadavg", 7) == 0) { - /* nop */ - } - else if (strncmp(entry->str, "vmstat", 6) == 0) { - ne_vmstat_exit(ctx); - } - else if (strncmp(entry->str, "netdev", 6) == 0) { - ne_netdev_exit(ctx); - } - else if (strncmp(entry->str, "filefd", 6) == 0) { - /* nop */ - } - else if (strncmp(entry->str, "textfile", 8) == 0) { - /* nop */ - } - else if (strncmp(entry->str, "systemd", 7) == 0) { - ne_systemd_exit(ctx); - } - else { - flb_plg_warn(ctx->ins, "Unknown metrics: %s", entry->str); - } - } - } - } - - /* destroy callback context */ - if (ctx->callback) { - flb_callback_destroy(ctx->callback); - } - - /* Teardown for timer tied up resources */ - if (ctx->coll_meminfo_fd != -1) { - ne_meminfo_exit(ctx); - } - if (ctx->coll_diskstats_fd != -1) { - ne_diskstats_exit(ctx); - } - if (ctx->coll_filesystem_fd != -1) { - ne_filesystem_exit(ctx); - } - if (ctx->coll_vmstat_fd != -1) { - ne_vmstat_exit(ctx); - } - if (ctx->coll_netdev_fd != -1) { - ne_netdev_exit(ctx); - } - if (ctx->coll_systemd_fd != -1) { - ne_systemd_exit(ctx); - } - - flb_ne_config_destroy(ctx); - - return 0; -} - -static void in_ne_pause(void *data, struct flb_config *config) -{ - struct flb_ne *ctx = data; - - flb_input_collector_pause(ctx->coll_fd, ctx->ins); - if (ctx->coll_cpu_fd != -1) { - flb_input_collector_pause(ctx->coll_cpu_fd, ctx->ins); - } - if (ctx->coll_cpufreq_fd != -1) { - flb_input_collector_pause(ctx->coll_cpufreq_fd, ctx->ins); - } - if (ctx->coll_meminfo_fd != -1) { - flb_input_collector_pause(ctx->coll_meminfo_fd, ctx->ins); - } - if (ctx->coll_diskstats_fd != -1) { - flb_input_collector_pause(ctx->coll_diskstats_fd, ctx->ins); - } - if (ctx->coll_filesystem_fd != -1) { - flb_input_collector_pause(ctx->coll_filesystem_fd, ctx->ins); - } - if (ctx->coll_uname_fd != -1) { - flb_input_collector_pause(ctx->coll_uname_fd, ctx->ins); - } - if (ctx->coll_stat_fd != -1) { - flb_input_collector_pause(ctx->coll_stat_fd, ctx->ins); - } - if (ctx->coll_time_fd != -1) { - flb_input_collector_pause(ctx->coll_time_fd, ctx->ins); - } - if (ctx->coll_loadavg_fd != -1) { - flb_input_collector_pause(ctx->coll_loadavg_fd, ctx->ins); - } - if (ctx->coll_vmstat_fd != -1) { - flb_input_collector_pause(ctx->coll_vmstat_fd, ctx->ins); - } - if (ctx->coll_netdev_fd != -1) { - flb_input_collector_pause(ctx->coll_netdev_fd, ctx->ins); - } - if (ctx->coll_filefd_fd != -1) { - flb_input_collector_pause(ctx->coll_filefd_fd, ctx->ins); - } - if (ctx->coll_textfile_fd != -1) { - flb_input_collector_pause(ctx->coll_textfile_fd, ctx->ins); - } - if (ctx->coll_systemd_fd != -1) { - flb_input_collector_pause(ctx->coll_systemd_fd, ctx->ins); - } -} - -static void in_ne_resume(void *data, struct flb_config *config) -{ - struct flb_ne *ctx = data; - - flb_input_collector_resume(ctx->coll_fd, ctx->ins); - if (ctx->coll_cpu_fd != -1) { - flb_input_collector_resume(ctx->coll_cpu_fd, ctx->ins); - } - if (ctx->coll_cpufreq_fd != -1) { - flb_input_collector_resume(ctx->coll_cpufreq_fd, ctx->ins); - } - if (ctx->coll_meminfo_fd != -1) { - flb_input_collector_resume(ctx->coll_meminfo_fd, ctx->ins); - } - if (ctx->coll_diskstats_fd != -1) { - flb_input_collector_resume(ctx->coll_diskstats_fd, ctx->ins); - } - if (ctx->coll_filesystem_fd != -1) { - flb_input_collector_resume(ctx->coll_filesystem_fd, ctx->ins); - } - if (ctx->coll_uname_fd != -1) { - flb_input_collector_resume(ctx->coll_uname_fd, ctx->ins); - } - if (ctx->coll_stat_fd != -1) { - flb_input_collector_resume(ctx->coll_stat_fd, ctx->ins); - } - if (ctx->coll_time_fd != -1) { - flb_input_collector_resume(ctx->coll_time_fd, ctx->ins); - } - if (ctx->coll_loadavg_fd != -1) { - flb_input_collector_resume(ctx->coll_loadavg_fd, ctx->ins); - } - if (ctx->coll_vmstat_fd != -1) { - flb_input_collector_resume(ctx->coll_vmstat_fd, ctx->ins); - } - if (ctx->coll_netdev_fd != -1) { - flb_input_collector_resume(ctx->coll_netdev_fd, ctx->ins); - } - if (ctx->coll_filefd_fd != -1) { - flb_input_collector_resume(ctx->coll_filefd_fd, ctx->ins); - } - if (ctx->coll_textfile_fd != -1) { - flb_input_collector_resume(ctx->coll_textfile_fd, ctx->ins); - } - if (ctx->coll_systemd_fd != -1) { - flb_input_collector_resume(ctx->coll_systemd_fd, ctx->ins); - } -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_TIME, "scrape_interval", "5", - 0, FLB_TRUE, offsetof(struct flb_ne, scrape_interval), - "scrape interval to collect metrics from the node." - }, - - { - FLB_CONFIG_MAP_TIME, "collector.cpu.scrape_interval", "0", - 0, FLB_TRUE, offsetof(struct flb_ne, cpu_scrape_interval), - "scrape interval to collect cpu metrics from the node." - }, - - { - FLB_CONFIG_MAP_TIME, "collector.cpufreq.scrape_interval", "0", - 0, FLB_TRUE, offsetof(struct flb_ne, cpufreq_scrape_interval), - "scrape interval to collect cpufreq metrics from the node." - }, - - { - FLB_CONFIG_MAP_TIME, "collector.meminfo.scrape_interval", "0", - 0, FLB_TRUE, offsetof(struct flb_ne, meminfo_scrape_interval), - "scrape interval to collect meminfo metrics from the node." - }, - - { - FLB_CONFIG_MAP_TIME, "collector.diskstats.scrape_interval", "0", - 0, FLB_TRUE, offsetof(struct flb_ne, diskstats_scrape_interval), - "scrape interval to collect diskstats metrics from the node." - }, - - { - FLB_CONFIG_MAP_TIME, "collector.filesystem.scrape_interval", "0", - 0, FLB_TRUE, offsetof(struct flb_ne, filesystem_scrape_interval), - "scrape interval to collect filesystem metrics from the node." - }, - - { - FLB_CONFIG_MAP_TIME, "collector.uname.scrape_interval", "0", - 0, FLB_TRUE, offsetof(struct flb_ne, uname_scrape_interval), - "scrape interval to collect uname metrics from the node." - }, - - { - FLB_CONFIG_MAP_TIME, "collector.stat.scrape_interval", "0", - 0, FLB_TRUE, offsetof(struct flb_ne, stat_scrape_interval), - "scrape interval to collect stat metrics from the node." - }, - - { - FLB_CONFIG_MAP_TIME, "collector.time.scrape_interval", "0", - 0, FLB_TRUE, offsetof(struct flb_ne, time_scrape_interval), - "scrape interval to collect time metrics from the node." - }, - - { - FLB_CONFIG_MAP_TIME, "collector.loadavg.scrape_interval", "0", - 0, FLB_TRUE, offsetof(struct flb_ne, loadavg_scrape_interval), - "scrape interval to collect loadavg metrics from the node." - }, - - { - FLB_CONFIG_MAP_TIME, "collector.vmstat.scrape_interval", "0", - 0, FLB_TRUE, offsetof(struct flb_ne, vmstat_scrape_interval), - "scrape interval to collect vmstat metrics from the node." - }, - - { - FLB_CONFIG_MAP_TIME, "collector.netdev.scrape_interval", "0", - 0, FLB_TRUE, offsetof(struct flb_ne, netdev_scrape_interval), - "scrape interval to collect netdev metrics from the node." - }, - - { - FLB_CONFIG_MAP_TIME, "collector.filefd.scrape_interval", "0", - 0, FLB_TRUE, offsetof(struct flb_ne, filefd_scrape_interval), - "scrape interval to collect filefd metrics from the node." - }, - - { - FLB_CONFIG_MAP_TIME, "collector.textfile.scrape_interval", "0", - 0, FLB_TRUE, offsetof(struct flb_ne, textfile_scrape_interval), - "scrape interval to collect textfile metrics from the node." - }, - - { - FLB_CONFIG_MAP_TIME, "collector.systemd.scrape_interval", "0", - 0, FLB_TRUE, offsetof(struct flb_ne, systemd_scrape_interval), - "scrape interval to collect systemd metrics from the node." - }, - - { - FLB_CONFIG_MAP_CLIST, "metrics", - "cpu,cpufreq,meminfo,diskstats,filesystem,uname,stat,time,loadavg,vmstat,netdev,filefd,systemd", - 0, FLB_TRUE, offsetof(struct flb_ne, metrics), - "Comma separated list of keys to enable metrics." - }, - - { - FLB_CONFIG_MAP_STR, "collector.textfile.path", NULL, - 0, FLB_TRUE, offsetof(struct flb_ne, path_textfile), - "Specify file path or directory to collect textfile metrics from the node." - }, - - { - FLB_CONFIG_MAP_STR, "path.procfs", "/proc", - 0, FLB_TRUE, offsetof(struct flb_ne, path_procfs), - "procfs mount point" - }, - - { - FLB_CONFIG_MAP_STR, "path.sysfs", "/sys", - 0, FLB_TRUE, offsetof(struct flb_ne, path_sysfs), - "sysfs mount point" - }, - - /* Systemd specific settings */ - { - FLB_CONFIG_MAP_BOOL, "systemd_service_restart_metrics", "false", - 0, FLB_TRUE, offsetof(struct flb_ne, systemd_include_service_restarts), - "include systemd service restart metrics" - }, - - { - FLB_CONFIG_MAP_BOOL, "systemd_unit_start_time_metrics", "false", - 0, FLB_TRUE, offsetof(struct flb_ne, systemd_include_unit_start_times), - "include systemd unit start time metrics" - }, - - { - FLB_CONFIG_MAP_BOOL, "systemd_include_service_task_metrics", "false", - 0, FLB_TRUE, offsetof(struct flb_ne, systemd_include_service_task_metrics), - "include systemd service task metrics" - }, - - { - FLB_CONFIG_MAP_STR, "systemd_include_pattern", NULL, - 0, FLB_TRUE, offsetof(struct flb_ne, systemd_regex_include_list_text), - "include list regular expression" - }, - - { - FLB_CONFIG_MAP_STR, "systemd_exclude_pattern", ".+\\.(automount|device|mount|scope|slice)", - 0, FLB_TRUE, offsetof(struct flb_ne, systemd_regex_exclude_list_text), - "exclude list regular expression" - }, - - /* filesystem specific settings */ - { - FLB_CONFIG_MAP_STR, "filesystem.ignore_mount_point_regex", IGNORED_MOUNT_POINTS, - 0, FLB_TRUE, offsetof(struct flb_ne, fs_regex_ingore_mount_point_text), - "ignore regular expression for mount points" - }, - - { - FLB_CONFIG_MAP_STR, "filesystem.ignore_filesystem_type_regex", IGNORED_FS_TYPES, - 0, FLB_TRUE, offsetof(struct flb_ne, fs_regex_ingore_filesystem_type_text), - "ignore regular expression for filesystem types" - }, - - /* diskstats specific settings */ - { - FLB_CONFIG_MAP_STR, "diskstats.ignore_device_regex", IGNORED_DEVICES, - 0, FLB_TRUE, offsetof(struct flb_ne, dt_regex_skip_devices_text), - "ignore regular expression for disk devices" - }, - /* EOF */ - {0} -}; - -struct flb_input_plugin in_node_exporter_metrics_plugin = { - .name = "node_exporter_metrics", - .description = "Node Exporter Metrics (Prometheus Compatible)", - .cb_init = in_ne_init, - .cb_pre_run = NULL, - .cb_collect = cb_ne_collect, - .cb_flush_buf = NULL, - .config_map = config_map, - .cb_pause = in_ne_pause, - .cb_resume = in_ne_resume, - .cb_exit = in_ne_exit, - .flags = FLB_INPUT_THREADED -}; diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne.h b/fluent-bit/plugins/in_node_exporter_metrics/ne.h deleted file mode 100644 index ba6e89caa..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne.h +++ /dev/null @@ -1,191 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_NODE_EXPORTER_H -#define FLB_NODE_EXPORTER_H - -/* utils: scan content type expected */ -#define NE_SCAN_FILE 1 -#define NE_SCAN_DIR 2 - -#include -#include -#include -#include -#include - -/* filesystem: regex for ignoring mount points and filesystem types */ - -#define IGNORED_MOUNT_POINTS "^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/)" -#define IGNORED_FS_TYPES "^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$" - -/* diskstats: regex for ignoring devices */ -#define IGNORED_DEVICES "^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\\d+n\\d+p)\\d+$" - -struct flb_ne { - /* configuration */ - flb_sds_t path_procfs; - flb_sds_t path_sysfs; - flb_sds_t path_textfile; - int scrape_interval; - - int coll_fd; /* collector fd */ - struct cmt *cmt; /* cmetrics context */ - struct flb_input_instance *ins; /* input instance */ - struct flb_callback *callback; /* metric callback */ - struct mk_list *metrics; /* enabled metrics */ - - /* Individual intervals for metrics */ - int cpu_scrape_interval; - int cpufreq_scrape_interval; - int meminfo_scrape_interval; - int diskstats_scrape_interval; - int filesystem_scrape_interval; - int uname_scrape_interval; - int stat_scrape_interval; - int time_scrape_interval; - int loadavg_scrape_interval; - int vmstat_scrape_interval; - int netdev_scrape_interval; - int filefd_scrape_interval; - int textfile_scrape_interval; - int systemd_scrape_interval; - - int coll_cpu_fd; /* collector fd (cpu) */ - int coll_cpufreq_fd; /* collector fd (cpufreq) */ - int coll_meminfo_fd; /* collector fd (meminfo) */ - int coll_diskstats_fd; /* collector fd (diskstat) */ - int coll_filesystem_fd; /* collector fd (filesystem) */ - int coll_uname_fd; /* collector fd (uname) */ - int coll_stat_fd; /* collector fd (stat) */ - int coll_time_fd; /* collector fd (time) */ - int coll_loadavg_fd; /* collector fd (loadavg) */ - int coll_vmstat_fd; /* collector fd (vmstat) */ - int coll_netdev_fd; /* collector fd (netdev) */ - int coll_filefd_fd; /* collector fd (filefd) */ - int coll_textfile_fd; /* collector fd (textfile) */ - int coll_systemd_fd ; /* collector fd (systemd) */ - - /* - * Metrics Contexts - * ---------------- - */ - - /* cpu_linux */ - struct cmt_counter *cpu_core_throttles; - struct cmt_counter *cpu_package_throttles; - - /* cpufreq_linux */ - struct cmt_gauge *cpu_freq_hertz; - struct cmt_gauge *cpu_freq_min_hertz; - struct cmt_gauge *cpu_freq_max_hertz; - - /* cpufreq scaling linux */ - struct cmt_gauge *cpu_scaling_freq_hertz; - struct cmt_gauge *cpu_scaling_freq_max_hertz; - struct cmt_gauge *cpu_scaling_freq_min_hertz; - - /* cpu seconds & guest seconds */ - struct cmt_counter *cpu_seconds; - struct cmt_counter *cpu_guest_seconds; - - /* meminfo hash table */ - struct flb_hash_table *meminfo_ht; - - /* diskstats: abbreviation 'dt' */ - void *dt_metrics; - struct flb_regex *dt_regex_skip_devices; - flb_sds_t dt_regex_skip_devices_text; - - /* uname */ - struct cmt_gauge *uname; - - /* stat_linux */ - struct cmt_counter *st_intr; - struct cmt_counter *st_context_switches; - struct cmt_gauge *st_boot_time; - struct cmt_counter *st_forks; - struct cmt_gauge *st_procs_running; - struct cmt_gauge *st_procs_blocked; - - /* vmstat_linux */ - struct flb_hash_table *vml_ht; - struct flb_regex *vml_regex_fields; - - /* netdev */ - struct flb_hash_table *netdev_ht; - - /* time */ - struct cmt_gauge *time; - - /* loadavg */ - struct cmt_gauge *lavg_1; - struct cmt_gauge *lavg_5; - struct cmt_gauge *lavg_15; - - /* filefd_linux */ - struct cmt_gauge *filefd_allocated; - struct cmt_gauge *filefd_maximum; - - /* filesystem: abbreviation 'fs' */ - struct cmt_gauge *fs_avail_bytes; - struct cmt_gauge *fs_device_error; - struct cmt_gauge *fs_files; - struct cmt_gauge *fs_files_free; - struct cmt_gauge *fs_free_bytes; - struct cmt_gauge *fs_readonly; - struct cmt_gauge *fs_size_bytes; - flb_sds_t fs_regex_ingore_mount_point_text; - flb_sds_t fs_regex_ingore_filesystem_type_text; - - struct flb_regex *fs_regex_read_only; - struct flb_regex *fs_regex_skip_mount; - struct flb_regex *fs_regex_skip_fs_types; - - /* testfile */ - struct cmt_counter *load_errors; - - /* systemd */ - - struct cmt_gauge *systemd_socket_accepted_connections; - struct cmt_gauge *systemd_socket_active_connections; - struct cmt_gauge *systemd_socket_refused_connections; - struct cmt_counter *systemd_service_restarts; - struct cmt_gauge *systemd_unit_start_times; - struct cmt_gauge *systemd_system_running; - struct cmt_gauge *systemd_timer_last_trigger_seconds; - struct cmt_gauge *systemd_unit_state; - struct cmt_gauge *systemd_unit_tasks; - struct cmt_gauge *systemd_unit_tasks_max; - struct cmt_gauge *systemd_units; - struct cmt_gauge *systemd_version; - void *systemd_dbus_handle; - int systemd_initialization_flag; - int systemd_include_unit_start_times; - int systemd_include_service_restarts; - int systemd_include_service_task_metrics; - flb_sds_t systemd_regex_include_list_text; - flb_sds_t systemd_regex_exclude_list_text; - struct flb_regex *systemd_regex_include_list; - struct flb_regex *systemd_regex_exclude_list; - double libsystemd_version; - char *libsystemd_version_text; -}; - -#endif diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_config.c b/fluent-bit/plugins/in_node_exporter_metrics/ne_config.c deleted file mode 100644 index ccf99d8ee..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_config.c +++ /dev/null @@ -1,69 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include "ne.h" - -struct flb_ne *flb_ne_config_create(struct flb_input_instance *ins, - struct flb_config *config) -{ - int ret; - struct flb_ne *ctx; - - ctx = flb_calloc(1, sizeof(struct flb_ne)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - - /* Load the config map */ - ret = flb_input_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return NULL; - } - - /* mount points */ - flb_plg_info(ins, "path.procfs = %s", ctx->path_procfs); - flb_plg_info(ins, "path.sysfs = %s", ctx->path_sysfs); - - ctx->cmt = cmt_create(); - if (!ctx->cmt) { - flb_plg_error(ins, "could not initialize CMetrics"); - flb_free(ctx); - return NULL; - } - - - return ctx; -} - -void flb_ne_config_destroy(struct flb_ne *ctx) -{ - if (!ctx) { - return; - } - - if (ctx->cmt) { - cmt_destroy(ctx->cmt); - } - - flb_free(ctx); -} diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_config.h b/fluent-bit/plugins/in_node_exporter_metrics/ne_config.h deleted file mode 100644 index 2bc1960e0..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_config.h +++ /dev/null @@ -1,31 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_NE_CONFIG_H -#define FLB_NE_CONFIG_H - -#include -#include "ne.h" - -struct flb_ne *flb_ne_config_create(struct flb_input_instance *ins, - struct flb_config *config); - -void flb_ne_config_destroy(struct flb_ne *ctx); - -#endif diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_cpu.c b/fluent-bit/plugins/in_node_exporter_metrics/ne_cpu.c deleted file mode 100644 index 70fd33ca1..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_cpu.c +++ /dev/null @@ -1,23 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifdef __linux__ -#include "ne_cpu_linux.c" -#include "ne_cpufreq_linux.c" -#endif diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_cpu.h b/fluent-bit/plugins/in_node_exporter_metrics/ne_cpu.h deleted file mode 100644 index 52c6b574f..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_cpu.h +++ /dev/null @@ -1,28 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_NE_CPU_H -#define FLB_IN_NE_CPU_H - -#include "ne.h" - -int ne_cpu_init(struct flb_ne *ctx); -int ne_cpu_update(struct flb_ne *ctx); - -#endif diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_cpu_linux.c b/fluent-bit/plugins/in_node_exporter_metrics/ne_cpu_linux.c deleted file mode 100644 index 8963f0c55..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_cpu_linux.c +++ /dev/null @@ -1,396 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include "ne.h" -#include "ne_utils.h" - -#include - -/* - * See kernel documentation for a description: - * https://www.kernel.org/doc/html/latest/filesystems/proc.html - * - * user: normal processes executing in user mode - * nice: niced processes executing in user mode - * system: processes executing in kernel mode - * idle: twiddling thumbs - * iowait: In a word, iowait stands for waiting for I/O to complete. But there are several problems: - * irq: servicing interrupts - * softirq: servicing softirqs - * steal: involuntary wait - * guest: running a normal guest - * guest_nice: running a niced guest - * - * Ensure to pick the correct version of the documentation, older versions here: - * https://github.com/torvalds/linux/tree/master/Documentation - */ -struct cpu_stat_info { - double user; - double nice; - double system; - double idle; - double iowait; - double irq; - double softirq; - double steal; - double guest; - double guest_nice; -}; - -/* - * Thermal throttle stats, reads /sys/devices/system/cpu/cpu* - * ---------------------------------------------------------- - */ -static inline int cpu_thermal_init(struct flb_ne *ctx) -{ - struct cmt_counter *c; - - c = cmt_counter_create(ctx->cmt, "node", "cpu", "core_throttles_total", - "Number of times this CPU core has been throttled.", - 2, (char *[]) {"core", "package"}); - if (!c) { - return -1; - } - ctx->cpu_core_throttles = c; - - - c = cmt_counter_create(ctx->cmt, "node", "cpu", "package_throttles_total", - "Number of times this CPU package has been throttled.", - 1, (char *[]) {"package"}); - if (!c) { - return -1; - } - ctx->cpu_package_throttles = c; - - return 0; -} - -static int cpu_thermal_update(struct flb_ne *ctx, uint64_t ts) -{ - int ret; - uint64_t core_id = 0; - uint64_t physical_package_id = 0; - uint64_t core_throttle_count; - uint64_t package_throttle_count; - char tmp1[32]; - char tmp2[32]; - struct mk_list *head; - struct mk_list list; - struct flb_slist_entry *entry; - const char *pattern = "/devices/system/cpu/cpu[0-9]*"; - /* Status arrays */ - uint64_t core_throttles_set[32][256]; - uint64_t package_throttles_set[32]; - - ret = ne_utils_path_scan(ctx, ctx->path_sysfs, pattern, NE_SCAN_DIR, &list); - if (ret != 0) { - return -1; - } - - if (mk_list_size(&list) == 0) { - return 0; - } - - /* Reset arrays status */ - memset(&core_throttles_set, 0, sizeof(core_throttles_set)); - memset(&package_throttles_set, 0, sizeof(package_throttles_set)); - - /* Process entries */ - mk_list_foreach(head, &list) { - entry = mk_list_entry(head, struct flb_slist_entry, _head); - - /* Core ID */ - ret = ne_utils_file_read_uint64(ctx->path_sysfs, - entry->str, - "topology", "core_id", - &core_id); - if (ret != 0) { - continue; - } - - /* Physical ID */ - ret = ne_utils_file_read_uint64(ctx->path_sysfs, - entry->str, - "topology", "physical_package_id", - &physical_package_id); - if (ret != 0) { - continue; - } - - /* Only update this kv pair once */ - if (core_throttles_set[physical_package_id][core_id] != 0) { - continue; - } - core_throttles_set[physical_package_id][core_id] = 1; - - /* Package Metric: node_cpu_core_throttles_total */ - ret = ne_utils_file_read_uint64(ctx->path_sysfs, - entry->str, - "thermal_throttle", "core_throttle_count", - &core_throttle_count); - if (ret != 0) { - flb_plg_debug(ctx->ins, - "CPU is missing core_throttle_count: %s", - entry->str); - } - else { - snprintf(tmp1, sizeof(tmp1) -1, "%" PRIu64, core_id); - snprintf(tmp2, sizeof(tmp2) -1, "%" PRIu64, physical_package_id); - - /* Set new value */ - cmt_counter_set(ctx->cpu_core_throttles, ts, - (double) core_throttle_count, - 2, (char *[]) {tmp1, tmp2}); - } - - /* Only update this entry once */ - if (package_throttles_set[physical_package_id] != 0) { - continue; - } - package_throttles_set[physical_package_id] = 1; - - /* Package Metric: node_cpu_package_throttles_total */ - ret = ne_utils_file_read_uint64(ctx->path_sysfs, - entry->str, - "thermal_throttle", "package_throttle_count", - &package_throttle_count); - if (ret != 0) { - flb_plg_debug(ctx->ins, - "CPU is missing package_throttle_count: %s", - entry->str); - } - else { - /* Set new value */ - cmt_counter_set(ctx->cpu_package_throttles, ts, - (double) package_throttle_count, - 1, (char *[]) {tmp2}); - } - } - flb_slist_destroy(&list); - - /* - * FIXME: continue fixing this: - * - * https://github.com/prometheus/node_exporter/blob/master/collector/cpu_linux.go#L194 - */ - - return 0; -} - -/* - * CPU stats, reads /proc/stat - * --------------------------- - */ -static inline int cpu_stat_init(struct flb_ne *ctx) -{ - struct cmt_counter *c; - - c = cmt_counter_create(ctx->cmt, "node", "cpu", "seconds_total", - "Seconds the CPUs spent in each mode.", - 2, (char *[]) {"cpu", "mode"}); - if (!c) { - return -1; - } - ctx->cpu_seconds = c; - - c = cmt_counter_create(ctx->cmt, "node", "cpu", "guest_seconds_total", - "Seconds the CPUs spent in guests (VMs) for each mode.", - 2, (char *[]) {"cpu", "mode"}); - if (!c) { - return -1; - } - ctx->cpu_guest_seconds = c; - - return 0; -} - -static int stat_line(char *line, struct cpu_stat_info *st) -{ - int ret; - double user_hz = sysconf(_SC_CLK_TCK); - const char *cpu_fmt = "%lf %lf %lf %lf %lf %lf %lf %lf %lf %lf"; - - ret = sscanf(line, cpu_fmt, - &st->user, - &st->nice, - &st->system, - &st->idle, - &st->iowait, - &st->irq, - &st->softirq, - &st->steal, - &st->guest, - &st->guest_nice); - - /* On some older kernels the 'guest_nice' value may be missing */ - if (ret < 9) { - return -1; - } - /* Ensure we zero initialise it */ - if ( ret == 9 ) { - st->guest_nice = 0; - } - - /* Convert to seconds based on USER_HZ kernel param */ - st->user /= user_hz; - st->nice /= user_hz; - st->system /= user_hz; - st->idle /= user_hz; - st->iowait /= user_hz; - st->irq /= user_hz; - st->softirq /= user_hz; - st->steal /= user_hz; - st->guest /= user_hz; - st->guest_nice /= user_hz; - - return 0; -} - -static int cpu_stat_set_metrics(struct flb_ne *ctx, char *cpu_id, - struct cpu_stat_info *st, uint64_t ts) -{ - - /* CPU seconds */ - cmt_counter_set(ctx->cpu_seconds, ts, - st->idle, - 2, (char *[]) {cpu_id, "idle"}); - - cmt_counter_set(ctx->cpu_seconds, ts, - st->iowait, - 2, (char *[]) {cpu_id, "iowait"}); - - cmt_counter_set(ctx->cpu_seconds, ts, - st->irq, - 2, (char *[]) {cpu_id, "irq"}); - - cmt_counter_set(ctx->cpu_seconds, ts, - st->nice, - 2, (char *[]) {cpu_id, "nice"}); - - cmt_counter_set(ctx->cpu_seconds, ts, - st->softirq, - 2, (char *[]) {cpu_id, "softirq"}); - - - cmt_counter_set(ctx->cpu_seconds, ts, - st->steal, - 2, (char *[]) {cpu_id, "steal"}); - - cmt_counter_set(ctx->cpu_seconds, ts, - st->system, - 2, (char *[]) {cpu_id, "system"}); - - cmt_counter_set(ctx->cpu_seconds, ts, - st->user, - 2, (char *[]) {cpu_id, "user"}); - - /* CPU Guest Seconds */ - cmt_counter_set(ctx->cpu_guest_seconds, ts, - st->guest, - 2, (char *[]) {cpu_id, "user"}); - - cmt_counter_set(ctx->cpu_guest_seconds, ts, - st->guest_nice, - 2, (char *[]) {cpu_id, "nice"}); - - return 0; -} - -static int cpu_stat_update(struct flb_ne *ctx, uint64_t ts) -{ - int len; - int ret; - char *p; - char tmp[32]; - struct mk_list list; - struct mk_list *head; - struct flb_slist_entry *line; - struct cpu_stat_info st = {0}; - - ret = ne_utils_file_read_lines(ctx->path_procfs, "/stat", &list); - if (ret == -1) { - return -1; - } - - mk_list_foreach(head, &list) { - line = mk_list_entry(head, struct flb_slist_entry, _head); - - if (strncmp(line->str, "cpu ", 4) == 0) { - /* CPU total, we skip this state since we care only about per core stats */ - continue; - } - else if (strncmp(line->str, "cpu", 3) == 0) { - /* CPU ID (per core) */ - p = strchr(line->str + 3, ' '); - len = p - (line->str + 3); - memcpy(tmp, line->str + 3, len); - tmp[len] = '\0'; - - /* Capture metrics */ - ret = stat_line(p, &st); - if (ret != 0) { - flb_plg_error(ctx->ins, - "could not process line: %s", line->str); - continue; - } - - /* Update our counters */ - cpu_stat_set_metrics(ctx, tmp, &st, ts); - } - } - - flb_slist_destroy(&list); - return 0; -} - -int ne_cpu_init(struct flb_ne *ctx) -{ - int ret; - - /* CPU Thermal */ - ret = cpu_thermal_init(ctx); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not initialize cpu_thermal metrics"); - return -1; - } - - /* CPU Stats */ - ret = cpu_stat_init(ctx); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not initialize cpu_stat metrics"); - return -1; - } - cpu_stat_init(ctx); - return 0; -} - -int ne_cpu_update(struct flb_ne *ctx) -{ - uint64_t ts; - - ts = cfl_time_now(); - - cpu_thermal_update(ctx, ts); - cpu_stat_update(ctx, ts); - - return 0; -} diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_cpufreq.h b/fluent-bit/plugins/in_node_exporter_metrics/ne_cpufreq.h deleted file mode 100644 index b6712dd5a..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_cpufreq.h +++ /dev/null @@ -1,28 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_NE_CPUFREQ_H -#define FLB_IN_NE_CPUFREQ_H - -#include "ne.h" - -int ne_cpufreq_init(struct flb_ne *ctx); -int ne_cpufreq_update(struct flb_ne *ctx); - -#endif diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_cpufreq_linux.c b/fluent-bit/plugins/in_node_exporter_metrics/ne_cpufreq_linux.c deleted file mode 100644 index e31b976d8..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_cpufreq_linux.c +++ /dev/null @@ -1,196 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include "ne.h" -#include "ne_utils.h" - -static int cpufreq_init(struct flb_ne *ctx) -{ - struct cmt_gauge *g; - - /* node_cpu_frequency_hertz */ - g = cmt_gauge_create(ctx->cmt, "node", "cpu", "frequency_hertz", - "Current cpu thread frequency in hertz.", - 1, (char *[]) {"cpu"}); - if (!g) { - return -1; - } - ctx->cpu_freq_hertz = g; - - /* node_cpu_frequency_max_hertz */ - g = cmt_gauge_create(ctx->cmt, "node", "cpu", "frequency_max_hertz", - "Maximum cpu thread frequency in hertz.", - 1, (char *[]) {"cpu"}); - if (!g) { - return -1; - } - ctx->cpu_freq_max_hertz = g; - - /* node_cpu_frequency_min_hertz */ - g = cmt_gauge_create(ctx->cmt, "node", "cpu", "frequency_min_hertz", - "Minimum cpu thread frequency in hertz.", - 1, (char *[]) {"cpu"}); - if (!g) { - return -1; - } - ctx->cpu_freq_min_hertz = g; - - /* node_cpu_scaling_frequency_hertz */ - g = cmt_gauge_create(ctx->cmt, "node", "cpu", "scaling_frequency_hertz", - "Current scaled CPU thread frequency in hertz.", - 1, (char *[]) {"cpu"}); - if (!g) { - return -1; - } - ctx->cpu_scaling_freq_hertz = g; - - /* node_cpu_scaling_frequency_max_hertz */ - g = cmt_gauge_create(ctx->cmt, "node", "cpu", "scaling_frequency_max_hertz", - "Maximum scaled CPU thread frequency in hertz.", - 1, (char *[]) {"cpu"}); - if (!g) { - return -1; - } - ctx->cpu_scaling_freq_max_hertz = g; - - /* node_cpu_scaling_frequency_min_hertz */ - g = cmt_gauge_create(ctx->cmt, "node", "cpu", "scaling_frequency_min_hertz", - "Minimum scaled CPU thread frequency in hertz.", - 1, (char *[]) {"cpu"}); - if (!g) { - return -1; - } - ctx->cpu_scaling_freq_min_hertz = g; - - return 0; -} - -static int cpufreq_update(struct flb_ne *ctx) -{ - int ret; - int len; - uint64_t ts; - uint64_t val; - char *cpu_id; - struct mk_list list; - struct mk_list *head; - struct flb_slist_entry *entry; - const char *pattern = "/devices/system/cpu/cpu[0-9]*"; - - ret = ne_utils_path_scan(ctx, ctx->path_sysfs, pattern, NE_SCAN_DIR, &list); - if (ret != 0) { - return -1; - } - - if (mk_list_size(&list) == 0) { - return 0; - } - - ts = cfl_time_now(); - - /* Process entries */ - mk_list_foreach(head, &list) { - entry = mk_list_entry(head, struct flb_slist_entry, _head); - - /* Locate CPU ID string */ - len = flb_sds_len(entry->str); - cpu_id = entry->str + len; - while (*cpu_id != 'u') cpu_id--; - cpu_id++; - - /* node_cpu_frequency_hertz */ - ret = ne_utils_file_read_uint64(ctx->path_sysfs, - entry->str, "cpufreq", "cpuinfo_cur_freq", - &val); - if (ret == 0) { - cmt_gauge_set(ctx->cpu_freq_hertz, ts, - (double) (val * 1000.0), - 1, (char *[]) {cpu_id}); - } - - /* node_cpu_frequency_max_hertz */ - ret = ne_utils_file_read_uint64(ctx->path_sysfs, - entry->str, "cpufreq", "cpuinfo_max_freq", - &val); - if (ret == 0) { - cmt_gauge_set(ctx->cpu_freq_max_hertz, ts, - (double) (val * 1000.0), - 1, (char *[]) {cpu_id}); - } - - /* node_cpu_frequency_min_hertz */ - ret = ne_utils_file_read_uint64(ctx->path_sysfs, - entry->str, "cpufreq", "cpuinfo_min_freq", - &val); - if (ret == 0) { - cmt_gauge_set(ctx->cpu_freq_min_hertz, ts, - (double) (val * 1000.0), - 1, (char *[]) {cpu_id}); - } - - - /* node_cpu_scaling_frequency_hertz */ - ret = ne_utils_file_read_uint64(ctx->path_sysfs, - entry->str, "cpufreq", "scaling_cur_freq", - &val); - if (ret == 0) { - cmt_gauge_set(ctx->cpu_scaling_freq_hertz, ts, - ((double) val) * 1000.0, - 1, (char *[]) {cpu_id}); - } - - /* node_cpu_scaling_frequency_max_hertz */ - ret = ne_utils_file_read_uint64(ctx->path_sysfs, - entry->str, "cpufreq", "scaling_max_freq", - &val); - if (ret == 0) { - cmt_gauge_set(ctx->cpu_scaling_freq_max_hertz, ts, - (double) (val * 1000.0), - 1, (char *[]) {cpu_id}); - } - - /* node_cpu_frequency_min_hertz */ - ret = ne_utils_file_read_uint64(ctx->path_sysfs, - entry->str, "cpufreq", "scaling_min_freq", - &val); - if (ret == 0) { - cmt_gauge_set(ctx->cpu_scaling_freq_min_hertz, ts, - (double) (val * 1000.0), - 1, (char *[]) {cpu_id}); - } - } - - flb_slist_destroy(&list); - return 0; -} - -int ne_cpufreq_init(struct flb_ne *ctx) -{ - cpufreq_init(ctx); - return 0; -} - -int ne_cpufreq_update(struct flb_ne *ctx) -{ - cpufreq_update(ctx); - return 0; -} diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_diskstats.c b/fluent-bit/plugins/in_node_exporter_metrics/ne_diskstats.c deleted file mode 100644 index 0b4ac57f1..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_diskstats.c +++ /dev/null @@ -1,22 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifdef __linux__ -#include "ne_diskstats_linux.c" -#endif diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_diskstats.h b/fluent-bit/plugins/in_node_exporter_metrics/ne_diskstats.h deleted file mode 100644 index 4904e33ef..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_diskstats.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_NE_DISKSTATS_H -#define FLB_IN_NE_DISKSTATS_H - -#include "ne.h" - -int ne_diskstats_init(struct flb_ne *ctx); -int ne_diskstats_update(struct flb_ne *ctx); -int ne_diskstats_exit(struct flb_ne *ctx); - -#endif diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_diskstats_linux.c b/fluent-bit/plugins/in_node_exporter_metrics/ne_diskstats_linux.c deleted file mode 100644 index 26ba9cb93..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_diskstats_linux.c +++ /dev/null @@ -1,449 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include "ne.h" -#include "ne_utils.h" - -#include -#include - -/* - * Diskstats interface references - * ------------------------------ - * https://www.kernel.org/doc/Documentation/ABI/testing/procfs-diskstats - * https://www.kernel.org/doc/Documentation/iostats.txt - * - * From the documentation, Kernel versions and expected fields: - * - * == =================================== - * 1 major number - * 2 minor mumber - * 3 device name - * 4 reads completed successfully - * 5 reads merged - * 6 sectors read - * 7 time spent reading (ms) - * 8 writes completed - * 9 writes merged - * 10 sectors written - * 11 time spent writing (ms) - * 12 I/Os currently in progress - * 13 time spent doing I/Os (ms) - * 14 weighted time spent doing I/Os (ms) - * == =================================== - * - * Kernel 4.18+ appends four more fields for discard - * tracking putting the total at 18: - * - * == =================================== - * 15 discards completed successfully - * 16 discards merged - * 17 sectors discarded - * 18 time spent discarding - * == =================================== - * - * Kernel 5.5+ appends two more fields for flush requests: - * - * == ===================================== - * 19 flush requests completed successfully - * 20 time spent flushing - * == ===================================== - */ - -#define KNOWN_FIELDS 17 -#define SECTOR_SIZE 512 - -struct dt_metric { - void *metric; - double factor; -}; - -static void metric_cache_set(struct flb_ne *ctx, void *metric, double factor, int *offset) -{ - int id; - struct dt_metric *m; - struct dt_metric **cache; - - id = *offset; - - cache = (struct dt_metric **) ctx->dt_metrics; - m = (struct dt_metric *) &cache[id]; - m->metric = metric; - m->factor = factor; - (*offset)++; -} - -static void metric_cache_update(struct flb_ne *ctx, int id, flb_sds_t device, - flb_sds_t str_val) -{ - int ret = -1; - uint64_t ts; - double val; - struct dt_metric *m; - struct dt_metric **cache; - struct cmt_gauge *g; - struct cmt_counter *c; - - cache = (struct dt_metric **) ctx->dt_metrics; - m = (struct dt_metric *) &cache[id]; - - ret = ne_utils_str_to_double(str_val, &val); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not represent string value '%s' for metric id '%i', " - "device '%s'", - str_val, id, device); - return; - } - - ts = cfl_time_now(); - - if (m->factor > DBL_EPSILON) { - val *= m->factor; - } - - if (id == 8) { - g = (struct cmt_gauge *) m->metric; - ret = cmt_gauge_set(g, ts, val, 1, (char *[]) {device}); - } - else { - c = (struct cmt_counter *) m->metric; - ret = cmt_counter_set(c, ts, val, 1, (char *[]) {device}); - } - - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not update metric id '%i', device '%s'", - id, device); - } - -} - -/* Setup metrics contexts */ -static int ne_diskstats_configure(struct flb_ne *ctx) -{ - int offset = 0; - struct cmt_counter *c; - struct cmt_gauge *g; - - /* Create cache for metrics */ - ctx->dt_metrics = flb_calloc(1, sizeof(struct dt_metric) * KNOWN_FIELDS); - if (!ctx->dt_metrics) { - flb_errno(); - return -1; - } - - /* Initialize regex for skipped devices */ - ctx->dt_regex_skip_devices = flb_regex_create(ctx->dt_regex_skip_devices_text); - if (!ctx->dt_regex_skip_devices) { - flb_plg_error(ctx->ins, - "could not initialize regex pattern for ignored " - "devices: '%s'", - IGNORED_DEVICES); - return -1; - } - - /* node_disk_reads_completed_total */ - c = cmt_counter_create(ctx->cmt, "node", "disk", "reads_completed_total", - "The total number of reads completed successfully.", - 1, (char *[]) {"device"}); - if (!c) { - return -1; - } - metric_cache_set(ctx, c, 0, &offset); - - /* node_disk_reads_merged_total */ - c = cmt_counter_create(ctx->cmt, "node", "disk", "reads_merged_total", - "The total number of reads merged.", - 1, (char *[]) {"device"}); - if (!c) { - return -1; - } - metric_cache_set(ctx, c, 0, &offset); - - /* node_disk_read_bytes_total */ - c = cmt_counter_create(ctx->cmt, "node", "disk", "read_bytes_total", - "The total number of bytes read successfully.", - 1, (char *[]) {"device"}); - if (!c) { - return -1; - } - metric_cache_set(ctx, c, SECTOR_SIZE, &offset); - - /* node_disk_read_time_seconds_total */ - c = cmt_counter_create(ctx->cmt, "node", "disk", "read_time_seconds_total", - "The total number of seconds spent by all reads.", - 1, (char *[]) {"device"}); - if (!c) { - return -1; - } - metric_cache_set(ctx, c, .001, &offset); - - /* node_disk_writes_completed_total */ - c = cmt_counter_create(ctx->cmt, "node", "disk", "writes_completed_total", - "The total number of writes completed successfully.", - 1, (char *[]) {"device"}); - if (!c) { - return -1; - } - metric_cache_set(ctx, c, 0, &offset); - - /* node_disk_writes_merged_total */ - c = cmt_counter_create(ctx->cmt, "node", "disk", "writes_merged_total", - "The number of writes merged.", - 1, (char *[]) {"device"}); - if (!c) { - return -1; - } - metric_cache_set(ctx, c, 0, &offset); - - /* node_disk_written_bytes_total */ - c = cmt_counter_create(ctx->cmt, "node", "disk", "written_bytes_total", - "The total number of bytes written successfully.", - 1, (char *[]) {"device"}); - if (!c) { - return -1; - } - metric_cache_set(ctx, c, SECTOR_SIZE, &offset); - - /* node_disk_write_time_seconds_total */ - c = cmt_counter_create(ctx->cmt, "node", "disk", "write_time_seconds_total", - "This is the total number of seconds spent by all writes.", - 1, (char *[]) {"device"}); - if (!c) { - return -1; - } - metric_cache_set(ctx, c, .001, &offset); - - /* node_disk_io_now */ - g = cmt_gauge_create(ctx->cmt, "node", "disk", "io_now", - "The number of I/Os currently in progress.", - 1, (char *[]) {"device"}); - if (!g) { - return -1; - } - metric_cache_set(ctx, g, 0, &offset); - - /* node_disk_io_time_seconds */ - c = cmt_counter_create(ctx->cmt, "node", "disk", "io_time_seconds_total", - "Total seconds spent doing I/Os.", - 1, (char *[]) {"device"}); - if (!c) { - return -1; - } - metric_cache_set(ctx, c, .001, &offset); - - /* node_disk_io_time_weighted_seconds */ - c = cmt_counter_create(ctx->cmt, "node", "disk", "io_time_weighted_seconds_total", - "The weighted # of seconds spent doing I/Os.", - 1, (char *[]) {"device"}); - if (!c) { - return -1; - } - metric_cache_set(ctx, c, .001, &offset); - - /* - * Linux Kernel >= 4.18 - * ==================== - */ - - /* node_disk_discards_completed */ - c = cmt_counter_create(ctx->cmt, "node", "disk", "discards_completed_total", - "The total number of discards completed successfully.", - 1, (char *[]) {"device"}); - if (!c) { - return -1; - } - metric_cache_set(ctx, c, 0, &offset); - - /* node_disk_discards_merged */ - c = cmt_counter_create(ctx->cmt, "node", "disk", "discards_merged_total", - "The total number of discards merged.", - 1, (char *[]) {"device"}); - if (!c) { - return -1; - } - metric_cache_set(ctx, c, 0, &offset); - - /* node_disk_discarded_sectors */ - c = cmt_counter_create(ctx->cmt, "node", "disk", "discarded_sectors_total", - "The total number of sectors discarded successfully.", - 1, (char *[]) {"device"}); - if (!c) { - return -1; - } - metric_cache_set(ctx, c, 0, &offset); - - /* node_disk_discard_time_seconds */ - c = cmt_counter_create(ctx->cmt, "node", "disk", "discard_time_seconds_total", - "This is the total number of seconds spent by all discards.", - 1, (char *[]) {"device"}); - if (!c) { - return -1; - } - metric_cache_set(ctx, c, .001, &offset); - - /* - * Linux Kernel >= 5.5 - * =================== - */ - - /* node_disk_flush_requests */ - c = cmt_counter_create(ctx->cmt, "node", "disk", "flush_requests_total", - "The total number of flush requests completed successfully", - 1, (char *[]) {"device"}); - if (!c) { - return -1; - } - metric_cache_set(ctx, c, 0, &offset); - - /* node_disk_flush_requests_time_seconds */ - c = cmt_counter_create(ctx->cmt, "node", "disk", "flush_requests_time_seconds_total", - "This is the total number of seconds spent by all flush " - "requests.", - 1, (char *[]) {"device"}); - if (!c) { - return -1; - } - metric_cache_set(ctx, c, .001, &offset); - - return 0; -} - -static flb_sds_t get_part_id(struct mk_list *list, int id) -{ - int i = 0; - struct mk_list *head; - struct flb_slist_entry *entry; - - mk_list_foreach(head, list) { - if (i == id) { - entry = mk_list_entry(head, struct flb_slist_entry, _head); - return entry->str; - } - i++; - } - return NULL; -} - -static int skip_device(struct flb_ne *ctx, flb_sds_t device) -{ - return flb_regex_match(ctx->dt_regex_skip_devices, - (unsigned char *) device, flb_sds_len(device)); -} - -static int update_stats(struct flb_ne *ctx, struct mk_list *list, int parts) -{ - int id = 0; - flb_sds_t device; - struct mk_list *head; - struct flb_slist_entry *entry; - - /* Get device name: third entry */ - device = get_part_id(list, 2); - if (!device) { - flb_plg_error(ctx->ins, "cannot retrieve device name"); - return -1; - } - - /* Check if we should process or skip this device */ - if (skip_device(ctx, device)) { - flb_plg_debug(ctx->ins, "skip device: %s", device); - return 0; - } - - mk_list_foreach(head, list) { - /* Skip: major number, minor number and device name */ - if (id <= 2) { - id++; - continue; - } - entry = mk_list_entry(head, struct flb_slist_entry, _head); - - /* update the metric */ - metric_cache_update(ctx, id - 3, device, entry->str); - id++; - - /* Do not process more than the known fields as of this version */ - if (id - 3 == KNOWN_FIELDS) { - break; - } - } - return 0; -} - -static int diskstats_update(struct flb_ne *ctx) -{ - int ret; - int parts; - struct mk_list *head; - struct mk_list list; - struct mk_list split_list; - struct flb_slist_entry *line; - - mk_list_init(&list); - mk_list_init(&split_list); - - ret = ne_utils_file_read_lines(ctx->path_procfs, "/diskstats", &list); - if (ret == -1) { - return -1; - } - - mk_list_foreach(head, &list) { - line = mk_list_entry(head, struct flb_slist_entry, _head); - - mk_list_init(&split_list); - ret = flb_slist_split_string(&split_list, line->str, ' ', -1); - if (ret == -1) { - continue; - } - parts = ret; - - update_stats(ctx, &split_list, parts); - flb_slist_destroy(&split_list); - } - - flb_slist_destroy(&list); - return 0; -} - -int ne_diskstats_init(struct flb_ne *ctx) -{ - ne_diskstats_configure(ctx); - return 0; -} - -int ne_diskstats_update(struct flb_ne *ctx) -{ - diskstats_update(ctx); - return 0; -} - -int ne_diskstats_exit(struct flb_ne *ctx) -{ - flb_free(ctx->dt_metrics); - if (ctx->dt_regex_skip_devices) { - flb_regex_destroy(ctx->dt_regex_skip_devices); - } - return 0; -} diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_filefd_linux.c b/fluent-bit/plugins/in_node_exporter_metrics/ne_filefd_linux.c deleted file mode 100644 index 5c0c0166d..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_filefd_linux.c +++ /dev/null @@ -1,115 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define _GNU_SOURCE - -#include -#include - -#include "ne.h" -#include "ne_utils.h" - -static int filefd_configure(struct flb_ne *ctx) -{ - struct cmt_gauge *g; - - /* node_filefd_allocated */ - g = cmt_gauge_create(ctx->cmt, "node", "filefd", "allocated", - "File descriptor statistics: allocated.", - 0, NULL); - ctx->filefd_allocated = g; - - /* node_filefd_maximum */ - g = cmt_gauge_create(ctx->cmt, "node", "filefd", "maximum", - "File descriptor statistics: maximum.", - 0, NULL); - ctx->filefd_maximum = g; - - return 0; -} - -static int filefd_update(struct flb_ne *ctx) -{ - int ret; - int parts; - uint64_t ts; - double d_val; - struct mk_list *head; - struct mk_list list; - struct mk_list split_list; - struct flb_slist_entry *line; - struct flb_slist_entry *alloc; - struct flb_slist_entry *max; - - mk_list_init(&list); - ret = ne_utils_file_read_lines(ctx->path_procfs, "/sys/fs/file-nr", &list); - if (ret == -1) { - return -1; - } - - ts = cfl_time_now(); - - mk_list_foreach(head, &list) { - line = mk_list_entry(head, struct flb_slist_entry, _head); - - mk_list_init(&split_list); - ret = flb_slist_split_string(&split_list, line->str, '\t', -1); - if (ret == -1) { - continue; - } - parts = ret; - if (parts == 0) { - flb_slist_destroy(&split_list); - continue; - } - else if (parts != 3) { - flb_plg_warn(ctx->ins, "/sys/fs/file-nr: invalid number of fields"); - flb_slist_destroy(&split_list); - break; - } - - /* allocated (0) */ - alloc = flb_slist_entry_get(&split_list, 0); - ne_utils_str_to_double(alloc->str, &d_val); - cmt_gauge_set(ctx->filefd_allocated, ts, d_val, 0, NULL); - - /* maximum (2) */ - max = flb_slist_entry_get(&split_list, 2); - ne_utils_str_to_double(max->str, &d_val); - cmt_gauge_set(ctx->filefd_maximum, ts, d_val, 0, NULL); - - flb_slist_destroy(&split_list); - break; - } - flb_slist_destroy(&list); - - return 0; -} - -int ne_filefd_init(struct flb_ne *ctx) -{ - filefd_configure(ctx); - return 0; -} - -int ne_filefd_update(struct flb_ne *ctx) -{ - filefd_update(ctx); - return 0; -} diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_filefd_linux.h b/fluent-bit/plugins/in_node_exporter_metrics/ne_filefd_linux.h deleted file mode 100644 index ef9df5d1c..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_filefd_linux.h +++ /dev/null @@ -1,28 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_NE_FILEFD_H -#define FLB_IN_NE_FILEFD_H - -#include "ne.h" - -int ne_filefd_init(struct flb_ne *ctx); -int ne_filefd_update(struct flb_ne *ctx); - -#endif diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_filesystem.c b/fluent-bit/plugins/in_node_exporter_metrics/ne_filesystem.c deleted file mode 100644 index 6cc2a1960..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_filesystem.c +++ /dev/null @@ -1,39 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifdef __linux__ -#include "ne_filesystem_linux.c" -#else - -int ne_filesystem_init(struct flb_ne *ctx) -{ - return 0; -} - -int ne_filesystem_update(struct flb_ne *ctx) -{ - return 0; -} - -int ne_filesystem_exit(struct flb_ne *ctx) -{ - return 0; -} - -#endif \ No newline at end of file diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_filesystem.h b/fluent-bit/plugins/in_node_exporter_metrics/ne_filesystem.h deleted file mode 100644 index 1e87b2825..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_filesystem.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_NE_FILESYSTEM_H -#define FLB_IN_NE_FILESYSTEM_H - -#include "ne.h" - -int ne_filesystem_init(struct flb_ne *ctx); -int ne_filesystem_update(struct flb_ne *ctx); -int ne_filesystem_exit(struct flb_ne *ctx); - -#endif diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_filesystem_linux.c b/fluent-bit/plugins/in_node_exporter_metrics/ne_filesystem_linux.c deleted file mode 100644 index 5f054aa78..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_filesystem_linux.c +++ /dev/null @@ -1,404 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "ne.h" -#include "ne_utils.h" - -#include -#include - -#define NE_ERROR_MOUNT_POINT_LIST_FETCH_SUCCESS 0 -#define NE_ERROR_MOUNT_POINT_LIST_FETCH_GENERIC_ERROR -1 -#define NE_ERROR_MOUNT_POINT_LIST_FETCH_FILE_ACCESS_ERROR -2 -#define NE_ERROR_MOUNT_POINT_LIST_FETCH_CORRUPTED_DATA -3 - -static void unescape_character(cfl_sds_t input_buffer, char character) -{ - size_t needle_length; - char needle[8]; - char *haystack; - char *match; - - needle_length = snprintf(needle, sizeof(needle), "\\0%02o", character); - - haystack = (char *) input_buffer; - - do { - match = strstr(haystack, needle); - - if (match != NULL) { - match[0] = character; - - memmove(&match[1], - &match[needle_length], - strlen(match) - needle_length + 1); - } - - haystack = match; - } - while (match != NULL); -} - -static cfl_sds_t greedy_read_file(char *path) -{ - char read_buffer[1024]; - cfl_sds_t temporary_buffer; - FILE *file_handle; - size_t read_size; - cfl_sds_t contents; - - file_handle = fopen(path, "rb"); - - if (file_handle == NULL) { - return NULL; - } - - contents = cfl_sds_create_size(0); - - if (contents == NULL) { - flb_errno(); - fclose(file_handle); - - return NULL; - } - - do { - read_size = fread(read_buffer, - 1, - sizeof(read_buffer), - file_handle); - - if (read_size > 0) { - temporary_buffer = cfl_sds_cat(contents, read_buffer, read_size); - - if (temporary_buffer == NULL) { - cfl_sds_set_len(contents, 0); - - read_size = 0; - } - else { - contents = temporary_buffer; - } - } - } - while (read_size > 0); - - fclose(file_handle); - - if (cfl_sds_len(contents) == 0) { - cfl_sds_destroy(contents); - - contents = NULL; - } - - return contents; -} - -static int greedy_read_file_lines(char *path, struct mk_list *lines) -{ - cfl_sds_t contents; - int result; - - contents = greedy_read_file(path); - - if (contents == NULL) { - return NE_ERROR_MOUNT_POINT_LIST_FETCH_FILE_ACCESS_ERROR; - } - - mk_list_init(lines); - - result = flb_slist_split_string(lines, contents, '\n', -1); - - cfl_sds_destroy(contents); - - if (result == -1) { - return NE_ERROR_MOUNT_POINT_LIST_FETCH_CORRUPTED_DATA; - } - - return NE_ERROR_MOUNT_POINT_LIST_FETCH_SUCCESS; -} - -static int filesystem_update(struct flb_ne *ctx, - char *mounts_file_path) -{ - struct statfs mount_point_info; - char *field_values[4]; - struct mk_list *field_iterator; - struct mk_list *line_iterator; - int readonly_flag; - int field_index; - int skip_flag; - uint64_t timestamp; - char *labels[3]; - int result; - struct mk_list fields; - struct mk_list lines; - struct flb_slist_entry *field; - struct flb_slist_entry *line; - - result = greedy_read_file_lines(mounts_file_path, &lines); - - if (result != NE_ERROR_MOUNT_POINT_LIST_FETCH_SUCCESS) { - return result; - } - - mk_list_foreach(line_iterator, &lines) { - line = mk_list_entry(line_iterator, struct flb_slist_entry, _head); - - mk_list_init(&fields); - - result = flb_slist_split_string(&fields, line->str, ' ', -1); - if (result == -1) { - continue; - } - - field_index = 0; - - memset(field_values, 0, sizeof(field_values)); - - mk_list_foreach(field_iterator, &fields) { - field = mk_list_entry(field_iterator, - struct flb_slist_entry, - _head); - - if (field_index < 4) { - field_values[field_index] = field->str; - } - else { - break; - } - - field_index++; - } - - if (field_values[0] != NULL && /* device */ - field_values[1] != NULL && /* path */ - field_values[2] != NULL && /* fs type */ - field_values[3] != NULL) { /* options */ - skip_flag = flb_regex_match(ctx->fs_regex_skip_fs_types, - (unsigned char *) field_values[2], - strlen(field_values[2])); - - if (!skip_flag) { - unescape_character(field_values[1], ' '); - unescape_character(field_values[1], '\t'); - - skip_flag = flb_regex_match(ctx->fs_regex_skip_mount, - (unsigned char *) field_values[1], - strlen(field_values[1])); - - if (!skip_flag) { - timestamp = cfl_time_now(); - - result = statfs(field_values[1], &mount_point_info); - - if (result == 0) { - labels[0] = field_values[0]; - labels[1] = field_values[2]; - labels[2] = field_values[1]; - - readonly_flag = mount_point_info.f_flags & ST_RDONLY; - readonly_flag = (readonly_flag != 0); - - cmt_gauge_set(ctx->fs_avail_bytes, - timestamp, - mount_point_info.f_bsize * - mount_point_info.f_bavail, - 3, labels); - - /* We don't support device error couting yet */ - cmt_gauge_set(ctx->fs_device_error, - timestamp, - 0, - 3, labels); - - cmt_gauge_set(ctx->fs_files, - timestamp, - mount_point_info.f_files, - 3, labels); - - cmt_gauge_set(ctx->fs_files_free, - timestamp, - mount_point_info.f_ffree, - 3, labels); - - cmt_gauge_set(ctx->fs_free_bytes, - timestamp, - mount_point_info.f_bsize * - mount_point_info.f_bfree, - 3, labels); - - cmt_gauge_set(ctx->fs_readonly, - timestamp, - readonly_flag, - 3, labels); - - cmt_gauge_set(ctx->fs_size_bytes, - timestamp, - mount_point_info.f_bsize * - mount_point_info.f_blocks, - 3, labels); - } - } - } - } - - flb_slist_destroy(&fields); - } - - flb_slist_destroy(&lines); - - return NE_ERROR_MOUNT_POINT_LIST_FETCH_SUCCESS; -} - -int ne_filesystem_init(struct flb_ne *ctx) -{ - ctx->fs_regex_skip_mount = flb_regex_create(ctx->fs_regex_ingore_mount_point_text); - ctx->fs_regex_skip_fs_types = flb_regex_create(ctx->fs_regex_ingore_filesystem_type_text); - - ctx->fs_avail_bytes = cmt_gauge_create(ctx->cmt, - "node", - "filesystem", - "avail_bytes", - "Filesystem space available to " \ - "non-root users in bytes.", - 3, (char *[]) {"device", - "fstype", - "mountpoint"}); - - if (ctx->fs_avail_bytes == NULL) { - return -1; - } - - ctx->fs_device_error = cmt_gauge_create(ctx->cmt, - "node", - "filesystem", - "device_error", - "Whether an error occurred while " \ - "getting statistics for the given " \ - "device.", - 3, (char *[]) {"device", - "fstype", - "mountpoint"}); - - if (ctx->fs_device_error == NULL) { - return -1; - } - - ctx->fs_files = cmt_gauge_create(ctx->cmt, - "node", - "filesystem", - "files", - "Filesystem total file nodes.", - 3, (char *[]) {"device", - "fstype", - "mountpoint"}); - - if (ctx->fs_files == NULL) { - return -1; - } - - ctx->fs_files_free = cmt_gauge_create(ctx->cmt, - "node", - "filesystem", - "files_free", - "Filesystem total free file nodes.", - 3, (char *[]) {"device", - "fstype", - "mountpoint"}); - - if (ctx->fs_files_free == NULL) { - return -1; - } - - ctx->fs_free_bytes = cmt_gauge_create(ctx->cmt, - "node", - "filesystem", - "free_bytes", - "Filesystem free space in bytes.", - 3, (char *[]) {"device", - "fstype", - "mountpoint"}); - - if (ctx->fs_free_bytes == NULL) { - return -1; - } - - ctx->fs_readonly = cmt_gauge_create(ctx->cmt, - "node", - "filesystem", - "readonly", - "Filesystem read-only status.", - 3, (char *[]) {"device", - "fstype", - "mountpoint"}); - - if (ctx->fs_readonly == NULL) { - return -1; - } - - ctx->fs_size_bytes = cmt_gauge_create(ctx->cmt, - "node", - "filesystem", - "size_bytes", - "Filesystem size in bytes.", - 3, (char *[]) {"device", - "fstype", - "mountpoint"}); - - if (ctx->fs_size_bytes == NULL) { - return -1; - } - - return 0; -} - -int ne_filesystem_update(struct flb_ne *ctx) -{ - int result; - - result = filesystem_update(ctx, "/proc/1/mounts"); - - if (result != NE_ERROR_MOUNT_POINT_LIST_FETCH_SUCCESS) { - result = filesystem_update(ctx, "/proc/self/mounts"); - } - - return 0; -} - -int ne_filesystem_exit(struct flb_ne *ctx) -{ - if (ctx->fs_regex_skip_mount != NULL) { - flb_regex_destroy(ctx->fs_regex_skip_mount); - } - - if (ctx->fs_regex_skip_fs_types != NULL) { - flb_regex_destroy(ctx->fs_regex_skip_fs_types); - } - - return 0; -} diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_loadavg.c b/fluent-bit/plugins/in_node_exporter_metrics/ne_loadavg.c deleted file mode 100644 index 123ab1dc8..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_loadavg.c +++ /dev/null @@ -1,22 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifdef __linux__ -#include "ne_loadavg_linux.c" -#endif diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_loadavg.h b/fluent-bit/plugins/in_node_exporter_metrics/ne_loadavg.h deleted file mode 100644 index 95540ac37..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_loadavg.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_NE_LOADAVG_H -#define FLB_IN_NE_LOADAVG_H - -#include "ne.h" - -int ne_loadavg_init(struct flb_ne *ctx); -int ne_loadavg_update(struct flb_ne *ctx); -int ne_loadavg_exit(struct flb_ne *ctx); - -#endif diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_loadavg_linux.c b/fluent-bit/plugins/in_node_exporter_metrics/ne_loadavg_linux.c deleted file mode 100644 index 13d872593..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_loadavg_linux.c +++ /dev/null @@ -1,126 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include "ne.h" -#include "ne_utils.h" - -#include - -/* Setup metrics contexts */ -static int ne_loadavg_configure(struct flb_ne *ctx) -{ - struct cmt_gauge *g; - - /* loadavg 1m */ - g = cmt_gauge_create(ctx->cmt, "node", "", "load1", - "1m load average.", - 0, NULL); - ctx->lavg_1 = g; - - /* loadavg 5m */ - g = cmt_gauge_create(ctx->cmt, "node", "", "load5", - "5m load average.", - 0, NULL); - ctx->lavg_5 = g; - - /* loadavg 15m */ - g = cmt_gauge_create(ctx->cmt, "node", "", "load15", - "15m load average.", - 0, NULL); - ctx->lavg_15 = g; - - return 0; -} - -static int loadavg_update(struct flb_ne *ctx) -{ - int ret; - int parts; - double val; - uint64_t ts; - struct mk_list *head; - struct mk_list list; - struct mk_list split_list; - struct flb_slist_entry *line; - struct flb_slist_entry *load; - - mk_list_init(&list); - mk_list_init(&split_list); - - ret = ne_utils_file_read_lines(ctx->path_procfs, "/loadavg", &list); - if (ret == -1) { - return -1; - } - - ts = cfl_time_now(); - mk_list_foreach(head, &list) { - line = mk_list_entry(head, struct flb_slist_entry, _head); - - mk_list_init(&split_list); - ret = flb_slist_split_string(&split_list, line->str, ' ', -1); - if (ret == -1) { - continue; - } - parts = ret; - - parts = ret; - if (parts == 0) { - flb_slist_destroy(&split_list); - continue; - } - - /* 1m */ - load = flb_slist_entry_get(&split_list, 0); - ne_utils_str_to_double(load->str, &val); - cmt_gauge_set(ctx->lavg_1, ts, val, 0, NULL); - - /* 5m */ - load = flb_slist_entry_get(&split_list, 1); - ne_utils_str_to_double(load->str, &val); - cmt_gauge_set(ctx->lavg_5, ts, val, 0, NULL); - - /* 15m */ - load = flb_slist_entry_get(&split_list, 2); - ne_utils_str_to_double(load->str, &val); - cmt_gauge_set(ctx->lavg_15, ts, val, 0, NULL); - - flb_slist_destroy(&split_list); - - break; - } - - flb_slist_destroy(&list); - return 0; -} - -int ne_loadavg_init(struct flb_ne *ctx) -{ - ne_loadavg_configure(ctx); - return 0; -} - -int ne_loadavg_update(struct flb_ne *ctx) -{ - loadavg_update(ctx); - return 0; -} diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_meminfo.c b/fluent-bit/plugins/in_node_exporter_metrics/ne_meminfo.c deleted file mode 100644 index 8dfeaa678..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_meminfo.c +++ /dev/null @@ -1,23 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#ifdef __linux__ -#include "ne_meminfo_linux.c" -#endif diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_meminfo.h b/fluent-bit/plugins/in_node_exporter_metrics/ne_meminfo.h deleted file mode 100644 index bd3edc195..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_meminfo.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_NE_MEMINFO_H -#define FLB_IN_NE_MEMINFO_H - -#include "ne.h" - -int ne_meminfo_init(struct flb_ne *ctx); -int ne_meminfo_update(struct flb_ne *ctx); -int ne_meminfo_exit(struct flb_ne *ctx); - -#endif diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_meminfo_linux.c b/fluent-bit/plugins/in_node_exporter_metrics/ne_meminfo_linux.c deleted file mode 100644 index 3189d53c0..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_meminfo_linux.c +++ /dev/null @@ -1,283 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include "ne.h" -#include "ne_utils.h" - -#include - -static int meminfo_configure(struct flb_ne *ctx) -{ - int ret; - int parts; - int len; - char *p; - char desc[] = "Memory information field "; - struct cmt_gauge *g; - struct mk_list *head; - struct mk_list list; - struct mk_list split_list; - struct flb_slist_entry *entry; - struct flb_slist_entry *line; - flb_sds_t metric_name; - flb_sds_t metric_desc; - - /* Initialize hash table */ - ctx->meminfo_ht = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, 16, 0); - if (!ctx->meminfo_ht) { - return -1; - } - - mk_list_init(&list); - mk_list_init(&split_list); - - ret = ne_utils_file_read_lines(ctx->path_procfs, "/meminfo", &list); - if (ret == -1) { - return -1; - } - metric_name = flb_sds_create_size(128); - if (!metric_name) { - flb_hash_table_destroy(ctx->meminfo_ht); - flb_slist_destroy(&list); - return -1; - } - - metric_desc = flb_sds_create_size(256); - if (!metric_desc) { - flb_hash_table_destroy(ctx->meminfo_ht); - flb_slist_destroy(&list); - return -1; - } - - mk_list_foreach(head, &list) { - line = mk_list_entry(head, struct flb_slist_entry, _head); - - mk_list_init(&split_list); - ret = flb_slist_split_string(&split_list, line->str, ' ', -1); - if (ret == -1) { - continue; - } - parts = ret; - - /* set metric name */ - entry = mk_list_entry_first(&split_list, struct flb_slist_entry, _head); - - if ((p = strstr(entry->str, "(anon)")) || - (p = strstr(entry->str, "(file)"))) { - *p = '_'; - len = flb_sds_len(entry->str) - 2; - flb_sds_len_set(entry->str, len); - } - else { - len = flb_sds_len(entry->str) - 1; - flb_sds_len_set(entry->str, len); - } - entry->str[len] = '\0'; - - flb_sds_len_set(metric_name, 0); - flb_sds_cat(metric_name, entry->str, flb_sds_len(entry->str)); - - /* Metric description */ - flb_sds_len_set(metric_desc, 0); - flb_sds_cat(metric_desc, desc, sizeof(desc) - 1); - - if (parts == 2) { - /* No unit */ - flb_sds_cat(metric_desc, metric_name, flb_sds_len(metric_name)); - flb_sds_cat(metric_desc, ".", 1); - - g = cmt_gauge_create(ctx->cmt, "node", "memory", metric_name, - metric_desc, - 0, NULL); - if (!g) { - flb_slist_destroy(&split_list); - goto error; - } - } - else if (parts == 3) { - /* It has an extra 'kB' string in the line */ - flb_sds_cat(metric_name, "_bytes", 6); - flb_sds_cat(metric_desc, metric_name, flb_sds_len(metric_name)); - flb_sds_cat(metric_desc, ".", 1); - g = cmt_gauge_create(ctx->cmt, "node", "memory", metric_name, - metric_desc, - 0, NULL); - if (!g) { - flb_slist_destroy(&split_list); - goto error; - } - } - else { - flb_slist_destroy(&split_list); - continue; - } - flb_slist_destroy(&split_list); - - /* - * Register the gauge context into the hash table: note that depending - * of the number of parts in the list, if it contains the extra 'kB' - * the metric name gets appended the '_bytes' string. - */ - ret = flb_hash_table_add(ctx->meminfo_ht, - metric_name, flb_sds_len(metric_name), g, 0); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not add hash for metric: %s", metric_name); - goto error; - } - } - - flb_sds_destroy(metric_name); - flb_sds_destroy(metric_desc); - flb_slist_destroy(&list); - return 0; - - error: - flb_sds_destroy(metric_name); - flb_sds_destroy(metric_desc); - flb_slist_destroy(&list); - return -1; -} - -static int meminfo_update(struct flb_ne *ctx) -{ - int i = 0; - int ret; - int len; - int parts; - uint64_t ts; - double val; - size_t out_size; - char *p; - flb_sds_t tmp; - flb_sds_t metric_name = NULL; - struct cmt_gauge *g; - struct mk_list *head; - struct mk_list list; - struct mk_list split_list; - struct flb_slist_entry *line; - struct flb_slist_entry *entry; - - mk_list_init(&list); - ret = ne_utils_file_read_lines(ctx->path_procfs, "/meminfo", &list); - if (ret == -1) { - return -1; - } - - ts = cfl_time_now(); - - mk_list_foreach(head, &list) { - line = mk_list_entry(head, struct flb_slist_entry, _head); - - mk_list_init(&split_list); - ret = flb_slist_split_string(&split_list, line->str, ' ', -1); - if (ret == -1) { - continue; - } - parts = ret; - if (parts == 0) { - flb_slist_destroy(&split_list); - continue; - } - - /* Metric name */ - entry = mk_list_entry_first(&split_list, struct flb_slist_entry, _head); - metric_name = entry->str; - - if ((p = strstr(entry->str, "(anon)")) || - (p = strstr(entry->str, "(file)"))) { - *p = '_'; - len = flb_sds_len(metric_name) - 1; - flb_sds_len_set(metric_name, len); - } - - /* Metric value */ - entry = mk_list_entry_next(&split_list, struct flb_slist_entry, _head, - &entry->_head); - - ret = ne_utils_str_to_double(entry->str, &val); - if (ret == -1) { - i++; - flb_slist_destroy(&split_list); - } - - g = NULL; - if (parts == 2) { - /* Metric name is the same, no extra bytes */ - ret = flb_hash_table_get(ctx->meminfo_ht, - metric_name, flb_sds_len(metric_name) - 1, - (void *) &g, &out_size); - } - else if (parts == 3) { - /* Compose new metric name */ - tmp = flb_sds_create_size(256); - flb_sds_cat_safe(&tmp, metric_name, flb_sds_len(metric_name) - 1); - flb_sds_cat_safe(&tmp, "_bytes", 6); - - /* Get metric context */ - ret = flb_hash_table_get(ctx->meminfo_ht, - tmp, flb_sds_len(tmp), - (void *) &g, &out_size); - flb_sds_destroy(tmp); - - /* Value is in kB, convert to bytes */ - val *= 1024; - } - - if (!g) { - flb_plg_error(ctx->ins, - "gauge content for metric '%s' not found", - metric_name); - flb_slist_destroy(&split_list); - continue; - } - - /* Update metric */ - cmt_gauge_set(g, ts, val, 0, NULL); - flb_slist_destroy(&split_list); - } - - flb_slist_destroy(&list); - return 0; -} - -int ne_meminfo_init(struct flb_ne *ctx) -{ - meminfo_configure(ctx); - return 0; -} - -int ne_meminfo_update(struct flb_ne *ctx) -{ - meminfo_update(ctx); - return 0; -} - -int ne_meminfo_exit(struct flb_ne *ctx) -{ - if (ctx->meminfo_ht) { - flb_hash_table_destroy(ctx->meminfo_ht); - } - return 0; -} diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_netdev.c b/fluent-bit/plugins/in_node_exporter_metrics/ne_netdev.c deleted file mode 100644 index 848ca0995..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_netdev.c +++ /dev/null @@ -1,22 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifdef __linux__ -#include "ne_netdev_linux.c" -#endif diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_netdev.h b/fluent-bit/plugins/in_node_exporter_metrics/ne_netdev.h deleted file mode 100644 index 8155bc3f5..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_netdev.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_NE_NETDEV_H -#define FLB_IN_NE_NETDEV_H - -#include "ne.h" - -int ne_netdev_init(struct flb_ne *ctx); -int ne_netdev_update(struct flb_ne *ctx); -int ne_netdev_exit(struct flb_ne *ctx); - -#endif diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_netdev_linux.c b/fluent-bit/plugins/in_node_exporter_metrics/ne_netdev_linux.c deleted file mode 100644 index 771a66189..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_netdev_linux.c +++ /dev/null @@ -1,363 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include "ne.h" -#include "ne_utils.h" - -#include - -static int netdev_hash_set(struct flb_ne *ctx, struct cmt_counter *c, - char *metric_name) -{ - int ret; - int len; - - len = strlen(metric_name); - ret = flb_hash_table_add(ctx->netdev_ht, - metric_name, len, c, 0); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not register hash entry"); - return -1; - } - - return 0; -} - -static struct cmt_counter *netdev_hash_get(struct flb_ne *ctx, - char *device, char *metric_name) -{ - int ret; - int len; - size_t out_size; - struct cmt_counter *c; - - len = strlen(metric_name); - ret = flb_hash_table_get(ctx->netdev_ht, - metric_name, len, - (void *) &c, &out_size); - if (ret == -1) { - flb_plg_error(ctx->ins, "hash entry '%s' not found", metric_name); - return NULL; - } - - return c; -} - -static int netdev_configure(struct flb_ne *ctx) -{ - int ret; - int parts; - int n = 0; - int len; - char tmp[256]; - char metric_name[256]; - struct mk_list *head; - struct mk_list *prop_head; - struct mk_list list; - struct mk_list head_list; - struct mk_list split_list; - struct mk_list rx_list; - struct mk_list tx_list; - struct flb_slist_entry *line; - struct flb_slist_entry *dev; - struct flb_slist_entry *rx_header; - struct flb_slist_entry *tx_header; - struct flb_slist_entry *prop; - - struct cmt_counter *c; - - /* Initialize hash table */ - ctx->netdev_ht = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, 16, 0); - if (!ctx->netdev_ht) { - return -1; - } - - mk_list_init(&list); - mk_list_init(&head_list); - mk_list_init(&split_list); - mk_list_init(&rx_list); - mk_list_init(&tx_list); - - ret = ne_utils_file_read_lines(ctx->path_procfs, "/net/dev", &list); - if (ret == -1) { - return -1; - } - - /* Validate file header (second header) */ - line = flb_slist_entry_get(&list, 1); - ret = flb_slist_split_string(&head_list, line->str, '|', -1); - if (ret != 3) { - flb_plg_error(ctx->ins, "invalid header line in net/dev: %s", - line->str); - flb_slist_destroy(&list); - return -1; - } - - /* column names */ - rx_header = flb_slist_entry_get(&head_list, 1); - tx_header = flb_slist_entry_get(&head_list, 2); - - flb_slist_split_string(&rx_list, rx_header->str, ' ', -1); - flb_slist_split_string(&tx_list, tx_header->str, ' ', -1); - - n = 0; - mk_list_foreach(head, &list) { - line = mk_list_entry(head, struct flb_slist_entry, _head); - - if (n < 2) { - /* skip first two lines */ - n++; - continue; - } - - mk_list_init(&split_list); - ret = flb_slist_split_string(&split_list, line->str, ' ', 1); - if (ret == -1) { - continue; - } - parts = ret; - - if (parts < 1) { - flb_slist_destroy(&split_list); - continue; - } - - /* device */ - dev = flb_slist_entry_get(&split_list, 0); - - /* sanitize device name */ - len = flb_sds_len(dev->str); - len--; - flb_sds_len_set(dev->str, len - 1); - dev->str[len] = '\0'; - - /* iterate all rx and tx fields to create a unique metric for each one */ - mk_list_foreach(prop_head, &rx_list) { - prop = mk_list_entry(prop_head, struct flb_slist_entry, _head); - - /* help string */ - snprintf(tmp, sizeof(tmp) - 1, - "Network device statistic %s.", - prop->str); - - /* metric name */ - snprintf(metric_name, sizeof(metric_name) - 1, - "receive_%s_total", prop->str); - - /* create the metric */ - c = cmt_counter_create(ctx->cmt, "node", "network", metric_name, - tmp, - 1, (char *[]) {"device"}); - - netdev_hash_set(ctx, c, metric_name); - } - - mk_list_foreach(prop_head, &tx_list) { - prop = mk_list_entry(prop_head, struct flb_slist_entry, _head); - - /* help string */ - snprintf(tmp, sizeof(tmp) - 1, "Network device statistic %s.", - prop->str); - - /* metric name */ - snprintf(metric_name, sizeof(metric_name) - 1, - "transmit_%s_total", prop->str); - - /* create the metric */ - c = cmt_counter_create(ctx->cmt, "node", "network", metric_name, - tmp, - 1, (char *[]) {"device"}); - - netdev_hash_set(ctx, c, metric_name); - } - - flb_slist_destroy(&split_list); - } - - flb_slist_destroy(&head_list); - flb_slist_destroy(&rx_list); - flb_slist_destroy(&tx_list); - flb_slist_destroy(&list); - - return 0; -} - -static int netdev_update(struct flb_ne *ctx) -{ - int ret; - int parts; - int n = 0; - int len; - int pos; - int rx_len; - uint64_t ts; - double val; - char metric_name[256]; - char *type; - struct mk_list *head; - struct mk_list *prop_head; - struct mk_list list; - struct mk_list head_list; - struct mk_list split_list; - struct mk_list rx_list; - struct mk_list tx_list; - struct flb_slist_entry *line; - struct flb_slist_entry *dev; - struct flb_slist_entry *rx_header; - struct flb_slist_entry *tx_header; - struct flb_slist_entry *prop; - struct flb_slist_entry *prop_name; - - struct cmt_counter *c; - - mk_list_init(&list); - mk_list_init(&head_list); - mk_list_init(&split_list); - mk_list_init(&rx_list); - mk_list_init(&tx_list); - - ret = ne_utils_file_read_lines(ctx->path_procfs, "/net/dev", &list); - if (ret == -1) { - return -1; - } - - /* Validate file header (second header) */ - line = flb_slist_entry_get(&list, 1); - ret = flb_slist_split_string(&head_list, line->str, '|', -1); - if (ret != 3) { - flb_plg_error(ctx->ins, "invalid header line in net/dev: %s", - line->str); - flb_slist_destroy(&list); - return -1; - } - - /* column names */ - rx_header = flb_slist_entry_get(&head_list, 1); - tx_header = flb_slist_entry_get(&head_list, 2); - - /* split rx properties */ - flb_slist_split_string(&rx_list, rx_header->str, ' ', -1); - rx_len = mk_list_size(&rx_list); - - /* split tx properties */ - flb_slist_split_string(&tx_list, tx_header->str, ' ', -1); - - n = 0; - ts = cfl_time_now(); - mk_list_foreach(head, &list) { - line = mk_list_entry(head, struct flb_slist_entry, _head); - - if (n < 2) { - /* skip first two lines */ - n++; - continue; - } - - mk_list_init(&split_list); - ret = flb_slist_split_string(&split_list, line->str, ' ', -1); - if (ret == -1) { - continue; - } - parts = ret; - - if (parts < 1) { - flb_slist_destroy(&split_list); - continue; - } - - /* device */ - dev = flb_slist_entry_get(&split_list, 0); - - /* sanitize device name */ - len = flb_sds_len(dev->str); - len--; - flb_sds_len_set(dev->str, len - 1); - dev->str[len] = '\0'; - - /* iterate line fields */ - n = 0; - mk_list_foreach(prop_head, &split_list) { - if (n == 0) { - /* skip device name */ - n++; - continue; - } - - prop = mk_list_entry(prop_head, struct flb_slist_entry, _head); - pos = n - 1; - if (pos < rx_len) { - prop_name = flb_slist_entry_get(&rx_list, pos); - type = "receive"; - } - else { - pos = (n - 1) - rx_len; - prop_name = flb_slist_entry_get(&tx_list, pos); - type = "transmit"; - } - - /* metric name */ - snprintf(metric_name, sizeof(metric_name) - 1, - "%s_%s_total", type, prop_name->str); - - c = netdev_hash_get(ctx, dev->str, metric_name); - if (!c) { - flb_plg_error(ctx->ins, "no hash metric found for %s:%s", - dev->str, prop->str); - continue; - } - - ne_utils_str_to_double(prop->str, &val); - ret = cmt_counter_set(c, ts, val, 1, (char *[]) {dev->str}); - n++; - } - flb_slist_destroy(&split_list); - } - - flb_slist_destroy(&head_list); - flb_slist_destroy(&rx_list); - flb_slist_destroy(&tx_list); - flb_slist_destroy(&list); - - return 0; -} - - -int ne_netdev_init(struct flb_ne *ctx) -{ - netdev_configure(ctx); - return 0; -} - -int ne_netdev_update(struct flb_ne *ctx) -{ - netdev_update(ctx); - return 0; -} - -int ne_netdev_exit(struct flb_ne *ctx) -{ - if (ctx->netdev_ht) { - flb_hash_table_destroy(ctx->netdev_ht); - } - return 0; -} diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_stat_linux.c b/fluent-bit/plugins/in_node_exporter_metrics/ne_stat_linux.c deleted file mode 100644 index 053fb2181..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_stat_linux.c +++ /dev/null @@ -1,152 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define _GNU_SOURCE - -#include -#include - -#include "ne.h" -#include "ne_utils.h" - -static int stat_configure(struct flb_ne *ctx) -{ - struct cmt_counter *c; - struct cmt_gauge *g; - - /* node_intr_total */ - c = cmt_counter_create(ctx->cmt, "node", "", "intr_total", - "Total number of interrupts serviced.", - 0, NULL); - ctx->st_intr = c; - - /* node_context_switches_total */ - c = cmt_counter_create(ctx->cmt, "node", "", "context_switches_total", - "Total number of context switches.", - 0, NULL); - ctx->st_context_switches = c; - - /* node_forks_total */ - c = cmt_counter_create(ctx->cmt, "node", "", "forks_total", - "Total number of forks.", - 0, NULL); - ctx->st_forks = c; - - /* node_boot_time_seconds */ - g = cmt_gauge_create(ctx->cmt, "node", "", "boot_time_seconds", - "Node boot time, in unixtime.", - 0, NULL); - ctx->st_boot_time = g; - - /* node_procs_running */ - g = cmt_gauge_create(ctx->cmt, "node", "", "procs_running", - "Number of processes in runnable state.", - 0, NULL); - ctx->st_procs_running = g; - - /* node_procs_blocked */ - g = cmt_gauge_create(ctx->cmt, "node", "", "procs_blocked", - "Number of processes blocked waiting for I/O to complete.", - 0, NULL); - ctx->st_procs_blocked = g; - - return 0; -} - -static int stat_update(struct flb_ne *ctx) -{ - int ret; - int parts; - uint64_t ts; - double d_val; - struct mk_list *head; - struct mk_list list; - struct mk_list split_list; - struct flb_slist_entry *line; - struct flb_slist_entry *entry; - struct flb_slist_entry *s_val; - - mk_list_init(&list); - ret = ne_utils_file_read_lines(ctx->path_procfs, "/stat", &list); - if (ret == -1) { - return -1; - } - - ts = cfl_time_now(); - - mk_list_foreach(head, &list) { - line = mk_list_entry(head, struct flb_slist_entry, _head); - - mk_list_init(&split_list); - ret = flb_slist_split_string(&split_list, line->str, ' ', -1); - if (ret == -1) { - continue; - } - parts = ret; - if (parts == 0) { - flb_slist_destroy(&split_list); - continue; - } - - /* metric name and value */ - entry = flb_slist_entry_get(&split_list, 0); - s_val = flb_slist_entry_get(&split_list, 1); - - if (strcmp(entry->str, "intr") == 0) { - ne_utils_str_to_double(s_val->str, &d_val); - cmt_counter_set(ctx->st_intr, ts, d_val, 0, NULL); - } - else if (strcmp(entry->str, "ctxt") == 0) { - ne_utils_str_to_double(s_val->str, &d_val); - cmt_counter_set(ctx->st_context_switches, ts, d_val, 0, NULL); - } - else if (strcmp(entry->str, "btime") == 0) { - ne_utils_str_to_double(s_val->str, &d_val); - cmt_gauge_set(ctx->st_boot_time, ts, d_val, 0, NULL); - } - else if (strcmp(entry->str, "processes") == 0) { - ne_utils_str_to_double(s_val->str, &d_val); - cmt_counter_set(ctx->st_forks, ts, d_val, 0, NULL); - } - else if (strcmp(entry->str, "procs_running") == 0) { - ne_utils_str_to_double(s_val->str, &d_val); - cmt_gauge_set(ctx->st_procs_running, ts, d_val, 0, NULL); - } - else if (strcmp(entry->str, "procs_blocked") == 0) { - ne_utils_str_to_double(s_val->str, &d_val); - cmt_gauge_set(ctx->st_procs_blocked, ts, d_val, 0, NULL); - } - flb_slist_destroy(&split_list); - } - flb_slist_destroy(&list); - - return 0; -} - -int ne_stat_init(struct flb_ne *ctx) -{ - stat_configure(ctx); - return 0; -} - -int ne_stat_update(struct flb_ne *ctx) -{ - stat_update(ctx); - return 0; -} diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_stat_linux.h b/fluent-bit/plugins/in_node_exporter_metrics/ne_stat_linux.h deleted file mode 100644 index 6d92a1492..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_stat_linux.h +++ /dev/null @@ -1,28 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_NE_STAT_H -#define FLB_IN_NE_STAT_H - -#include "ne.h" - -int ne_stat_init(struct flb_ne *ctx); -int ne_stat_update(struct flb_ne *ctx); - -#endif diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_systemd.c b/fluent-bit/plugins/in_node_exporter_metrics/ne_systemd.c deleted file mode 100644 index ec4df2455..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_systemd.c +++ /dev/null @@ -1,807 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include - -#include "ne.h" -#include "ne_utils.h" -#include "ne_systemd.h" - -#include -#include - -static int str_ends_with(char *haystack, char *needle, int caseless) { - size_t haystack_length; - size_t trailer_offset; - size_t needle_length; - int result; - - haystack_length = strlen(haystack); - needle_length = strlen(needle); - - if (needle_length > haystack_length) { - return FLB_FALSE; - } - - trailer_offset = haystack_length - needle_length; - - if (caseless) { - result = strcasecmp(&haystack[trailer_offset], - needle); - } - else { - result = strcmp(&haystack[trailer_offset], - needle); - } - - if (result == 0) { - return FLB_TRUE; - } - - return FLB_FALSE; -} - -static void clear_property_variable(char property_type, void *property_value) -{ - if (property_type == 'y') { - *((uint8_t *) property_value) = 0; - } - else if (property_type == 'b') { - *((int *) property_value) = 0; - } - else if (property_type == 'n') { - *((int16_t *) property_value) = 0; - } - else if (property_type == 'q') { - *((uint16_t *) property_value) = 0; - } - else if (property_type == 'i') { - *((int32_t *) property_value) = 0; - } - else if (property_type == 'u') { - *((uint32_t *) property_value) = 0; - } - else if (property_type == 'x') { - *((int64_t *) property_value) = 0; - } - else if (property_type == 't') { - *((uint64_t *) property_value) = 0; - } - else if (property_type == 'd') { - *((double *) property_value) = 0; - } - else if (property_type == 's') { - *((char **) property_value) = NULL; - } - else if (property_type == 'o') { - *((char **) property_value) = NULL; - } - else if (property_type == 'g') { - *((char **) property_value) = NULL; - } - else if (property_type == 'h') { - *((int32_t *) property_value) = -1; - } -} - -static int get_system_property(struct flb_ne *ctx, - char *interface, - char *property_name, - char property_type, - void *property_value) -{ - int result; - - clear_property_variable(property_type, property_value); - - if (interface == NULL) { - interface = "org.freedesktop.systemd1.Manager"; - } - - if (property_type == 's' || - property_type == 'o' || - property_type == 'g') { - result = sd_bus_get_property_string((sd_bus *) ctx->systemd_dbus_handle, - "org.freedesktop.systemd1", - "/org/freedesktop/systemd1", - interface, - property_name, - NULL, - property_value); - } - else { - result = sd_bus_get_property_trivial((sd_bus *) ctx->systemd_dbus_handle, - "org.freedesktop.systemd1", - "/org/freedesktop/systemd1", - interface, - property_name, - NULL, - property_type, - property_value); - } - - if (result < 0) { - return -1; - } - - return 0; -} - -static int get_unit_property(struct flb_ne *ctx, - struct ne_systemd_unit *unit, - char *interface, - char *property_name, - char property_type, - void *property_value) -{ - int result; - - clear_property_variable(property_type, property_value); - - if (interface == NULL) { - if (unit->unit_type == SYSTEMD_UNIT_TYPE_SERVICE) { - interface = "org.freedesktop.systemd1.Service"; - } - else if (unit->unit_type == SYSTEMD_UNIT_TYPE_MOUNT) { - interface = "org.freedesktop.systemd1.Mount"; - } - else if (unit->unit_type == SYSTEMD_UNIT_TYPE_SOCKET) { - interface = "org.freedesktop.systemd1.Socket"; - } - else if (unit->unit_type == SYSTEMD_UNIT_TYPE_TIMER) { - interface = "org.freedesktop.systemd1.Timer"; - } - else { - interface = unit->name; - } - } - - if (property_type == 's' || - property_type == 'o' || - property_type == 'g') { - result = sd_bus_get_property_string((sd_bus *) ctx->systemd_dbus_handle, - "org.freedesktop.systemd1", - unit->path, - interface, - property_name, - NULL, - property_value); - } - else { - result = sd_bus_get_property_trivial((sd_bus *) ctx->systemd_dbus_handle, - "org.freedesktop.systemd1", - unit->path, - interface, - property_name, - NULL, - property_type, - property_value); - } - - if (result < 0) { - return -1; - } - - return 0; -} - -static int ne_systemd_update_unit_state(struct flb_ne *ctx) -{ - char *unit_states[] = { "activating", "active", - "deactivating", "inactive", - "failed" }; - double timer_trigger_timestamp; - uint64_t deactivating_units; - uint64_t activating_units; - double unit_start_time; - uint64_t inactive_units; - uint64_t active_units; - uint64_t failed_units; - int include_flag; - uint64_t timestamp; - int result; - size_t index; - sd_bus_message *reply; - struct ne_systemd_unit unit; - sd_bus *bus; - - bus = (sd_bus *) ctx->systemd_dbus_handle; - - result = sd_bus_call_method(bus, - "org.freedesktop.systemd1", - "/org/freedesktop/systemd1", - "org.freedesktop.systemd1.Manager", - "ListUnits", - NULL, - &reply, - ""); - - if (result < 0) { - return -1; - } - - result = sd_bus_message_enter_container(reply, 'a', "(ssssssouso)"); - - if (result < 0) { - sd_bus_message_unref(reply); - - return -2; - } - - timestamp = cfl_time_now(); - - deactivating_units = 0; - activating_units = 0; - inactive_units = 0; - active_units = 0; - failed_units = 0; - - do { - result = sd_bus_message_read(reply, - "(ssssssouso)", - &unit.name, - &unit.description, - &unit.load_state, - &unit.active_state, - &unit.sub_state, - &unit.followed, - &unit.path, - &unit.job_id, - &unit.job_type, - &unit.object_path); - - - if (result < 0) { - sd_bus_message_unref(reply); - - return -3; - } - else if(result > 0) { - unit.type = NULL; - - if (strcasecmp(unit.active_state, "activating") == 0) { - activating_units++; - } - else if (strcasecmp(unit.active_state, "deactivating") == 0) { - deactivating_units++; - } - else if (strcasecmp(unit.active_state, "inactive") == 0) { - inactive_units++; - } - else if (strcasecmp(unit.active_state, "active") == 0) { - active_units++; - } - else if (strcasecmp(unit.active_state, "failed") == 0) { - failed_units++; - } - - if (ctx->systemd_regex_include_list != NULL) { - include_flag = flb_regex_match(ctx->systemd_regex_include_list, - (unsigned char *) unit.name, - strlen(unit.name)); - } - else { - include_flag = FLB_TRUE; - } - - if (!include_flag) { - continue; - } - - if (ctx->systemd_regex_exclude_list != NULL) { - include_flag = !flb_regex_match(ctx->systemd_regex_exclude_list, - (unsigned char *) unit.name, - strlen(unit.name)); - } - else { - include_flag = FLB_TRUE; - } - - if (!include_flag) { - continue; - } - - if (strcasecmp(unit.load_state, "loaded") != 0) { - continue; - } - - if (str_ends_with(unit.name, ".service", FLB_TRUE)) { - unit.unit_type = SYSTEMD_UNIT_TYPE_SERVICE; - - result = get_service_type(ctx, - &unit, - &unit.type); - - if (ctx->systemd_include_service_restarts) { - result = get_service_restart_count(ctx, - &unit, - &unit.restart_count); - - cmt_counter_set(ctx->systemd_service_restarts, - timestamp, - unit.restart_count, - 1, - (char *[]){ unit.name }); - - } - - if (ctx->systemd_include_service_task_metrics) { - result = get_service_active_tasks(ctx, - &unit, - &unit.active_tasks); - - if (unit.active_tasks != UINT64_MAX) { - cmt_gauge_set(ctx->systemd_unit_tasks, - timestamp, - unit.active_tasks, - 1, - (char *[]){ unit.name }); - } - - result = get_service_max_tasks(ctx, - &unit, - &unit.max_tasks); - - if (unit.max_tasks != UINT64_MAX) { - cmt_gauge_set(ctx->systemd_unit_tasks_max, - timestamp, - unit.max_tasks, - 1, - (char *[]){ unit.name }); - } - } - - result = 1; - } - else if (str_ends_with(unit.name, ".mount", FLB_TRUE)) { - unit.unit_type = SYSTEMD_UNIT_TYPE_MOUNT; - } - else if (str_ends_with(unit.name, ".socket", FLB_TRUE)) { - unit.unit_type = SYSTEMD_UNIT_TYPE_SOCKET; - - result = get_socket_accepted_connection_count( - ctx, - &unit, - &unit.accepted_connections); - - result = get_socket_active_connection_count( - ctx, - &unit, - &unit.active_connections); - - result = get_socket_refused_connection_count( - ctx, - &unit, - &unit.refused_connections); - - cmt_gauge_set(ctx->systemd_socket_accepted_connections, - timestamp, - unit.accepted_connections, - 1, - (char *[]){ unit.name }); - - cmt_gauge_set(ctx->systemd_socket_active_connections, - timestamp, - unit.active_connections, - 1, - (char *[]){ unit.name }); - - cmt_gauge_set(ctx->systemd_socket_refused_connections, - timestamp, - unit.refused_connections, - 1, - (char *[]){ unit.name }); - - result = 1; - } - else if (str_ends_with(unit.name, ".timer", FLB_TRUE)) { - unit.unit_type = SYSTEMD_UNIT_TYPE_TIMER; - - result = get_timer_last_trigger_timestamp( - ctx, - &unit, - &unit.last_trigger_timestamp); - - timer_trigger_timestamp = (double) unit.last_trigger_timestamp; - timer_trigger_timestamp /= 1000000.0; - - cmt_gauge_set(ctx->systemd_timer_last_trigger_seconds, - timestamp, - timer_trigger_timestamp, - 1, - (char *[]){ unit.name }); - - result = 1; - } - else { - unit.unit_type = SYSTEMD_UNIT_TYPE_UNDEFINED; - } - - if (ctx->systemd_include_unit_start_times) { - if (strcasecmp(unit.active_state, "active") == 0) { - result = get_unit_start_time(ctx, &unit, &unit.start_time); - - unit_start_time = (double) unit.start_time; - unit_start_time /= 1000000.0; - } - else { - unit_start_time = 0; - } - - cmt_gauge_set(ctx->systemd_unit_start_times, - timestamp, - unit_start_time, - 1, - (char *[]){ unit.name }); - - result = 1; - } - - for(index = 0 ; index < 5 ; index++) { - cmt_gauge_add(ctx->systemd_unit_state, - timestamp, - 0, - 3, - (char *[]){ unit.name, - unit_states[index], - unit.type - }); - } - - cmt_gauge_inc(ctx->systemd_unit_state, - timestamp, - 3, - (char *[]){ unit.name, - unit.active_state, - unit.type - }); - - - if (unit.type != NULL) { - free(unit.type); - } - } - } - while (result > 0); - - sd_bus_message_exit_container(reply); - - sd_bus_message_unref(reply); - - cmt_gauge_set(ctx->systemd_units, - timestamp, - activating_units, - 1, - (char *[]){ "activating" }); - - cmt_gauge_set(ctx->systemd_units, - timestamp, - deactivating_units, - 1, - (char *[]){ "deactivating" }); - - cmt_gauge_set(ctx->systemd_units, - timestamp, - inactive_units, - 1, - (char *[]){ "inactive" }); - - cmt_gauge_set(ctx->systemd_units, - timestamp, - active_units, - 1, - (char *[]){ "active" }); - - cmt_gauge_set(ctx->systemd_units, - timestamp, - failed_units, - 1, - (char *[]){ "failed" }); - - return 0; -} - -static int ne_systemd_update_system_state(struct flb_ne *ctx) -{ - int system_running; - uint64_t timestamp; - char *version; - int result; - char *state; - - timestamp = cfl_time_now(); - - if (!ctx->systemd_initialization_flag) { - result = get_system_version(ctx, &version); - - if (result != 0) { - return -1; - } - - ctx->libsystemd_version_text = version; - ctx->libsystemd_version = strtod(version, NULL); - - cmt_gauge_set(ctx->systemd_version, - timestamp, - ctx->libsystemd_version, - 1, - (char *[]){ ctx->libsystemd_version_text }); - } - else { - cmt_gauge_add(ctx->systemd_version, - timestamp, - 0, - 1, - (char *[]){ ctx->libsystemd_version_text }); - } - - result = get_system_state(ctx, &state); - - if (result != 0) { - return -2; - } - - system_running = 0; - - if (strcasecmp(state, "running") == 0) { - system_running = 1; - } - - cmt_gauge_set(ctx->systemd_system_running, - timestamp, - system_running, - 0, - NULL); - free(state); - - return 0; -} - -int ne_systemd_init(struct flb_ne *ctx) -{ - int result; - - ctx->systemd_dbus_handle = NULL; - - result = sd_bus_open_system((sd_bus **) &ctx->systemd_dbus_handle); - - if (result < 0) { - return -1; - } - - ctx->systemd_socket_accepted_connections = cmt_gauge_create(ctx->cmt, - "node", - "systemd", - "socket_accepted_connections_total", - "Total number of accepted " \ - "socket connections.", - 1, - (char *[]) {"name"}); - - if (ctx->systemd_socket_accepted_connections == NULL) { - return -1; - } - - ctx->systemd_socket_active_connections = cmt_gauge_create(ctx->cmt, - "node", - "systemd", - "socket_current_connections", - "Current number of socket " \ - "connections.", - 1, - (char *[]) {"name"}); - - if (ctx->systemd_socket_active_connections == NULL) { - return -1; - } - - ctx->systemd_socket_refused_connections = cmt_gauge_create(ctx->cmt, - "node", - "systemd", - "socket_refused_connections_total", - "Total number of refused " \ - "socket connections.", - 1, - (char *[]) {"name"}); - - if (ctx->systemd_socket_refused_connections == NULL) { - return -1; - } - - ctx->systemd_system_running = cmt_gauge_create(ctx->cmt, - "node", - "systemd", - "system_running", - "Whether the system is " \ - "operational (see 'systemctl" \ - " is-system-running')", - 0, NULL); - - if (ctx->systemd_system_running == NULL) { - return -1; - } - - ctx->systemd_timer_last_trigger_seconds = cmt_gauge_create(ctx->cmt, - "node", - "systemd", - "timer_last_trigger_seconds", - "Seconds since epoch of " \ - "last trigger.", - 1, - (char *[]) {"name"}); - - if (ctx->systemd_timer_last_trigger_seconds == NULL) { - return -1; - } - - ctx->systemd_service_restarts = cmt_counter_create(ctx->cmt, - "node", - "systemd", - "service_restart_total", - "Service unit count of " \ - "Restart triggers", - 1, (char *[]) {"name"}); - - if (ctx->systemd_service_restarts == NULL) { - return -1; - } - - cmt_counter_allow_reset(ctx->systemd_service_restarts); - - ctx->systemd_unit_tasks = cmt_gauge_create(ctx->cmt, - "node", - "systemd", - "unit_tasks_current", - "Current number of tasks " \ - "per Systemd unit.", - 1, (char *[]) {"name"}); - - if (ctx->systemd_unit_tasks == NULL) { - return -1; - } - - ctx->systemd_unit_tasks_max = cmt_gauge_create(ctx->cmt, - "node", - "systemd", - "unit_tasks_max", - "Maximum number of tasks " \ - "per Systemd unit.", - 1, (char *[]) {"name"}); - - if (ctx->systemd_unit_tasks == NULL) { - return -1; - } - - ctx->systemd_unit_start_times = cmt_gauge_create(ctx->cmt, - "node", - "systemd", - "unit_start_time_seconds", - "Start time of the unit since " \ - "unix epoch in seconds.", - 1, (char *[]) {"name"}); - - if (ctx->systemd_unit_start_times == NULL) { - return -1; - } - - ctx->systemd_unit_state = cmt_gauge_create(ctx->cmt, - "node", - "systemd", - "unit_state", - "Systemd unit", - 3, (char *[]) {"name", - "state", - "type"}); - - if (ctx->systemd_unit_state == NULL) { - return -1; - } - - ctx->systemd_units = cmt_gauge_create(ctx->cmt, - "node", - "systemd", - "units", - "Summary of systemd unit states", - 1, (char *[]) {"state"}); - - if (ctx->systemd_units == NULL) { - return -1; - } - - ctx->systemd_version = cmt_gauge_create(ctx->cmt, - "node", - "systemd", - "version", - "Detected systemd version", - 1, (char *[]) {"version"}); - - if (ctx->systemd_version == NULL) { - return -1; - } - - if (ctx->systemd_regex_include_list_text != NULL) { - ctx->systemd_regex_include_list = \ - flb_regex_create(ctx->systemd_regex_include_list_text); - - if (ctx->systemd_regex_include_list == NULL) { - return -1; - } - } - - if (ctx->systemd_regex_exclude_list_text != NULL) { - ctx->systemd_regex_exclude_list = \ - flb_regex_create(ctx->systemd_regex_exclude_list_text); - - if (ctx->systemd_regex_exclude_list == NULL) { - return -1; - } - } - - return 0; -} - -int ne_systemd_update(struct flb_ne *ctx) -{ - int result; - - result = ne_systemd_update_system_state(ctx); - - if (result != 0) { - return result; - } - - result = ne_systemd_update_unit_state(ctx); - - if (result != 0) { - return result; - } - - if (!ctx->systemd_initialization_flag) { - ctx->systemd_initialization_flag = FLB_TRUE; - } - - return 0; -} - -int ne_systemd_exit(struct flb_ne *ctx) -{ - if (ctx->systemd_dbus_handle != NULL) { - sd_bus_unref((sd_bus *) ctx->systemd_dbus_handle); - - ctx->systemd_dbus_handle = NULL; - } - - if (ctx->systemd_regex_include_list != NULL) { - flb_regex_destroy(ctx->systemd_regex_include_list); - } - - if (ctx->systemd_regex_exclude_list != NULL) { - flb_regex_destroy(ctx->systemd_regex_exclude_list); - } - - if (ctx->libsystemd_version_text != NULL) { - flb_free(ctx->libsystemd_version_text); - } - return 0; -} diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_systemd.h b/fluent-bit/plugins/in_node_exporter_metrics/ne_systemd.h deleted file mode 100644 index 8c6fb26c1..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_systemd.h +++ /dev/null @@ -1,127 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_NE_SYSTEMD_H -#define FLB_IN_NE_SYSTEMD_H - -#include "ne.h" - -#define SYSTEMD_UNIT_TYPE_UNDEFINED 0 -#define SYSTEMD_UNIT_TYPE_SERVICE 1 -#define SYSTEMD_UNIT_TYPE_SOCKET 2 -#define SYSTEMD_UNIT_TYPE_MOUNT 3 -#define SYSTEMD_UNIT_TYPE_TIMER 4 - -struct ne_systemd_unit { - char *name; - char *description; - char *load_state; - char *active_state; - char *sub_state; - char *followed; - char *path; - uint32_t job_id; - char *job_type; - char *object_path; - - /* not part of the unit list result */ - uint64_t start_time; - int unit_type; - char *type; - - /* services */ - uint32_t restart_count; - uint64_t active_tasks; - uint64_t max_tasks; - - /* sockets */ - uint32_t accepted_connections; - uint32_t active_connections; - uint32_t refused_connections; - - /* timers */ - uint64_t last_trigger_timestamp; -}; - -#ifdef FLB_HAVE_SYSTEMD_SDBUS -int ne_systemd_init(struct flb_ne *ctx); -int ne_systemd_update(struct flb_ne *ctx); -int ne_systemd_exit(struct flb_ne *ctx); -#else -static int ne_systemd_init(struct flb_ne *ctx) -{ - return 0; -} -static int ne_systemd_update(struct flb_ne *ctx) -{ - return 0; -} -static int ne_systemd_exit(struct flb_ne *ctx) -{ - return 0; -} -#endif - -#define get_system_state(context, output_variable) \ - get_system_property(context, NULL, "SystemState", \ - 's', (void *) (output_variable)) - -#define get_system_version(context, output_variable) \ - get_system_property(context, NULL, "Version", \ - 's', (void *) (output_variable)) - -#define get_service_type(context, unit, output_variable) \ - get_unit_property(context, unit, NULL, "Type", \ - 's', (void *) (output_variable)) - -#define get_service_active_tasks(context, unit, output_variable) \ - get_unit_property(context, unit, NULL, "TasksCurrent", \ - 't', (void *) (output_variable)) - -#define get_service_max_tasks(context, unit, output_variable) \ - get_unit_property(context, unit, NULL, "TasksMax", \ - 't', (void *) (output_variable)) - -#define get_service_restart_count(context, unit, output_variable) \ - get_unit_property(context, unit, NULL, "NRestarts", \ - 'u', (void *) (output_variable)) - -#define get_socket_accepted_connection_count(context, unit, output_variable) \ - get_unit_property(context, unit, NULL, "NAccepted", \ - 'u', (void *) (output_variable)) - -#define get_socket_active_connection_count(context, unit, output_variable) \ - get_unit_property(context, unit, NULL, "NConnections", \ - 'u', (void *) (output_variable)) - -#define get_socket_refused_connection_count(context, unit, output_variable) \ - get_unit_property(context, unit, NULL, "NRefused", \ - 'u', (void *) (output_variable)) - -#define get_timer_last_trigger_timestamp(context, unit, output_variable) \ - get_unit_property(context, unit, NULL, "LastTriggerUSec", \ - 't', (void *) (output_variable)) - -#define get_unit_start_time(context, unit, output_variable) \ - get_unit_property(context, \ - unit, \ - "org.freedesktop.systemd1.Unit", \ - "ActiveEnterTimestamp", \ - 't', (void *) (output_variable)) -#endif diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_textfile.c b/fluent-bit/plugins/in_node_exporter_metrics/ne_textfile.c deleted file mode 100644 index f9c584453..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_textfile.c +++ /dev/null @@ -1,22 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifdef __linux__ -#include "ne_textfile_linux.c" -#endif diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_textfile.h b/fluent-bit/plugins/in_node_exporter_metrics/ne_textfile.h deleted file mode 100644 index 7f67d817f..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_textfile.h +++ /dev/null @@ -1,28 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_NE_TEXTFILE_H -#define FLB_IN_NE_TEXTFILE_H - -#include "ne.h" - -int ne_textfile_init(struct flb_ne *ctx); -int ne_textfile_update(struct flb_ne *ctx); - -#endif diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_textfile_linux.c b/fluent-bit/plugins/in_node_exporter_metrics/ne_textfile_linux.c deleted file mode 100644 index d65e01441..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_textfile_linux.c +++ /dev/null @@ -1,204 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include "ne.h" -#include "ne_utils.h" - -#include -#include -#include -#include - -/* Prometheus decoder */ -#include -#include "cmt_decode_prometheus_parser.h" - -static char *error_reason(int cmt_error) -{ - static char *reason = NULL; - - switch(cmt_error) { - case CMT_DECODE_PROMETHEUS_SYNTAX_ERROR: - reason = "syntax error"; - break; - case CMT_DECODE_PROMETHEUS_ALLOCATION_ERROR: - reason = "allocation error"; - break; - case CMT_DECODE_PROMETHEUS_MAX_LABEL_COUNT_EXCEEDED: - reason = "max label count exceeded"; - break; - case CMT_DECODE_PROMETHEUS_CMT_SET_ERROR: - reason = "cmt set error"; - break; - case CMT_DECODE_PROMETHEUS_CMT_CREATE_ERROR: - reason = "cmt create error"; - break; - case CMT_DECODE_PROMETHEUS_PARSE_VALUE_FAILED: - reason = "parse value failed"; - break; - case CMT_DECODE_PROMETHEUS_PARSE_TIMESTAMP_FAILED: - reason = "parse timestamp failed"; - break; - default: - reason = "unknown reason"; - } - - return reason; -} - -static int textfile_update(struct flb_ne *ctx) -{ - int ret; - char errbuf[256]; - flb_sds_t contents; - struct cmt_decode_prometheus_parse_opts opts; - uint64_t timestamp; - struct cmt *cmt; - struct mk_list *head; - struct mk_list list; - struct flb_slist_entry *entry; - const char *nop_pattern = ""; - const char *dir_pattern = "/*.prom"; - char *ext; - struct stat st; - int use_directory_pattern = FLB_FALSE; - - timestamp = cfl_time_now(); - - memset(&opts, 0, sizeof(opts)); - opts.errbuf = errbuf; - opts.errbuf_size = sizeof(errbuf); - opts.default_timestamp = timestamp; - - flb_plg_debug(ctx->ins, "scanning path %s", ctx->path_textfile); - - if (ctx->path_textfile == NULL) { - flb_plg_warn(ctx->ins, "No valid path for textfile metric is registered"); - return -1; - } - - ext = strrchr(ctx->path_textfile, '.'); - if (ext != NULL) { - if (strncmp(ext, ".prom", 5) == 0) { - flb_plg_debug(ctx->ins, "specified path %s has \".prom\" extension", - ctx->path_textfile); - use_directory_pattern = FLB_FALSE; - } - else { - ret = stat(ctx->path_textfile, &st); - if (ret != 0) { - flb_plg_warn(ctx->ins, "specified path %s is not accesible", - ctx->path_textfile); - } - if (S_ISREG(st.st_mode)) { - flb_plg_warn(ctx->ins, "specified path %s does not have \".prom\" extension. Assuming directory", - ctx->path_textfile); - use_directory_pattern = FLB_TRUE; - } - } - } - else { - flb_plg_debug(ctx->ins, "specified file path %s does not have extension part. Globbing directory with \"%s\" suffix", - ctx->path_textfile, dir_pattern); - use_directory_pattern = FLB_TRUE; - } - - if (use_directory_pattern == FLB_TRUE) { - /* Scan the given directory path */ - ret = ne_utils_path_scan(ctx, ctx->path_textfile, dir_pattern, NE_SCAN_FILE, &list); - if (ret != 0) { - return -1; - } - } - else { - /* Scan the given file path */ - ret = ne_utils_path_scan(ctx, ctx->path_textfile, nop_pattern, NE_SCAN_FILE, &list); - if (ret != 0) { - return -1; - } - } - - /* Process entries */ - mk_list_foreach(head, &list) { - entry = mk_list_entry(head, struct flb_slist_entry, _head); - /* Update metrics from text file */ - contents = flb_file_read(entry->str); - if (contents == NULL) { - flb_plg_debug(ctx->ins, "skip invalid file of prometheus: %s", - entry->str); - continue; - } - - if (flb_sds_len(contents) == 0) { - flb_plg_debug(ctx->ins, "skip empty payload of prometheus: %s", - entry->str); - continue; - } - - ret = cmt_decode_prometheus_create(&cmt, contents, flb_sds_len(contents), &opts); - if (ret == 0) { - flb_plg_debug(ctx->ins, "parse a payload of prometheus: %s", - entry->str); - cmt_cat(ctx->cmt, cmt); - cmt_decode_prometheus_destroy(cmt); - } - else { - flb_plg_debug(ctx->ins, "parse a payload of prometheus: dismissed: %s, error: %d", - entry->str, ret); - cmt_counter_set(ctx->load_errors, timestamp, 1.0, 1, (char*[]){error_reason(ret)}); - } - flb_sds_destroy(contents); - } - flb_slist_destroy(&list); - - return 0; -} - -int ne_textfile_init(struct flb_ne *ctx) -{ - ctx->load_errors = cmt_counter_create(ctx->cmt, - "node", - "textfile", - "node_textfile_scrape_error", - "Greater equal than 1 if there was an error opening, reading, or parsing a file, 0 otherwise.", - 1, (char *[]) {"reason"}); - - if (ctx->load_errors == NULL) { - return -1; - } - - return 0; -} - -int ne_textfile_update(struct flb_ne *ctx) -{ - textfile_update(ctx); - - return 0; -} - -int ne_textfile_exit(struct flb_ne *ctx) -{ - return 0; -} diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_time.c b/fluent-bit/plugins/in_node_exporter_metrics/ne_time.c deleted file mode 100644 index 81bf57ed9..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_time.c +++ /dev/null @@ -1,59 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include "ne.h" -#include "ne_utils.h" - -static int time_configure(struct flb_ne *ctx) -{ - struct cmt_gauge *g; - - g = cmt_gauge_create(ctx->cmt, "node", "", "time_seconds", - "System time in seconds since epoch (1970).", - 0, NULL); - ctx->time = g; - return 0; -} - -static int time_update(struct flb_ne *ctx) -{ - double val; - uint64_t ts; - - ts = cfl_time_now(); - val = ((double) ts) / 1e9; - cmt_gauge_set(ctx->time, ts, val, 0, NULL); - - return 0; -} - -int ne_time_init(struct flb_ne *ctx) -{ - time_configure(ctx); - return 0; -} - -int ne_time_update(struct flb_ne *ctx) -{ - time_update(ctx); - return 0; -} diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_time.h b/fluent-bit/plugins/in_node_exporter_metrics/ne_time.h deleted file mode 100644 index e594c935c..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_time.h +++ /dev/null @@ -1,28 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_NE_TIME_H -#define FLB_IN_NE_TIME_H - -#include "ne.h" - -int ne_time_init(struct flb_ne *ctx); -int ne_time_update(struct flb_ne *ctx); - -#endif diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_uname.c b/fluent-bit/plugins/in_node_exporter_metrics/ne_uname.c deleted file mode 100644 index 7ab7b1613..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_uname.c +++ /dev/null @@ -1,22 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifdef __linux__ -#include "ne_uname_linux.c" -#endif diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_uname.h b/fluent-bit/plugins/in_node_exporter_metrics/ne_uname.h deleted file mode 100644 index dc94bc42b..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_uname.h +++ /dev/null @@ -1,28 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_NE_UNAME_H -#define FLB_IN_NE_UNAME_H - -#include "ne.h" - -int ne_uname_init(struct flb_ne *ctx); -int ne_uname_update(struct flb_ne *ctx); - -#endif diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_uname_linux.c b/fluent-bit/plugins/in_node_exporter_metrics/ne_uname_linux.c deleted file mode 100644 index e42cab6a3..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_uname_linux.c +++ /dev/null @@ -1,84 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define _GNU_SOURCE - -#include -#include - -#include "ne.h" -#include "ne_utils.h" - -#include -#include - -static int uname_configure(struct flb_ne *ctx) -{ - struct cmt_gauge *g; - - g = cmt_gauge_create(ctx->cmt, "node", "uname", "info", - "Labeled system information as provided by the uname system call.", - 6, (char *[]) - { - "sysname", - "release", - "version", - "machine", - "nodename", - "domainname" - }); - if (!g) { - return -1; - } - ctx->uname = g; - return 0; -} - -static int uname_update(struct flb_ne *ctx) -{ - int ret; - uint64_t ts; - struct utsname u = {0}; - - - uname(&u); - - ts = cfl_time_now(); - ret = cmt_gauge_set(ctx->uname, ts, 1, 6, - (char *[]) { - u.sysname, - u.release, - u.version, - u.machine, - u.nodename, - u.domainname}); - return ret; -} - -int ne_uname_init(struct flb_ne *ctx) -{ - uname_configure(ctx); - return 0; -} - -int ne_uname_update(struct flb_ne *ctx) -{ - uname_update(ctx); - return 0; -} diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_utils.c b/fluent-bit/plugins/in_node_exporter_metrics/ne_utils.c deleted file mode 100644 index 54cb2e2da..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_utils.c +++ /dev/null @@ -1,256 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include "ne.h" - -/* required by stat(2), open(2) */ -#include -#include -#include -#include - -#include - -int ne_utils_str_to_double(char *str, double *out_val) -{ - double val; - char *end; - - errno = 0; - val = strtod(str, &end); - if (errno != 0 || *end != '\0') { - return -1; - } - *out_val = val; - return 0; -} - -int ne_utils_str_to_uint64(char *str, uint64_t *out_val) -{ - uint64_t val; - char *end; - - errno = 0; - val = strtoll(str, &end, 10); - if ((errno == ERANGE && (val == LONG_MAX || val == LONG_MIN)) - || (errno != 0 && val == 0)) { - flb_errno(); - return -1; - } - - if (end == str) { - return -1; - } - - *out_val = val; - return 0; -} - -int ne_utils_file_read_uint64(const char *mount, - const char *path, - const char *join_a, const char *join_b, - uint64_t *out_val) -{ - int fd; - int len; - int ret; - flb_sds_t p; - uint64_t val; - ssize_t bytes; - char tmp[32]; - - /* Check the path starts with the mount point to prevent duplication. */ - if (strncasecmp(path, mount, strlen(mount)) == 0 && - path[strlen(mount)] == '/') { - mount = ""; - } - - /* Compose the final path */ - p = flb_sds_create(mount); - if (!p) { - return -1; - } - - len = strlen(path); - flb_sds_cat_safe(&p, path, len); - - if (join_a) { - flb_sds_cat_safe(&p, "/", 1); - len = strlen(join_a); - flb_sds_cat_safe(&p, join_a, len); - } - - if (join_b) { - flb_sds_cat_safe(&p, "/", 1); - len = strlen(join_b); - flb_sds_cat_safe(&p, join_b, len); - } - - fd = open(p, O_RDONLY); - if (fd == -1) { - flb_sds_destroy(p); - return -1; - } - flb_sds_destroy(p); - - bytes = read(fd, &tmp, sizeof(tmp)); - if (bytes == -1) { - flb_errno(); - close(fd); - return -1; - } - close(fd); - - ret = ne_utils_str_to_uint64(tmp, &val); - if (ret == -1) { - return -1; - } - - *out_val = val; - return 0; -} - -/* - * Read a file and every non-empty line is stored as a flb_slist_entry in the - * given list. - */ -int ne_utils_file_read_lines(const char *mount, const char *path, struct mk_list *list) -{ - int len; - int ret; - FILE *f; - char line[512]; - char real_path[2048]; - - mk_list_init(list); - - /* Check the path starts with the mount point to prevent duplication. */ - if (strncasecmp(path, mount, strlen(mount)) == 0 && - path[strlen(mount)] == '/') { - mount = ""; - } - - snprintf(real_path, sizeof(real_path) - 1, "%s%s", mount, path); - f = fopen(real_path, "r"); - if (f == NULL) { - flb_errno(); - return -1; - } - - /* Read the content */ - while (fgets(line, sizeof(line) - 1, f)) { - len = strlen(line); - if (line[len - 1] == '\n') { - line[--len] = 0; - if (len && line[len - 1] == '\r') { - line[--len] = 0; - } - } - - ret = flb_slist_add(list, line); - if (ret == -1) { - fclose(f); - flb_slist_destroy(list); - return -1; - } - } - - fclose(f); - return 0; -} - -int ne_utils_path_scan(struct flb_ne *ctx, const char *mount, const char *path, - int expected, struct mk_list *list) -{ - int i; - int ret; - glob_t globbuf; - struct stat st; - char real_path[2048]; - - if (!path) { - return -1; - } - - /* Safe reset for globfree() */ - globbuf.gl_pathv = NULL; - - /* Scan the real path */ - snprintf(real_path, sizeof(real_path) - 1, "%s%s", mount, path); - ret = glob(real_path, GLOB_TILDE | GLOB_ERR, NULL, &globbuf); - if (ret != 0) { - switch (ret) { - case GLOB_NOSPACE: - flb_plg_error(ctx->ins, "no memory space available"); - return -1; - case GLOB_ABORTED: - flb_plg_error(ctx->ins, "read error, check permissions: %s", path); - return -1;; - case GLOB_NOMATCH: - ret = stat(path, &st); - if (ret == -1) { - flb_plg_debug(ctx->ins, "cannot read info from: %s", path); - } - else { - ret = access(path, R_OK); - if (ret == -1 && errno == EACCES) { - flb_plg_error(ctx->ins, "NO read access for path: %s", path); - } - else { - flb_plg_debug(ctx->ins, "NO matches for path: %s", path); - } - } - return -1; - } - } - - if (globbuf.gl_pathc <= 0) { - globfree(&globbuf); - return -1; - } - - /* Initialize list */ - flb_slist_create(list); - - /* For every entry found, generate an output list */ - for (i = 0; i < globbuf.gl_pathc; i++) { - ret = stat(globbuf.gl_pathv[i], &st); - if (ret != 0) { - continue; - } - - if ((expected == NE_SCAN_FILE && S_ISREG(st.st_mode)) || - (expected == NE_SCAN_DIR && S_ISDIR(st.st_mode))) { - - /* Compose the path */ - ret = flb_slist_add(list, globbuf.gl_pathv[i]); - if (ret != 0) { - globfree(&globbuf); - flb_slist_destroy(list); - return -1; - } - } - } - - globfree(&globbuf); - return 0; -} diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_utils.h b/fluent-bit/plugins/in_node_exporter_metrics/ne_utils.h deleted file mode 100644 index 448293a03..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_utils.h +++ /dev/null @@ -1,39 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_NODE_EXPORTER_UTILS_H -#define FLB_NODE_EXPORTER_UTILS_H - -#include -#include -#include -#include "ne.h" - -int ne_utils_str_to_double(char *str, double *out_val); -int ne_utils_str_to_uint64(char *str, uint64_t *out_val); - -int ne_utils_file_read_uint64(const char *mount, - const char *path, - const char *join_a, const char *join_b, - uint64_t *out_val); - -int ne_utils_file_read_lines(const char *mount, const char *path, struct mk_list *list); -int ne_utils_path_scan(struct flb_ne *ctx, const char *mount, const char *path, - int expected, struct mk_list *list); -#endif diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_vmstat_linux.c b/fluent-bit/plugins/in_node_exporter_metrics/ne_vmstat_linux.c deleted file mode 100644 index a0240d4ee..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_vmstat_linux.c +++ /dev/null @@ -1,216 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include - -#include "ne.h" -#include "ne_utils.h" - -#define VMSTAT_ENTRIES "^(oom_kill|pgpg|pswp|pg.*fault).*" - -static int keep_field(struct flb_ne *ctx, flb_sds_t field) -{ - return flb_regex_match(ctx->vml_regex_fields, - (unsigned char *) field, flb_sds_len(field)); -} - -static int vmstat_configure(struct flb_ne *ctx) -{ - int ret; - int parts; - char tmp[256]; - struct mk_list *head; - struct mk_list list; - struct mk_list split_list; - struct flb_slist_entry *line; - struct flb_slist_entry *key; - struct cmt_counter *c; - - /* Initialize regex for skipped devices */ - ctx->vml_regex_fields = flb_regex_create(VMSTAT_ENTRIES); - if (!ctx->vml_regex_fields) { - flb_plg_error(ctx->ins, - "could not initialize regex pattern for matching " - "fields: '%s'", - VMSTAT_ENTRIES); - return -1; - } - - /* Initialize hash table */ - ctx->vml_ht = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, 16, 0); - if (!ctx->vml_ht) { - return -1; - } - - mk_list_init(&list); - mk_list_init(&split_list); - - ret = ne_utils_file_read_lines(ctx->path_procfs, "/vmstat", &list); - if (ret == -1) { - return -1; - } - - mk_list_foreach(head, &list) { - line = mk_list_entry(head, struct flb_slist_entry, _head); - - mk_list_init(&split_list); - ret = flb_slist_split_string(&split_list, line->str, ' ', 2); - if (ret == -1) { - continue; - } - parts = ret; - - parts = ret; - if (parts < 2) { - flb_slist_destroy(&split_list); - continue; - } - - /* retrieve key and value */ - key = flb_slist_entry_get(&split_list, 0); - - /* keep field ? */ - if (!keep_field(ctx, key->str)) { - flb_slist_destroy(&split_list); - continue; - } - - snprintf(tmp, sizeof(tmp) - 1, - "/proc/vmstat information field %s.", key->str); - c = cmt_counter_create(ctx->cmt, "node", "vmstat", key->str, - tmp, 0, NULL); - if (!c) { - flb_slist_destroy(&split_list); - flb_slist_destroy(&list); - return -1; - } - - ret = flb_hash_table_add(ctx->vml_ht, - key->str, flb_sds_len(key->str), c, 0); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not add hash for vmstat metric: %s", key->str); - flb_slist_destroy(&split_list); - flb_slist_destroy(&list); - return -1; - } - - flb_slist_destroy(&split_list); - } - - flb_slist_destroy(&list); - return 0; -} - -static int vmstat_update(struct flb_ne *ctx) -{ - int ret; - int parts; - double v; - uint64_t ts; - size_t out_size = 0; - struct mk_list *head; - struct mk_list list; - struct mk_list split_list; - struct flb_slist_entry *line; - struct flb_slist_entry *key; - struct flb_slist_entry *val; - struct cmt_untyped *u; - - mk_list_init(&list); - mk_list_init(&split_list); - - ret = ne_utils_file_read_lines(ctx->path_procfs, "/vmstat", &list); - if (ret == -1) { - return -1; - } - - ts = cfl_time_now(); - mk_list_foreach(head, &list) { - line = mk_list_entry(head, struct flb_slist_entry, _head); - - mk_list_init(&split_list); - ret = flb_slist_split_string(&split_list, line->str, ' ', 2); - if (ret == -1) { - continue; - } - - parts = ret; - if (parts == 0) { - flb_slist_destroy(&split_list); - continue; - } - - /* retrieve key and value */ - key = flb_slist_entry_get(&split_list, 0); - val = flb_slist_entry_get(&split_list, 1); - - /* keep field ? */ - if (!keep_field(ctx, key->str)) { - flb_slist_destroy(&split_list); - continue; - } - - ret = flb_hash_table_get(ctx->vml_ht, - key->str, flb_sds_len(key->str), - (void *) &u, &out_size); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not retrieve vmstat hash metric: '%s'", key->str); - flb_slist_destroy(&split_list); - continue; - } - - /* set metric */ - ne_utils_str_to_double(val->str, &v); - cmt_untyped_set(u, ts, v, 0, NULL); - - flb_slist_destroy(&split_list); - } - - flb_slist_destroy(&list); - return 0; -} - -int ne_vmstat_init(struct flb_ne *ctx) -{ - vmstat_configure(ctx); - return 0; -} - -int ne_vmstat_update(struct flb_ne *ctx) -{ - vmstat_update(ctx); - return 0; -} - -int ne_vmstat_exit(struct flb_ne *ctx) -{ - if (ctx->vml_regex_fields) { - flb_regex_destroy(ctx->vml_regex_fields); - } - - if (ctx->vml_ht) { - flb_hash_table_destroy(ctx->vml_ht); - } - return 0; -} diff --git a/fluent-bit/plugins/in_node_exporter_metrics/ne_vmstat_linux.h b/fluent-bit/plugins/in_node_exporter_metrics/ne_vmstat_linux.h deleted file mode 100644 index 08b78f4f4..000000000 --- a/fluent-bit/plugins/in_node_exporter_metrics/ne_vmstat_linux.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_NE_VMSTAT_LINUX_H -#define FLB_IN_NE_VMSTAT_LINUX_H - -#include "ne.h" - -int ne_vmstat_init(struct flb_ne *ctx); -int ne_vmstat_update(struct flb_ne *ctx); -int ne_vmstat_exit(struct flb_ne *ctx); - -#endif diff --git a/fluent-bit/plugins/in_opentelemetry/CMakeLists.txt b/fluent-bit/plugins/in_opentelemetry/CMakeLists.txt deleted file mode 100644 index 4c3d6db32..000000000 --- a/fluent-bit/plugins/in_opentelemetry/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -if(NOT FLB_METRICS) - message(FATAL_ERROR "OpenTelemetry input plugin requires FLB_HTTP_SERVER=On.") -endif() - -set(src - http_conn.c - opentelemetry.c - opentelemetry_prot.c - opentelemetry_config.c - ) - -FLB_PLUGIN(in_opentelemetry "${src}" "monkey-core-static") diff --git a/fluent-bit/plugins/in_opentelemetry/http_conn.c b/fluent-bit/plugins/in_opentelemetry/http_conn.c deleted file mode 100644 index a402295b1..000000000 --- a/fluent-bit/plugins/in_opentelemetry/http_conn.c +++ /dev/null @@ -1,301 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include "opentelemetry.h" -#include "http_conn.h" -#include "opentelemetry_prot.h" - -static void opentelemetry_conn_request_init(struct mk_http_session *session, - struct mk_http_request *request); - -static int opentelemetry_conn_event(void *data) -{ - int status; - size_t size; - ssize_t available; - ssize_t bytes; - char *tmp; - char *request_end; - size_t request_len; - struct http_conn *conn; - struct mk_event *event; - struct flb_opentelemetry *ctx; - struct flb_connection *connection; - - connection = (struct flb_connection *) data; - - conn = connection->user_data; - - ctx = conn->ctx; - - event = &connection->event; - - if (event->mask & MK_EVENT_READ) { - available = (conn->buf_size - conn->buf_len) - 1; - if (available < 1) { - if (conn->buf_size + ctx->buffer_chunk_size > ctx->buffer_max_size) { - flb_plg_trace(ctx->ins, - "fd=%i incoming data exceed limit (%zu KB)", - event->fd, (ctx->buffer_max_size / 1024)); - opentelemetry_conn_del(conn); - return -1; - } - - size = conn->buf_size + ctx->buffer_chunk_size; - tmp = flb_realloc(conn->buf_data, size); - if (!tmp) { - flb_errno(); - return -1; - } - flb_plg_trace(ctx->ins, "fd=%i buffer realloc %i -> %zu", - event->fd, conn->buf_size, size); - - conn->buf_data = tmp; - conn->buf_size = size; - available = (conn->buf_size - conn->buf_len) - 1; - } - - /* Read data */ - bytes = flb_io_net_read(connection, - (void *) &conn->buf_data[conn->buf_len], - available); - - if (bytes <= 0) { - flb_plg_trace(ctx->ins, "fd=%i closed connection", event->fd); - opentelemetry_conn_del(conn); - return -1; - } - - flb_plg_trace(ctx->ins, "read()=%zi pre_len=%i now_len=%zi", - bytes, conn->buf_len, conn->buf_len + bytes); - conn->buf_len += bytes; - conn->buf_data[conn->buf_len] = '\0'; - - status = mk_http_parser(&conn->request, &conn->session.parser, - conn->buf_data, conn->buf_len, conn->session.server); - - if (status == MK_HTTP_PARSER_OK) { - /* Do more logic parsing and checks for this request */ - opentelemetry_prot_handle(ctx, conn, &conn->session, &conn->request); - - /* Evict the processed request from the connection buffer and reinitialize - * the HTTP parser. - */ - - request_end = NULL; - - if (NULL != conn->request.data.data) { - request_end = &conn->request.data.data[conn->request.data.len]; - } - else { - request_end = strstr(conn->buf_data, "\r\n\r\n"); - - if(NULL != request_end) { - request_end = &request_end[4]; - } - } - - if (NULL != request_end) { - request_len = (size_t)(request_end - conn->buf_data); - - if (0 < (conn->buf_len - request_len)) { - memmove(conn->buf_data, &conn->buf_data[request_len], - conn->buf_len - request_len); - - conn->buf_data[conn->buf_len - request_len] = '\0'; - conn->buf_len -= request_len; - } - else { - memset(conn->buf_data, 0, request_len); - - conn->buf_len = 0; - } - - /* Reinitialize the parser so the next request is properly - * handled, the additional memset intends to wipe any left over data - * from the headers parsed in the previous request. - */ - memset(&conn->session.parser, 0, sizeof(struct mk_http_parser)); - mk_http_parser_init(&conn->session.parser); - opentelemetry_conn_request_init(&conn->session, &conn->request); - } - } - else if (status == MK_HTTP_PARSER_ERROR) { - opentelemetry_prot_handle_error(ctx, conn, &conn->session, &conn->request); - - /* Reinitialize the parser so the next request is properly - * handled, the additional memset intends to wipe any left over data - * from the headers parsed in the previous request. - */ - memset(&conn->session.parser, 0, sizeof(struct mk_http_parser)); - mk_http_parser_init(&conn->session.parser); - opentelemetry_conn_request_init(&conn->session, &conn->request); - } - - /* FIXME: add Protocol handler here */ - return bytes; - } - - if (event->mask & MK_EVENT_CLOSE) { - flb_plg_trace(ctx->ins, "fd=%i hangup", event->fd); - opentelemetry_conn_del(conn); - return -1; - } - - return 0; - -} - -static void opentelemetry_conn_session_init(struct mk_http_session *session, - struct mk_server *server, - int client_fd) -{ - /* Alloc memory for node */ - session->_sched_init = MK_TRUE; - session->pipelined = MK_FALSE; - session->counter_connections = 0; - session->close_now = MK_FALSE; - session->status = MK_REQUEST_STATUS_INCOMPLETE; - session->server = server; - session->socket = client_fd; - - /* creation time in unix time */ - session->init_time = time(NULL); - - session->channel = mk_channel_new(MK_CHANNEL_SOCKET, session->socket); - session->channel->io = session->server->network; - - /* Init session request list */ - mk_list_init(&session->request_list); - - /* Initialize the parser */ - mk_http_parser_init(&session->parser); -} - -static void opentelemetry_conn_request_init(struct mk_http_session *session, - struct mk_http_request *request) -{ - memset(request, 0, sizeof(struct mk_http_request)); - - mk_http_request_init(session, request, session->server); - - request->in_headers.type = MK_STREAM_IOV; - request->in_headers.dynamic = MK_FALSE; - request->in_headers.cb_consumed = NULL; - request->in_headers.cb_finished = NULL; - request->in_headers.stream = &request->stream; - - mk_list_add(&request->in_headers._head, &request->stream.inputs); - - request->session = session; -} - -struct http_conn *opentelemetry_conn_add(struct flb_connection *connection, - struct flb_opentelemetry *ctx) -{ - struct http_conn *conn; - int ret; - - conn = flb_calloc(1, sizeof(struct http_conn)); - if (!conn) { - flb_errno(); - return NULL; - } - conn->connection = connection; - - /* Set data for the event-loop */ - MK_EVENT_NEW(&connection->event); - - connection->user_data = conn; - connection->event.type = FLB_ENGINE_EV_CUSTOM; - connection->event.handler = opentelemetry_conn_event; - - /* Connection info */ - conn->ctx = ctx; - conn->buf_len = 0; - - conn->buf_data = flb_malloc(ctx->buffer_chunk_size); - if (!conn->buf_data) { - flb_errno(); - flb_plg_error(ctx->ins, "could not allocate new connection"); - flb_free(conn); - return NULL; - } - conn->buf_size = ctx->buffer_chunk_size; - - /* Register instance into the event loop */ - ret = mk_event_add(flb_engine_evl_get(), - connection->fd, - FLB_ENGINE_EV_CUSTOM, - MK_EVENT_READ, - &connection->event); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not register new connection"); - flb_free(conn->buf_data); - flb_free(conn); - return NULL; - } - - /* Initialize HTTP Session: this is a custom context for Monkey HTTP */ - opentelemetry_conn_session_init(&conn->session, ctx->server, connection->fd); - - /* Initialize HTTP Request: this is the initial request and it will be reinitialized - * automatically after the request is handled so it can be used for the next one. - */ - opentelemetry_conn_request_init(&conn->session, &conn->request); - - /* Link connection node to parent context list */ - mk_list_add(&conn->_head, &ctx->connections); - return conn; -} - -int opentelemetry_conn_del(struct http_conn *conn) -{ - if (conn->session.channel != NULL) { - mk_channel_release(conn->session.channel); - } - - /* The downstream unregisters the file descriptor from the event-loop - * so there's nothing to be done by the plugin - */ - flb_downstream_conn_release(conn->connection); - - mk_list_del(&conn->_head); - - flb_free(conn->buf_data); - flb_free(conn); - - return 0; -} - -void opentelemetry_conn_release_all(struct flb_opentelemetry *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - struct http_conn *conn; - - mk_list_foreach_safe(head, tmp, &ctx->connections) { - conn = mk_list_entry(head, struct http_conn, _head); - opentelemetry_conn_del(conn); - } -} diff --git a/fluent-bit/plugins/in_opentelemetry/http_conn.h b/fluent-bit/plugins/in_opentelemetry/http_conn.h deleted file mode 100644 index 60627d860..000000000 --- a/fluent-bit/plugins/in_opentelemetry/http_conn.h +++ /dev/null @@ -1,57 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_HTTP_CONN -#define FLB_IN_HTTP_CONN - -#include -#include -#include -#include - -#include "opentelemetry.h" - -struct http_conn { - struct mk_event event; /* Built-in event data for mk_events */ - - /* Buffer */ - char *buf_data; /* Buffer data */ - int buf_len; /* Data length */ - int buf_size; /* Buffer size */ - - /* - * Parser context: we only held one parser per connection - * which is re-used everytime we have a new request. - */ - struct mk_http_parser parser; - struct mk_http_request request; - struct mk_http_session session; - struct flb_connection *connection; - - void *ctx; /* Plugin parent context */ - struct mk_list _head; /* link to flb_opentelemetry->connections */ -}; - -struct http_conn *opentelemetry_conn_add(struct flb_connection *connection, - struct flb_opentelemetry *ctx); -int opentelemetry_conn_del(struct http_conn *conn); -void opentelemetry_conn_release_all(struct flb_opentelemetry *ctx); - - -#endif diff --git a/fluent-bit/plugins/in_opentelemetry/opentelemetry.c b/fluent-bit/plugins/in_opentelemetry/opentelemetry.c deleted file mode 100644 index 5cd26f8e6..000000000 --- a/fluent-bit/plugins/in_opentelemetry/opentelemetry.c +++ /dev/null @@ -1,200 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#include -#include -#include -#include - -#include "http_conn.h" -#include "opentelemetry.h" -#include "opentelemetry_config.h" - -/* - * For a server event, the collection event means a new client have arrived, we - * accept the connection and create a new TCP instance which will wait for - * JSON map messages. - */ -static int in_opentelemetry_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_connection *connection; - struct http_conn *conn; - struct flb_opentelemetry *ctx; - - ctx = in_context; - - connection = flb_downstream_conn_get(ctx->downstream); - - if (connection == NULL) { - flb_plg_error(ctx->ins, "could not accept new connection"); - - return -1; - } - - flb_plg_trace(ctx->ins, "new TCP connection arrived FD=%i", connection->fd); - - conn = opentelemetry_conn_add(connection, ctx); - - if (conn == NULL) { - return -1; - } - - return 0; -} - -static int in_opentelemetry_init(struct flb_input_instance *ins, - struct flb_config *config, void *data) -{ - unsigned short int port; - int ret; - struct flb_opentelemetry *ctx; - - (void) data; - - /* Create context and basic conf */ - ctx = opentelemetry_config_create(ins); - if (!ctx) { - return -1; - } - ctx->collector_id = -1; - - /* Populate context with config map defaults and incoming properties */ - ret = flb_input_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_plg_error(ctx->ins, "configuration error"); - opentelemetry_config_destroy(ctx); - return -1; - } - - /* Set the context */ - flb_input_set_context(ins, ctx); - - port = (unsigned short int) strtoul(ctx->tcp_port, NULL, 10); - - ctx->downstream = flb_downstream_create(FLB_TRANSPORT_TCP, - ins->flags, - ctx->listen, - port, - ins->tls, - config, - &ins->net_setup); - - if (ctx->downstream == NULL) { - flb_plg_error(ctx->ins, - "could not initialize downstream on %s:%s. Aborting", - ctx->listen, ctx->tcp_port); - - opentelemetry_config_destroy(ctx); - - return -1; - } - - flb_input_downstream_set(ctx->downstream, ctx->ins); - - flb_plg_info(ctx->ins, "listening on %s:%s", ctx->listen, ctx->tcp_port); - - if (ctx->successful_response_code != 200 && - ctx->successful_response_code != 201 && - ctx->successful_response_code != 204) { - flb_plg_error(ctx->ins, "%d is not supported response code. Use default 201", - ctx->successful_response_code); - ctx->successful_response_code = 201; - } - - /* Collect upon data available on the standard input */ - ret = flb_input_set_collector_socket(ins, - in_opentelemetry_collect, - ctx->downstream->server_fd, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "Could not set collector for IN_TCP input plugin"); - opentelemetry_config_destroy(ctx); - return -1; - } - - ctx->collector_id = ret; - - return 0; -} - -static int in_opentelemetry_exit(void *data, struct flb_config *config) -{ - struct flb_opentelemetry *ctx; - - (void) config; - - ctx = data; - - if (ctx != NULL) { - opentelemetry_config_destroy(ctx); - } - - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_SIZE, "buffer_max_size", HTTP_BUFFER_MAX_SIZE, - 0, FLB_TRUE, offsetof(struct flb_opentelemetry, buffer_max_size), - "" - }, - - { - FLB_CONFIG_MAP_SIZE, "buffer_chunk_size", HTTP_BUFFER_CHUNK_SIZE, - 0, FLB_TRUE, offsetof(struct flb_opentelemetry, buffer_chunk_size), - "" - }, - - { - FLB_CONFIG_MAP_STR, "tag_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_opentelemetry, tag_key), - "" - }, - { - FLB_CONFIG_MAP_INT, "successful_response_code", "201", - 0, FLB_TRUE, offsetof(struct flb_opentelemetry, successful_response_code), - "Set successful response code. 200, 201 and 204 are supported." - }, - { - FLB_CONFIG_MAP_BOOL, "raw_traces", "false", - 0, FLB_TRUE, offsetof(struct flb_opentelemetry, raw_traces), - "Forward traces without processing" - }, - - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_input_plugin in_opentelemetry_plugin = { - .name = "opentelemetry", - .description = "OpenTelemetry", - .cb_init = in_opentelemetry_init, - .cb_pre_run = NULL, - .cb_collect = in_opentelemetry_collect, - .cb_flush_buf = NULL, - .cb_pause = NULL, - .cb_resume = NULL, - .cb_exit = in_opentelemetry_exit, - .config_map = config_map, - .flags = FLB_INPUT_NET_SERVER | FLB_IO_OPT_TLS -}; diff --git a/fluent-bit/plugins/in_opentelemetry/opentelemetry.h b/fluent-bit/plugins/in_opentelemetry/opentelemetry.h deleted file mode 100644 index 512f2ab6f..000000000 --- a/fluent-bit/plugins/in_opentelemetry/opentelemetry.h +++ /dev/null @@ -1,51 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_OPENTELEMETRY_H -#define FLB_IN_OPENTELEMETRY_H - -#include -#include -#include - -#include - -#define HTTP_BUFFER_MAX_SIZE "4M" -#define HTTP_BUFFER_CHUNK_SIZE "512K" - -struct flb_opentelemetry { - int successful_response_code; - flb_sds_t listen; - flb_sds_t tcp_port; - const char *tag_key; - bool raw_traces; - - size_t buffer_max_size; /* Maximum buffer size */ - size_t buffer_chunk_size; /* Chunk allocation size */ - - int collector_id; /* Listener collector id */ - struct flb_downstream *downstream; /* Client manager */ - struct mk_list connections; /* linked list of connections */ - - struct mk_server *server; - struct flb_input_instance *ins; -}; - - -#endif diff --git a/fluent-bit/plugins/in_opentelemetry/opentelemetry_config.c b/fluent-bit/plugins/in_opentelemetry/opentelemetry_config.c deleted file mode 100644 index b57596f94..000000000 --- a/fluent-bit/plugins/in_opentelemetry/opentelemetry_config.c +++ /dev/null @@ -1,92 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include "opentelemetry.h" -#include "http_conn.h" - -/* default HTTP port for OTLP/HTTP is 4318 */ -#define OTLP_HTTP_PORT 4318 - -struct flb_opentelemetry *opentelemetry_config_create(struct flb_input_instance *ins) -{ - int ret; - char port[8]; - struct flb_opentelemetry *ctx; - - ctx = flb_calloc(1, sizeof(struct flb_opentelemetry)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - mk_list_init(&ctx->connections); - - /* Load the config map */ - ret = flb_input_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return NULL; - } - - /* Listen interface (if not set, defaults to 0.0.0.0:4318) */ - flb_input_net_default_listener("0.0.0.0", OTLP_HTTP_PORT, ins); - - ctx->listen = flb_strdup(ins->host.listen); - snprintf(port, sizeof(port) - 1, "%d", ins->host.port); - ctx->tcp_port = flb_strdup(port); - - /* HTTP Server specifics */ - ctx->server = flb_calloc(1, sizeof(struct mk_server)); - ctx->server->keep_alive = MK_TRUE; - - /* monkey detects server->workers == 0 as the server not being initialized at the - * moment so we want to make sure that it stays that way! - */ - - return ctx; -} - -int opentelemetry_config_destroy(struct flb_opentelemetry *ctx) -{ - /* release all connections */ - opentelemetry_conn_release_all(ctx); - - if (ctx->collector_id != -1) { - flb_input_collector_delete(ctx->collector_id, ctx->ins); - - ctx->collector_id = -1; - } - - if (ctx->downstream != NULL) { - flb_downstream_destroy(ctx->downstream); - } - - if (ctx->server) { - flb_free(ctx->server); - } - - flb_free(ctx->listen); - flb_free(ctx->tcp_port); - flb_free(ctx); - - return 0; -} diff --git a/fluent-bit/plugins/in_opentelemetry/opentelemetry_config.h b/fluent-bit/plugins/in_opentelemetry/opentelemetry_config.h deleted file mode 100644 index 0d980c7aa..000000000 --- a/fluent-bit/plugins/in_opentelemetry/opentelemetry_config.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_OPENTELEMETRY_CONFIG_H -#define FLB_IN_OPENTELEMETRY_CONFIG_H - -#include -#include "opentelemetry.h" - -struct flb_opentelemetry *opentelemetry_config_create(struct flb_input_instance *ins); -int opentelemetry_config_destroy(struct flb_opentelemetry *ctx); - -#endif diff --git a/fluent-bit/plugins/in_opentelemetry/opentelemetry_prot.c b/fluent-bit/plugins/in_opentelemetry/opentelemetry_prot.c deleted file mode 100644 index c9ccba7f9..000000000 --- a/fluent-bit/plugins/in_opentelemetry/opentelemetry_prot.c +++ /dev/null @@ -1,1674 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include -#include "opentelemetry.h" -#include "http_conn.h" - -#define HTTP_CONTENT_JSON 0 - -static int json_payload_append_converted_value( - struct flb_log_event_encoder *encoder, - int target_field, - msgpack_object *object); - -static int json_payload_append_converted_array( - struct flb_log_event_encoder *encoder, - int target_field, - msgpack_object *object); - -static int json_payload_append_converted_kvlist( - struct flb_log_event_encoder *encoder, - int target_field, - msgpack_object *object); - -static int json_payload_to_msgpack(struct flb_opentelemetry *ctx, - struct flb_log_event_encoder *encoder, - const char *body, - size_t len); - -static int otlp_pack_any_value(msgpack_packer *mp_pck, - Opentelemetry__Proto__Common__V1__AnyValue *body); - -static int send_response(struct http_conn *conn, int http_status, char *message) -{ - int len; - flb_sds_t out; - size_t sent; - - out = flb_sds_create_size(256); - if (!out) { - return -1; - } - - if (message) { - len = strlen(message); - } - else { - len = 0; - } - - if (http_status == 201) { - flb_sds_printf(&out, - "HTTP/1.1 201 Created \r\n" - "Server: Fluent Bit v%s\r\n" - "Content-Length: 0\r\n\r\n", - FLB_VERSION_STR); - } - else if (http_status == 200) { - flb_sds_printf(&out, - "HTTP/1.1 200 OK\r\n" - "Server: Fluent Bit v%s\r\n" - "Content-Length: 0\r\n\r\n", - FLB_VERSION_STR); - } - else if (http_status == 204) { - flb_sds_printf(&out, - "HTTP/1.1 204 No Content\r\n" - "Server: Fluent Bit v%s\r\n" - "\r\n", - FLB_VERSION_STR); - } - else if (http_status == 400) { - flb_sds_printf(&out, - "HTTP/1.1 400 Forbidden\r\n" - "Server: Fluent Bit v%s\r\n" - "Content-Length: %i\r\n\r\n%s", - FLB_VERSION_STR, - len, message); - } - - /* We should check the outcome of this operation */ - flb_io_net_write(conn->connection, - (void *) out, - flb_sds_len(out), - &sent); - - flb_sds_destroy(out); - - return 0; -} - -static int process_payload_metrics(struct flb_opentelemetry *ctx, struct http_conn *conn, - flb_sds_t tag, - struct mk_http_session *session, - struct mk_http_request *request) -{ - struct cfl_list decoded_contexts; - struct cfl_list *iterator; - struct cmt *context; - size_t offset; - int result; - - offset = 0; - - result = cmt_decode_opentelemetry_create(&decoded_contexts, - request->data.data, - request->data.len, - &offset); - - if (result == CMT_DECODE_OPENTELEMETRY_SUCCESS) { - cfl_list_foreach(iterator, &decoded_contexts) { - context = cfl_list_entry(iterator, struct cmt, _head); - - result = flb_input_metrics_append(ctx->ins, NULL, 0, context); - - if (result != 0) { - flb_plg_debug(ctx->ins, "could not ingest metrics context : %d", result); - } - } - - cmt_decode_opentelemetry_destroy(&decoded_contexts); - } - - return 0; -} - -static int process_payload_traces_proto(struct flb_opentelemetry *ctx, struct http_conn *conn, - flb_sds_t tag, - struct mk_http_session *session, - struct mk_http_request *request) -{ - struct ctrace *decoded_context; - size_t offset; - int result; - - offset = 0; - result = ctr_decode_opentelemetry_create(&decoded_context, - request->data.data, - request->data.len, - &offset); - if (result == 0) { - result = flb_input_trace_append(ctx->ins, NULL, 0, decoded_context); - ctr_decode_opentelemetry_destroy(decoded_context); - } - - return result; -} - -static int process_payload_raw_traces(struct flb_opentelemetry *ctx, struct http_conn *conn, - flb_sds_t tag, - struct mk_http_session *session, - struct mk_http_request *request) -{ - int ret; - int root_type; - char *out_buf = NULL; - size_t out_size; - - msgpack_packer mp_pck; - msgpack_sbuffer mp_sbuf; - - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - msgpack_pack_array(&mp_pck, 2); - flb_pack_time_now(&mp_pck); - - /* Check if the incoming payload is a valid JSON message and convert it to msgpack */ - ret = flb_pack_json(request->data.data, request->data.len, - &out_buf, &out_size, &root_type, NULL); - - if (ret == 0 && root_type == JSMN_OBJECT) { - /* JSON found, pack it msgpack representation */ - msgpack_sbuffer_write(&mp_sbuf, out_buf, out_size); - } - else { - /* the content might be a binary payload or invalid JSON */ - msgpack_pack_map(&mp_pck, 1); - msgpack_pack_str_with_body(&mp_pck, "trace", 5); - msgpack_pack_str_with_body(&mp_pck, request->data.data, request->data.len); - } - - /* release 'out_buf' if it was allocated */ - if (out_buf) { - flb_free(out_buf); - } - - flb_input_log_append(ctx->ins, tag, flb_sds_len(tag), mp_sbuf.data, mp_sbuf.size); - msgpack_sbuffer_destroy(&mp_sbuf); - - return 0; -} - -static int process_payload_traces(struct flb_opentelemetry *ctx, struct http_conn *conn, - flb_sds_t tag, - struct mk_http_session *session, - struct mk_http_request *request) -{ - int result; - - if (ctx->raw_traces) { - result = process_payload_raw_traces(ctx, conn, tag, session, request); - } - else { - result = process_payload_traces_proto(ctx, conn, tag, session, request); - } - - return result; -} - -static int otel_pack_string(msgpack_packer *mp_pck, char *str) -{ - return msgpack_pack_str_with_body(mp_pck, str, strlen(str)); -} - -static int otel_pack_bool(msgpack_packer *mp_pck, bool val) -{ - if (val) { - return msgpack_pack_true(mp_pck); - } - else { - return msgpack_pack_false(mp_pck); - } -} - -static int otel_pack_int(msgpack_packer *mp_pck, int val) -{ - return msgpack_pack_int64(mp_pck, val); -} - -static int otel_pack_double(msgpack_packer *mp_pck, double val) -{ - return msgpack_pack_double(mp_pck, val); -} - -static int otel_pack_kvarray(msgpack_packer *mp_pck, - Opentelemetry__Proto__Common__V1__KeyValue **kv_array, - size_t kv_count) -{ - int result; - int index; - - result = msgpack_pack_map(mp_pck, kv_count); - - if (result != 0) { - return result; - } - - for (index = 0; index < kv_count && result == 0; index++) { - result = otel_pack_string(mp_pck, kv_array[index]->key); - - if(result == 0) { - result = otlp_pack_any_value(mp_pck, kv_array[index]->value); - } - } - - return result; -} - -static int otel_pack_kvlist(msgpack_packer *mp_pck, - Opentelemetry__Proto__Common__V1__KeyValueList *kv_list) -{ - int kv_index; - int ret; - char *key; - Opentelemetry__Proto__Common__V1__AnyValue *value; - - ret = msgpack_pack_map(mp_pck, kv_list->n_values); - if (ret != 0) { - return ret; - } - - for (kv_index = 0; kv_index < kv_list->n_values && ret == 0; kv_index++) { - key = kv_list->values[kv_index]->key; - value = kv_list->values[kv_index]->value; - - ret = otel_pack_string(mp_pck, key); - - if(ret == 0) { - ret = otlp_pack_any_value(mp_pck, value); - } - } - - return ret; -} - -static int otel_pack_array(msgpack_packer *mp_pck, - Opentelemetry__Proto__Common__V1__ArrayValue *array) -{ - int ret; - int array_index; - - ret = msgpack_pack_array(mp_pck, array->n_values); - - if (ret != 0) { - return ret; - } - - for (array_index = 0; array_index < array->n_values && ret == 0; array_index++) { - ret = otlp_pack_any_value(mp_pck, array->values[array_index]); - } - - return ret; -} - -static int otel_pack_bytes(msgpack_packer *mp_pck, - ProtobufCBinaryData bytes) -{ - return msgpack_pack_bin_with_body(mp_pck, bytes.data, bytes.len); -} - -static int otlp_pack_any_value(msgpack_packer *mp_pck, - Opentelemetry__Proto__Common__V1__AnyValue *body) -{ - int result; - - result = -2; - - switch(body->value_case){ - case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE: - result = otel_pack_string(mp_pck, body->string_value); - break; - - case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BOOL_VALUE: - result = otel_pack_bool(mp_pck, body->bool_value); - break; - - case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_INT_VALUE: - result = otel_pack_int(mp_pck, body->int_value); - break; - - case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_DOUBLE_VALUE: - result = otel_pack_double(mp_pck, body->double_value); - break; - - case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_ARRAY_VALUE: - result = otel_pack_array(mp_pck, body->array_value); - break; - - case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_KVLIST_VALUE: - result = otel_pack_kvlist(mp_pck, body->kvlist_value); - break; - - case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BYTES_VALUE: - result = otel_pack_bytes(mp_pck, body->bytes_value); - break; - - default: - break; - } - - if (result == -2) { - flb_error("[otel]: invalid value type in pack_any_value"); - result = -1; - } - - return result; -} - -static int binary_payload_to_msgpack(struct flb_log_event_encoder *encoder, - uint8_t *in_buf, - size_t in_size) -{ - int ret; - msgpack_packer packer; - msgpack_sbuffer buffer; - int resource_logs_index; - int scope_log_index; - int log_record_index; - - Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest *input_logs; - Opentelemetry__Proto__Logs__V1__ScopeLogs **scope_logs; - Opentelemetry__Proto__Logs__V1__ScopeLogs *scope_log; - Opentelemetry__Proto__Logs__V1__ResourceLogs **resource_logs; - Opentelemetry__Proto__Logs__V1__ResourceLogs *resource_log; - Opentelemetry__Proto__Logs__V1__LogRecord **log_records; - - msgpack_sbuffer_init(&buffer); - msgpack_packer_init(&packer, &buffer, msgpack_sbuffer_write); - - input_logs = opentelemetry__proto__collector__logs__v1__export_logs_service_request__unpack(NULL, in_size, in_buf); - if (input_logs == NULL) { - flb_error("[otel] Failed to unpack input logs"); - return -1; - } - - resource_logs = input_logs->resource_logs; - if (resource_logs == NULL) { - flb_error("[otel] No resource logs found"); - return -1; - } - - for (resource_logs_index = 0; resource_logs_index < input_logs->n_resource_logs; resource_logs_index++) { - resource_log = resource_logs[resource_logs_index]; - scope_logs = resource_log->scope_logs; - - if (resource_log->n_scope_logs > 0 && scope_logs == NULL) { - flb_error("[otel] No scope logs found"); - return -1; - } - - for (scope_log_index = 0; scope_log_index < resource_log->n_scope_logs; scope_log_index++) { - scope_log = scope_logs[scope_log_index]; - log_records = scope_log->log_records; - - if (log_records == NULL) { - flb_error("[otel] No log records found"); - return -1; - } - - for (log_record_index=0; log_record_index < scope_log->n_log_records; log_record_index++) { - ret = flb_log_event_encoder_begin_record(encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp(encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = otel_pack_kvarray( - &packer, - log_records[log_record_index]->attributes, - log_records[log_record_index]->n_attributes); - - if (ret != 0) { - flb_error("[otel] Failed to convert log record attributes"); - - ret = FLB_EVENT_ENCODER_ERROR_SERIALIZATION_FAILURE; - } - else { - ret = flb_log_event_encoder_set_metadata_from_raw_msgpack( - encoder, - buffer.data, - buffer.size); - } - - msgpack_sbuffer_clear(&buffer); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = otlp_pack_any_value( - &packer, - log_records[log_record_index]->body); - - if (ret != 0) { - flb_error("[otel] Failed to convert log record body"); - - ret = FLB_EVENT_ENCODER_ERROR_SERIALIZATION_FAILURE; - } - else { - if (log_records[log_record_index]->body->value_case == - OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_KVLIST_VALUE) { - ret = flb_log_event_encoder_set_body_from_raw_msgpack( - encoder, - buffer.data, - buffer.size); - } - else { - ret = flb_log_event_encoder_append_body_values( - encoder, - FLB_LOG_EVENT_CSTRING_VALUE("message"), - FLB_LOG_EVENT_MSGPACK_RAW_VALUE(buffer.data, buffer.size)); - } - } - - msgpack_sbuffer_clear(&buffer); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(encoder); - } - else { - flb_error("[otel] marshalling error"); - - msgpack_sbuffer_destroy(&buffer); - - return -1; - } - } - } - } - - msgpack_sbuffer_destroy(&buffer); - - return 0; -} - -static int find_map_entry_by_key(msgpack_object_map *map, - char *key, - size_t match_index, - int case_insensitive) -{ - size_t match_count; - int result; - int index; - - match_count = 0; - - for (index = 0 ; index < (int) map->size ; index++) { - if (map->ptr[index].key.type == MSGPACK_OBJECT_STR) { - if (case_insensitive) { - result = strncasecmp(map->ptr[index].key.via.str.ptr, - key, - map->ptr[index].key.via.str.size); - } - else { - result = strncmp(map->ptr[index].key.via.str.ptr, - key, - map->ptr[index].key.via.str.size); - } - - if (result == 0) { - if (match_count == match_index) { - return index; - } - - match_count++; - } - } - } - - return -1; -} - -static int json_payload_get_wrapped_value(msgpack_object *wrapper, - msgpack_object **value, - int *type) -{ - int internal_type; - msgpack_object *kv_value; - msgpack_object_str *kv_key; - msgpack_object_map *map; - - if (wrapper->type != MSGPACK_OBJECT_MAP) { - return -1; - } - - map = &wrapper->via.map; - kv_value = NULL; - internal_type = -1; - - if (map->size == 1) { - if (map->ptr[0].key.type == MSGPACK_OBJECT_STR) { - kv_value = &map->ptr[0].val; - kv_key = &map->ptr[0].key.via.str; - - if (strncasecmp(kv_key->ptr, "stringValue", kv_key->size) == 0 || - strncasecmp(kv_key->ptr, "string_value", kv_key->size) == 0) { - internal_type = MSGPACK_OBJECT_STR; - } - else if (strncasecmp(kv_key->ptr, "boolValue", kv_key->size) == 0 || - strncasecmp(kv_key->ptr, "bool_value", kv_key->size) == 0) { - internal_type = MSGPACK_OBJECT_BOOLEAN; - } - else if (strncasecmp(kv_key->ptr, "intValue", kv_key->size) == 0 || - strncasecmp(kv_key->ptr, "int_value", kv_key->size) == 0) { - internal_type = MSGPACK_OBJECT_POSITIVE_INTEGER; - } - else if (strncasecmp(kv_key->ptr, "doubleValue", kv_key->size) == 0 || - strncasecmp(kv_key->ptr, "double_value", kv_key->size) == 0) { - internal_type = MSGPACK_OBJECT_FLOAT; - } - else if (strncasecmp(kv_key->ptr, "bytesValue", kv_key->size) == 0 || - strncasecmp(kv_key->ptr, "bytes_value", kv_key->size) == 0) { - internal_type = MSGPACK_OBJECT_BIN; - } - else if (strncasecmp(kv_key->ptr, "arrayValue", kv_key->size) == 0 || - strncasecmp(kv_key->ptr, "array_value", kv_key->size) == 0) { - internal_type = MSGPACK_OBJECT_ARRAY; - } - else if (strncasecmp(kv_key->ptr, "kvlistValue", kv_key->size) == 0 || - strncasecmp(kv_key->ptr, "kvlist_value", kv_key->size) == 0) { - internal_type = MSGPACK_OBJECT_MAP; - } - } - } - - if (internal_type != -1) { - if (type != NULL) { - *type = internal_type; - } - - if (value != NULL) { - *value = kv_value; - } - - if (kv_value->type == MSGPACK_OBJECT_MAP) { - map = &kv_value->via.map; - - if (map->size == 1) { - kv_value = &map->ptr[0].val; - kv_key = &map->ptr[0].key.via.str; - - if (strncasecmp(kv_key->ptr, "values", kv_key->size) == 0) { - if (value != NULL) { - *value = kv_value; - } - } - else { - return -3; - } - } - } - } - else { - return -2; - } - - return 0; -} - -static int json_payload_append_unwrapped_value( - struct flb_log_event_encoder *encoder, - int target_field, - msgpack_object *object, - int *encoder_result) -{ - char temporary_buffer[33]; - int unwrap_value; - int result; - msgpack_object *value; - int type; - - result = json_payload_get_wrapped_value(object, - &value, - &type); - - if (result == 0) { - unwrap_value = FLB_FALSE; - - if (type == MSGPACK_OBJECT_STR) { - unwrap_value = FLB_TRUE; - } - else if (type == MSGPACK_OBJECT_BOOLEAN) { - unwrap_value = FLB_TRUE; - } - else if (type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - if (value->type == MSGPACK_OBJECT_STR) { - memset(temporary_buffer, 0, sizeof(temporary_buffer)); - - if (value->via.str.size < sizeof(temporary_buffer)) { - strncpy(temporary_buffer, - value->via.str.ptr, - value->via.str.size); - } - else { - strncpy(temporary_buffer, - value->via.str.ptr, - sizeof(temporary_buffer) - 1); - } - - result = flb_log_event_encoder_append_int64( - encoder, - target_field, - strtoll(temporary_buffer, NULL, 10)); - } - else { - unwrap_value = FLB_TRUE; - } - } - else if (type == MSGPACK_OBJECT_FLOAT) { - unwrap_value = FLB_TRUE; - } - else if (type == MSGPACK_OBJECT_BIN) { - unwrap_value = FLB_TRUE; - } - else if (type == MSGPACK_OBJECT_ARRAY) { - result = json_payload_append_converted_array(encoder, - target_field, - value); - } - else if (type == MSGPACK_OBJECT_MAP) { - result = json_payload_append_converted_kvlist(encoder, - target_field, - value); - } - else { - return -2; - } - - if (unwrap_value) { - result = json_payload_append_converted_value(encoder, - target_field, - value); - } - - *encoder_result = result; - - return 0; - } - else { - return -1; - } - - return -1; -} - - -static int json_payload_append_converted_map( - struct flb_log_event_encoder *encoder, - int target_field, - msgpack_object *object) -{ - int encoder_result; - int result; - size_t index; - msgpack_object_map *map; - - map = &object->via.map; - - result = json_payload_append_unwrapped_value( - encoder, - target_field, - object, - &encoder_result); - - if (result == 0 && encoder_result == FLB_EVENT_ENCODER_SUCCESS) { - return result; - } - - result = flb_log_event_encoder_begin_map(encoder, target_field); - - for (index = 0 ; - index < map->size && - result == FLB_EVENT_ENCODER_SUCCESS; - index++) { - result = json_payload_append_converted_value( - encoder, - target_field, - &map->ptr[index].key); - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = json_payload_append_converted_value( - encoder, - target_field, - &map->ptr[index].val); - } - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_commit_map(encoder, target_field); - } - else { - flb_log_event_encoder_rollback_map(encoder, target_field); - } - - return result; -} - -static int json_payload_append_converted_array( - struct flb_log_event_encoder *encoder, - int target_field, - msgpack_object *object) -{ - int result; - size_t index; - msgpack_object_array *array; - - array = &object->via.array; - - result = flb_log_event_encoder_begin_array(encoder, target_field); - - for (index = 0 ; - index < array->size && - result == FLB_EVENT_ENCODER_SUCCESS; - index++) { - result = json_payload_append_converted_value( - encoder, - target_field, - &array->ptr[index]); - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_commit_array(encoder, target_field); - } - else { - flb_log_event_encoder_rollback_array(encoder, target_field); - } - - return result; -} - -static int json_payload_append_converted_kvlist( - struct flb_log_event_encoder *encoder, - int target_field, - msgpack_object *object) -{ - int value_index; - int key_index; - int result; - size_t index; - msgpack_object_array *array; - msgpack_object_map *entry; - - array = &object->via.array; - - result = flb_log_event_encoder_begin_map(encoder, target_field); - - for (index = 0 ; - index < array->size && - result == FLB_EVENT_ENCODER_SUCCESS; - index++) { - - if (array->ptr[index].type != MSGPACK_OBJECT_MAP) { - result = FLB_EVENT_ENCODER_ERROR_INVALID_ARGUMENT; - } - else { - entry = &array->ptr[index].via.map; - - key_index = find_map_entry_by_key(entry, "key", 0, FLB_TRUE); - - if (key_index == -1) { - result = FLB_EVENT_ENCODER_ERROR_INVALID_ARGUMENT; - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - value_index = find_map_entry_by_key(entry, "value", 0, FLB_TRUE); - } - - if (value_index == -1) { - result = FLB_EVENT_ENCODER_ERROR_INVALID_ARGUMENT; - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = json_payload_append_converted_value( - encoder, - target_field, - &entry->ptr[key_index].val); - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = json_payload_append_converted_value( - encoder, - target_field, - &entry->ptr[value_index].val); - } - } - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_commit_map(encoder, target_field); - } - else { - flb_log_event_encoder_rollback_map(encoder, target_field); - } - - return result; -} - -static int json_payload_append_converted_value( - struct flb_log_event_encoder *encoder, - int target_field, - msgpack_object *object) -{ - int result; - - result = FLB_EVENT_ENCODER_SUCCESS; - - switch (object->type) { - case MSGPACK_OBJECT_BOOLEAN: - result = flb_log_event_encoder_append_boolean( - encoder, - target_field, - object->via.boolean); - break; - - case MSGPACK_OBJECT_POSITIVE_INTEGER: - result = flb_log_event_encoder_append_uint64( - encoder, - target_field, - object->via.u64); - break; - case MSGPACK_OBJECT_NEGATIVE_INTEGER: - result = flb_log_event_encoder_append_int64( - encoder, - target_field, - object->via.i64); - break; - - case MSGPACK_OBJECT_FLOAT32: - case MSGPACK_OBJECT_FLOAT64: - result = flb_log_event_encoder_append_double( - encoder, - target_field, - object->via.f64); - break; - - case MSGPACK_OBJECT_STR: - result = flb_log_event_encoder_append_string( - encoder, - target_field, - (char *) object->via.str.ptr, - object->via.str.size); - - break; - - case MSGPACK_OBJECT_BIN: - result = flb_log_event_encoder_append_binary( - encoder, - target_field, - (char *) object->via.bin.ptr, - object->via.bin.size); - break; - - case MSGPACK_OBJECT_ARRAY: - result = json_payload_append_converted_array( - encoder, - target_field, - object); - break; - - case MSGPACK_OBJECT_MAP: - result = json_payload_append_converted_map( - encoder, - target_field, - object); - - break; - - default: - break; - } - - return result; -} - -static int process_json_payload_log_records_entry( - struct flb_opentelemetry *ctx, - struct flb_log_event_encoder *encoder, - msgpack_object *log_records_object) -{ - msgpack_object_map *log_records_entry; - char timestamp_str[32]; - msgpack_object *timestamp_object; - uint64_t timestamp_uint64; - msgpack_object *metadata_object; - msgpack_object *body_object; - int body_type; - struct flb_time timestamp; - int result; - - if (log_records_object->type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "unexpected logRecords entry type"); - - return -4; - } - - log_records_entry = &log_records_object->via.map; - - result = find_map_entry_by_key(log_records_entry, "timeUnixNano", 0, FLB_TRUE); - - if (result == -1) { - result = find_map_entry_by_key(log_records_entry, "time_unix_nano", 0, FLB_TRUE); - } - - if (result == -1) { - result = find_map_entry_by_key(log_records_entry, "observedTimeUnixNano", 0, FLB_TRUE); - } - - if (result == -1) { - result = find_map_entry_by_key(log_records_entry, "observed_time_unix_nano", 0, FLB_TRUE); - } - - if (result == -1) { - flb_plg_info(ctx->ins, "neither timeUnixNano nor observedTimeUnixNano found"); - - flb_time_get(×tamp); - } - else { - timestamp_object = &log_records_entry->ptr[result].val; - - if (timestamp_object->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - timestamp_uint64 = timestamp_object->via.u64; - } - else if (timestamp_object->type == MSGPACK_OBJECT_STR) { - memset(timestamp_str, 0, sizeof(timestamp_str)); - - if (timestamp_object->via.str.size < sizeof(timestamp_str)) { - strncpy(timestamp_str, - timestamp_object->via.str.ptr, - timestamp_object->via.str.size); - } - else { - strncpy(timestamp_str, - timestamp_object->via.str.ptr, - sizeof(timestamp_str) - 1); - } - - timestamp_uint64 = strtoul(timestamp_str, NULL, 10); - } - else { - flb_plg_error(ctx->ins, "unexpected timeUnixNano type"); - - return -4; - } - - flb_time_from_uint64(×tamp, timestamp_uint64); - } - - - result = find_map_entry_by_key(log_records_entry, "attributes", 0, FLB_TRUE); - - if (result == -1) { - flb_plg_debug(ctx->ins, "attributes missing"); - - metadata_object = NULL; - } - else { - if (log_records_entry->ptr[result].val.type != MSGPACK_OBJECT_ARRAY) { - flb_plg_error(ctx->ins, "unexpected attributes type"); - - return -4; - } - - metadata_object = &log_records_entry->ptr[result].val; - } - - result = find_map_entry_by_key(log_records_entry, "body", 0, FLB_TRUE); - - if (result == -1) { - flb_plg_info(ctx->ins, "body missing"); - - body_object = NULL; - } - else { - if (log_records_entry->ptr[result].val.type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "unexpected body type"); - - return -4; - } - - body_object = &log_records_entry->ptr[result].val; - } - - result = flb_log_event_encoder_begin_record(encoder); - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_set_timestamp(encoder, ×tamp); - } - - if (result == FLB_EVENT_ENCODER_SUCCESS && - metadata_object != NULL) { - flb_log_event_encoder_dynamic_field_reset(&encoder->metadata); - - result = json_payload_append_converted_kvlist( - encoder, - FLB_LOG_EVENT_METADATA, - metadata_object); - } - - if (result == FLB_EVENT_ENCODER_SUCCESS && - body_object != NULL) { - result = json_payload_get_wrapped_value(body_object, NULL, &body_type); - - if (result != 0 || body_type == MSGPACK_OBJECT_MAP) { - flb_log_event_encoder_dynamic_field_reset(&encoder->body); - } - else { - flb_log_event_encoder_append_cstring( - encoder, - FLB_LOG_EVENT_BODY, - "log"); - } - - result = json_payload_append_converted_value( - encoder, - FLB_LOG_EVENT_BODY, - body_object); - } - - result = flb_log_event_encoder_dynamic_field_flush(&encoder->body); - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_commit_record(encoder); - } - else { - flb_plg_error(ctx->ins, "log event encoder failure : %d", result); - - flb_log_event_encoder_rollback_record(encoder); - - result = -4; - } - - return result; -} - -static int process_json_payload_scope_logs_entry( - struct flb_opentelemetry *ctx, - struct flb_log_event_encoder *encoder, - msgpack_object *scope_logs_object) -{ - msgpack_object_map *scope_logs_entry; - msgpack_object_array *log_records; - int result; - size_t index; - - if (scope_logs_object->type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "unexpected scopeLogs entry type"); - - return -3; - } - - scope_logs_entry = &scope_logs_object->via.map; - - result = find_map_entry_by_key(scope_logs_entry, "logRecords", 0, FLB_TRUE); - - if (result == -1) { - result = find_map_entry_by_key(scope_logs_entry, "logRecords", 0, FLB_TRUE); - - if (result == -1) { - flb_plg_error(ctx->ins, "scopeLogs missing"); - - return -3; - } - } - - if (scope_logs_entry->ptr[result].val.type != MSGPACK_OBJECT_ARRAY) { - flb_plg_error(ctx->ins, "unexpected logRecords type"); - - return -3; - } - - log_records = &scope_logs_entry->ptr[result].val.via.array; - - result = 0; - - for (index = 0 ; index < log_records->size ; index++) { - result = process_json_payload_log_records_entry( - ctx, - encoder, - &log_records->ptr[index]); - } - - return result; -} - - -static int process_json_payload_resource_logs_entry( - struct flb_opentelemetry *ctx, - struct flb_log_event_encoder *encoder, - msgpack_object *resource_logs_object) -{ - msgpack_object_map *resource_logs_entry; - msgpack_object_array *scope_logs; - int result; - size_t index; - - - if (resource_logs_object->type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "unexpected resourceLogs entry type"); - - return -2; - } - - resource_logs_entry = &resource_logs_object->via.map; - - result = find_map_entry_by_key(resource_logs_entry, "scopeLogs", 0, FLB_TRUE); - - if (result == -1) { - result = find_map_entry_by_key(resource_logs_entry, "scope_logs", 0, FLB_TRUE); - - if (result == -1) { - flb_plg_error(ctx->ins, "scopeLogs missing"); - - return -2; - } - } - - if (resource_logs_entry->ptr[result].val.type != MSGPACK_OBJECT_ARRAY) { - flb_plg_error(ctx->ins, "unexpected scopeLogs type"); - - return -2; - } - - scope_logs = &resource_logs_entry->ptr[result].val.via.array; - - result = 0; - - for (index = 0 ; index < scope_logs->size ; index++) { - result = process_json_payload_scope_logs_entry( - ctx, - encoder, - &scope_logs->ptr[index]); - } - - return result; -} - -static int process_json_payload_root(struct flb_opentelemetry *ctx, - struct flb_log_event_encoder *encoder, - msgpack_object *root_object) -{ - msgpack_object_array *resource_logs; - int result; - size_t index; - msgpack_object_map *root; - - if (root_object->type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "unexpected root object type"); - - return -1; - } - - root = &root_object->via.map; - - result = find_map_entry_by_key(root, "resourceLogs", 0, FLB_TRUE); - - if (result == -1) { - result = find_map_entry_by_key(root, "resource_logs", 0, FLB_TRUE); - - if (result == -1) { - flb_plg_error(ctx->ins, "resourceLogs missing"); - - return -1; - } - } - - if (root->ptr[result].val.type != MSGPACK_OBJECT_ARRAY) { - flb_plg_error(ctx->ins, "unexpected resourceLogs type"); - - return -1; - } - - resource_logs = &root->ptr[result].val.via.array; - - result = 0; - - for (index = 0 ; index < resource_logs->size ; index++) { - result = process_json_payload_resource_logs_entry( - ctx, - encoder, - &resource_logs->ptr[index]); - } - - return result; -} - -/* This code is definitely not complete and beyond fishy, it needs to be - * refactored. - */ -static int json_payload_to_msgpack(struct flb_opentelemetry *ctx, - struct flb_log_event_encoder *encoder, - const char *body, - size_t len) -{ - size_t msgpack_body_length; - msgpack_unpacked unpacked_root; - char *msgpack_body; - int root_type; - size_t offset; - int result; - - result = flb_pack_json(body, len, &msgpack_body, &msgpack_body_length, - &root_type, NULL); - - if (result != 0) { - flb_plg_error(ctx->ins, "json to msgpack conversion error"); - } - else { - msgpack_unpacked_init(&unpacked_root); - - offset = 0; - result = msgpack_unpack_next(&unpacked_root, - msgpack_body, - msgpack_body_length, - &offset); - - if (result == MSGPACK_UNPACK_SUCCESS) { - result = process_json_payload_root(ctx, - encoder, - &unpacked_root.data); - } - else { - result = -1; - } - - msgpack_unpacked_destroy(&unpacked_root); - - flb_free(msgpack_body); - } - - return result; -} - -static int process_payload_logs(struct flb_opentelemetry *ctx, struct http_conn *conn, - flb_sds_t tag, - struct mk_http_session *session, - struct mk_http_request *request) -{ - struct flb_log_event_encoder *encoder; - int ret; - - encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2); - - if (encoder == NULL) { - return -1; - } - - /* Check if the incoming payload is a valid JSON message and convert it to msgpack */ - if (strncasecmp(request->content_type.data, - "application/json", - request->content_type.len) == 0) { - ret = json_payload_to_msgpack(ctx, - encoder, - request->data.data, - request->data.len); - } - else if (strncasecmp(request->content_type.data, - "application/x-protobuf", - request->content_type.len) == 0) { - ret = binary_payload_to_msgpack(encoder, (uint8_t *) request->data.data, request->data.len); - } - else { - flb_error("[otel] Unsupported content type %.*s", (int)request->content_type.len, request->content_type.data); - - ret = -1; - } - - if (ret == 0) { - ret = flb_input_log_append(ctx->ins, - tag, - flb_sds_len(tag), - encoder->output_buffer, - encoder->output_length); - } - - flb_log_event_encoder_destroy(encoder); - - return ret; -} - -static inline int mk_http_point_header(mk_ptr_t *h, - struct mk_http_parser *parser, int key) -{ - struct mk_http_header *header; - - header = &parser->headers[key]; - if (header->type == key) { - h->data = header->val.data; - h->len = header->val.len; - return 0; - } - else { - h->data = NULL; - h->len = -1; - } - - return -1; -} - -static \ -int uncompress_zlib(char **output_buffer, - size_t *output_size, - char *input_buffer, - size_t input_size) -{ - flb_error("[opentelemetry] unsupported compression format"); - - return -1; -} - -static \ -int uncompress_zstd(char **output_buffer, - size_t *output_size, - char *input_buffer, - size_t input_size) -{ - flb_error("[opentelemetry] unsupported compression format"); - - return -1; -} - -static \ -int uncompress_deflate(char **output_buffer, - size_t *output_size, - char *input_buffer, - size_t input_size) -{ - flb_error("[opentelemetry] unsupported compression format"); - - return -1; -} - -static \ -int uncompress_snappy(char **output_buffer, - size_t *output_size, - char *input_buffer, - size_t input_size) -{ - int ret; - - ret = flb_snappy_uncompress_framed_data(input_buffer, - input_size, - output_buffer, - output_size); - - if (ret != 0) { - flb_error("[opentelemetry] snappy decompression failed"); - - return -1; - } - - return 1; -} - -static \ -int uncompress_gzip(char **output_buffer, - size_t *output_size, - char *input_buffer, - size_t input_size) -{ - int ret; - - ret = flb_gzip_uncompress(input_buffer, - input_size, - (void *) output_buffer, - output_size); - - if (ret == -1) { - flb_error("[opentelemetry] gzip decompression failed"); - - return -1; - } - - return 1; -} - -int opentelemetry_prot_uncompress(struct mk_http_session *session, - struct mk_http_request *request, - char **output_buffer, - size_t *output_size) -{ - struct mk_http_header *header; - size_t index; - - *output_buffer = NULL; - *output_size = 0; - - for (index = 0; - index < session->parser.headers_extra_count; - index++) { - header = &session->parser.headers_extra[index]; - - if (strncasecmp(header->key.data, "Content-Encoding", 16) == 0) { - if (strncasecmp(header->val.data, "gzip", 4) == 0) { - return uncompress_gzip(output_buffer, - output_size, - request->data.data, - request->data.len); - } - else if (strncasecmp(header->val.data, "zlib", 4) == 0) { - return uncompress_zlib(output_buffer, - output_size, - request->data.data, - request->data.len); - } - else if (strncasecmp(header->val.data, "zstd", 4) == 0) { - return uncompress_zstd(output_buffer, - output_size, - request->data.data, - request->data.len); - } - else if (strncasecmp(header->val.data, "snappy", 6) == 0) { - return uncompress_snappy(output_buffer, - output_size, - request->data.data, - request->data.len); - } - else if (strncasecmp(header->val.data, "deflate", 4) == 0) { - return uncompress_deflate(output_buffer, - output_size, - request->data.data, - request->data.len); - } - else { - return -2; - } - } - } - - return 0; -} - - -/* - * Handle an incoming request. It perform extra checks over the request, if - * everything is OK, it enqueue the incoming payload. - */ -int opentelemetry_prot_handle(struct flb_opentelemetry *ctx, struct http_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request) -{ - int i; - int ret = -1; - int len; - char *uri; - char *qs; - off_t diff; - flb_sds_t tag; - struct mk_http_header *header; - char *original_data; - size_t original_data_size; - char *uncompressed_data; - size_t uncompressed_data_size; - - if (request->uri.data[0] != '/') { - send_response(conn, 400, "error: invalid request\n"); - return -1; - } - - /* Decode URI */ - uri = mk_utils_url_decode(request->uri); - if (!uri) { - uri = mk_mem_alloc_z(request->uri.len + 1); - if (!uri) { - return -1; - } - memcpy(uri, request->uri.data, request->uri.len); - uri[request->uri.len] = '\0'; - } - - if (strcmp(uri, "/v1/metrics") != 0 && - strcmp(uri, "/v1/traces") != 0 && - strcmp(uri, "/v1/logs") != 0) { - - send_response(conn, 400, "error: invalid endpoint\n"); - mk_mem_free(uri); - - return -1; - } - - /* Try to match a query string so we can remove it */ - qs = strchr(uri, '?'); - if (qs) { - /* remove the query string part */ - diff = qs - uri; - uri[diff] = '\0'; - } - - /* Compose the query string using the URI */ - len = strlen(uri); - - if (len == 1) { - tag = NULL; /* use default tag */ - } - else { - tag = flb_sds_create_size(len); - if (!tag) { - mk_mem_free(uri); - return -1; - } - - /* New tag skipping the URI '/' */ - flb_sds_cat(tag, uri + 1, len - 1); - - /* Sanitize, only allow alphanum chars */ - for (i = 0; i < flb_sds_len(tag); i++) { - if (!isalnum(tag[i]) && tag[i] != '_' && tag[i] != '.') { - tag[i] = '_'; - } - } - } - - /* Check if we have a Host header: Hostname ; port */ - mk_http_point_header(&request->host, &session->parser, MK_HEADER_HOST); - - /* Header: Connection */ - mk_http_point_header(&request->connection, &session->parser, - MK_HEADER_CONNECTION); - - /* HTTP/1.1 needs Host header */ - if (!request->host.data && request->protocol == MK_HTTP_PROTOCOL_11) { - flb_sds_destroy(tag); - mk_mem_free(uri); - return -1; - } - - /* Should we close the session after this request ? */ - mk_http_keepalive_check(session, request, ctx->server); - - /* Content Length */ - header = &session->parser.headers[MK_HEADER_CONTENT_LENGTH]; - if (header->type == MK_HEADER_CONTENT_LENGTH) { - request->_content_length.data = header->val.data; - request->_content_length.len = header->val.len; - } - else { - request->_content_length.data = NULL; - } - - mk_http_point_header(&request->content_type, &session->parser, MK_HEADER_CONTENT_TYPE); - - if (request->method != MK_METHOD_POST) { - flb_sds_destroy(tag); - mk_mem_free(uri); - send_response(conn, 400, "error: invalid HTTP method\n"); - return -1; - } - - original_data = request->data.data; - original_data_size = request->data.len; - - ret = opentelemetry_prot_uncompress(session, request, - &uncompressed_data, - &uncompressed_data_size); - - if (ret > 0) { - request->data.data = uncompressed_data; - request->data.len = uncompressed_data_size; - } - - if (strcmp(uri, "/v1/metrics") == 0) { - ret = process_payload_metrics(ctx, conn, tag, session, request); - } - else if (strcmp(uri, "/v1/traces") == 0) { - ret = process_payload_traces(ctx, conn, tag, session, request); - } - else if (strcmp(uri, "/v1/logs") == 0) { - ret = process_payload_logs(ctx, conn, tag, session, request); - } - - if (uncompressed_data != NULL) { - flb_free(uncompressed_data); - } - - request->data.data = original_data; - request->data.len = original_data_size; - - mk_mem_free(uri); - flb_sds_destroy(tag); - - send_response(conn, ctx->successful_response_code, NULL); - - return ret; -} - -/* - * Handle an incoming request which has resulted in an http parser error. - */ -int opentelemetry_prot_handle_error(struct flb_opentelemetry *ctx, struct http_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request) -{ - send_response(conn, 400, "error: invalid request\n"); - return -1; -} diff --git a/fluent-bit/plugins/in_opentelemetry/opentelemetry_prot.h b/fluent-bit/plugins/in_opentelemetry/opentelemetry_prot.h deleted file mode 100644 index bbfd8332f..000000000 --- a/fluent-bit/plugins/in_opentelemetry/opentelemetry_prot.h +++ /dev/null @@ -1,31 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_OPENTELEMETRY_PROT -#define FLB_IN_OPENTELEMETRY_PROT - -int opentelemetry_prot_handle(struct flb_opentelemetry *ctx, struct http_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request); - -int opentelemetry_prot_handle_error(struct flb_opentelemetry *ctx, struct http_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request); - -#endif diff --git a/fluent-bit/plugins/in_podman_metrics/CMakeLists.txt b/fluent-bit/plugins/in_podman_metrics/CMakeLists.txt deleted file mode 100644 index 9de0e5331..000000000 --- a/fluent-bit/plugins/in_podman_metrics/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(src - podman_metrics.c - podman_metrics_data.c - ) - -FLB_PLUGIN(in_podman_metrics "${src}" "") diff --git a/fluent-bit/plugins/in_podman_metrics/podman_metrics.c b/fluent-bit/plugins/in_podman_metrics/podman_metrics.c deleted file mode 100644 index df64452ff..000000000 --- a/fluent-bit/plugins/in_podman_metrics/podman_metrics.c +++ /dev/null @@ -1,515 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include - -#include "podman_metrics.h" -#include "podman_metrics_config.h" -#include "podman_metrics_data.h" - -/* - * Collect information about podman containers (ID and Name) from podman configuration - * file (default is /var/lib/containers/storage/overlay-containers/containers.json). - * Since flb_jsmn library show JSON as a tree, search for objects with parent 0 (objects - * that are children to root array, and in them, search for ID and name (which is also - * an array. - */ -static int collect_container_data(struct flb_in_metrics *ctx) -{ - /* Buffers for reading data from JSON */ - char *buffer; - char name[CONTAINER_NAME_SIZE]; - char id[CONTAINER_ID_SIZE]; - char image_name[IMAGE_NAME_SIZE]; - char metadata[CONTAINER_METADATA_SIZE]; - char *metadata_token_start; - char *metadata_token_stop; - int metadata_token_size; - - int array_id; - int r, i, j; - size_t read_bytes = 0; - int collected_containers = 0; - int token_len; - - jsmn_parser p; - jsmntok_t t[JSON_TOKENS]; - - flb_utils_read_file(ctx->config, &buffer, &read_bytes); - if (!read_bytes) { - flb_plg_warn(ctx->ins, "Failed to open %s", ctx->config); - return -1; - } - buffer[read_bytes] = 0; - flb_plg_debug(ctx->ins, "Read %zu bytes", read_bytes); - - jsmn_init(&p); - r = jsmn_parse(&p, buffer, strlen(buffer), t, sizeof(t) / sizeof(t[0])); - if (r < 0) { - flb_plg_warn(ctx->ins, "Failed to parse JSON %d: %s", r, buffer); - free(buffer); - return -1; - } - - flb_plg_debug(ctx->ins, "Got %d nested tokens", t[0].size); - - if (r < 1 || t[0].type != JSMN_ARRAY) { - flb_plg_warn(ctx->ins, "Expected array at the json root"); - free(buffer); - return -1; - } - - for (i=0; iins, "Found id %s", id); - } - else if (sizeof(JSON_FIELD_NAMES)-1 == t[i].end - t[i].start && - strncmp(buffer + t[i].start, JSON_FIELD_NAMES, t[i].end - t[i].start) == 0) { - array_id = i + 1; - if (t[array_id].type == JSMN_ARRAY) { - j = array_id + 1; - while (t[j].parent == array_id) - { - strncpy(name, buffer + t[j].start, t[j].end - t[j].start); - name[t[j].end - t[j].start] = '\0'; - flb_plg_trace(ctx->ins, "Found name %s", name); - j++; - } - } - } - else if (sizeof(JSON_FIELD_METADATA)-1 == t[i].end - t[i].start && - strncmp(buffer + t[i].start, JSON_FIELD_METADATA, t[i].end - t[i].start) == 0) { - token_len = t[i + 1].end - t[i + 1].start; - strncpy(metadata, buffer + t[i+1].start, t[i + 1].end - t[i + 1].start); - metadata[token_len] = '\0'; - - metadata_token_start = strstr(metadata, JSON_SUBFIELD_IMAGE_NAME); - if (metadata_token_start) { - metadata_token_stop = strstr(metadata_token_start + JSON_SUBFIELD_SIZE_IMAGE_NAME+1, "\\\""); - metadata_token_size = metadata_token_stop - metadata_token_start - JSON_SUBFIELD_SIZE_IMAGE_NAME; - - strncpy(image_name, metadata_token_start+JSON_SUBFIELD_SIZE_IMAGE_NAME, metadata_token_size); - image_name[metadata_token_size] = '\0'; - - flb_plg_trace(ctx->ins, "Found image name %s", image_name); - add_container_to_list(ctx, id, name, image_name); - } - else { - flb_plg_warn(ctx->ins, "Image name was not found for %s", id); - add_container_to_list(ctx, id, name, "unknown"); - } - collected_containers++; - } - } - } - - flb_plg_debug(ctx->ins, "Collected %d containers from podman config file", collected_containers); - free(buffer); - return collected_containers; -} - -/* - * Create structure instance based on previously found id, name and image name. Set all its values (like - * memory or cpu to UINT64_MAX, in case it won't be found later. This function also adds this structure - * to internal list, so it can be found by iteration later on. - */ -static int add_container_to_list(struct flb_in_metrics *ctx, flb_sds_t id, flb_sds_t name, flb_sds_t image_name) -{ - struct container *cnt; - cnt = flb_malloc(sizeof(struct container)); - if (!cnt) { - flb_errno(); - return -1; - } - cnt->id = flb_sds_create(id); - cnt->name = flb_sds_create(name); - cnt->image_name = flb_sds_create(image_name); - - cnt->memory_usage = UINT64_MAX; - cnt->memory_max_usage = UINT64_MAX; - cnt->memory_limit = UINT64_MAX; - cnt->rss = UINT64_MAX; - cnt->cpu_user = UINT64_MAX; - cnt->cpu = UINT64_MAX; - - mk_list_init(&cnt->net_data); - - mk_list_add(&cnt->_head, &ctx->items); - return 0; -} - -/* - * Iterate over container list and remove collected data - */ -static int destroy_container_list(struct flb_in_metrics *ctx) -{ - struct container *cnt; - struct net_iface *iface; - struct sysfs_path *pth; - struct mk_list *head; - struct mk_list *tmp; - struct mk_list *inner_head; - struct mk_list *inner_tmp; - - mk_list_foreach_safe(head, tmp, &ctx->items) { - cnt = mk_list_entry(head, struct container, _head); - flb_plg_debug(ctx->ins, "Destroying container data (id: %s, name: %s", cnt->id, cnt->name); - - flb_sds_destroy(cnt->id); - flb_sds_destroy(cnt->name); - flb_sds_destroy(cnt->image_name); - mk_list_foreach_safe(inner_head, inner_tmp, &cnt->net_data) { - iface = mk_list_entry(inner_head, struct net_iface, _head); - flb_sds_destroy(iface->name); - mk_list_del(&iface->_head); - flb_free(iface); - } - mk_list_del(&cnt->_head); - flb_free(cnt); - } - - mk_list_foreach_safe(head, tmp, &ctx->sysfs_items) { - pth = mk_list_entry(head, struct sysfs_path, _head); - flb_plg_trace(ctx->ins, "Destroying sysfs data (name: %s", pth->path); - flb_sds_destroy(pth->path); - mk_list_del(&pth->_head); - flb_free(pth); - } - return 0; -} - - -/* - * Create counter for given metric name, using name, image name and value as counter labels. Counters - * are created per counter name, so they are "shared" between multiple containers - counter - * name remains the same, only labels like ID are changed. - * This function creates counter only once per counter name - every next call only sets counter - * value for specific labels. - */ -static int create_counter(struct flb_in_metrics *ctx, struct cmt_counter **counter, flb_sds_t id, flb_sds_t name, flb_sds_t image_name, flb_sds_t metric_prefix, - flb_sds_t *fields, flb_sds_t metric_name, flb_sds_t description, flb_sds_t interface, uint64_t value) -{ - flb_sds_t *labels; - uint64_t fvalue = value; - - int label_count; - if (value == UINT64_MAX) { - flb_plg_debug(ctx->ins, "Ignoring invalid counter for %s, %s_%s_%s", name, COUNTER_PREFIX, metric_prefix, metric_name); - return -1; - } - - if (strcmp(metric_name, COUNTER_CPU) == 0 || strcmp(metric_name, COUNTER_CPU_USER) == 0) { - fvalue = fvalue / 1000000000; - flb_plg_trace(ctx->ins, "Converting %s from nanoseconds to seconds (%lu -> %lu)", metric_name, value, fvalue); - - } - - if (interface == NULL) { - labels = (char *[]){id, name, image_name}; - label_count = 3; - } - else { - labels = (char *[]){id, name, image_name, interface}; - label_count = 4; - } - - /* if counter was not yet created, it means that this function is called for the first time per counter type */ - if (*counter == NULL) { - flb_plg_debug(ctx->ins, "Creating counter for %s, %s_%s_%s", name, COUNTER_PREFIX, metric_prefix, metric_name); - *counter = cmt_counter_create(ctx->ins->cmt, COUNTER_PREFIX, metric_prefix, metric_name, description, label_count, fields); - } - - /* Allow setting value that is not grater that current one (if, for example, memory usage stays exactly the same) */ - cmt_counter_allow_reset(*counter); - flb_plg_debug(ctx->ins, "Set counter for %s, %s_%s_%s: %lu", name, COUNTER_PREFIX, metric_prefix, metric_name, fvalue); - if (cmt_counter_set(*counter, cfl_time_now(), fvalue, label_count, labels) == -1) { - flb_plg_warn(ctx->ins, "Failed to set counter for %s, %s_%s_%s", name, COUNTER_PREFIX, metric_prefix, metric_name); - return -1; - } - return 0; -} - -/* - * Create gauge for given metric name, using name, image name and value as counter labels. Gauges - * are created per counter name, so they are "shared" between multiple containers - counter - * name remains the same, only labels like ID are changed. - * This function creates gauge only once per counter name - every next call only sets gauge - * value for specific labels. - */ -static int create_gauge(struct flb_in_metrics *ctx, struct cmt_gauge **gauge, flb_sds_t id, flb_sds_t name, flb_sds_t image_name, flb_sds_t metric_prefix, - flb_sds_t *fields, flb_sds_t metric_name, flb_sds_t description, flb_sds_t interface, uint64_t value) -{ - flb_sds_t *labels; - int label_count; - if (value == UINT64_MAX) { - flb_plg_debug(ctx->ins, "Ignoring invalid gauge for %s, %s_%s_%s", name, COUNTER_PREFIX, metric_prefix, metric_name); - return -1; - } - - labels = (char *[]){id, name, image_name}; - label_count = 3; - - /* if gauge was not yet created, it means that this function is called for the first time per counter type */ - if (*gauge == NULL) { - flb_plg_debug(ctx->ins, "Creating gauge for %s, %s_%s_%s", name, COUNTER_PREFIX, metric_prefix, metric_name); - *gauge = cmt_gauge_create(ctx->ins->cmt, COUNTER_PREFIX, metric_prefix, metric_name, description, label_count, fields); - } - - flb_plg_debug(ctx->ins, "Set gauge for %s, %s_%s_%s: %lu", name, COUNTER_PREFIX, metric_prefix, metric_name, value); - if (cmt_gauge_set(*gauge, cfl_time_now(), value, label_count, labels) == -1) { - flb_plg_warn(ctx->ins, "Failed to set gauge for %s, %s_%s_%s", name, COUNTER_PREFIX, metric_prefix, metric_name); - return -1; - } - return 0; -} - -/* - * Call create_counter for every counter type defined in this plugin. - * - * Currently supported counters are: - * - container_memory_usage_bytes - * - container_memory_max_usage_bytes - * - container_memory_rss - * - container_spec_memory_limit_bytes - * - container_cpu_user_seconds_total - * - container_cpu_usage_seconds_total - * - container_network_receive_bytes_total - * - container_network_receive_errors_total - * - container_network_transmit_bytes_total - * - container_network_transmit_errors_total - */ -static int create_counters(struct flb_in_metrics *ctx) -{ - struct container *cnt; - struct net_iface *iface; - struct mk_list *head; - struct mk_list *tmp; - struct mk_list *inner_head; - struct mk_list *inner_tmp; - - mk_list_foreach_safe(head, tmp, &ctx->items) - { - cnt = mk_list_entry(head, struct container, _head); - create_counter(ctx, &ctx->c_memory_usage, cnt->id, cnt->name, cnt->image_name, COUNTER_MEMORY_PREFIX, FIELDS_METRIC, COUNTER_MEMORY_USAGE, - DESCRIPTION_MEMORY_USAGE, NULL, cnt->memory_usage); - create_counter(ctx, &ctx->c_memory_max_usage, cnt->id, cnt->name, cnt->image_name, COUNTER_MEMORY_PREFIX, FIELDS_METRIC, COUNTER_MEMORY_MAX_USAGE, - DESCRIPTION_MEMORY_MAX_USAGE, NULL, cnt->memory_max_usage); - create_counter(ctx, &ctx->c_memory_limit, cnt->id, cnt->name, cnt->image_name, COUNTER_SPEC_MEMORY_PREFIX, FIELDS_METRIC, COUNTER_MEMORY_LIMIT, - DESCRIPTION_MEMORY_LIMIT, NULL, cnt->memory_limit); - create_gauge(ctx, &ctx->g_rss, cnt->id, cnt->name, cnt->image_name, COUNTER_MEMORY_PREFIX, FIELDS_METRIC, GAUGE_MEMORY_RSS, - DESCRIPTION_MEMORY_RSS, NULL, cnt->rss); - create_counter(ctx, &ctx->c_cpu_user, cnt->id, cnt->name, cnt->image_name, COUNTER_CPU_PREFIX, FIELDS_METRIC, COUNTER_CPU_USER, - DESCRIPTION_CPU_USER, NULL, cnt->cpu_user); - create_counter(ctx, &ctx->c_cpu, cnt->id, cnt->name, cnt->image_name, COUNTER_CPU_PREFIX, FIELDS_METRIC, COUNTER_CPU, - DESCRIPTION_CPU, NULL, cnt->cpu); - mk_list_foreach_safe(inner_head, inner_tmp, &cnt->net_data) - { - iface = mk_list_entry(inner_head, struct net_iface, _head); - create_counter(ctx, &ctx->rx_bytes, cnt->id, cnt->name, cnt->image_name, COUNTER_NETWORK_PREFIX, FIELDS_METRIC_WITH_IFACE, COUNTER_RX_BYTES, - DESCRIPTION_RX_BYTES, iface->name, iface->rx_bytes); - create_counter(ctx, &ctx->rx_errors, cnt->id, cnt->name, cnt->image_name, COUNTER_NETWORK_PREFIX, FIELDS_METRIC_WITH_IFACE, COUNTER_RX_ERRORS, - DESCRIPTION_RX_ERRORS, iface->name, iface->rx_errors); - create_counter(ctx, &ctx->tx_bytes, cnt->id, cnt->name, cnt->image_name, COUNTER_NETWORK_PREFIX, FIELDS_METRIC_WITH_IFACE, COUNTER_TX_BYTES, - DESCRIPTION_TX_BYTES, iface->name, iface->tx_bytes); - create_counter(ctx, &ctx->tx_errors, cnt->id, cnt->name, cnt->image_name, COUNTER_NETWORK_PREFIX, FIELDS_METRIC_WITH_IFACE, COUNTER_TX_ERRORS, - DESCRIPTION_TX_ERRORS, iface->name, iface->tx_errors); - } - } - return 0; -} - -/* Main function. Destroy (optionally) previous data, gather container data and - * create counters. - */ -static int scrape_metrics(struct flb_config *config, struct flb_in_metrics *ctx) -{ - uint64_t start_ts = cfl_time_now(); - flb_plg_debug(ctx->ins, "Starting to scrape podman metrics"); - if (destroy_container_list(ctx) == -1) { - flb_plg_error(ctx->ins, "Could not destroy previous container data"); - return -1; - } - - if (collect_container_data(ctx) == -1) { - flb_plg_error(ctx->ins, "Could not collect container ids"); - return -1; - } - - if (collect_sysfs_directories(ctx, ctx->sysfs_path) == -1) - { - flb_plg_error(ctx->ins, "Could not collect sysfs data"); - return -1; - } - - if (ctx->cgroup_version == CGROUP_V1) { - if (fill_counters_with_sysfs_data_v1(ctx) == -1) { - flb_plg_error(ctx->ins, "Could not collect V1 sysfs data"); - return -1; - } - } - else if (ctx->cgroup_version == CGROUP_V2) { - if (fill_counters_with_sysfs_data_v2(ctx) == -1) { - flb_plg_error(ctx->ins, "Could not collect V2 sysfs data"); - return -1; - } - } - - if (create_counters(ctx) == -1) { - flb_plg_error(ctx->ins, "Could not create container counters"); - return -1; - } - - if (flb_input_metrics_append(ctx->ins, NULL, 0, ctx->ins->cmt) == -1) { - flb_plg_error(ctx->ins, "Could not append metrics"); - return -1; - } - - flb_plg_info(ctx->ins, "Scraping metrics took %luns", cfl_time_now() - start_ts); - return 0; -} - -/* - * Call scrape_metrics function every `scrape interval`. - */ -static int cb_metrics_collect_runtime(struct flb_input_instance *ins, struct flb_config *config, void *in_context) -{ - return scrape_metrics(config, in_context); -} - -/* - * Initialize plugin, setup config file path and (optionally) scrape container - * data (if `scrape_at_start` is set). - */ -static int in_metrics_init(struct flb_input_instance *in, struct flb_config *config, void *data) -{ - struct flb_in_metrics *ctx; - int coll_fd_runtime; - - ctx = flb_calloc(1, sizeof(struct flb_in_metrics)); - if (!ctx) { - return -1; - } - ctx->ins = in; - - ctx->c_memory_usage = NULL; - ctx->c_memory_max_usage = NULL; - ctx->g_rss = NULL; - ctx->c_memory_limit = NULL; - ctx->c_cpu_user = NULL; - ctx->c_cpu = NULL; - ctx->rx_bytes = NULL; - ctx->rx_errors = NULL; - ctx->tx_bytes = NULL; - ctx->tx_errors = NULL; - - if (flb_input_config_map_set(in, (void *) ctx) == -1) { - flb_free(ctx); - return -1; - } - - flb_input_set_context(in, ctx); - coll_fd_runtime = flb_input_set_collector_time(in, cb_metrics_collect_runtime, ctx->scrape_interval, 0, config); - if (coll_fd_runtime == -1) { - flb_plg_error(ctx->ins, "Could not set collector for podman metrics plugin"); - return -1; - } - ctx->coll_fd_runtime = coll_fd_runtime; - - if (ctx->podman_config_path) { - flb_plg_info(ctx->ins, "Using config file %s", ctx->podman_config_path); - ctx->config = flb_sds_create(ctx->podman_config_path); - } - else { - flb_plg_info(ctx->ins, "Using default config file %s", PODMAN_CONFIG_DEFAULT_PATH); - ctx->config = flb_sds_create(PODMAN_CONFIG_DEFAULT_PATH); - } - - if (get_cgroup_version(ctx) == CGROUP_V2) { - flb_plg_info(ctx->ins, "Detected cgroups v2"); - ctx->cgroup_version = CGROUP_V2; - } - else { - flb_plg_info(ctx->ins, "Detected cgroups v1"); - ctx->cgroup_version = CGROUP_V1; - } - - mk_list_init(&ctx->items); - mk_list_init(&ctx->sysfs_items); - - if (ctx->scrape_interval >= 2 && ctx->scrape_on_start) { - flb_plg_info(ctx->ins, "Generating podman metrics (initial scrape)"); - if (scrape_metrics(config, ctx) == -1) { - flb_plg_error(ctx->ins, "Could not start collector for podman metrics plugin"); - flb_sds_destroy(ctx->config); - destroy_container_list(ctx); - flb_free(ctx); - return -1; - } - } - - flb_plg_info(ctx->ins, "Generating podman metrics"); - - return 0; -} - -/* - * Function called at plugin exit - destroy collected container data list. - */ -static int in_metrics_exit(void *data, struct flb_config *config) -{ - struct flb_in_metrics *ctx = data; - - if (!ctx) { - return 0; - } - - flb_sds_destroy(ctx->config); - destroy_container_list(ctx); - flb_free(ctx); - return 0; -} - -/* - * Function called at plugin pause. - */ -static void in_metrics_pause(void *data, struct flb_config *config) -{ - struct flb_in_metrics *ctx = data; - flb_input_collector_pause(ctx->coll_fd_runtime, ctx->ins); -} - -/* - * Function called at plugin resume. - */ -static void in_metrics_resume(void *data, struct flb_config *config) -{ - struct flb_in_metrics *ctx = data; - flb_input_collector_resume(ctx->coll_fd_runtime, ctx->ins); -} diff --git a/fluent-bit/plugins/in_podman_metrics/podman_metrics.h b/fluent-bit/plugins/in_podman_metrics/podman_metrics.h deleted file mode 100644 index 3b02d24ed..000000000 --- a/fluent-bit/plugins/in_podman_metrics/podman_metrics.h +++ /dev/null @@ -1,98 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_PODMAN_METRICS_H -#define FLB_IN_PODMAN_METRICS_H - -#include -#include -#include -#include -#include - -#include - -#include "podman_metrics_config.h" - -static int collect_container_data(struct flb_in_metrics *ctx); -static int add_container_to_list(struct flb_in_metrics *ctx, flb_sds_t id, flb_sds_t name, flb_sds_t image_name); -static int destroy_container_list(struct flb_in_metrics *ctx); - -static int create_counter(struct flb_in_metrics *ctx, struct cmt_counter **counter, flb_sds_t id, flb_sds_t name, flb_sds_t image_name, flb_sds_t metric_prefix, - flb_sds_t *fieds, flb_sds_t metric_name, flb_sds_t description, flb_sds_t interface, uint64_t value); -static int create_gauge(struct flb_in_metrics *ctx, struct cmt_gauge **gauge, flb_sds_t id, flb_sds_t name, flb_sds_t image_name, flb_sds_t metric_prefix, - flb_sds_t *fields, flb_sds_t metric_name, flb_sds_t description, flb_sds_t interface, uint64_t value); -static int create_counters(struct flb_in_metrics *ctx); - -static int scrape_metrics(struct flb_config *config, struct flb_in_metrics *ctx); - -static int cb_metrics_collect_runtime(struct flb_input_instance *ins, struct flb_config *config, void *in_context); -static int in_metrics_init(struct flb_input_instance *in, struct flb_config *config, void *data); -static int in_metrics_exit(void *data, struct flb_config *config); -static void in_metrics_pause(void *data, struct flb_config *config); -static void in_metrics_resume(void *data, struct flb_config *config); - - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_TIME, "scrape_interval", "30", - 0, FLB_TRUE, offsetof(struct flb_in_metrics, scrape_interval), - "Scrape interval to collect the metrics of podman containers" - "(defaults to 30s)" - }, - - { - FLB_CONFIG_MAP_BOOL, "scrape_on_start", "false", - 0, FLB_TRUE, offsetof(struct flb_in_metrics, scrape_on_start), - "Scrape metrics upon start, useful to avoid waiting for 'scrape_interval' " - "for the first round of metrics." - }, - { - FLB_CONFIG_MAP_STR, "path.config", NULL, - 0, FLB_TRUE, offsetof(struct flb_in_metrics, podman_config_path), - "Path to podman config file" - }, - { - FLB_CONFIG_MAP_STR, "path.sysfs", SYSFS_PATH, - 0, FLB_TRUE, offsetof(struct flb_in_metrics, sysfs_path), - "Path to sysfs subsystem directory" - }, - { - FLB_CONFIG_MAP_STR, "path.procfs", PROCFS_PATH, - 0, FLB_TRUE, offsetof(struct flb_in_metrics, procfs_path), - "Path to proc subsystem directory" - }, - - /* EOF */ - {0} -}; - -struct flb_input_plugin in_podman_metrics_plugin = { - .name = "podman_metrics", - .description = "Podman metrics", - .cb_init = in_metrics_init, - .cb_pre_run = NULL, - .cb_flush_buf = NULL, - .config_map = config_map, - .cb_pause = in_metrics_pause, - .cb_resume = in_metrics_resume, - .cb_exit = in_metrics_exit -}; - -#endif diff --git a/fluent-bit/plugins/in_podman_metrics/podman_metrics_config.h b/fluent-bit/plugins/in_podman_metrics/podman_metrics_config.h deleted file mode 100644 index fabdc0a8d..000000000 --- a/fluent-bit/plugins/in_podman_metrics/podman_metrics_config.h +++ /dev/null @@ -1,211 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_PODMAN_METRICS_CONFIG_H -#define FLB_IN_PODMAN_METRICS_CONFIG_H - -#include -#include -#include -#include -#include - -#include - -/* Buffers and sizes */ -#define JSON_TOKENS 2048 -#define CONTAINER_NAME_SIZE 50 -#define CONTAINER_ID_SIZE 80 -#define CONTAINER_METADATA_SIZE 512 -#define IMAGE_NAME_SIZE 512 -#define PID_BUFFER_SIZE 21 -#define SYSFS_FILE_PATH_SIZE 512 -#define PROCFS_FILE_PATH_SIZE 512 -#define CGROUP_PATH_SIZE 25 - -/* Special paths for sysfs traversal */ -#define CURRENT_DIR "." -#define PREV_DIR ".." - -/* Ignored network interfaces */ -#define VETH_INTERFACE "veth" - -#define JSON_FIELD_NAMES "names" -#define JSON_FIELD_ID "id" -#define JSON_FIELD_METADATA "metadata" - -#define JSON_SUBFIELD_IMAGE_NAME "image-name\\\":\\\"" -#define JSON_SUBFIELD_SIZE_IMAGE_NAME 15 - -#define CGROUP_V2_PATH "cgroup.controllers" -#define CGROUP_V1 1 -#define CGROUP_V2 2 - -/* Paths in /proc subsystem */ -#define PROCFS_PATH "/proc" -#define PROC_NET_SUFFIX "net/dev" - -/* Paths in /sys subsystem */ -#define SYSFS_PATH "/sys/fs/cgroup" -#define V1_SYSFS_MEMORY "memory" -#define V1_SYSFS_CPU "cpuacct" -#define V1_SYSFS_SYSTEMD "systemd" -#define SYSFS_CONTAINER_PREFIX "libpod" -#define SYSFS_LIBPOD_PARENT "libpod_parent" -#define SYSFS_CONMON "conmon" - -/* Default podman config file path, in case of not provided one */ -#define PODMAN_CONFIG_DEFAULT_PATH "/var/lib/containers/storage/overlay-containers/containers.json" - -/* Markers of network values in /proc//dev/net */ -#define DEV_NET_IGNORE_LINES 2 -#define DEV_NET_NAME 0 -#define DEV_NET_RX_BYTES 1 -#define DEV_NET_RX_ERRORS 3 -#define DEV_NET_TX_BYTES 9 -#define DEV_NET_TX_ERRORS 11 - -/* Key names in .stat files */ -#define STAT_KEY_RSS "rss" -#define STAT_KEY_CPU "usage_usec" -#define STAT_KEY_CPU_USER "user_usec" - -/* Static lists of fields in counters or gauges */ -#define FIELDS_METRIC (char*[3]){"id", "name", "image" } -#define FIELDS_METRIC_WITH_IFACE (char*[4]){"id", "name", "image", "interface" } - -/* Files from sysfs containing required data (cgroups v1) */ -#define V1_SYSFS_FILE_MEMORY "memory.usage_in_bytes" -#define V1_SYSFS_FILE_MAX_MEMORY "memory.max_usage_in_bytes" -#define V1_SYSFS_FILE_MEMORY_STAT "memory.stat" -#define V1_SYSFS_FILE_MEMORY_LIMIT "memory.limit_in_bytes" -#define V1_SYSFS_FILE_CPU_USER "cpuacct.usage_user" -#define V1_SYSFS_FILE_CPU "cpuacct.usage" -#define V1_SYSFS_FILE_PIDS "cgroup.procs" - -/* Files from sysfs containing required data (cgroups v2) */ -#define V2_SYSFS_FILE_MEMORY "memory.current" -#define V2_SYSFS_FILE_MAX_MEMORY "memory.peak" -#define V2_SYSFS_FILE_MEMORY_STAT "memory.stat" -#define V2_SYSFS_FILE_MEMORY_LIMIT "memory.max" -#define V2_SYSFS_FILE_CPU_STAT "cpu.stat" -#define V2_SYSFS_FILE_PIDS "cgroup.procs" -#define V2_SYSFS_FILE_PIDS_ALT "containers/cgroup.procs" - -/* Values used to construct counters/gauges names and descriptions */ -#define COUNTER_PREFIX "container" - -#define COUNTER_MEMORY_PREFIX "memory" -#define COUNTER_SPEC_MEMORY_PREFIX "spec_memory" -#define COUNTER_MEMORY_USAGE "usage_bytes" -#define DESCRIPTION_MEMORY_USAGE "Container memory usage in bytes" -#define COUNTER_MEMORY_MAX_USAGE "max_usage_bytes" -#define DESCRIPTION_MEMORY_MAX_USAGE "Container max memory usage in bytes" -#define COUNTER_MEMORY_LIMIT "limit_bytes" -#define DESCRIPTION_MEMORY_LIMIT "Container memory limit in bytes" -#define GAUGE_MEMORY_RSS "rss" -#define DESCRIPTION_MEMORY_RSS "Container RSS in bytes" - -#define COUNTER_CPU_PREFIX "cpu" -#define COUNTER_CPU_USER "user_seconds_total" -#define DESCRIPTION_CPU_USER "Container cpu usage in seconds in user mode" -#define COUNTER_CPU "usage_seconds_total" -#define DESCRIPTION_CPU "Container cpu usage in seconds" - -#define COUNTER_NETWORK_PREFIX "network" -#define COUNTER_RX_BYTES "receive_bytes_total" -#define DESCRIPTION_RX_BYTES "Network received bytes" -#define COUNTER_RX_ERRORS "receive_errors_total" -#define DESCRIPTION_RX_ERRORS "Network received errors" -#define COUNTER_TX_BYTES "transmit_bytes_total" -#define DESCRIPTION_TX_BYTES "Network transmited bytes" -#define COUNTER_TX_ERRORS "transmit_errors_total" -#define DESCRIPTION_TX_ERRORS "Network transmitedd errors" - - -struct net_iface { - flb_sds_t name; - uint64_t rx_bytes; - uint64_t rx_errors; - uint64_t tx_bytes; - uint64_t tx_errors; - struct mk_list _head; -}; - -struct container { - flb_sds_t name; - flb_sds_t id; - flb_sds_t image_name; - struct mk_list _head; - - uint64_t memory_usage; - uint64_t memory_max_usage; - uint64_t memory_limit; - uint64_t cpu; - uint64_t cpu_user; - uint64_t rss; - - struct mk_list net_data; -}; - -struct sysfs_path { - flb_sds_t path; - struct mk_list _head; -}; - -struct flb_in_metrics { - /* config map options */ - int scrape_on_start; - int scrape_interval; - flb_sds_t podman_config_path; - - /* container list */ - struct mk_list items; - - /* sysfs path list */ - struct mk_list sysfs_items; - - /* counters */ - struct cmt_counter *c_memory_usage; - struct cmt_counter *c_memory_max_usage; - struct cmt_counter *c_memory_limit; - struct cmt_gauge *g_rss; - struct cmt_counter *c_cpu_user; - struct cmt_counter *c_cpu; - struct cmt_counter *rx_bytes; - struct cmt_counter *rx_errors; - struct cmt_counter *tx_bytes; - struct cmt_counter *tx_errors; - - /* cgroup version used by host */ - int cgroup_version; - - /* podman config file path */ - flb_sds_t config; - - /* proc and sys paths, overwriting mostly for testing */ - flb_sds_t sysfs_path; - flb_sds_t procfs_path; - - /* internal */ - int coll_fd_runtime; - struct flb_input_instance *ins; -}; - -#endif diff --git a/fluent-bit/plugins/in_podman_metrics/podman_metrics_data.c b/fluent-bit/plugins/in_podman_metrics/podman_metrics_data.c deleted file mode 100644 index e747fe4b8..000000000 --- a/fluent-bit/plugins/in_podman_metrics/podman_metrics_data.c +++ /dev/null @@ -1,407 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include - -#include "podman_metrics_data.h" -#include "podman_metrics_config.h" - -/* - * Read uint64_t value from given path. If this function fails, it - * returns UINT64_MAX, which will be later interpeted as invalid counter value - * (it cannot return 0, because it is a valid value for counter - */ -uint64_t read_from_file(struct flb_in_metrics *ctx, flb_sds_t path) -{ - int c; - uint64_t value = UINT64_MAX; - FILE *fp; - - fp = fopen(path, "r"); - if (!fp) { - flb_plg_warn(ctx->ins, "Failed to read %s", path); - return value; - } - - c = fscanf(fp, "%lu", &value); - fclose(fp); - if (c != 1) { - flb_plg_warn(ctx->ins, "Failed to read a number from %s", path); - return value; - } - return value; -} - -/* - * Read uint64_t value from given path. Check for key: and return it. - * If this function fails, it - * returns UINT64_MAX, which will be later interpeted as invalid counter value - * (it cannot return 0, because it is a valid value for counter - */ -uint64_t read_key_value_from_file(struct flb_in_metrics *ctx, flb_sds_t path, flb_sds_t key) -{ - uint64_t value = UINT64_MAX; - FILE *fp; - flb_sds_t line = NULL; - flb_sds_t field = NULL; - flb_sds_t line2 = NULL; - size_t len = 0; - ssize_t read = 0; - int key_found = 0; - - fp = fopen(path, "r"); - if (!fp) { - flb_plg_warn(ctx->ins, "Failed to read %s", path); - return value; - } - - while ((read = getline(&line, &len, fp)) != -1) { - line2 = line; - - while( (field = strsep(&line2, " :")) != NULL ) { - if( *field == '\0' ) { - continue; - } - if (strcmp(field, key) == 0) { - key_found = 1; - continue; - } - if (key_found) { - value = strtoull(field, NULL, 10); - flb_plg_trace(ctx->ins, "Found key %s: %lu", key, value); - fclose(fp); - flb_free(line); - flb_free(line2); - return value; - } - - } - flb_free(line2); - } - flb_free(line); - flb_plg_warn(ctx->ins, "%s not found in %s", key, path); - fclose(fp); - return value; -} - -/* - * Read uint64_t value from path previously picked from sysfs directory list. - * If key is not NULL, it will be used to search a file instead of reading single value. - */ -uint64_t get_data_from_sysfs(struct flb_in_metrics *ctx, flb_sds_t dir, flb_sds_t name, flb_sds_t key) -{ - char path[SYSFS_FILE_PATH_SIZE]; - uint64_t data = UINT64_MAX; - path[0]=0; - - if (dir == NULL) { - return data; - } - - snprintf(path, sizeof(path), "%s/%s", dir, name); - - if (key == NULL) { - data = read_from_file(ctx, path); - } - else { - data = read_key_value_from_file(ctx, path, key); - } - flb_plg_debug(ctx->ins, "%s: %lu", path, data); - return data; -} - -/* - * Check if container sysfs data is pressent in previously generated list of sysfs directories. - * For cgroups v1, use subsystem (directory, for example memory) to search full path. - */ -int get_container_sysfs_subdirectory(struct flb_in_metrics *ctx, flb_sds_t id, flb_sds_t subsystem, flb_sds_t *path) -{ - struct sysfs_path *pth; - struct mk_list *head; - struct mk_list *tmp; - - mk_list_foreach_safe(head, tmp, &ctx->sysfs_items) { - pth = mk_list_entry(head, struct sysfs_path, _head); - if (strstr(pth->path, id) != 0) { - if (subsystem != NULL && strstr(pth->path, subsystem) == 0) { - continue; - } - *path = pth->path; - flb_plg_trace(ctx->ins, "Found path for %s: %s", id, pth->path); - return 0; - } - } - *path = NULL; - return -1; -} - -/* -* Read data from /proc/ subsystem containing all data about network usage for pid (so, in this case, -* for container). These fields seem to be in constant positions, so check only specific fields in each -* row. -*/ -int get_net_data_from_proc(struct flb_in_metrics *ctx, struct container *cnt, uint64_t pid) { - char path[PROCFS_FILE_PATH_SIZE]; - char pid_buff[PID_BUFFER_SIZE]; - - FILE * fp; - flb_sds_t line = NULL; - flb_sds_t field = NULL; - flb_sds_t line2 = NULL; - - size_t len = 0; - ssize_t read = 0; - int curr_line = 0; - int curr_field = 0; - - struct net_iface *iface; - - path[0]=0; - sprintf(pid_buff, "%" PRIu64, pid); - snprintf(path, sizeof(path), "%s/%s/%s", ctx->procfs_path, pid_buff, PROC_NET_SUFFIX); - - fp = fopen(path, "r"); - if (fp == NULL) { - flb_plg_warn(ctx->ins, "Failed to open %s", path); - return -1; - } - - while ((read = getline(&line, &len, fp)) != -1) { - line2 = line; - if (curr_line++ <= DEV_NET_IGNORE_LINES) { - flb_plg_trace(ctx->ins, "Ignoring line %d in %s", curr_line, path); - continue; - } - - iface = flb_malloc(sizeof(struct net_iface)); - if (!iface) { - flb_errno(); - return -1; - } - iface->name = NULL; - iface->rx_bytes = UINT64_MAX; - iface->rx_errors = UINT64_MAX; - iface->tx_bytes = UINT64_MAX; - iface->tx_errors = UINT64_MAX; - - - while( (field = strsep(&line2, " ")) != NULL ) { - if( *field == '\0' ) { - continue; - } - switch (curr_field++) - { - case DEV_NET_NAME: - /* Remove ':' from the end of name */ - iface->name = flb_sds_create_len(field, strlen(field)-1); - flb_plg_trace(ctx->ins, "Reading name from %s: %s", path, iface->name); - break; - - case DEV_NET_RX_BYTES: - iface->rx_bytes = strtoull(field, NULL, 10); - flb_plg_trace(ctx->ins, "Reading rx_bytes from %s: %lu", path, iface->rx_bytes); - break; - - case DEV_NET_RX_ERRORS: - iface->rx_errors = strtoull(field, NULL, 10); - flb_plg_trace(ctx->ins, "Reading rx_errors from %s: %lu", path, iface->rx_errors); - break; - - case DEV_NET_TX_BYTES: - iface->tx_bytes = strtoull(field, NULL, 10); - flb_plg_trace(ctx->ins, "Reading tx_bytes from %s: %lu", path, iface->tx_bytes); - break; - - case DEV_NET_TX_ERRORS: - iface->tx_errors = strtoull(field, NULL, 10); - flb_plg_trace(ctx->ins, "Reading tx_errors from %s: %lu", path, iface->tx_errors); - break; - } - } - flb_free(line2); - curr_field = 0; - - /* Ignore virtual interfaces connected to podman containers */ - if (name_starts_with(iface->name, VETH_INTERFACE) == 0) { - flb_plg_trace(ctx->ins, "Ignoring virtual interface %s", iface->name); - flb_sds_destroy(iface->name); - flb_free(iface); - continue; - } - mk_list_add(&iface->_head, &cnt->net_data); - } - - flb_free(line); - fclose(fp); - return 0; -} - -/* - * Iterate over directories in sysfs system and collect all libpod-* directories - */ -int collect_sysfs_directories(struct flb_in_metrics *ctx, flb_sds_t name) -{ - char path[SYSFS_FILE_PATH_SIZE]; - path[0] = 0; - DIR *dir; - struct dirent *entry; - struct sysfs_path *pth; - - if (!(dir = opendir(name))) { - flb_plg_warn(ctx->ins, "Failed to open %s", name); - return -1; - } - - while ((entry = readdir(dir)) != NULL) { - if (entry->d_type == DT_DIR) { - if (strcmp(entry->d_name, CURRENT_DIR) == 0 || strcmp(entry->d_name, PREV_DIR) == 0) { - continue; - } - snprintf(path, sizeof(path), "%s/%s", name, entry->d_name); - - if (name_starts_with(entry->d_name, SYSFS_CONTAINER_PREFIX) == 0 && - strcmp(entry->d_name, SYSFS_LIBPOD_PARENT) != 0 && - strstr(entry->d_name, SYSFS_CONMON) == 0) { - pth = flb_malloc(sizeof(struct sysfs_path)); - if (!pth) { - flb_errno(); - return -1; - } - pth->path = flb_sds_create(path); - flb_plg_debug(ctx->ins, "Collected sysfs directory: %s", pth->path); - mk_list_add(&pth->_head, &ctx->sysfs_items); - } - - collect_sysfs_directories(ctx, path); - } - } - closedir(dir); - return 0; -} - -/* - * Iterate over previously created container list. For each entry, generate its - * paths in sysfs system directory. From this path, grab data about container metrics - * and put it this entry. - * This function is used in cgroups v1 - meaning different directories for files. - */ -int fill_counters_with_sysfs_data_v1(struct flb_in_metrics *ctx) -{ - uint64_t pid; - flb_sds_t mem_path; - flb_sds_t cpu_path; - flb_sds_t systemd_path; - struct container *cnt; - struct mk_list *head; - struct mk_list *tmp; - - mk_list_foreach_safe(head, tmp, &ctx->items) { - cnt = mk_list_entry(head, struct container, _head); - - get_container_sysfs_subdirectory(ctx, cnt->id, V1_SYSFS_MEMORY, &mem_path); - get_container_sysfs_subdirectory(ctx, cnt->id, V1_SYSFS_CPU, &cpu_path); - get_container_sysfs_subdirectory(ctx, cnt->id, V1_SYSFS_SYSTEMD, &systemd_path); - - cnt->memory_usage = get_data_from_sysfs(ctx, mem_path, V1_SYSFS_FILE_MEMORY, NULL); - cnt->memory_max_usage = get_data_from_sysfs(ctx, mem_path, V1_SYSFS_FILE_MAX_MEMORY, NULL); - cnt->rss = get_data_from_sysfs(ctx, mem_path, V1_SYSFS_FILE_MEMORY_STAT, STAT_KEY_RSS); - cnt->memory_limit = get_data_from_sysfs(ctx, mem_path, V1_SYSFS_FILE_MEMORY_LIMIT, NULL); - cnt->cpu_user = get_data_from_sysfs(ctx, cpu_path, V1_SYSFS_FILE_CPU_USER, NULL); - cnt->cpu = get_data_from_sysfs(ctx, cpu_path, V1_SYSFS_FILE_CPU, NULL); - pid = get_data_from_sysfs(ctx, systemd_path, V1_SYSFS_FILE_PIDS, NULL); - if (pid && pid != UINT64_MAX) { - get_net_data_from_proc(ctx, cnt, pid); - } - else { - flb_plg_warn(ctx->ins, "Failed to collect PID for %s", cnt->name); - } - } - return 0; -} - -/* - * Iterate over previously created container list. For each entry, generate its - * path in sysfs system directory. From this path, grab data about container metrics - * and put it this entry. - * This function is used in cgroups v2 - meaning same directory for all files. - */ -int fill_counters_with_sysfs_data_v2(struct flb_in_metrics *ctx) -{ - uint64_t pid; - flb_sds_t path; - struct container *cnt; - struct mk_list *head; - struct mk_list *tmp; - - mk_list_foreach_safe(head, tmp, &ctx->items) { - cnt = mk_list_entry(head, struct container, _head); - - get_container_sysfs_subdirectory(ctx, cnt->id, NULL, &path); - - cnt->memory_usage = get_data_from_sysfs(ctx, path, V2_SYSFS_FILE_MEMORY, NULL); - cnt->memory_max_usage = get_data_from_sysfs(ctx, path, V2_SYSFS_FILE_MAX_MEMORY, NULL); - cnt->rss = get_data_from_sysfs(ctx, path, V2_SYSFS_FILE_MEMORY_STAT, STAT_KEY_RSS); - cnt->memory_limit = get_data_from_sysfs(ctx, path, V2_SYSFS_FILE_MEMORY_LIMIT, NULL); - cnt->cpu_user = get_data_from_sysfs(ctx, path, V2_SYSFS_FILE_CPU_STAT, STAT_KEY_CPU_USER); - cnt->cpu = get_data_from_sysfs(ctx, path, V2_SYSFS_FILE_CPU_STAT, STAT_KEY_CPU); - pid = get_data_from_sysfs(ctx, path, V2_SYSFS_FILE_PIDS, NULL); - if (!pid || pid == UINT64_MAX) { - pid = get_data_from_sysfs(ctx, path, V2_SYSFS_FILE_PIDS_ALT, NULL); - } - if (pid && pid != UINT64_MAX) { - get_net_data_from_proc(ctx, cnt, pid); - } - else { - flb_plg_warn(ctx->ins, "Failed to collect PID for %s", cnt->name); - } - } - return 0; -} - -/* - * Check if flb_sds_t starts with given string - */ -int name_starts_with(flb_sds_t s, const char *str) -{ - size_t len = strlen(str); - size_t flen = flb_sds_len(s); - - if (s == NULL || len > flen) { - return -1; - } - - return strncmp(s, str, len); -} - -/* - * Calculate which cgroup version is used on host by checing existence of - * cgroup.controllers file (if it exists, it is V2). - */ -int get_cgroup_version(struct flb_in_metrics *ctx) -{ - char path[SYSFS_FILE_PATH_SIZE]; - snprintf(path, sizeof(path), "%s/%s", ctx->sysfs_path, CGROUP_V2_PATH); - return (access(path, F_OK) == 0) ? CGROUP_V2 : CGROUP_V1; -} diff --git a/fluent-bit/plugins/in_podman_metrics/podman_metrics_data.h b/fluent-bit/plugins/in_podman_metrics/podman_metrics_data.h deleted file mode 100644 index 93fa6de00..000000000 --- a/fluent-bit/plugins/in_podman_metrics/podman_metrics_data.h +++ /dev/null @@ -1,51 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_PODMAN_METRICS_DATA_H -#define FLB_IN_PODMAN_METRICS_DATA_H - -#include -#include -#include -#include -#include - -#include -#include - -#include "podman_metrics_config.h" - -int destroy_counter(struct flb_in_metrics *ctx, struct cmt_counter **c); -int destroy_gauge(struct flb_in_metrics *ctx, struct cmt_gauge **g); - -uint64_t read_from_file(struct flb_in_metrics *ctx, flb_sds_t path); -uint64_t read_key_value_from_file(struct flb_in_metrics *ctx, flb_sds_t path, flb_sds_t key); -uint64_t get_data_from_sysfs(struct flb_in_metrics *ctx, flb_sds_t dir, flb_sds_t name, flb_sds_t key); - -int get_container_sysfs_subdirectory(struct flb_in_metrics *ctx, flb_sds_t id, flb_sds_t subsystem, flb_sds_t *path); -int get_net_data_from_proc(struct flb_in_metrics *ctx, struct container *cnt, uint64_t pid); - -int collect_sysfs_directories(struct flb_in_metrics *ctx, flb_sds_t name); -int fill_counters_with_sysfs_data_v1(struct flb_in_metrics *ctx); -int fill_counters_with_sysfs_data_v2(struct flb_in_metrics *ctx); - -int name_starts_with(flb_sds_t s, const char *str); -int get_cgroup_version(struct flb_in_metrics *ctx); - -#endif diff --git a/fluent-bit/plugins/in_proc/CMakeLists.txt b/fluent-bit/plugins/in_proc/CMakeLists.txt deleted file mode 100644 index 92f1071fc..000000000 --- a/fluent-bit/plugins/in_proc/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - in_proc.c) - -FLB_PLUGIN(in_proc "${src}" "") diff --git a/fluent-bit/plugins/in_proc/in_proc.c b/fluent-bit/plugins/in_proc/in_proc.c deleted file mode 100644 index ff1df585c..000000000 --- a/fluent-bit/plugins/in_proc/in_proc.c +++ /dev/null @@ -1,534 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "in_proc.h" - -struct flb_in_proc_mem_offset mem_linux[] = { - { - "Peak", - "mem.VmPeak", - offsetof(struct flb_in_proc_mem_linux, vmpeak) - }, - { - "Size", - "mem.VmSize", - offsetof(struct flb_in_proc_mem_linux, vmsize) - }, - { - "Lck", - "mem.VmLck", - offsetof(struct flb_in_proc_mem_linux, vmlck) - }, - { - "HWM", - "mem.VmHWM", - offsetof(struct flb_in_proc_mem_linux, vmhwm) - }, - { - "RSS", - "mem.VmRSS", - offsetof(struct flb_in_proc_mem_linux, vmrss) - }, - { - "Data", - "mem.VmData", - offsetof(struct flb_in_proc_mem_linux, vmdata) - }, - { - "Stk", - "mem.VmStk", - offsetof(struct flb_in_proc_mem_linux, vmstk) - }, - { - "Exe", - "mem.VmExe", - offsetof(struct flb_in_proc_mem_linux, vmexe) - }, - { - "Lib", - "mem.VmLib", - offsetof(struct flb_in_proc_mem_linux, vmlib) - }, - { - "PTE", - "mem.VmPTE", - offsetof(struct flb_in_proc_mem_linux, vmpte) - }, - { - "Swap", - "mem.VmSwap", - offsetof(struct flb_in_proc_mem_linux, vmswap) - }, - {NULL, NULL, 0}/* end of array */ -}; - - - -static pid_t get_pid_from_procname_linux(struct flb_in_proc_config *ctx, - const char* proc) -{ - pid_t ret = -1; - glob_t glb; - int i; - int fd = -1; - long ret_scan = -1; - int ret_glb = -1; - ssize_t count; - - char cmdname[FLB_CMD_LEN]; - char* bname = NULL; - - ret_glb = glob("/proc/*/cmdline", 0 ,NULL, &glb); - if (ret_glb != 0) { - switch(ret_glb){ - case GLOB_NOSPACE: - flb_plg_warn(ctx->ins, "glob: no space"); - break; - case GLOB_NOMATCH: - flb_plg_warn(ctx->ins, "glob: no match"); - break; - case GLOB_ABORTED: - flb_plg_warn(ctx->ins, "glob: aborted"); - break; - default: - flb_plg_warn(ctx->ins, "glob: other error"); - } - return ret; - } - - for (i = 0; i < glb.gl_pathc; i++) { - fd = open(glb.gl_pathv[i], O_RDONLY); - if (fd < 0) { - continue; - } - count = read(fd, &cmdname, FLB_CMD_LEN); - if (count <= 0){ - close(fd); - continue; - } - cmdname[FLB_CMD_LEN-1] = '\0'; - bname = basename(cmdname); - - if (strncmp(proc, bname, FLB_CMD_LEN) == 0) { - sscanf(glb.gl_pathv[i],"/proc/%ld/cmdline",&ret_scan); - ret = (pid_t)ret_scan; - close(fd); - break; - } - close(fd); - } - globfree(&glb); - return ret; -} - -static int configure(struct flb_in_proc_config *ctx, - struct flb_input_instance *in) -{ - int ret; - - /* Load the config map */ - ret = flb_input_config_map_set(in, (void *)ctx); - if (ret == -1) { - flb_plg_error(in, "unable to load configuration"); - return -1; - } - - if (ctx->interval_sec <= 0 && ctx->interval_nsec <= 0) { - /* Illegal settings. Override them. */ - ctx->interval_sec = atoi(DEFAULT_INTERVAL_SEC); - ctx->interval_nsec = atoi(DEFAULT_INTERVAL_NSEC); - } - - if (ctx->proc_name != NULL && strcmp(ctx->proc_name, "") != 0) { - ctx->len_proc_name = strlen(ctx->proc_name); - } - - return 0; -} - -static int get_pid_status(pid_t pid) -{ - int ret = kill(pid, 0); - return ((ret != ESRCH) && (ret != EPERM) && (ret != ESRCH)); -} - -static int generate_record_linux(struct flb_input_instance *i_ins, - struct flb_config *config, void *in_context, - struct flb_in_proc_mem_linux *mem_stat, - uint64_t fds) -{ - int i; - int ret; - struct flb_in_proc_config *ctx = in_context; - - if (ctx->alive == FLB_TRUE && ctx->alert == FLB_TRUE) { - return 0; - } - - ret = flb_log_event_encoder_begin_record(ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp(ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE("alive"), - FLB_LOG_EVENT_BOOLEAN_VALUE(ctx->alive), - /* proc name */ - FLB_LOG_EVENT_CSTRING_VALUE("proc_name"), - FLB_LOG_EVENT_CSTRING_VALUE(ctx->proc_name), - /* pid */ - FLB_LOG_EVENT_CSTRING_VALUE("pid"), - FLB_LOG_EVENT_INT64_VALUE(ctx->pid)); - } - - /* memory */ - if (ctx->mem == FLB_TRUE) { - char *str = NULL; - uint64_t *val = NULL; - for (i = 0; - mem_linux[i].key != NULL && - ret == FLB_EVENT_ENCODER_SUCCESS; - i++) { - str = mem_linux[i].msgpack_key; - val = (uint64_t*)((char*)mem_stat + mem_linux[i].offset); - - ret = flb_log_event_encoder_append_body_values( - ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE(str), - FLB_LOG_EVENT_UINT64_VALUE(*val)); - } - } - - /* file descriptor */ - if (ctx->fds) { - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE("fd"), - FLB_LOG_EVENT_UINT64_VALUE(fds)); - } - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(i_ins, NULL, 0, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); - ret = 0; - } - else { - flb_plg_error(i_ins, "log event encoding error : %d", ret); - - ret = -1; - } - - flb_log_event_encoder_reset(ctx->log_encoder); - - return ret; -} - -static void update_alive(struct flb_in_proc_config *ctx) -{ - if (ctx->pid >= 0 && get_pid_status(ctx->pid)) { - ctx->alive = FLB_TRUE; - } - else { - ctx->alive = FLB_FALSE; - } -} - -static void mem_linux_clear(struct flb_in_proc_mem_linux *mem_stat) -{ - int i; - uint64_t *temp = NULL; - for (i=0;mem_linux[i].key != NULL;i++) { - temp = (uint64_t*)((char*)mem_stat + mem_linux[i].offset); - *temp = 0; - } -} - -static int update_mem_linux(struct flb_in_proc_config *ctx, - struct flb_in_proc_mem_linux *mem_stat) -{ - int ret = -1; - int i; - char path[PATH_MAX] = {0}; - char str_name[32] = {0}; - char *line = NULL; - char *fmt = NULL; - char *buf = NULL; - ssize_t count; - size_t len = 256; - uint64_t mem_size; - uint64_t *temp = NULL; - FILE *fp = NULL; - - snprintf(path, sizeof(path), "/proc/%d/status",ctx->pid); - fp = fopen(path, "r"); - - if (fp == NULL) { - flb_plg_error(ctx->ins, "open error: %s", path); - mem_linux_clear(mem_stat); - return -1; - } - - line = (char*)flb_malloc(len); - while(1) { - count = getline(&line, &len, fp); - if (count < 0) { - break; - } - - /* VmPeak: 14860 kB */ - fmt = "Vm%s"; /* e.g. "Peak:" */ - memset(str_name, '\0', sizeof(str_name)); - ret = sscanf(line, fmt, str_name); - if (ret < 1) { - continue; - } - /* replace : -> NULL char*/ - if ((buf = strchr(str_name, ':')) != NULL) { - *buf = '\0'; - } - - /* calcurate size */ - mem_size = 0; - for (i=0;line[i] != '\0';i++) { - if (line[i] >= 0x30 && line[i] <= 0x39 /* is number*/) { - mem_size *= 10; - mem_size += line[i] - 0x30; - } - } - - for (i=0;mem_linux[i].key != NULL;i++) { - if (!strcmp(str_name, mem_linux[i].key)) { - temp = (uint64_t*)((char*)mem_stat + mem_linux[i].offset); - *temp = mem_size * 1000; /* kB size */ - break; - } - } - } - flb_free(line); - fclose(fp); - return ret; -} - -static int update_fds_linux(struct flb_in_proc_config *ctx, - uint64_t *fds) -{ - DIR *dirp = NULL; - struct dirent *entry = NULL; - char path[PATH_MAX] = {0}; - - *fds = 0; - - snprintf(path, sizeof(path), "/proc/%d/fd", ctx->pid); - dirp = opendir(path); - if (dirp == NULL) { - perror("opendir"); - flb_plg_error(ctx->ins, "opendir error %s", path); - return -1; - } - - entry = readdir(dirp); - while (entry != NULL) { - *fds += 1;/* should we check entry->d_name ? */ - entry = readdir(dirp); - } - *fds -= 2; /* '.' and '..' */ - closedir(dirp); - - return 0; -} - -static int in_proc_collect_linux(struct flb_input_instance *i_ins, - struct flb_config *config, void *in_context) -{ - uint64_t fds = 0; - struct flb_in_proc_config *ctx = in_context; - struct flb_in_proc_mem_linux mem; - - if (ctx->proc_name != NULL){ - ctx->pid = get_pid_from_procname_linux(ctx, ctx->proc_name); - update_alive(ctx); - - if (ctx->mem == FLB_TRUE && ctx->alive == FLB_TRUE) { - mem_linux_clear(&mem); - update_mem_linux(ctx, &mem); - } - if (ctx->fds == FLB_TRUE && ctx->alive == FLB_TRUE) { - update_fds_linux(ctx, &fds); - } - generate_record_linux(i_ins, config, in_context, &mem, fds); - } - - return 0; -} - -static int in_proc_collect(struct flb_input_instance *i_ins, - struct flb_config *config, void *in_context) -{ - return in_proc_collect_linux(i_ins, config, in_context); -} - -static int in_proc_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - int ret; - struct flb_in_proc_config *ctx = NULL; - (void) data; - - /* Allocate space for the configuration */ - ctx = flb_calloc(1, sizeof(struct flb_in_proc_config)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->alert = FLB_FALSE; - ctx->mem = FLB_TRUE; - ctx->fds = FLB_TRUE; - ctx->proc_name = NULL; - ctx->pid = -1; - ctx->ins = in; - - ctx->log_encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ctx->log_encoder == NULL) { - flb_plg_error(in, "event encoder initialization error"); - flb_free(ctx); - - return -1; - } - - configure(ctx, in); - - if (ctx->proc_name == NULL) { - flb_plg_error(ctx->ins, "'proc_name' is not set"); - flb_free(ctx); - return -1; - } - - /* Set the context */ - flb_input_set_context(in, ctx); - - /* Set our collector based on time */ - ret = flb_input_set_collector_time(in, - in_proc_collect, - ctx->interval_sec, - ctx->interval_nsec, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not set collector for Proc input plugin"); - flb_free(ctx); - return -1; - } - - return 0; -} - -static int in_proc_exit(void *data, struct flb_config *config) -{ - (void) *config; - struct flb_in_proc_config *ctx = data; - - if (!ctx) { - return 0; - } - - if (ctx->log_encoder != NULL) { - flb_log_event_encoder_destroy(ctx->log_encoder); - } - - /* Destroy context */ - flb_free(ctx); - - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_INT, "interval_sec", DEFAULT_INTERVAL_SEC, - 0, FLB_TRUE, offsetof(struct flb_in_proc_config, interval_sec), - "Set the collector interval" - }, - { - FLB_CONFIG_MAP_INT, "interval_nsec", DEFAULT_INTERVAL_NSEC, - 0, FLB_TRUE, offsetof(struct flb_in_proc_config, interval_nsec), - "Set the collector interval (nanoseconds)" - }, - { - FLB_CONFIG_MAP_BOOL, "alert", "false", - 0, FLB_TRUE, offsetof(struct flb_in_proc_config, alert), - "Only generate alerts if process is down" - }, - { - FLB_CONFIG_MAP_BOOL, "mem", "true", - 0, FLB_TRUE, offsetof(struct flb_in_proc_config, mem), - "Append memory usage to record" - }, - { - FLB_CONFIG_MAP_BOOL, "fd", "true", - 0, FLB_TRUE, offsetof(struct flb_in_proc_config, fds), - "Append fd count to record" - }, - { - FLB_CONFIG_MAP_STR, "proc_name", NULL, - 0, FLB_TRUE, offsetof(struct flb_in_proc_config, proc_name), - "Define process name to health check" - }, - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_input_plugin in_proc_plugin = { - .name = "proc", - .description = "Check Process health", - .cb_init = in_proc_init, - .cb_pre_run = NULL, - .cb_collect = in_proc_collect, - .cb_flush_buf = NULL, - .cb_exit = in_proc_exit, - .config_map = config_map, - .flags = 0, -}; diff --git a/fluent-bit/plugins/in_proc/in_proc.h b/fluent-bit/plugins/in_proc/in_proc.h deleted file mode 100644 index 8aa155660..000000000 --- a/fluent-bit/plugins/in_proc/in_proc.h +++ /dev/null @@ -1,78 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_PROC_H -#define FLB_IN_PROC_H - -#include -#include -#include -#include -#include - -#define DEFAULT_INTERVAL_SEC "1" -#define DEFAULT_INTERVAL_NSEC "0" - -#define FLB_CMD_LEN 256 -#define FLB_IN_PROC_NAME "in_proc" - -struct flb_in_proc_mem_linux { - uint64_t vmpeak; - uint64_t vmsize; - uint64_t vmlck; - uint64_t vmhwm; - uint64_t vmrss; - uint64_t vmdata; - uint64_t vmstk; - uint64_t vmexe; - uint64_t vmlib; - uint64_t vmpte; - uint64_t vmswap; -}; - -struct flb_in_proc_mem_offset { - char *key; - char *msgpack_key; - size_t offset; -}; - -struct flb_in_proc_config { - uint8_t alert; - uint8_t alive; - - /* Checking process */ - flb_sds_t proc_name; - pid_t pid; - size_t len_proc_name; - - /* Time interval check */ - int interval_sec; - int interval_nsec; - - /* Memory */ - uint8_t mem; - - /* File descriptor */ - uint8_t fds; - - struct flb_input_instance *ins; - struct flb_log_event_encoder *log_encoder; -}; - -#endif /*FLB_IN_PROC_H*/ diff --git a/fluent-bit/plugins/in_prometheus_scrape/CMakeLists.txt b/fluent-bit/plugins/in_prometheus_scrape/CMakeLists.txt deleted file mode 100644 index 73ae0fbd4..000000000 --- a/fluent-bit/plugins/in_prometheus_scrape/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - prom_scrape.c) - -FLB_PLUGIN(in_prometheus_scrape "${src}" "") diff --git a/fluent-bit/plugins/in_prometheus_scrape/prom_scrape.c b/fluent-bit/plugins/in_prometheus_scrape/prom_scrape.c deleted file mode 100644 index 68d4540bc..000000000 --- a/fluent-bit/plugins/in_prometheus_scrape/prom_scrape.c +++ /dev/null @@ -1,261 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include - -#include "prom_scrape.h" - -static struct prom_scrape *prom_scrape_create(struct flb_input_instance *ins, - struct flb_config *config) -{ - int ret; - int upstream_flags; - struct prom_scrape *ctx; - struct flb_upstream *upstream; - - if (ins->host.name == NULL) { - ins->host.name = flb_sds_create("localhost"); - } - if (ins->host.port == 0) { - ins->host.port = 9100; - } - - ctx = flb_calloc(1, sizeof(struct prom_scrape)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - - /* Load the config map */ - ret = flb_input_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return NULL; - } - - upstream_flags = FLB_IO_TCP; - - if (ins->use_tls) { - upstream_flags |= FLB_IO_TLS; - } - - upstream = flb_upstream_create(config, ins->host.name, ins->host.port, - upstream_flags, ins->tls); - - if (!upstream) { - flb_plg_error(ins, "upstream initialization error"); - return NULL; - } - ctx->upstream = upstream; - - return ctx; -} - -static int collect_metrics(struct prom_scrape *ctx) -{ - int ret = -1; - char errbuf[1024]; - size_t b_sent; - struct flb_http_client *c; - struct flb_connection *u_conn; - struct cmt *cmt = NULL; - struct cmt_decode_prometheus_parse_opts opts = {0}; - - /* get upstream connection */ - u_conn = flb_upstream_conn_get(ctx->upstream); - if (!u_conn) { - flb_plg_error(ctx->ins, "could not get an upstream connection to %s:%u", - ctx->ins->host.name, ctx->ins->host.port); - return -1; - } - - c = flb_http_client(u_conn, FLB_HTTP_GET, ctx->metrics_path, - NULL, 0, - ctx->ins->host.name, ctx->ins->host.port, NULL, 0); - if (!c) { - flb_plg_error(ctx->ins, "unable to create http client"); - goto client_error; - } - - flb_http_buffer_size(c, ctx->buffer_max_size); - - /* Auth headers */ - if (ctx->http_user && ctx->http_passwd) { /* Basic */ - flb_http_basic_auth(c, ctx->http_user, ctx->http_passwd); - } else if (ctx->bearer_token) { /* Bearer token */ - flb_http_bearer_auth(c, ctx->bearer_token); - } - - ret = flb_http_do(c, &b_sent); - if (ret != 0) { - flb_plg_error(ctx->ins, "http do error"); - goto http_error; - } - - if (c->resp.status != 200) { - flb_plg_error(ctx->ins, "http status code error: [%s] %d", - ctx->metrics_path, c->resp.status); - goto http_error; - } - - if (c->resp.payload_size <= 0) { - flb_plg_error(ctx->ins, "empty response"); - goto http_error; - } - - /* configure prometheus decoder options */ - opts.default_timestamp = cfl_time_now(); - opts.errbuf = errbuf; - opts.errbuf_size = sizeof(errbuf); - - /* convert Prometheus Text to CMetrics */ - ret = cmt_decode_prometheus_create(&cmt, - c->resp.payload, - c->resp.payload_size, - &opts); - if (ret == 0) { - /* Append the updated metrics */ - ret = flb_input_metrics_append(ctx->ins, NULL, 0, cmt); - if (ret != 0) { - flb_plg_error(ctx->ins, "could not append metrics"); - } - cmt_destroy(cmt); - } - else { - flb_plg_error(ctx->ins, "error decoding Prometheus Text format"); - } - -http_error: - flb_http_client_destroy(c); -client_error: - flb_upstream_conn_release(u_conn); - - return ret; -} - -static int cb_prom_scrape_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - int rc; - struct prom_scrape *ctx = (struct prom_scrape *) in_context; - - rc = collect_metrics(ctx); - FLB_INPUT_RETURN(rc); -} - -static int cb_prom_scrape_init(struct flb_input_instance *ins, - struct flb_config *config, void *data) -{ - struct prom_scrape *ctx; - - /* Allocate space for the configuration */ - ctx = prom_scrape_create(ins, config); - if (!ctx) { - return -1; - } - - flb_input_set_context(ins, ctx); - ctx->coll_id = flb_input_set_collector_time(ins, - cb_prom_scrape_collect, - ctx->scrape_interval, - 0, config); - return 0; -} - -static int prom_scrape_destroy(struct prom_scrape *ctx) -{ - if (ctx->upstream) { - flb_upstream_destroy(ctx->upstream); - } - flb_free(ctx); - - return 0; -} - -static int cb_prom_scrape_exit(void *data, struct flb_config *config) -{ - struct prom_scrape *ctx = (struct prom_scrape *) data; - - if (!ctx) { - return 0; - } - - prom_scrape_destroy(ctx); - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_TIME, "scrape_interval", "10s", - 0, FLB_TRUE, offsetof(struct prom_scrape, scrape_interval), - "Scraping interval." - }, - - { - FLB_CONFIG_MAP_SIZE, "buffer_max_size", HTTP_BUFFER_MAX_SIZE, - 0, FLB_TRUE, offsetof(struct prom_scrape, buffer_max_size), - "" - }, - - { - FLB_CONFIG_MAP_STR, "metrics_path", DEFAULT_URI, - 0, FLB_TRUE, offsetof(struct prom_scrape, metrics_path), - "Set the metrics URI endpoint, it must start with a forward slash." - }, - - { - FLB_CONFIG_MAP_STR, "http_user", NULL, - 0, FLB_TRUE, offsetof(struct prom_scrape, http_user), - "Set HTTP auth user" - }, - - { - FLB_CONFIG_MAP_STR, "http_passwd", "", - 0, FLB_TRUE, offsetof(struct prom_scrape, http_passwd), - "Set HTTP auth password" - }, - - { - FLB_CONFIG_MAP_STR, "bearer_token", NULL, - 0, FLB_TRUE, offsetof(struct prom_scrape, bearer_token), - "Set bearer token auth" - }, - - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_input_plugin in_prometheus_scrape_plugin = { - .name = "prometheus_scrape", - .description = "Scrape metrics from Prometheus Endpoint", - .cb_init = cb_prom_scrape_init, - .cb_pre_run = NULL, - .cb_collect = cb_prom_scrape_collect, - .cb_flush_buf = NULL, - .cb_exit = cb_prom_scrape_exit, - .config_map = config_map, - .flags = FLB_INPUT_NET | FLB_INPUT_CORO, -}; diff --git a/fluent-bit/plugins/in_prometheus_scrape/prom_scrape.h b/fluent-bit/plugins/in_prometheus_scrape/prom_scrape.h deleted file mode 100644 index 9510abfef..000000000 --- a/fluent-bit/plugins/in_prometheus_scrape/prom_scrape.h +++ /dev/null @@ -1,45 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_PROMETHEUS_SCRAPE_H -#define FLB_IN_PROMETHEUS_SCRAPE_H - -#include - -#define DEFAULT_URI "/metrics" -#define HTTP_BUFFER_MAX_SIZE "10M" - -struct prom_scrape -{ - int coll_id; /* collector id */ - uint64_t scrape_interval; - flb_sds_t metrics_path; - struct flb_upstream *upstream; - struct flb_input_instance *ins; /* input plugin instance */ - size_t buffer_max_size; /* Maximum buffer size */ - - /* HTTP Auth */ - flb_sds_t http_user; - flb_sds_t http_passwd; - - /* Bearer Token Auth */ - flb_sds_t bearer_token; -}; - -#endif diff --git a/fluent-bit/plugins/in_random/CMakeLists.txt b/fluent-bit/plugins/in_random/CMakeLists.txt deleted file mode 100644 index 2cb59e83d..000000000 --- a/fluent-bit/plugins/in_random/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - random.c) - -FLB_PLUGIN(in_random "${src}" "") diff --git a/fluent-bit/plugins/in_random/random.c b/fluent-bit/plugins/in_random/random.c deleted file mode 100644 index ab6e59ffd..000000000 --- a/fluent-bit/plugins/in_random/random.c +++ /dev/null @@ -1,245 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#define DEFAULT_INTERVAL_SEC "1" -#define DEFAULT_INTERVAL_NSEC "0" - -struct flb_in_random_config { - /* Config properties */ - int interval_sec; - int interval_nsec; - int samples; - - /* Internal */ - int samples_count; - int coll_fd; - - struct flb_input_instance *ins; - struct flb_log_event_encoder *log_encoder; -}; - -/* cb_collect callback */ -static int in_random_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - int ret; - uint64_t val; - struct flb_in_random_config *ctx = in_context; - - if (ctx->samples == 0) { - return -1; - } - - if (ctx->samples > 0 && (ctx->samples_count >= ctx->samples)) { - return -1; - } - - if (flb_random_bytes((unsigned char *) &val, sizeof(uint64_t))) { - val = time(NULL); - } - - ret = flb_log_event_encoder_begin_record(ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp(ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE("rand_value"), - FLB_LOG_EVENT_UINT64_VALUE(val)); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(ctx->ins, NULL, 0, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); - ret = 0; - } - else { - flb_plg_error(ctx->ins, "log event encoding error : %d", ret); - - ret = -1; - } - - flb_log_event_encoder_reset(ctx->log_encoder); - - ctx->samples_count++; - - return 0; -} - -/* Set plugin configuration */ -static int in_random_config_read(struct flb_in_random_config *ctx, - struct flb_input_instance *in) -{ - int ret; - /* Load the config map */ - ret = flb_input_config_map_set(in, (void *)ctx); - if (ret == -1) { - return -1; - } - - /* interval settings */ - if (ctx->interval_sec <= 0 && ctx->interval_nsec <= 0) { - /* Illegal settings. Override them. */ - ctx->interval_sec = atoi(DEFAULT_INTERVAL_SEC); - ctx->interval_nsec = atoi(DEFAULT_INTERVAL_NSEC); - } - - - flb_plg_debug(ctx->ins, "interval_sec=%d interval_nsec=%d", - ctx->interval_sec, ctx->interval_nsec); - - return 0; -} - -/* Initialize plugin */ -static int in_random_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - int ret = -1; - struct flb_in_random_config *ctx = NULL; - - /* Allocate space for the configuration */ - ctx = flb_calloc(1, sizeof(struct flb_in_random_config)); - if (!ctx) { - return -1; - } - ctx->samples_count = 0; - ctx->ins = in; - - ctx->log_encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ctx->log_encoder == NULL) { - flb_plg_error(in, "could not initialize event encoder"); - flb_free(ctx); - - return -1; - } - - /* Initialize head config */ - ret = in_random_config_read(ctx, in); - if (ret < 0) { - flb_free(ctx); - return -1; - } - - flb_input_set_context(in, ctx); - ret = flb_input_set_collector_time(in, - in_random_collect, - ctx->interval_sec, - ctx->interval_nsec, config); - if (ret < 0) { - flb_plg_error(ctx->ins, "could not set collector for head input plugin"); - flb_free(ctx); - return -1; - } - ctx->coll_fd = ret; - return 0; -} - -static void in_random_pause(void *data, struct flb_config *config) -{ - struct flb_in_random_config *ctx = data; - - flb_input_collector_pause(ctx->coll_fd, ctx->ins); - -} - -static void in_random_resume(void *data, struct flb_config *config) -{ - struct flb_in_random_config *ctx = data; - - flb_input_collector_resume(ctx->coll_fd, ctx->ins); -} - -static int in_random_exit(void *data, struct flb_config *config) -{ - struct flb_in_random_config *ctx = data; - (void) *config; - - if (!ctx) { - return 0; - } - - if (ctx->log_encoder != NULL) { - flb_log_event_encoder_destroy(ctx->log_encoder); - } - - flb_free(ctx); - return 0; -} - - -static struct flb_config_map config_map[] = { - // samples - // interval_sec - // interval_nsec - { - FLB_CONFIG_MAP_INT, "samples", "-1", - 0, FLB_TRUE, offsetof(struct flb_in_random_config, samples), - "Number of samples to send, -1 for infinite" - }, - { - FLB_CONFIG_MAP_INT, "interval_sec", DEFAULT_INTERVAL_SEC, - 0, FLB_TRUE, offsetof(struct flb_in_random_config, interval_sec), - "Set the collector interval" - }, - { - FLB_CONFIG_MAP_INT, "interval_nsec", DEFAULT_INTERVAL_NSEC, - 0, FLB_TRUE, offsetof(struct flb_in_random_config, interval_nsec), - "Set the collector interval (sub seconds)" - }, - /* EOF */ - {0} - -}; - -struct flb_input_plugin in_random_plugin = { - .name = "random", - .description = "Random", - .cb_init = in_random_init, - .cb_pre_run = NULL, - .cb_collect = in_random_collect, - .cb_flush_buf = NULL, - .cb_pause = in_random_pause, - .cb_resume = in_random_resume, - .cb_exit = in_random_exit, - .config_map = config_map -}; diff --git a/fluent-bit/plugins/in_serial/CMakeLists.txt b/fluent-bit/plugins/in_serial/CMakeLists.txt deleted file mode 100644 index cf15742c2..000000000 --- a/fluent-bit/plugins/in_serial/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - in_serial.c in_serial_config.c) - -FLB_PLUGIN(in_serial "${src}" "") diff --git a/fluent-bit/plugins/in_serial/in_serial.c b/fluent-bit/plugins/in_serial/in_serial.c deleted file mode 100644 index 3675f1e92..000000000 --- a/fluent-bit/plugins/in_serial/in_serial.c +++ /dev/null @@ -1,443 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Serial input plugin for Fluent Bit - * ================================== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * Copyright (C) 2015-2016 Takeshi HASEGAWA - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "in_serial.h" -#include "in_serial_config.h" - -static inline int process_line(const char *line, int len, - struct flb_in_serial_config *ctx) -{ - int ret; - - ret = flb_log_event_encoder_begin_record(ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp(ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE("msg"), - FLB_LOG_EVENT_STRING_VALUE(line, len)); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(ctx->log_encoder); - } - - flb_debug("[in_serial] message '%s'", line); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = 0; - } - else { - ret = -1; - } - - return ret; -} - -static inline int process_pack(struct flb_in_serial_config *ctx, - char *pack, size_t size) -{ - int ret; - size_t off = 0; - msgpack_unpacked result; - msgpack_object entry; - - ret = FLB_EVENT_ENCODER_SUCCESS; - - /* First pack the results, iterate concatenated messages */ - msgpack_unpacked_init(&result); - while (msgpack_unpack_next(&result, pack, size, &off) == MSGPACK_UNPACK_SUCCESS) { - entry = result.data; - - ret = flb_log_event_encoder_begin_record(ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp(ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE("msg"), - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&entry)); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(ctx->log_encoder); - } - } - - msgpack_unpacked_destroy(&result); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = 0; - } - else { - ret = -1; - } - - return ret; -} - -static inline void consume_bytes(char *buf, int bytes, int length) -{ - memmove(buf, buf + bytes, length - bytes); -} - -/* Callback triggered when some serial msgs are available */ -static int cb_serial_collect(struct flb_input_instance *in, - struct flb_config *config, void *in_context) -{ - int ret; - int bytes = 0; - int available; - int len; - int hits; - char *sep; - char *buf; - struct flb_in_serial_config *ctx = in_context; - - flb_log_event_encoder_reset(ctx->log_encoder); - - ret = 0; - - while (1) { - available = (sizeof(ctx->buf_data) -1) - ctx->buf_len; - if (available > 1) { - bytes = read(ctx->fd, ctx->buf_data + ctx->buf_len, available); - - if (bytes == -1) { - if (errno == EPIPE || errno == EINTR) { - ret = -1; - } - else { - ret = 0; - } - - break; - } - else if (bytes == 0) { - ret = 0; - - break; - } - } - ctx->buf_len += bytes; - - /* Always set a delimiter to avoid buffer trash */ - ctx->buf_data[ctx->buf_len] = '\0'; - - /* Check if our buffer is full */ - if (ctx->buffer_id + 1 == SERIAL_BUFFER_SIZE) { - ret = flb_engine_flush(config, &in_serial_plugin); - if (ret == -1) { - ctx->buffer_id = 0; - } - } - - sep = NULL; - buf = ctx->buf_data; - len = ctx->buf_len; - hits = 0; - - /* Handle FTDI handshake */ - if (ctx->buf_data[0] == '\0') { - consume_bytes(ctx->buf_data, 1, ctx->buf_len); - ctx->buf_len--; - } - - /* Strip CR or LF if found at first byte */ - if (ctx->buf_data[0] == '\r' || ctx->buf_data[0] == '\n') { - /* Skip message with one byte with CR or LF */ - flb_trace("[in_serial] skip one byte message with ASCII code=%i", - ctx->buf_data[0]); - consume_bytes(ctx->buf_data, 1, ctx->buf_len); - ctx->buf_len--; - } - - /* Handle the case when a Separator is set */ - if (ctx->separator) { - while ((sep = strstr(ctx->buf_data, ctx->separator))) { - len = (sep - ctx->buf_data); - if (len > 0) { - /* process the line based in the separator position */ - process_line(buf, len, ctx); - consume_bytes(ctx->buf_data, len + ctx->sep_len, ctx->buf_len); - ctx->buf_len -= (len + ctx->sep_len); - hits++; - } - else { - consume_bytes(ctx->buf_data, ctx->sep_len, ctx->buf_len); - ctx->buf_len -= ctx->sep_len; - } - ctx->buf_data[ctx->buf_len] = '\0'; - } - - if (hits == 0 && available <= 1) { - flb_debug("[in_serial] no separator found, no more space"); - ctx->buf_len = 0; - ret = 0; - - break; - } - } - else if (ctx->format == FLB_SERIAL_FORMAT_JSON) { - /* JSON Format handler */ - char *pack; - int out_size; - - ret = flb_pack_json_state(ctx->buf_data, ctx->buf_len, - &pack, &out_size, &ctx->pack_state); - if (ret == FLB_ERR_JSON_PART) { - flb_debug("[in_serial] JSON incomplete, waiting for more data..."); - - ret = 0; - - break; - } - else if (ret == FLB_ERR_JSON_INVAL) { - flb_debug("[in_serial] invalid JSON message, skipping"); - flb_pack_state_reset(&ctx->pack_state); - flb_pack_state_init(&ctx->pack_state); - ctx->pack_state.multiple = FLB_TRUE; - - ret = -1; - - break; - } - - /* - * Given the Tokens used for the packaged message, append - * the records and then adjust buffer. - */ - process_pack(ctx, pack, out_size); - flb_free(pack); - - consume_bytes(ctx->buf_data, ctx->pack_state.last_byte, ctx->buf_len); - ctx->buf_len -= ctx->pack_state.last_byte; - ctx->buf_data[ctx->buf_len] = '\0'; - - flb_pack_state_reset(&ctx->pack_state); - flb_pack_state_init(&ctx->pack_state); - ctx->pack_state.multiple = FLB_TRUE; - } - else { - /* Process and enqueue the received line */ - process_line(ctx->buf_data, ctx->buf_len, ctx); - ctx->buf_len = 0; - } - } - - if (ctx->log_encoder->output_length > 0) { - flb_input_log_append(in, NULL, 0, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); - } - - flb_log_event_encoder_reset(ctx->log_encoder); - - return ret; -} - -/* Cleanup serial input */ -static int cb_serial_exit(void *in_context, struct flb_config *config) -{ - struct flb_in_serial_config *ctx = in_context; - - flb_trace("[in_serial] Restoring original termios..."); - tcsetattr(ctx->fd, TCSANOW, &ctx->tio_orig); - - if (ctx->log_encoder != NULL) { - flb_log_event_encoder_destroy(ctx->log_encoder); - } - - flb_pack_state_reset(&ctx->pack_state); - flb_free(ctx); - - return 0; -} - -/* Init serial input */ -static int cb_serial_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - int fd; - int ret; - int br; - struct flb_in_serial_config *ctx; - (void) data; - - ctx = flb_calloc(1, sizeof(struct flb_in_serial_config)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->format = FLB_SERIAL_FORMAT_NONE; - - ctx->log_encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ctx->log_encoder == NULL) { - flb_plg_error(in, "could not initialize event encoder"); - flb_free(ctx); - - return -1; - } - - if (!serial_config_read(ctx, in)) { - flb_log_event_encoder_destroy(ctx->log_encoder); - flb_free(ctx); - return -1; - } - - /* Initialize JSON pack state */ - if (ctx->format == FLB_SERIAL_FORMAT_JSON) { - flb_pack_state_init(&ctx->pack_state); - ctx->pack_state.multiple = FLB_TRUE; - } - - /* Input instance */ - ctx->i_ins = in; - - /* set context */ - flb_input_set_context(in, ctx); - - /* open device */ - fd = open(ctx->file, O_RDWR | O_NOCTTY | O_NONBLOCK); - if (fd == -1) { - perror("open"); - flb_error("[in_serial] Could not open serial port device"); - flb_log_event_encoder_destroy(ctx->log_encoder); - flb_free(ctx); - return -1; - } - ctx->fd = fd; - - /* Store original settings */ - tcgetattr(fd, &ctx->tio_orig); - - /* Reset for new... */ - memset(&ctx->tio, 0, sizeof(ctx->tio)); - tcgetattr(fd, &ctx->tio); - - br = atoi(ctx->bitrate); - cfsetospeed(&ctx->tio, (speed_t) flb_serial_speed(br)); - cfsetispeed(&ctx->tio, (speed_t) flb_serial_speed(br)); - - /* Settings */ - ctx->tio.c_cflag &= ~PARENB; /* 8N1 */ - ctx->tio.c_cflag &= ~CSTOPB; - ctx->tio.c_cflag &= ~CSIZE; - ctx->tio.c_cflag |= CS8; - ctx->tio.c_cflag &= ~CRTSCTS; /* No flow control */ - ctx->tio.c_cc[VMIN] = ctx->min_bytes; /* Min number of bytes to read */ - ctx->tio.c_cflag |= CREAD | CLOCAL; /* Enable READ & ign ctrl lines */ - - tcflush(fd, TCIFLUSH); - tcsetattr(fd, TCSANOW, &ctx->tio); - -#if __linux__ - /* Set our collector based on a file descriptor event */ - ret = flb_input_set_collector_event(in, - cb_serial_collect, - ctx->fd, - config); -#else - /* Set our collector based on a timer event */ - ret = flb_input_set_collector_time(in, - cb_serial_collect, - IN_SERIAL_COLLECT_SEC, - IN_SERIAL_COLLECT_NSEC, - config); -#endif - - if (ret == -1) { - flb_log_event_encoder_destroy(ctx->log_encoder); - return -1; - } - - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "file", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_in_serial_config, file), - "Set the serial character device file name" - }, - { - FLB_CONFIG_MAP_STR, "bitrate", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_in_serial_config, bitrate), - "Set the serial bitrate (baudrate)" - }, - { - FLB_CONFIG_MAP_STR, "separator", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_in_serial_config, separator), - "Set the record separator" - }, - { - FLB_CONFIG_MAP_STR, "format", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_in_serial_config, format_str), - "Set the serial format: json or none" - }, - { - FLB_CONFIG_MAP_INT, "min_bytes", "0", - 0, FLB_TRUE, offsetof(struct flb_in_serial_config, min_bytes), - "Set the serial minimum bytes" - }, - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_input_plugin in_serial_plugin = { - .name = "serial", - .description = "Serial input", - .cb_init = cb_serial_init, - .cb_pre_run = NULL, - .cb_collect = cb_serial_collect, - .cb_flush_buf = NULL, - .cb_exit = cb_serial_exit, - .config_map = config_map, -}; diff --git a/fluent-bit/plugins/in_serial/in_serial.h b/fluent-bit/plugins/in_serial/in_serial.h deleted file mode 100644 index 2bed91c00..000000000 --- a/fluent-bit/plugins/in_serial/in_serial.h +++ /dev/null @@ -1,63 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Serial input plugin for Fluent Bit - * ================================== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * Copyright (C) 2015-2016 Takeshi HASEGAWA - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_SERIAL -#define FLB_IN_SERIAL - -#include - -#define SERIAL_BUFFER_SIZE 256 -#define IN_SERIAL_COLLECT_SEC 1 -#define IN_SERIAL_COLLECT_NSEC 0 - -static inline speed_t flb_serial_speed(int br) -{ - switch (br) { - case 0: return B0; - case 50: return B50; - case 75: return B75; - case 110: return B110; - case 134: return B134; - case 150: return B150; - case 200: return B200; - case 300: return B300; - case 600: return B600; - case 1200: return B1200; - case 1800: return B1800; - case 2400: return B2400; - case 4800: return B4800; - case 9600: return B9600; - case 19200: return B19200; - case 38400: return B38400; - case 57600: return B57600; - case 115200: return B115200; - case 230400: return B230400; - default: return B9600; - }; - - return 0; -} - -int in_serial_start(); - - -extern struct flb_input_plugin in_serial_plugin; - -#endif diff --git a/fluent-bit/plugins/in_serial/in_serial_config.c b/fluent-bit/plugins/in_serial/in_serial_config.c deleted file mode 100644 index 152caee3d..000000000 --- a/fluent-bit/plugins/in_serial/in_serial_config.c +++ /dev/null @@ -1,82 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Serial input plugin for Fluent Bit - * ================================== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * Copyright (C) 2015-2016 Takeshi HASEGAWA - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "in_serial_config.h" - -struct flb_in_serial_config *serial_config_read(struct flb_in_serial_config *config, - struct flb_input_instance *i_ins) -{ - int ret; - - /* Load the config map */ - ret = flb_input_config_map_set(i_ins, (void *)config); - if (ret == -1) { - flb_plg_error(i_ins, "unable to load configuration"); - return NULL; - } - - if (!config->file) { - flb_error("[serial] error reading filename from " - "configuration"); - return NULL; - } - - if (!config->bitrate) { - flb_error("[serial] error reading bitrate from " - "configuration"); - return NULL; - } - - if (config->min_bytes <= 0) { - config->min_bytes = 1; - } - - config->fd = -1; - config->buf_len = 0; - - if (config->format_str && config->separator) { - flb_error("[in_serial] specify 'format' or 'separator', not both"); - return NULL; - } - - if (config->separator) { - config->sep_len = strlen(config->separator); - } - else { - config->sep_len = 0; - } - - if (config->format_str) { - if (strcasecmp(config->format_str, "json") == 0) { - config->format = FLB_SERIAL_FORMAT_JSON; - } - } - - flb_debug("[in_serial] file='%s' bitrate='%s' min_bytes=%i format=%i", - config->file, config->bitrate, config->min_bytes, config->format); - - return config; -} diff --git a/fluent-bit/plugins/in_serial/in_serial_config.h b/fluent-bit/plugins/in_serial/in_serial_config.h deleted file mode 100644 index e625635e7..000000000 --- a/fluent-bit/plugins/in_serial/in_serial_config.h +++ /dev/null @@ -1,77 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Serial input plugin for Fluent Bit - * ================================== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * Copyright (C) 2015-2016 Takeshi HASEGAWA - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_SERIAL_CONFIG_H -#define FLB_IN_SERIAL_CONFIG_H - -#define FLB_SERIAL_FORMAT_NONE 0 -#define FLB_SERIAL_FORMAT_JSON 1 - -#include -#include - -#include -#include - -struct flb_in_serial_config { - int fd; /* Socket to destination/backend */ - - /* Buffer */ - int buf_len; - char buf_data[8192]; - - /* config */ - int min_bytes; - flb_sds_t file; - flb_sds_t bitrate; - - /* separator */ - int sep_len; - flb_sds_t separator; - - /* Incoming format: JSON only for now */ - int format; - flb_sds_t format_str; - - struct termios tio; - struct termios tio_orig; - - /* Tag: used to extend original tag */ - int tag_len; /* The real string length */ - char tag[32]; /* Custom Tag for this input */ - - /* Line processing */ - int buffer_id; - - /* Input instance reference */ - struct flb_input_instance *i_ins; - struct flb_log_event_encoder *log_encoder; - - /* - * If (format == FLB_SERIAL_FORMAT_JSON), we use this pack_state - * to perform validation of the incomming JSON message. - */ - struct flb_pack_state pack_state; -}; - -struct flb_in_serial_config *serial_config_read(struct flb_in_serial_config *config, - struct flb_input_instance *i_ins); - -#endif diff --git a/fluent-bit/plugins/in_splunk/CMakeLists.txt b/fluent-bit/plugins/in_splunk/CMakeLists.txt deleted file mode 100644 index 42ecf2e31..000000000 --- a/fluent-bit/plugins/in_splunk/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -if(NOT FLB_METRICS) - message(FATAL_ERROR "Splunk input plugin requires FLB_HTTP_SERVER=On.") -endif() - -set(src - splunk.c - splunk_conn.c - splunk_prot.c - splunk_config.c - ) - -FLB_PLUGIN(in_splunk "${src}" "monkey-core-static") diff --git a/fluent-bit/plugins/in_splunk/splunk.c b/fluent-bit/plugins/in_splunk/splunk.c deleted file mode 100644 index 78589037c..000000000 --- a/fluent-bit/plugins/in_splunk/splunk.c +++ /dev/null @@ -1,213 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#include -#include -#include - -#include "splunk.h" -#include "splunk_conn.h" -#include "splunk_config.h" - -/* - * For a server event, the collection event means a new client have arrived, we - * accept the connection and create a new TCP instance which will wait for - * JSON map messages. - */ -static int in_splunk_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_connection *connection; - struct splunk_conn *conn; - struct flb_splunk *ctx; - - ctx = in_context; - - connection = flb_downstream_conn_get(ctx->downstream); - - if (connection == NULL) { - flb_plg_error(ctx->ins, "could not accept new connection"); - - return -1; - } - - flb_plg_trace(ctx->ins, "new TCP connection arrived FD=%i", - connection->fd); - - conn = splunk_conn_add(connection, ctx); - - if (conn == NULL) { - flb_downstream_conn_release(connection); - - return -1; - } - - return 0; -} - -static int in_splunk_init(struct flb_input_instance *ins, - struct flb_config *config, void *data) -{ - unsigned short int port; - int ret; - struct flb_splunk *ctx; - - (void) data; - - /* Create context and basic conf */ - ctx = splunk_config_create(ins); - if (!ctx) { - return -1; - } - - ctx->collector_id = -1; - - /* Populate context with config map defaults and incoming properties */ - ret = flb_input_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_plg_error(ctx->ins, "configuration error"); - splunk_config_destroy(ctx); - return -1; - } - - /* Set the context */ - flb_input_set_context(ins, ctx); - - port = (unsigned short int) strtoul(ctx->tcp_port, NULL, 10); - - ctx->downstream = flb_downstream_create(FLB_TRANSPORT_TCP, - ins->flags, - ctx->listen, - port, - ins->tls, - config, - &ins->net_setup); - - if (ctx->downstream == NULL) { - flb_plg_error(ctx->ins, - "could not initialize downstream on %s:%s. Aborting", - ctx->listen, ctx->tcp_port); - - splunk_config_destroy(ctx); - - return -1; - } - - flb_input_downstream_set(ctx->downstream, ctx->ins); - - /* Collect upon data available on the standard input */ - ret = flb_input_set_collector_socket(ins, - in_splunk_collect, - ctx->downstream->server_fd, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "Could not set collector for IN_TCP input plugin"); - splunk_config_destroy(ctx); - - return -1; - } - - ctx->collector_id = ret; - - return 0; -} - -static int in_splunk_exit(void *data, struct flb_config *config) -{ - struct flb_splunk *ctx; - - (void) config; - - ctx = data; - - if (ctx != NULL) { - splunk_config_destroy(ctx); - } - - return 0; -} - - -static void in_splunk_pause(void *data, struct flb_config *config) -{ - struct flb_splunk *ctx = data; - - flb_input_collector_pause(ctx->collector_id, ctx->ins); - -} - -static void in_splunk_resume(void *data, struct flb_config *config) -{ - struct flb_splunk *ctx = data; - - flb_input_collector_resume(ctx->collector_id, ctx->ins); -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_SIZE, "buffer_max_size", HTTP_BUFFER_MAX_SIZE, - 0, FLB_TRUE, offsetof(struct flb_splunk, buffer_max_size), - "" - }, - - { - FLB_CONFIG_MAP_SIZE, "buffer_chunk_size", HTTP_BUFFER_CHUNK_SIZE, - 0, FLB_TRUE, offsetof(struct flb_splunk, buffer_chunk_size), - "" - }, - - { - FLB_CONFIG_MAP_SLIST_1, "success_header", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct flb_splunk, success_headers), - "Add an HTTP header key/value pair on success. Multiple headers can be set" - }, - - { - FLB_CONFIG_MAP_STR, "splunk_token", NULL, - 0, FLB_FALSE, 0, - "Set valid Splunk HEC tokens for the requests" - }, - - { - FLB_CONFIG_MAP_STR, "tag_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_splunk, tag_key), - "" - }, - - - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_input_plugin in_splunk_plugin = { - .name = "splunk", - .description = "Input plugin for Splunk HEC payloads", - .cb_init = in_splunk_init, - .cb_pre_run = NULL, - .cb_collect = in_splunk_collect, - .cb_flush_buf = NULL, - .cb_pause = in_splunk_pause, - .cb_resume = in_splunk_resume, - .cb_exit = in_splunk_exit, - .config_map = config_map, - .flags = FLB_INPUT_NET_SERVER | FLB_IO_OPT_TLS -}; diff --git a/fluent-bit/plugins/in_splunk/splunk.h b/fluent-bit/plugins/in_splunk/splunk.h deleted file mode 100644 index bf935ea22..000000000 --- a/fluent-bit/plugins/in_splunk/splunk.h +++ /dev/null @@ -1,60 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_SPLUNK_H -#define FLB_IN_SPLUNK_H - -#include -#include -#include -#include -#include - -#include - -#define HTTP_BUFFER_MAX_SIZE "4M" -#define HTTP_BUFFER_CHUNK_SIZE "512K" - -struct flb_splunk { - flb_sds_t listen; - flb_sds_t tcp_port; - const char *tag_key; - - int collector_id; - - /* Success HTTP headers */ - struct mk_list *success_headers; - flb_sds_t success_headers_str; - - size_t buffer_max_size; /* Maximum buffer size */ - size_t buffer_chunk_size; /* Chunk allocation size */ - - /* Token Auth */ - flb_sds_t auth_header; - - struct flb_log_event_encoder log_encoder; - struct flb_downstream *downstream; /* Client manager */ - struct mk_list connections; /* linked list of connections */ - - struct mk_server *server; - struct flb_input_instance *ins; -}; - - -#endif diff --git a/fluent-bit/plugins/in_splunk/splunk_config.c b/fluent-bit/plugins/in_splunk/splunk_config.c deleted file mode 100644 index f61070153..000000000 --- a/fluent-bit/plugins/in_splunk/splunk_config.c +++ /dev/null @@ -1,184 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include "splunk.h" -#include "splunk_config.h" -#include "splunk_conn.h" -#include "splunk_config.h" - -struct flb_splunk *splunk_config_create(struct flb_input_instance *ins) -{ - struct mk_list *header_iterator; - struct flb_slist_entry *header_value; - struct flb_slist_entry *header_name; - struct flb_config_map_val *header_pair; - char port[8]; - int ret; - struct flb_splunk *ctx; - const char *tmp; - - ctx = flb_calloc(1, sizeof(struct flb_splunk)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - mk_list_init(&ctx->connections); - - /* Load the config map */ - ret = flb_input_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return NULL; - } - - ctx->auth_header = NULL; - tmp = flb_input_get_property("splunk_token", ins); - if (tmp) { - ctx->auth_header = flb_sds_create("Splunk "); - if (ctx->auth_header == NULL) { - flb_plg_error(ctx->ins, "error on prefix of auth_header generation"); - splunk_config_destroy(ctx); - return NULL; - } - ret = flb_sds_cat_safe(&ctx->auth_header, tmp, strlen(tmp)); - if (ret < 0) { - flb_plg_error(ctx->ins, "error on token generation"); - splunk_config_destroy(ctx); - return NULL; - } - } - - /* Listen interface (if not set, defaults to 0.0.0.0:8088) */ - flb_input_net_default_listener("0.0.0.0", 8088, ins); - - ctx->listen = flb_strdup(ins->host.listen); - snprintf(port, sizeof(port) - 1, "%d", ins->host.port); - ctx->tcp_port = flb_strdup(port); - - /* HTTP Server specifics */ - ctx->server = flb_calloc(1, sizeof(struct mk_server)); - if (ctx->server == NULL) { - flb_plg_error(ctx->ins, "error on mk_server allocation"); - splunk_config_destroy(ctx); - return NULL; - } - ctx->server->keep_alive = MK_TRUE; - - /* monkey detects server->workers == 0 as the server not being initialized at the - * moment so we want to make sure that it stays that way! - */ - - ret = flb_log_event_encoder_init(&ctx->log_encoder, - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, "error initializing event encoder : %d", ret); - - splunk_config_destroy(ctx); - - return NULL; - } - - ctx->success_headers_str = flb_sds_create_size(1); - - if (ctx->success_headers_str == NULL) { - splunk_config_destroy(ctx); - - return NULL; - } - - flb_config_map_foreach(header_iterator, header_pair, ctx->success_headers) { - header_name = mk_list_entry_first(header_pair->val.list, - struct flb_slist_entry, - _head); - - header_value = mk_list_entry_last(header_pair->val.list, - struct flb_slist_entry, - _head); - - ret = flb_sds_cat_safe(&ctx->success_headers_str, - header_name->str, - flb_sds_len(header_name->str)); - - if (ret == 0) { - ret = flb_sds_cat_safe(&ctx->success_headers_str, - ": ", - 2); - } - - if (ret == 0) { - ret = flb_sds_cat_safe(&ctx->success_headers_str, - header_value->str, - flb_sds_len(header_value->str)); - } - - if (ret == 0) { - ret = flb_sds_cat_safe(&ctx->success_headers_str, - "\r\n", - 2); - } - - if (ret != 0) { - splunk_config_destroy(ctx); - - return NULL; - } - } - - return ctx; -} - -int splunk_config_destroy(struct flb_splunk *ctx) -{ - /* release all connections */ - splunk_conn_release_all(ctx); - - flb_log_event_encoder_destroy(&ctx->log_encoder); - - if (ctx->collector_id != -1) { - flb_input_collector_delete(ctx->collector_id, ctx->ins); - - ctx->collector_id = -1; - } - - if (ctx->auth_header != NULL) { - flb_sds_destroy(ctx->auth_header); - } - - if (ctx->downstream != NULL) { - flb_downstream_destroy(ctx->downstream); - } - - if (ctx->server) { - flb_free(ctx->server); - } - - if (ctx->success_headers_str != NULL) { - flb_sds_destroy(ctx->success_headers_str); - } - - - flb_free(ctx->listen); - flb_free(ctx->tcp_port); - flb_free(ctx); - return 0; -} diff --git a/fluent-bit/plugins/in_splunk/splunk_config.h b/fluent-bit/plugins/in_splunk/splunk_config.h deleted file mode 100644 index 24d2008f2..000000000 --- a/fluent-bit/plugins/in_splunk/splunk_config.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_SPLUNK_CONFIG_H -#define FLB_IN_SPLUNK_CONFIG_H - -#include -#include "splunk.h" - -struct flb_splunk *splunk_config_create(struct flb_input_instance *ins); -int splunk_config_destroy(struct flb_splunk *ctx); - -#endif diff --git a/fluent-bit/plugins/in_splunk/splunk_conn.c b/fluent-bit/plugins/in_splunk/splunk_conn.c deleted file mode 100644 index f605a16c7..000000000 --- a/fluent-bit/plugins/in_splunk/splunk_conn.c +++ /dev/null @@ -1,306 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for thet specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include "splunk.h" -#include "splunk_conn.h" -#include "splunk_prot.h" - -static void splunk_conn_request_init(struct mk_http_session *session, - struct mk_http_request *request); - -static int splunk_conn_event(void *data) -{ - int status; - size_t size; - ssize_t available; - ssize_t bytes; - char *tmp; - char *request_end; - size_t request_len; - struct flb_connection *connection; - struct splunk_conn *conn; - struct mk_event *event; - struct flb_splunk *ctx; - - connection = (struct flb_connection *) data; - - conn = connection->user_data; - - ctx = conn->ctx; - - event = &connection->event; - - if (event->mask & MK_EVENT_READ) { - available = (conn->buf_size - conn->buf_len) - 1; - if (available < 1) { - if (conn->buf_size + ctx->buffer_chunk_size > ctx->buffer_max_size) { - flb_plg_trace(ctx->ins, - "fd=%i incoming data exceed limit (%zu KB)", - event->fd, (ctx->buffer_max_size / 1024)); - splunk_conn_del(conn); - return -1; - } - - size = conn->buf_size + ctx->buffer_chunk_size; - tmp = flb_realloc(conn->buf_data, size); - if (!tmp) { - flb_errno(); - return -1; - } - flb_plg_trace(ctx->ins, "fd=%i buffer realloc %i -> %zu", - event->fd, conn->buf_size, size); - - conn->buf_data = tmp; - conn->buf_size = size; - available = (conn->buf_size - conn->buf_len) - 1; - } - - /* Read data */ - bytes = flb_io_net_read(connection, - (void *) &conn->buf_data[conn->buf_len], - available); - - if (bytes <= 0) { - flb_plg_trace(ctx->ins, "fd=%i closed connection", event->fd); - splunk_conn_del(conn); - return -1; - } - - flb_plg_trace(ctx->ins, "read()=%zi pre_len=%i now_len=%zi", - bytes, conn->buf_len, conn->buf_len + bytes); - conn->buf_len += bytes; - conn->buf_data[conn->buf_len] = '\0'; - - status = mk_http_parser(&conn->request, &conn->session.parser, - conn->buf_data, conn->buf_len, conn->session.server); - - if (status == MK_HTTP_PARSER_OK) { - /* Do more logic parsing and checks for this request */ - splunk_prot_handle(ctx, conn, &conn->session, &conn->request); - - /* Evict the processed request from the connection buffer and reinitialize - * the HTTP parser. - */ - - request_end = NULL; - - if (NULL != conn->request.data.data) { - request_end = &conn->request.data.data[conn->request.data.len]; - } - else { - request_end = strstr(conn->buf_data, "\r\n\r\n"); - - if(NULL != request_end) { - request_end = &request_end[4]; - } - } - - if (NULL != request_end) { - request_len = (size_t)(request_end - conn->buf_data); - - if (0 < (conn->buf_len - request_len)) { - memmove(conn->buf_data, &conn->buf_data[request_len], - conn->buf_len - request_len); - - conn->buf_data[conn->buf_len - request_len] = '\0'; - conn->buf_len -= request_len; - } - else { - memset(conn->buf_data, 0, request_len); - - conn->buf_len = 0; - } - - /* Reinitialize the parser so the next request is properly - * handled, the additional memset intends to wipe any left over data - * from the headers parsed in the previous request. - */ - memset(&conn->session.parser, 0, sizeof(struct mk_http_parser)); - mk_http_parser_init(&conn->session.parser); - splunk_conn_request_init(&conn->session, &conn->request); - } - } - else if (status == MK_HTTP_PARSER_ERROR) { - splunk_prot_handle_error(ctx, conn, &conn->session, &conn->request); - - /* Reinitialize the parser so the next request is properly - * handled, the additional memset intends to wipe any left over data - * from the headers parsed in the previous request. - */ - memset(&conn->session.parser, 0, sizeof(struct mk_http_parser)); - mk_http_parser_init(&conn->session.parser); - splunk_conn_request_init(&conn->session, &conn->request); - } - - /* FIXME: add Protocol handler here */ - return bytes; - } - - if (event->mask & MK_EVENT_CLOSE) { - flb_plg_trace(ctx->ins, "fd=%i hangup", event->fd); - splunk_conn_del(conn); - return -1; - } - - return 0; - -} - -static void splunk_conn_session_init(struct mk_http_session *session, - struct mk_server *server, - int client_fd) -{ - /* Alloc memory for node */ - session->_sched_init = MK_TRUE; - session->pipelined = MK_FALSE; - session->counter_connections = 0; - session->close_now = MK_FALSE; - session->status = MK_REQUEST_STATUS_INCOMPLETE; - session->server = server; - session->socket = client_fd; - - /* creation time in unix time */ - session->init_time = time(NULL); - - session->channel = mk_channel_new(MK_CHANNEL_SOCKET, session->socket); - session->channel->io = session->server->network; - - /* Init session request list */ - mk_list_init(&session->request_list); - - /* Initialize the parser */ - mk_http_parser_init(&session->parser); -} - -static void splunk_conn_request_init(struct mk_http_session *session, - struct mk_http_request *request) -{ - memset(request, 0, sizeof(struct mk_http_request)); - - mk_http_request_init(session, request, session->server); - - request->in_headers.type = MK_STREAM_IOV; - request->in_headers.dynamic = MK_FALSE; - request->in_headers.cb_consumed = NULL; - request->in_headers.cb_finished = NULL; - request->in_headers.stream = &request->stream; - - mk_list_add(&request->in_headers._head, &request->stream.inputs); - - request->session = session; -} - -struct splunk_conn *splunk_conn_add(struct flb_connection *connection, - struct flb_splunk *ctx) -{ - struct splunk_conn *conn; - int ret; - - conn = flb_calloc(1, sizeof(struct splunk_conn)); - if (!conn) { - flb_errno(); - return NULL; - } - - conn->connection = connection; - - /* Set data for the event-loop */ - MK_EVENT_NEW(&connection->event); - - connection->user_data = conn; - connection->event.type = FLB_ENGINE_EV_CUSTOM; - connection->event.handler = splunk_conn_event; - - /* Connection info */ - conn->ctx = ctx; - conn->buf_len = 0; - - conn->buf_data = flb_malloc(ctx->buffer_chunk_size); - if (!conn->buf_data) { - flb_errno(); - - flb_plg_error(ctx->ins, "could not allocate new connection"); - flb_free(conn); - - return NULL; - } - conn->buf_size = ctx->buffer_chunk_size; - - /* Register instance into the event loop */ - ret = mk_event_add(flb_engine_evl_get(), - connection->fd, - FLB_ENGINE_EV_CUSTOM, - MK_EVENT_READ, - &connection->event); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not register new connection"); - - flb_free(conn->buf_data); - flb_free(conn); - - return NULL; - } - - /* Initialize HTTP Session: this is a custom context for Monkey HTTP */ - splunk_conn_session_init(&conn->session, ctx->server, conn->connection->fd); - - /* Initialize HTTP Request: this is the initial request and it will be reinitialized - * automatically after the request is handled so it can be used for the next one. - */ - splunk_conn_request_init(&conn->session, &conn->request); - - /* Link connection node to parent context list */ - mk_list_add(&conn->_head, &ctx->connections); - - return conn; -} - -int splunk_conn_del(struct splunk_conn *conn) -{ - if (conn->session.channel != NULL) { - mk_channel_release(conn->session.channel); - } - - /* The downstream unregisters the file descriptor from the event-loop - * so there's nothing to be done by the plugin - */ - flb_downstream_conn_release(conn->connection); - - mk_list_del(&conn->_head); - - flb_free(conn->buf_data); - flb_free(conn); - - return 0; -} - -void splunk_conn_release_all(struct flb_splunk *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - struct splunk_conn *conn; - - mk_list_foreach_safe(head, tmp, &ctx->connections) { - conn = mk_list_entry(head, struct splunk_conn, _head); - splunk_conn_del(conn); - } -} diff --git a/fluent-bit/plugins/in_splunk/splunk_conn.h b/fluent-bit/plugins/in_splunk/splunk_conn.h deleted file mode 100644 index f4c955fc0..000000000 --- a/fluent-bit/plugins/in_splunk/splunk_conn.h +++ /dev/null @@ -1,54 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_SPLUNK_CONN -#define FLB_IN_SPLUNK_CONN - -#include -#include - -#include -#include -#include - -struct splunk_conn { - /* Buffer */ - char *buf_data; /* Buffer data */ - int buf_len; /* Data length */ - int buf_size; /* Buffer size */ - - /* - * Parser context: we only held one parser per connection - * which is re-used everytime we have a new request. - */ - struct mk_http_parser parser; - struct mk_http_request request; - struct mk_http_session session; - struct flb_connection *connection; - - void *ctx; /* Plugin parent context */ - struct mk_list _head; /* link to flb_http->connections */ -}; - -struct splunk_conn *splunk_conn_add(struct flb_connection *connection, struct flb_splunk *ctx); -int splunk_conn_del(struct splunk_conn *conn); -void splunk_conn_release_all(struct flb_splunk *ctx); - - -#endif diff --git a/fluent-bit/plugins/in_splunk/splunk_prot.c b/fluent-bit/plugins/in_splunk/splunk_prot.c deleted file mode 100644 index 5b0606083..000000000 --- a/fluent-bit/plugins/in_splunk/splunk_prot.c +++ /dev/null @@ -1,779 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include -#include - -#include "splunk.h" -#include "splunk_conn.h" -#include "splunk_prot.h" - -#define HTTP_CONTENT_JSON 0 -#define HTTP_CONTENT_TEXT 1 -#define HTTP_CONTENT_UNKNOWN 2 - -static int send_response(struct splunk_conn *conn, int http_status, char *message) -{ - struct flb_splunk *context; - size_t sent; - int len; - flb_sds_t out; - - context = (struct flb_splunk *) conn->ctx; - - out = flb_sds_create_size(256); - if (!out) { - return -1; - } - - if (message) { - len = strlen(message); - } - else { - len = 0; - } - - if (http_status == 201) { - flb_sds_printf(&out, - "HTTP/1.1 201 Created \r\n" - "Server: Fluent Bit v%s\r\n" - "%s" - "Content-Length: 0\r\n\r\n", - FLB_VERSION_STR, - context->success_headers_str); - } - else if (http_status == 200) { - flb_sds_printf(&out, - "HTTP/1.1 200 OK\r\n" - "Server: Fluent Bit v%s\r\n" - "%s" - "Content-Length: 0\r\n\r\n", - FLB_VERSION_STR, - context->success_headers_str); - } - else if (http_status == 204) { - flb_sds_printf(&out, - "HTTP/1.1 204 No Content\r\n" - "Server: Fluent Bit v%s\r\n" - "%s" - "\r\n\r\n", - FLB_VERSION_STR, - context->success_headers_str); - } - else if (http_status == 400) { - flb_sds_printf(&out, - "HTTP/1.1 400 Forbidden\r\n" - "Server: Fluent Bit v%s\r\n" - "Content-Length: %i\r\n\r\n%s", - FLB_VERSION_STR, - len, message); - } - else if (http_status == 401) { - flb_sds_printf(&out, - "HTTP/1.1 401 Unauthorized\r\n" - "Server: Fluent Bit v%s\r\n" - "Content-Length: %i\r\n\r\n%s", - FLB_VERSION_STR, - len, message); - } - /* We should check this operations result */ - flb_io_net_write(conn->connection, - (void *) out, - flb_sds_len(out), - &sent); - - flb_sds_destroy(out); - - return 0; -} - -static int send_json_message_response(struct splunk_conn *conn, int http_status, char *message) -{ - size_t sent; - int len; - flb_sds_t out; - - out = flb_sds_create_size(256); - if (!out) { - return -1; - } - - if (message) { - len = strlen(message); - } - else { - len = 0; - } - - if (http_status == 200) { - flb_sds_printf(&out, - "HTTP/1.1 200 OK\r\n" - "Content-Type: application/json\r\n" - "Content-Length: %i\r\n\r\n%s", - len, message); - } - - /* We should check this operations result */ - flb_io_net_write(conn->connection, - (void *) out, - flb_sds_len(out), - &sent); - - flb_sds_destroy(out); - - return 0; -} - -/* implements functionality to get tag from key in record */ -static flb_sds_t tag_key(struct flb_splunk *ctx, msgpack_object *map) -{ - size_t map_size = map->via.map.size; - msgpack_object_kv *kv; - msgpack_object key; - msgpack_object val; - char *key_str = NULL; - char *val_str = NULL; - size_t key_str_size = 0; - size_t val_str_size = 0; - int j; - int check = FLB_FALSE; - int found = FLB_FALSE; - flb_sds_t tag; - - kv = map->via.map.ptr; - - for(j=0; j < map_size; j++) { - check = FLB_FALSE; - found = FLB_FALSE; - key = (kv+j)->key; - if (key.type == MSGPACK_OBJECT_BIN) { - key_str = (char *) key.via.bin.ptr; - key_str_size = key.via.bin.size; - check = FLB_TRUE; - } - if (key.type == MSGPACK_OBJECT_STR) { - key_str = (char *) key.via.str.ptr; - key_str_size = key.via.str.size; - check = FLB_TRUE; - } - - if (check == FLB_TRUE) { - if (strncmp(ctx->tag_key, key_str, key_str_size) == 0) { - val = (kv+j)->val; - if (val.type == MSGPACK_OBJECT_BIN) { - val_str = (char *) val.via.bin.ptr; - val_str_size = val.via.str.size; - found = FLB_TRUE; - break; - } - if (val.type == MSGPACK_OBJECT_STR) { - val_str = (char *) val.via.str.ptr; - val_str_size = val.via.str.size; - found = FLB_TRUE; - break; - } - } - } - } - - if (found == FLB_TRUE) { - tag = flb_sds_create_len(val_str, val_str_size); - if (!tag) { - flb_errno(); - return NULL; - } - return tag; - } - - - flb_plg_error(ctx->ins, "Could not find tag_key %s in record", ctx->tag_key); - return NULL; -} - -/* - * Process a raw text payload for Splunk HEC requests, uses the delimited character to split records, - * return the number of processed bytes - */ -static int process_raw_payload_pack(struct flb_splunk *ctx, flb_sds_t tag, char *buf, size_t size) -{ - int ret = FLB_EVENT_ENCODER_SUCCESS; - - ret = flb_log_event_encoder_begin_record(&ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp(&ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - &ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE("log"), - FLB_LOG_EVENT_STRING_VALUE(buf, size)); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(&ctx->log_encoder); - } - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_log_event_encoder_rollback_record(&ctx->log_encoder); - return -1; - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - if (tag) { - flb_input_log_append(ctx->ins, tag, flb_sds_len(tag), - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - } - else { - /* use default plugin Tag (it internal name, e.g: http.0 */ - flb_input_log_append(ctx->ins, NULL, 0, - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - } - } - else { - flb_plg_error(ctx->ins, "log event encoding error : %d", ret); - } - - return 0; -} - -static void process_flb_log_append(struct flb_splunk *ctx, msgpack_object *record, - flb_sds_t tag, flb_sds_t tag_from_record, - struct flb_time tm) { - int ret; - - ret = flb_log_event_encoder_begin_record(&ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_timestamp( - &ctx->log_encoder, - &tm); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_body_from_msgpack_object( - &ctx->log_encoder, - record); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(&ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - if (tag_from_record) { - flb_input_log_append(ctx->ins, - tag_from_record, - flb_sds_len(tag_from_record), - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - - flb_sds_destroy(tag_from_record); - } - else if (tag) { - flb_input_log_append(ctx->ins, tag, flb_sds_len(tag), - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - } - else { - /* use default plugin Tag (it internal name, e.g: http.0 */ - flb_input_log_append(ctx->ins, NULL, 0, - ctx->log_encoder.output_buffer, - ctx->log_encoder.output_length); - } - } - else { - flb_plg_error(ctx->ins, "Error encoding record : %d", ret); - } -} - -static int process_json_payload_pack(struct flb_splunk *ctx, flb_sds_t tag, char *buf, size_t size) -{ - size_t off = 0; - msgpack_unpacked result; - struct flb_time tm; - int i = 0; - msgpack_object *obj; - msgpack_object record; - flb_sds_t tag_from_record = NULL; - - flb_time_get(&tm); - - msgpack_unpacked_init(&result); - while (msgpack_unpack_next(&result, buf, size, &off) == MSGPACK_UNPACK_SUCCESS) { - if (result.data.type == MSGPACK_OBJECT_MAP) { - tag_from_record = NULL; - if (ctx->tag_key) { - tag_from_record = tag_key(ctx, &result.data); - } - - process_flb_log_append(ctx, &result.data, tag, tag_from_record, tm); - - flb_log_event_encoder_reset(&ctx->log_encoder); - } - else if (result.data.type == MSGPACK_OBJECT_ARRAY) { - obj = &result.data; - for (i = 0; i < obj->via.array.size; i++) - { - record = obj->via.array.ptr[i]; - - tag_from_record = NULL; - if (ctx->tag_key) { - tag_from_record = tag_key(ctx, &record); - } - - process_flb_log_append(ctx, &record, tag, tag_from_record, tm); - - /* TODO : Optimize this - * - * This is wasteful, considering that we are emitting a series - * of records we should start and commit each one and then - * emit them all at once after the loop. - */ - - flb_log_event_encoder_reset(&ctx->log_encoder); - } - - break; - } - else { - flb_plg_error(ctx->ins, "skip record from invalid type: %i", - result.data.type); - - msgpack_unpacked_destroy(&result); - - return -1; - } - } - - msgpack_unpacked_destroy(&result); - - return 0; -} - -static ssize_t parse_hec_payload_json(struct flb_splunk *ctx, flb_sds_t tag, - char *payload, size_t size) -{ - int ret; - int out_size; - char *pack; - struct flb_pack_state pack_state; - - /* Initialize packer */ - flb_pack_state_init(&pack_state); - - /* Pack JSON as msgpack */ - ret = flb_pack_json_state(payload, size, - &pack, &out_size, &pack_state); - flb_pack_state_reset(&pack_state); - - /* Handle exceptions */ - if (ret == FLB_ERR_JSON_PART) { - flb_plg_warn(ctx->ins, "JSON data is incomplete, skipping"); - return -1; - } - else if (ret == FLB_ERR_JSON_INVAL) { - flb_plg_warn(ctx->ins, "invalid JSON message, skipping"); - return -1; - } - else if (ret == -1) { - return -1; - } - - /* Process the packaged JSON and return the last byte used */ - process_json_payload_pack(ctx, tag, pack, out_size); - flb_free(pack); - - return 0; -} - -static int validate_auth_header(struct flb_splunk *ctx, struct mk_http_request *request) -{ - struct mk_http_header *auth_header = NULL; - - if (ctx->auth_header == NULL) { - return SPLUNK_AUTH_UNAUTH; - } - - auth_header = mk_http_header_get(MK_HEADER_AUTHORIZATION, request, NULL, 0); - - if (auth_header == NULL) { - return SPLUNK_AUTH_MISSING_CRED; - } - - if (auth_header != NULL && auth_header->val.len > 0) { - if (strncmp(ctx->auth_header, - auth_header->val.data, - strlen(ctx->auth_header)) == 0) { - return SPLUNK_AUTH_SUCCESS; - } - else { - return SPLUNK_AUTH_UNAUTHORIZED; - } - } - else { - return SPLUNK_AUTH_MISSING_CRED; - } - - return SPLUNK_AUTH_SUCCESS; -} - -static int handle_hec_payload(struct flb_splunk *ctx, int content_type, - flb_sds_t tag, char *buf, size_t size) -{ - int ret = -1; - - if (content_type == HTTP_CONTENT_JSON) { - ret = parse_hec_payload_json(ctx, tag, buf, size); - } - else if (content_type == HTTP_CONTENT_TEXT) { - ret = process_raw_payload_pack(ctx, tag, buf, size); - } - else if (content_type == HTTP_CONTENT_UNKNOWN) { - if (buf[0] == '{') { - ret = parse_hec_payload_json(ctx, tag, buf, size); - } - else { - ret = process_raw_payload_pack(ctx, tag, buf, size); - } - } - - return ret; -} - -static int process_hec_payload(struct flb_splunk *ctx, struct splunk_conn *conn, - flb_sds_t tag, - struct mk_http_session *session, - struct mk_http_request *request) -{ - int i = 0; - int ret = 0; - int type = -1; - struct mk_http_header *header; - int extra_size = -1; - struct mk_http_header *headers_extra; - int gzip_compressed = FLB_FALSE; - void *gz_data = NULL; - size_t gz_size = -1; - - header = &session->parser.headers[MK_HEADER_CONTENT_TYPE]; - if (header->key.data == NULL) { - send_response(conn, 400, "error: header 'Content-Type' is not set\n"); - return -1; - } - - if (header->val.len == 16 && - strncasecmp(header->val.data, "application/json", 16) == 0) { - type = HTTP_CONTENT_JSON; - } - else if (header->val.len == 10 && - strncasecmp(header->val.data, "text/plain", 10) == 0) { - type = HTTP_CONTENT_TEXT; - } - else { - /* Not neccesary to specify content-type for Splunk HEC. */ - flb_plg_debug(ctx->ins, "Mark as unknown type for ingested payloads"); - type = HTTP_CONTENT_UNKNOWN; - } - - if (request->data.len <= 0) { - send_response(conn, 400, "error: no payload found\n"); - return -1; - } - - extra_size = session->parser.headers_extra_count; - if (extra_size > 0) { - for (i = 0; i < extra_size; i++) { - headers_extra = &session->parser.headers_extra[i]; - if (headers_extra->key.len == 16 && - strncasecmp(headers_extra->key.data, "Content-Encoding", 16) == 0) { - if (headers_extra->val.len == 4 && - strncasecmp(headers_extra->val.data, "gzip", 4) == 0) { - flb_plg_debug(ctx->ins, "body is gzipped"); - gzip_compressed = FLB_TRUE; - } - } - } - } - - if (gzip_compressed == FLB_TRUE) { - ret = flb_gzip_uncompress((void *) request->data.data, request->data.len, - &gz_data, &gz_size); - if (ret == -1) { - flb_plg_error(ctx->ins, "gzip uncompress is failed"); - return -1; - } - - ret = handle_hec_payload(ctx, type, tag, gz_data, gz_size); - flb_free(gz_data); - } - else { - ret = handle_hec_payload(ctx, type, tag, request->data.data, request->data.len); - } - - return 0; -} - -static int process_hec_raw_payload(struct flb_splunk *ctx, struct splunk_conn *conn, - flb_sds_t tag, - struct mk_http_session *session, - struct mk_http_request *request) -{ - int ret = -1; - struct mk_http_header *header; - - header = &session->parser.headers[MK_HEADER_CONTENT_TYPE]; - if (header->key.data == NULL) { - send_response(conn, 400, "error: header 'Content-Type' is not set\n"); - return -1; - } - else if (header->val.len != 10 || - strncasecmp(header->val.data, "text/plain", 10) != 0) { - /* Not neccesary to specify content-type for Splunk HEC. */ - flb_plg_debug(ctx->ins, "Mark as unknown type for ingested payloads"); - } - - if (request->data.len <= 0) { - send_response(conn, 400, "error: no payload found\n"); - return -1; - } - - /* Always handle as raw type of payloads here */ - ret = process_raw_payload_pack(ctx, tag, request->data.data, request->data.len); - - return ret; -} - -static inline int mk_http_point_header(mk_ptr_t *h, - struct mk_http_parser *parser, int key) -{ - struct mk_http_header *header; - - header = &parser->headers[key]; - if (header->type == key) { - h->data = header->val.data; - h->len = header->val.len; - return 0; - } - else { - h->data = NULL; - h->len = -1; - } - - return -1; -} - -/* - * Handle an incoming request. It perform extra checks over the request, if - * everything is OK, it enqueue the incoming payload. - */ -int splunk_prot_handle(struct flb_splunk *ctx, struct splunk_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request) -{ - int i; - int ret; - int len; - char *uri; - char *qs; - off_t diff; - flb_sds_t tag; - struct mk_http_header *header; - - if (request->uri.data[0] != '/') { - send_response(conn, 400, "error: invalid request\n"); - return -1; - } - - /* Decode URI */ - uri = mk_utils_url_decode(request->uri); - if (!uri) { - uri = mk_mem_alloc_z(request->uri.len + 1); - if (!uri) { - return -1; - } - memcpy(uri, request->uri.data, request->uri.len); - uri[request->uri.len] = '\0'; - } - - /* Try to match a query string so we can remove it */ - qs = strchr(uri, '?'); - if (qs) { - /* remove the query string part */ - diff = qs - uri; - uri[diff] = '\0'; - } - - /* Refer the tag at first*/ - if (ctx->ins->tag && !ctx->ins->tag_default) { - tag = flb_sds_create(ctx->ins->tag); - if (tag == NULL) { - return -1; - } - } - else { - /* Compose the query string using the URI */ - len = strlen(uri); - - if (len == 1) { - tag = NULL; /* use default tag */ - } - else { - /* New tag skipping the URI '/' */ - tag = flb_sds_create_len(&uri[1], len - 1); - if (!tag) { - mk_mem_free(uri); - return -1; - } - - /* Sanitize, only allow alphanum chars */ - for (i = 0; i < flb_sds_len(tag); i++) { - if (!isalnum(tag[i]) && tag[i] != '_' && tag[i] != '.') { - tag[i] = '_'; - } - } - } - } - - /* Check if we have a Host header: Hostname ; port */ - mk_http_point_header(&request->host, &session->parser, MK_HEADER_HOST); - - /* Header: Connection */ - mk_http_point_header(&request->connection, &session->parser, - MK_HEADER_CONNECTION); - - /* HTTP/1.1 needs Host header */ - if (request->host.data == NULL && request->protocol == MK_HTTP_PROTOCOL_11) { - flb_sds_destroy(tag); - mk_mem_free(uri); - - return -1; - } - - /* Should we close the session after this request ? */ - mk_http_keepalive_check(session, request, ctx->server); - - /* Content Length */ - header = &session->parser.headers[MK_HEADER_CONTENT_LENGTH]; - if (header->type == MK_HEADER_CONTENT_LENGTH) { - request->_content_length.data = header->val.data; - request->_content_length.len = header->val.len; - } - else { - request->_content_length.data = NULL; - } - - if (request->method == MK_METHOD_GET) { - /* Handle health minotoring of splunk hec endpoint for load balancers */ - if (strcasecmp(uri, "/services/collector/health") == 0) { - send_json_message_response(conn, 200, "{\"text\":\"Success\",\"code\":200}"); - } - else { - send_response(conn, 400, "error: invalid HTTP endpoint\n"); - } - - flb_sds_destroy(tag); - mk_mem_free(uri); - - return 0; - } - - /* Under services/collector endpoints are required for - * authentication if provided splunk_token */ - ret = validate_auth_header(ctx, request); - if (ret < 0){ - send_response(conn, 401, "error: unauthroized\n"); - if (ret == SPLUNK_AUTH_MISSING_CRED) { - flb_plg_warn(ctx->ins, "missing credentials in request headers"); - } - else if (ret == SPLUNK_AUTH_UNAUTHORIZED) { - flb_plg_warn(ctx->ins, "wrong credentials in request headers"); - } - - flb_sds_destroy(tag); - mk_mem_free(uri); - - return -1; - } - - /* Handle every ingested payload cleanly */ - flb_log_event_encoder_reset(&ctx->log_encoder); - - if (request->method == MK_METHOD_POST) { - if (strcasecmp(uri, "/services/collector/raw") == 0) { - ret = process_hec_raw_payload(ctx, conn, tag, session, request); - - if (!ret) { - send_json_message_response(conn, 400, "{\"text\":\"Invalid data format\",\"code\":6}"); - } - send_json_message_response(conn, 200, "{\"text\":\"Success\",\"code\":0}"); - } - else if (strcasecmp(uri, "/services/collector/event") == 0 || - strcasecmp(uri, "/services/collector") == 0) { - ret = process_hec_payload(ctx, conn, tag, session, request); - - if (!ret) { - send_json_message_response(conn, 400, "{\"text\":\"Invalid data format\",\"code\":6}"); - } - send_json_message_response(conn, 200, "{\"text\":\"Success\",\"code\":0}"); - } - else { - send_response(conn, 400, "error: invalid HTTP endpoint\n"); - - flb_sds_destroy(tag); - mk_mem_free(uri); - - return -1; - } - } - else { - /* HEAD, PUT, PATCH, and DELETE methods are prohibited to use.*/ - - flb_sds_destroy(tag); - mk_mem_free(uri); - - send_response(conn, 400, "error: invalid HTTP method\n"); - return -1; - } - - flb_sds_destroy(tag); - mk_mem_free(uri); - - return ret; -} - -/* - * Handle an incoming request which has resulted in an http parser error. - */ -int splunk_prot_handle_error(struct flb_splunk *ctx, struct splunk_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request) -{ - send_response(conn, 400, "error: invalid request\n"); - return -1; -} diff --git a/fluent-bit/plugins/in_splunk/splunk_prot.h b/fluent-bit/plugins/in_splunk/splunk_prot.h deleted file mode 100644 index 100f12d2e..000000000 --- a/fluent-bit/plugins/in_splunk/splunk_prot.h +++ /dev/null @@ -1,36 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_SPLUNK_PROT -#define FLB_IN_SPLUNK_PROT - -#define SPLUNK_AUTH_UNAUTH 1 -#define SPLUNK_AUTH_SUCCESS 0 -#define SPLUNK_AUTH_MISSING_CRED -1 -#define SPLUNK_AUTH_UNAUTHORIZED -2 - -int splunk_prot_handle(struct flb_splunk *ctx, struct splunk_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request); - -int splunk_prot_handle_error(struct flb_splunk *ctx, struct splunk_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request); - -#endif diff --git a/fluent-bit/plugins/in_statsd/CMakeLists.txt b/fluent-bit/plugins/in_statsd/CMakeLists.txt deleted file mode 100644 index 5b9dde230..000000000 --- a/fluent-bit/plugins/in_statsd/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - statsd.c) - -FLB_PLUGIN(in_statsd "${src}" "") diff --git a/fluent-bit/plugins/in_statsd/statsd.c b/fluent-bit/plugins/in_statsd/statsd.c deleted file mode 100644 index 0cccb679a..000000000 --- a/fluent-bit/plugins/in_statsd/statsd.c +++ /dev/null @@ -1,386 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#define MAX_PACKET_SIZE 65536 -#define DEFAULT_LISTEN "0.0.0.0" -#define DEFAULT_PORT 8125 - -#define STATSD_TYPE_COUNTER 1 -#define STATSD_TYPE_GAUGE 2 -#define STATSD_TYPE_TIMER 3 -#define STATSD_TYPE_SET 4 - -struct flb_statsd { - char *buf; /* buffer */ - char listen[256]; /* listening address (RFC-2181) */ - char port[6]; /* listening port (RFC-793) */ - flb_sockfd_t server_fd; /* server socket */ - flb_pipefd_t coll_fd; /* server handler */ - struct flb_input_instance *ins; /* input instance */ - struct flb_log_event_encoder *log_encoder; -}; - -/* - * The "statsd_message" represents a single line in UDP packet. - * It's just a bunch of pointers to ephemeral buffer. - */ -struct statsd_message { - char *bucket; - int bucket_len; - char *value; - int value_len; - int type; - double sample_rate; -}; - -static int get_statsd_type(char *str) -{ - switch (*str) { - case 'g': - return STATSD_TYPE_GAUGE; - case 's': - return STATSD_TYPE_SET; - case 'c': - return STATSD_TYPE_COUNTER; - case 'm': - if (*(str + 1) == 's') { - return STATSD_TYPE_TIMER; - } - } - return STATSD_TYPE_COUNTER; -} - -static int is_incremental(char *str) -{ - return (*str == '+' || *str == '-'); -} - -static int statsd_process_message(struct flb_statsd *ctx, - struct statsd_message *m) -{ - int ret; - - ret = flb_log_event_encoder_begin_record(ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp(ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - switch (m->type) { - case STATSD_TYPE_COUNTER: - ret = flb_log_event_encoder_append_body_values( - ctx->log_encoder, - - FLB_LOG_EVENT_CSTRING_VALUE("type"), - FLB_LOG_EVENT_CSTRING_VALUE("counter"), - FLB_LOG_EVENT_CSTRING_VALUE("bucket"), - FLB_LOG_EVENT_STRING_VALUE(m->bucket, m->bucket_len), - FLB_LOG_EVENT_CSTRING_VALUE("value"), - FLB_LOG_EVENT_DOUBLE_VALUE(strtod(m->value, NULL)), - FLB_LOG_EVENT_CSTRING_VALUE("sample_rate"), - FLB_LOG_EVENT_DOUBLE_VALUE(m->sample_rate)); - - break; - case STATSD_TYPE_GAUGE: - ret = flb_log_event_encoder_append_body_values( - ctx->log_encoder, - - FLB_LOG_EVENT_CSTRING_VALUE("type"), - FLB_LOG_EVENT_CSTRING_VALUE("gauge"), - FLB_LOG_EVENT_CSTRING_VALUE("bucket"), - FLB_LOG_EVENT_STRING_VALUE(m->bucket, m->bucket_len), - FLB_LOG_EVENT_CSTRING_VALUE("value"), - FLB_LOG_EVENT_DOUBLE_VALUE(strtod(m->value, NULL)), - FLB_LOG_EVENT_CSTRING_VALUE("incremental"), - FLB_LOG_EVENT_INT64_VALUE(is_incremental(m->value))); - break; - case STATSD_TYPE_TIMER: - ret = flb_log_event_encoder_append_body_values( - ctx->log_encoder, - - FLB_LOG_EVENT_CSTRING_VALUE("type"), - FLB_LOG_EVENT_CSTRING_VALUE("timer"), - FLB_LOG_EVENT_CSTRING_VALUE("bucket"), - FLB_LOG_EVENT_STRING_VALUE(m->bucket, m->bucket_len), - FLB_LOG_EVENT_CSTRING_VALUE("value"), - FLB_LOG_EVENT_DOUBLE_VALUE(strtod(m->value, NULL)), - FLB_LOG_EVENT_CSTRING_VALUE("sample_rate"), - FLB_LOG_EVENT_DOUBLE_VALUE(m->sample_rate)); - - case STATSD_TYPE_SET: - ret = flb_log_event_encoder_append_body_values( - ctx->log_encoder, - - FLB_LOG_EVENT_CSTRING_VALUE("type"), - FLB_LOG_EVENT_CSTRING_VALUE("set"), - FLB_LOG_EVENT_CSTRING_VALUE("bucket"), - FLB_LOG_EVENT_STRING_VALUE(m->bucket, m->bucket_len), - FLB_LOG_EVENT_CSTRING_VALUE("value"), - FLB_LOG_EVENT_STRING_VALUE(m->value, m->value_len)); - break; - } - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(ctx->log_encoder); - } - - return ret; -} - -static int statsd_process_line(struct flb_statsd *ctx, char *line) -{ - char *colon, *bar, *atmark; - struct statsd_message m; - - /* - * bucket:value|type|@sample_rate - * ------ - */ - colon = strchr(line, ':'); - if (colon == NULL) { - flb_plg_error(ctx->ins, "no bucket name found"); - return -1; - } - m.bucket = line; - m.bucket_len = (colon - line); - - /* - * bucket:value|type|@sample_rate - * ---- - */ - bar = strchr(colon + 1, '|'); - if (bar == NULL) { - flb_plg_error(ctx->ins, "no metric type found"); - return -1; - } - m.type = get_statsd_type(bar + 1); - - /* - * bucket:value|type|@sample_rate - * ----- - */ - m.value = colon + 1; - m.value_len = (bar - colon - 1); - - /* - * bucket:value|type|@sample_rate - * ------------ - */ - atmark = strstr(bar + 1, "|@"); - if (atmark == NULL || atof(atmark + 2) == 0) { - m.sample_rate = 1.0; - } - else { - m.sample_rate = atof(atmark + 2); - } - - return statsd_process_message(ctx, &m); -} - - -static int cb_statsd_receive(struct flb_input_instance *ins, - struct flb_config *config, void *data) -{ - int ret; - char *line; - int len; - struct flb_statsd *ctx = data; - - /* Receive a UDP datagram */ - len = recv(ctx->server_fd, ctx->buf, MAX_PACKET_SIZE - 1, 0); - if (len < 0) { - flb_errno(); - return -1; - } - ctx->buf[len] = '\0'; - - ret = FLB_EVENT_ENCODER_SUCCESS; - /* Process all messages in buffer */ - line = strtok(ctx->buf, "\n"); - while (line != NULL) { - flb_plg_trace(ctx->ins, "received a line: '%s'", line); - - ret = statsd_process_line(ctx, line); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, "failed to process line: '%s'", line); - - break; - } - - line = strtok(NULL, "\n"); - } - - if (ctx->log_encoder->output_length > 0) { - flb_input_log_append(ctx->ins, NULL, 0, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); - } - else { - flb_plg_error(ctx->ins, "log event encoding error : %d", ret); - } - - flb_log_event_encoder_reset(ctx->log_encoder); - - return 0; -} - -static int cb_statsd_init(struct flb_input_instance *ins, - struct flb_config *config, void *data) -{ - struct flb_statsd *ctx; - char *listen; - int port; - int ret; - - ctx = flb_calloc(1, sizeof(struct flb_statsd)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->ins = ins; - - ctx->log_encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ctx->log_encoder == NULL) { - flb_plg_error(ins, "could not initialize event encoder"); - flb_free(ctx); - - return -1; - } - - ctx->buf = flb_malloc(MAX_PACKET_SIZE); - if (!ctx->buf) { - flb_errno(); - flb_log_event_encoder_destroy(ctx->log_encoder); - flb_free(ctx); - return -1; - } - - /* Load the config map */ - ret = flb_input_config_map_set(ins, (void *)ctx); - if (ret == -1) { - flb_plg_error(ins, "unable to load configuration"); - flb_log_event_encoder_destroy(ctx->log_encoder); - flb_free(ctx); - return -1; - } - - /* Listening address */ - if (ins->host.listen) { - listen = ins->host.listen; - } - else { - listen = DEFAULT_LISTEN; - } - strncpy(ctx->listen, listen, sizeof(ctx->listen) - 1); - - /* Listening port */ - if (ins->host.port) { - port = ins->host.port; - } - else { - port = DEFAULT_PORT; - } - snprintf(ctx->port, sizeof(ctx->port), "%hu", (unsigned short) port); - - /* Export plugin context */ - flb_input_set_context(ins, ctx); - - /* Accepts metrics from UDP connections. */ - ctx->server_fd = flb_net_server_udp(ctx->port, ctx->listen); - if (ctx->server_fd == -1) { - flb_plg_error(ctx->ins, "can't bind to %s:%s", ctx->listen, ctx->port); - flb_log_event_encoder_destroy(ctx->log_encoder); - flb_free(ctx->buf); - flb_free(ctx); - return -1; - } - - /* Set up the UDP connection callback */ - ctx->coll_fd = flb_input_set_collector_socket(ins, cb_statsd_receive, - ctx->server_fd, config); - if (ctx->coll_fd == -1) { - flb_plg_error(ctx->ins, "cannot set up connection callback "); - flb_log_event_encoder_destroy(ctx->log_encoder); - flb_socket_close(ctx->server_fd); - flb_free(ctx->buf); - flb_free(ctx); - return -1; - } - - flb_plg_info(ctx->ins, "start UDP server on %s:%s", ctx->listen, ctx->port); - return 0; -} - -static void cb_statsd_pause(void *data, struct flb_config *config) -{ - struct flb_statsd *ctx = data; - flb_input_collector_pause(ctx->coll_fd, ctx->ins); -} - -static void cb_statsd_resume(void *data, struct flb_config *config) -{ - struct flb_statsd *ctx = data; - flb_input_collector_resume(ctx->coll_fd, ctx->ins); -} - -static int cb_statsd_exit(void *data, struct flb_config *config) -{ - struct flb_statsd *ctx = data; - - if (ctx->log_encoder != NULL) { - flb_log_event_encoder_destroy(ctx->log_encoder); - } - - flb_socket_close(ctx->server_fd); - flb_free(ctx->buf); - flb_free(ctx); - - return 0; -} - -static struct flb_config_map config_map[] = { - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_input_plugin in_statsd_plugin = { - .name = "statsd", - .description = "StatsD input plugin", - .cb_init = cb_statsd_init, - .cb_pre_run = NULL, - .cb_collect = NULL, - .cb_ingest = NULL, - .cb_flush_buf = NULL, - .cb_pause = cb_statsd_pause, - .cb_resume = cb_statsd_resume, - .cb_exit = cb_statsd_exit, - .config_map = config_map, - .flags = FLB_INPUT_NET_SERVER, -}; diff --git a/fluent-bit/plugins/in_stdin/CMakeLists.txt b/fluent-bit/plugins/in_stdin/CMakeLists.txt deleted file mode 100644 index 3c2e2bdfe..000000000 --- a/fluent-bit/plugins/in_stdin/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -# FIXME: there is something wrong when linking objects and this -# static plugin, I should not require to link to a specific symbol -# if the object was already linked from fluent-bit core on src/, also -# jsmn should not be required. - -set(src - in_stdin.c - ) - -FLB_PLUGIN(in_stdin "${src}" "") diff --git a/fluent-bit/plugins/in_stdin/in_stdin.c b/fluent-bit/plugins/in_stdin/in_stdin.c deleted file mode 100644 index ff3114067..000000000 --- a/fluent-bit/plugins/in_stdin/in_stdin.c +++ /dev/null @@ -1,472 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include - -#include "in_stdin.h" - -static inline void consume_bytes(char *buf, int bytes, int length) -{ - memmove(buf, buf + bytes, length - bytes); -} - -static inline int process_pack(struct flb_in_stdin_config *ctx, - char *data, size_t data_size) -{ - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - msgpack_unpacked result; - msgpack_object entry; - int ret; - size_t off; - - ret = flb_log_event_decoder_init(&log_decoder, NULL, 0); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - return -1; - } - - /* Queue the data with time field */ - msgpack_unpacked_init(&result); - - off = 0; - while (msgpack_unpack_next(&result, data, data_size, &off) == MSGPACK_UNPACK_SUCCESS) { - entry = result.data; - - if (entry.type == MSGPACK_OBJECT_MAP) { - ret = flb_log_event_encoder_begin_record(ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp(ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_body_from_msgpack_object( - ctx->log_encoder, &entry); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = 0; - } - else { - ret = -1; - - break; - } - } - else if (entry.type == MSGPACK_OBJECT_ARRAY) { - ret = flb_event_decoder_decode_object(&log_decoder, - &log_event, - &entry); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - ret = -1; - - break; - } - - ret = flb_log_event_encoder_begin_record(ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_timestamp(ctx->log_encoder, - &log_event.timestamp); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_metadata_from_msgpack_object( - ctx->log_encoder, log_event.metadata); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_body_from_msgpack_object( - ctx->log_encoder, log_event.body); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = 0; - } - else { - ret = -1; - - break; - } - } - else { - /* - * Upon exception, acknowledge the user about the problem but continue - * working, do not discard valid JSON entries. - */ - flb_plg_error(ctx->ins, "invalid record found, " - "it's not a JSON map or array"); - ret = -1; - break; - } - } - - flb_log_event_decoder_destroy(&log_decoder); - - msgpack_unpacked_destroy(&result); - - return ret; -} - -static inline int pack_regex(struct flb_in_stdin_config *ctx, - struct flb_time *t, char *data, size_t data_size) -{ - int ret; - - ret = flb_log_event_encoder_begin_record(ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_timestamp(ctx->log_encoder, t); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_body_from_raw_msgpack( - ctx->log_encoder, data, data_size); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = 0; - } - else { - ret = -1; - } - - return ret; -} - -static int in_stdin_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - int bytes = 0; - int pack_size; - int ret; - char *pack; - void *out_buf; - size_t out_size; - struct flb_time out_time; - struct flb_in_stdin_config *ctx = in_context; - - bytes = read(ctx->fd, - ctx->buf + ctx->buf_len, - ctx->buf_size - ctx->buf_len - 1); - flb_plg_trace(ctx->ins, "stdin read() = %i", bytes); - - if (bytes == 0) { - flb_plg_warn(ctx->ins, "end of file (stdin closed by remote end)"); - } - - if (bytes <= 0) { - flb_input_collector_pause(ctx->coll_fd, ctx->ins); - flb_engine_exit(config); - return -1; - } - ctx->buf_len += bytes; - ctx->buf[ctx->buf_len] = '\0'; - - while (ctx->buf_len > 0) { - /* Try built-in JSON parser */ - if (!ctx->parser) { - ret = flb_pack_json_state(ctx->buf, ctx->buf_len, - &pack, &pack_size, &ctx->pack_state); - if (ret == FLB_ERR_JSON_PART) { - flb_plg_debug(ctx->ins, "data incomplete, waiting for more..."); - return 0; - } - else if (ret == FLB_ERR_JSON_INVAL) { - flb_plg_debug(ctx->ins, "invalid JSON message, skipping"); - flb_pack_state_reset(&ctx->pack_state); - flb_pack_state_init(&ctx->pack_state); - ctx->pack_state.multiple = FLB_TRUE; - ctx->buf_len = 0; - return -1; - } - - /* Process valid packaged records */ - process_pack(ctx, pack, pack_size); - - /* Move out processed bytes */ - consume_bytes(ctx->buf, ctx->pack_state.last_byte, ctx->buf_len); - ctx->buf_len -= ctx->pack_state.last_byte; - ctx->buf[ctx->buf_len] = '\0'; - - flb_pack_state_reset(&ctx->pack_state); - flb_pack_state_init(&ctx->pack_state); - ctx->pack_state.multiple = FLB_TRUE; - - flb_free(pack); - - if (ctx->log_encoder->output_length > 0) { - flb_input_log_append(ctx->ins, NULL, 0, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); - } - - flb_log_event_encoder_reset(ctx->log_encoder); - - return 0; - } - else { - /* Reset time for each line */ - flb_time_zero(&out_time); - - /* Use the defined parser */ - ret = flb_parser_do(ctx->parser, ctx->buf, ctx->buf_len, - &out_buf, &out_size, &out_time); - - if (ret >= 0) { - if (flb_time_to_nanosec(&out_time) == 0L) { - flb_time_get(&out_time); - } - pack_regex(ctx, &out_time, out_buf, out_size); - flb_free(out_buf); - - if (ctx->log_encoder->output_length > 0) { - flb_input_log_append(ctx->ins, NULL, 0, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); - } - - flb_log_event_encoder_reset(ctx->log_encoder); - } - else { - /* we need more data ? */ - flb_plg_trace(ctx->ins, "data mismatch or incomplete : %d", ret); - return 0; - } - } - - if (ret == ctx->buf_len) { - ctx->buf_len = 0; - break; - } - else if (ret >= 0) { - /* - * 'ret' is the last byte consumed by the regex engine, we need - * to advance it position. - */ - ret++; - consume_bytes(ctx->buf, ret, ctx->buf_len); - ctx->buf_len -= ret; - ctx->buf[ctx->buf_len] = '\0'; - } - } - - return 0; -} - -/* Read stdin config*/ -static int in_stdin_config_init(struct flb_in_stdin_config *ctx, - struct flb_input_instance *in, - struct flb_config *config) -{ - int ret; - - ctx->buf_size = DEFAULT_BUF_SIZE; - ctx->buf = NULL; - ctx->buf_len = 0; - ctx->ins = in; - - ret = flb_input_config_map_set(in, (void *)ctx); - if (ret == -1) { - return -1; - } - - /* parser settings */ - if (ctx->parser_name) { - ctx->parser = flb_parser_get(ctx->parser_name, config); - if (!ctx->parser) { - flb_plg_error(ctx->ins, "requested parser '%s' not found", ctx->parser_name); - return -1; - } - } - - /* buffer size setting */ - if (ctx->buf_size == -1) { - flb_plg_error(ctx->ins, "buffer_size is invalid"); - return -1; - } - else if (ctx->buf_size < DEFAULT_BUF_SIZE) { - flb_plg_error(ctx->ins, "buffer_size '%zu' must be at least %i bytes", - ctx->buf_size, DEFAULT_BUF_SIZE); - return -1; - } - - flb_plg_debug(ctx->ins, "buf_size=%zu", ctx->buf_size); - return 0; -} - -static void in_stdin_config_destroy(struct flb_in_stdin_config *ctx) -{ - if (!ctx) { - return; - } - - if (ctx->log_encoder != NULL) { - flb_log_event_encoder_destroy(ctx->log_encoder); - } - - /* release buffer */ - if (ctx->buf) { - flb_free(ctx->buf); - } - flb_free(ctx); -} - -/* Initialize plugin */ -static int in_stdin_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - int fd; - int ret; - struct flb_in_stdin_config *ctx; - - /* Allocate space for the configuration context */ - ctx = flb_calloc(1, sizeof(struct flb_in_stdin_config)); - if (!ctx) { - return -1; - } - - ctx->log_encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ctx->log_encoder == NULL) { - flb_plg_error(in, "could not initialize event encoder"); - - goto init_error; - } - - /* Initialize stdin config */ - ret = in_stdin_config_init(ctx, in, config); - if (ret < 0) { - goto init_error; - } - - ctx->buf = flb_malloc(ctx->buf_size); - if (!ctx->buf) { - flb_errno(); - goto init_error; - } - - /* Clone the standard input file descriptor */ - fd = dup(STDIN_FILENO); - if (fd == -1) { - flb_errno(); - flb_plg_error(ctx->ins, "Could not open standard input!"); - goto init_error; - } - ctx->fd = fd; - - /* Always initialize built-in JSON pack state */ - flb_pack_state_init(&ctx->pack_state); - ctx->pack_state.multiple = FLB_TRUE; - - /* Set the context */ - flb_input_set_context(in, ctx); - - /* Collect upon data available on the standard input */ - ret = flb_input_set_collector_event(in, - in_stdin_collect, - ctx->fd, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "Could not set collector for STDIN input plugin"); - goto init_error; - } - ctx->coll_fd = ret; - - return 0; - -init_error: - in_stdin_config_destroy(ctx); - - return -1; -} - -/* Cleanup serial input */ -static int in_stdin_exit(void *in_context, struct flb_config *config) -{ - struct flb_in_stdin_config *ctx = in_context; - - if (!ctx) { - return 0; - } - - if (ctx->fd >= 0) { - close(ctx->fd); - } - flb_pack_state_reset(&ctx->pack_state); - in_stdin_config_destroy(ctx); - - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "parser", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_in_stdin_config, parser_name), - "Set and use a fluent-bit parser" - }, - { - FLB_CONFIG_MAP_SIZE, "buffer_size", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_in_stdin_config, buf_size), - "Set the read buffer size" - }, - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_input_plugin in_stdin_plugin = { - .name = "stdin", - .description = "Standard Input", - .cb_init = in_stdin_init, - .cb_pre_run = NULL, - .cb_collect = in_stdin_collect, - .cb_flush_buf = NULL, - .cb_exit = in_stdin_exit, - .config_map = config_map -}; diff --git a/fluent-bit/plugins/in_stdin/in_stdin.h b/fluent-bit/plugins/in_stdin/in_stdin.h deleted file mode 100644 index 0c165809b..000000000 --- a/fluent-bit/plugins/in_stdin/in_stdin.h +++ /dev/null @@ -1,48 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_STDIN_H -#define FLB_IN_STDIN_H - -#include -#include -#include -#include - -#define DEFAULT_BUF_SIZE 16000 - -/* STDIN Input configuration & context */ -struct flb_in_stdin_config { - int fd; /* stdin file descriptor */ - int coll_fd; /* collector fd */ - size_t buf_size; /* size of a buffer */ - int buf_len; /* read buffer length */ - char *buf; /* read buffer */ - flb_sds_t parser_name; /* name of the parser */ - - /* Parser / Format */ - struct flb_parser *parser; - struct flb_pack_state pack_state; - struct flb_input_instance *ins; - struct flb_log_event_encoder *log_encoder; -}; - -extern struct flb_input_plugin in_stdin_plugin; - -#endif diff --git a/fluent-bit/plugins/in_storage_backlog/CMakeLists.txt b/fluent-bit/plugins/in_storage_backlog/CMakeLists.txt deleted file mode 100644 index ae2da6341..000000000 --- a/fluent-bit/plugins/in_storage_backlog/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -set(src - sb.c - ) - -FLB_PLUGIN(in_storage_backlog "${src}" "chunkio-static") diff --git a/fluent-bit/plugins/in_storage_backlog/sb.c b/fluent-bit/plugins/in_storage_backlog/sb.c deleted file mode 100644 index 1380caf8a..000000000 --- a/fluent-bit/plugins/in_storage_backlog/sb.c +++ /dev/null @@ -1,713 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#ifndef FLB_SYSTEM_WINDOWS -#include -#endif - -struct sb_out_chunk { - struct cio_chunk *chunk; - struct cio_stream *stream; - size_t size; - struct mk_list _head; -}; - -struct sb_out_queue { - struct flb_output_instance *ins; - struct mk_list chunks; /* head for every sb_out_chunk */ - struct mk_list _head; -}; - -struct flb_sb { - int coll_fd; /* collector id */ - size_t mem_limit; /* memory limit */ - struct flb_input_instance *ins; /* input instance */ - struct cio_ctx *cio; /* chunk i/o instance */ - struct mk_list backlogs; /* list of all pending chunks segregated by output plugin */ -}; - - -static inline struct flb_sb *sb_get_context(struct flb_config *config); - -static struct sb_out_chunk *sb_allocate_chunk(struct cio_chunk *chunk, - struct cio_stream *stream, - size_t size); - -static void sb_destroy_chunk(struct sb_out_chunk *chunk); - -static void sb_destroy_backlog(struct sb_out_queue *backlog, struct flb_sb *context); - -static int sb_allocate_backlogs(struct flb_sb *ctx); - -static void sb_destroy_backlogs(struct flb_sb *ctx); - -static struct sb_out_queue *sb_find_segregated_backlog_by_output_plugin_instance( - struct flb_output_instance *output_plugin, - struct flb_sb *context); - -static void sb_remove_chunk_from_segregated_backlog(struct cio_chunk *target_chunk, - struct sb_out_queue *backlog, - int destroy); - -static void sb_remove_chunk_from_segregated_backlogs(struct cio_chunk *chunk, - struct flb_sb *context); - -static int sb_append_chunk_to_segregated_backlog(struct cio_chunk *target_chunk, - struct cio_stream *stream, - size_t target_chunk_size, - struct sb_out_queue *backlog); - -static int sb_append_chunk_to_segregated_backlogs(struct cio_chunk *target_chunk, - struct cio_stream *stream, - struct flb_sb *context); - -int sb_segregate_chunks(struct flb_config *config); - -int sb_release_output_queue_space(struct flb_output_instance *output_plugin, - ssize_t *required_space); - -ssize_t sb_get_releasable_output_queue_space(struct flb_output_instance *output_plugin, - size_t required_space); - - -static inline struct flb_sb *sb_get_context(struct flb_config *config) -{ - if (config == NULL) { - return NULL; - } - - if (config->storage_input_plugin == NULL) { - return NULL; - } - - return ((struct flb_input_instance *) config->storage_input_plugin)->context; -} - -static struct sb_out_chunk *sb_allocate_chunk(struct cio_chunk *chunk, - struct cio_stream *stream, - size_t size) -{ - struct sb_out_chunk *result; - - result = (struct sb_out_chunk *) flb_calloc(1, sizeof(struct sb_out_chunk)); - - if (result != NULL) { - result->chunk = chunk; - result->stream = stream; - result->size = size; - } - - return result; -} - -static void sb_destroy_chunk(struct sb_out_chunk *chunk) -{ - flb_free(chunk); -} - -static void sb_destroy_backlog(struct sb_out_queue *backlog, struct flb_sb *context) -{ - struct mk_list *chunk_iterator_tmp; - struct mk_list *chunk_iterator; - struct sb_out_chunk *chunk; - - mk_list_foreach_safe(chunk_iterator, chunk_iterator_tmp, &backlog->chunks) { - chunk = mk_list_entry(chunk_iterator, struct sb_out_chunk, _head); - - sb_remove_chunk_from_segregated_backlogs(chunk->chunk, context); - } -} - -static int sb_allocate_backlogs(struct flb_sb *context) -{ - struct mk_list *output_plugin_iterator; - struct flb_output_instance *output_plugin; - struct sb_out_queue *backlog; - - mk_list_foreach(output_plugin_iterator, &context->ins->config->outputs) { - output_plugin = mk_list_entry(output_plugin_iterator, - struct flb_output_instance, - _head); - - backlog = (struct sb_out_queue *) \ - flb_calloc(1, sizeof(struct sb_out_queue)); - - if (backlog == NULL) { - sb_destroy_backlogs(context); - - return -1; - } - - backlog->ins = output_plugin; - - mk_list_init(&backlog->chunks); - - mk_list_add(&backlog->_head, &context->backlogs); - } - - return 0; -} - -static void sb_destroy_backlogs(struct flb_sb *context) -{ - struct mk_list *backlog_iterator_tmp; - struct mk_list *backlog_iterator; - struct sb_out_queue *backlog; - - mk_list_foreach_safe(backlog_iterator, backlog_iterator_tmp, &context->backlogs) { - backlog = mk_list_entry(backlog_iterator, struct sb_out_queue, _head); - - mk_list_del(&backlog->_head); - - sb_destroy_backlog(backlog, context); - - flb_free(backlog); - } -} - -static struct sb_out_queue *sb_find_segregated_backlog_by_output_plugin_instance( - struct flb_output_instance *output_plugin, - struct flb_sb *context) -{ - struct mk_list *backlog_iterator; - struct sb_out_queue *backlog; - - mk_list_foreach(backlog_iterator, &context->backlogs) { - backlog = mk_list_entry(backlog_iterator, struct sb_out_queue, _head); - - if (output_plugin == backlog->ins) { - return backlog; - } - } - - return NULL; -} - -static void sb_remove_chunk_from_segregated_backlog(struct cio_chunk *target_chunk, - struct sb_out_queue *backlog, - int destroy) -{ - struct mk_list *chunk_iterator_tmp; - struct mk_list *chunk_iterator; - struct sb_out_chunk *chunk; - - mk_list_foreach_safe(chunk_iterator, chunk_iterator_tmp, &backlog->chunks) { - chunk = mk_list_entry(chunk_iterator, struct sb_out_chunk, _head); - - if (chunk->chunk == target_chunk) { - mk_list_del(&chunk->_head); - - backlog->ins->fs_backlog_chunks_size -= cio_chunk_get_real_size(target_chunk); - - if (destroy) { - sb_destroy_chunk(chunk); - } - - break; - } - } -} - -static void sb_remove_chunk_from_segregated_backlogs(struct cio_chunk *target_chunk, - struct flb_sb *context) -{ - struct mk_list *backlog_iterator; - struct sb_out_queue *backlog; - - mk_list_foreach(backlog_iterator, &context->backlogs) { - backlog = mk_list_entry(backlog_iterator, struct sb_out_queue, _head); - - sb_remove_chunk_from_segregated_backlog(target_chunk, backlog, FLB_TRUE); - } -} - -static int sb_append_chunk_to_segregated_backlog(struct cio_chunk *target_chunk, - struct cio_stream *stream, - size_t target_chunk_size, - struct sb_out_queue *backlog) -{ - struct sb_out_chunk *chunk; - - chunk = sb_allocate_chunk(target_chunk, stream, target_chunk_size); - - if (chunk == NULL) { - flb_errno(); - return -1; - } - - mk_list_add(&chunk->_head, &backlog->chunks); - - backlog->ins->fs_backlog_chunks_size += target_chunk_size; - - return 0; -} - -static int sb_append_chunk_to_segregated_backlogs(struct cio_chunk *target_chunk, - struct cio_stream *stream, - struct flb_sb *context) -{ - struct flb_input_chunk dummy_input_chunk; - struct mk_list *tmp; - struct mk_list *head; - size_t chunk_size; - struct sb_out_queue *backlog; - int tag_len; - const char * tag_buf; - int result; - - memset(&dummy_input_chunk, 0, sizeof(struct flb_input_chunk)); - - dummy_input_chunk.in = context->ins; - dummy_input_chunk.chunk = target_chunk; - - chunk_size = cio_chunk_get_real_size(target_chunk); - - if (chunk_size < 0) { - flb_warn("[storage backlog] could not get real size of chunk %s/%s", - stream->name, target_chunk->name); - return -1; - } - - result = flb_input_chunk_get_tag(&dummy_input_chunk, &tag_buf, &tag_len); - if (result == -1) { - flb_error("[storage backlog] could not retrieve chunk tag from %s/%s, " - "removing it from the queue", - stream->name, target_chunk->name); - return -2; - } - - flb_routes_mask_set_by_tag(dummy_input_chunk.routes_mask, tag_buf, tag_len, - context->ins); - - mk_list_foreach_safe(head, tmp, &context->backlogs) { - backlog = mk_list_entry(head, struct sb_out_queue, _head); - if (flb_routes_mask_get_bit(dummy_input_chunk.routes_mask, - backlog->ins->id)) { - result = sb_append_chunk_to_segregated_backlog(target_chunk, stream, - chunk_size, backlog); - if (result) { - return -3; - } - } - } - - return 0; -} - -int sb_segregate_chunks(struct flb_config *config) -{ - int ret; - size_t size; - struct mk_list *tmp; - struct mk_list *stream_iterator; - struct mk_list *chunk_iterator; - int chunk_error; - struct flb_sb *context; - struct cio_stream *stream; - struct cio_chunk *chunk; - - context = sb_get_context(config); - - if (context == NULL) { - return 0; - } - - ret = sb_allocate_backlogs(context); - if (ret) { - return -2; - } - - mk_list_foreach(stream_iterator, &context->cio->streams) { - stream = mk_list_entry(stream_iterator, struct cio_stream, _head); - - mk_list_foreach_safe(chunk_iterator, tmp, &stream->chunks) { - chunk = mk_list_entry(chunk_iterator, struct cio_chunk, _head); - - if (!cio_chunk_is_up(chunk)) { - ret = cio_chunk_up_force(chunk); - if (ret == CIO_CORRUPTED) { - if (config->storage_del_bad_chunks) { - chunk_error = cio_error_get(chunk); - - if (chunk_error == CIO_ERR_BAD_FILE_SIZE || - chunk_error == CIO_ERR_BAD_LAYOUT) - { - flb_plg_error(context->ins, "discarding irrecoverable chunk %s/%s", stream->name, chunk->name); - - cio_chunk_close(chunk, CIO_TRUE); - } - } - - continue; - } - } - - if (!cio_chunk_is_up(chunk)) { - return -3; - } - - /* try to segregate a chunk */ - ret = sb_append_chunk_to_segregated_backlogs(chunk, stream, context); - if (ret) { - /* - * if the chunk could not be segregated, just remove it from the - * queue and continue. - * - * if content size is zero, it's safe to 'delete it'. - */ - size = cio_chunk_get_content_size(chunk); - if (size <= 0) { - cio_chunk_close(chunk, CIO_TRUE); - } - else { - cio_chunk_close(chunk, CIO_FALSE); - } - continue; - } - - /* lock the chunk */ - flb_plg_info(context->ins, "register %s/%s", stream->name, chunk->name); - - cio_chunk_lock(chunk); - cio_chunk_down(chunk); - } - } - - return 0; -} - -ssize_t sb_get_releasable_output_queue_space(struct flb_output_instance *output_plugin, - size_t required_space) -{ - ssize_t releasable_space; - struct mk_list *chunk_iterator; - struct flb_sb *context; - struct sb_out_queue *backlog; - struct sb_out_chunk *chunk; - - context = sb_get_context(output_plugin->config); - - if (context == NULL) { - return 0; - } - - backlog = sb_find_segregated_backlog_by_output_plugin_instance( - output_plugin, context); - - if (backlog == NULL) { - return 0; - } - - releasable_space = 0; - - mk_list_foreach(chunk_iterator, &backlog->chunks) { - chunk = mk_list_entry(chunk_iterator, struct sb_out_chunk, _head); - - releasable_space += chunk->size; - - if (releasable_space >= required_space) { - break; - } - } - - return releasable_space; -} - -int sb_release_output_queue_space(struct flb_output_instance *output_plugin, - ssize_t *required_space) -{ - struct mk_list *chunk_iterator_tmp; - struct cio_chunk *underlying_chunk; - struct mk_list *chunk_iterator; - size_t released_space; - struct flb_sb *context; - struct sb_out_queue *backlog; - struct sb_out_chunk *chunk; - - context = sb_get_context(output_plugin->config); - - if (context == NULL) { - return -1; - } - - backlog = sb_find_segregated_backlog_by_output_plugin_instance( - output_plugin, context); - - if (backlog == NULL) { - return -2; - } - - released_space = 0; - - mk_list_foreach_safe(chunk_iterator, chunk_iterator_tmp, &backlog->chunks) { - chunk = mk_list_entry(chunk_iterator, struct sb_out_chunk, _head); - - released_space += chunk->size; - underlying_chunk = chunk->chunk; - - sb_remove_chunk_from_segregated_backlogs(underlying_chunk, context); - cio_chunk_close(underlying_chunk, FLB_TRUE); - - if (released_space >= *required_space) { - break; - } - } - - *required_space -= released_space; - - return 0; -} - -/* Collection callback */ -static int cb_queue_chunks(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - size_t empty_output_queue_count; - struct mk_list *output_queue_iterator; - struct sb_out_queue *output_queue_instance; - struct sb_out_chunk *chunk_instance; - struct flb_sb *ctx; - struct flb_input_chunk *ic; - struct flb_input_chunk tmp_ic; - void *ch; - size_t total = 0; - ssize_t size; - int ret; - int event_type; - - /* Get context */ - ctx = (struct flb_sb *) data; - - /* Get the total number of bytes already enqueued */ - total = flb_input_chunk_total_size(in); - - /* If we already hitted our limit, just wait and re-check later */ - if (total >= ctx->mem_limit) { - return 0; - } - - empty_output_queue_count = 0; - - while (total < ctx->mem_limit && - empty_output_queue_count < mk_list_size(&ctx->backlogs)) { - empty_output_queue_count = 0; - - mk_list_foreach(output_queue_iterator, &ctx->backlogs) { - output_queue_instance = mk_list_entry(output_queue_iterator, - struct sb_out_queue, - _head); - - if (mk_list_is_empty(&output_queue_instance->chunks) != 0) { - chunk_instance = mk_list_entry_first(&output_queue_instance->chunks, - struct sb_out_chunk, - _head); - - /* Try to enqueue one chunk */ - /* - * All chunks on this backlog are 'file' based, always try to set - * them up. We validate the status. - */ - ret = cio_chunk_is_up(chunk_instance->chunk); - - if (ret == CIO_FALSE) { - ret = cio_chunk_up_force(chunk_instance->chunk); - - if (ret == CIO_CORRUPTED) { - flb_plg_error(ctx->ins, "removing corrupted chunk from the " - "queue %s:%s", - chunk_instance->stream->name, chunk_instance->chunk->name); - cio_chunk_close(chunk_instance->chunk, FLB_FALSE); - sb_remove_chunk_from_segregated_backlogs(chunk_instance->chunk, ctx); - /* This function will indirecly release chunk_instance so it has to be - * called last. - */ - continue; - } - else if (ret == CIO_ERROR || ret == CIO_RETRY) { - continue; - } - } - - /* - * Map the chunk file context into a temporary buffer since the - * flb_input_chunk_get_event_type() interface needs an - * struct fb_input_chunk argument. - */ - tmp_ic.chunk = chunk_instance->chunk; - - /* Retrieve the event type: FLB_INPUT_LOGS, FLB_INPUT_METRICS of FLB_INPUT_TRACES */ - ret = flb_input_chunk_get_event_type(&tmp_ic); - if (ret == -1) { - flb_plg_error(ctx->ins, "removing chunk with wrong metadata " - "from the queue %s:%s", - chunk_instance->stream->name, - chunk_instance->chunk->name); - cio_chunk_close(chunk_instance->chunk, FLB_TRUE); - sb_remove_chunk_from_segregated_backlogs(chunk_instance->chunk, - ctx); - continue; - } - event_type = ret; - - /* get the number of bytes being used by the chunk */ - size = cio_chunk_get_content_size(chunk_instance->chunk); - if (size <= 0) { - flb_plg_error(ctx->ins, "removing empty chunk from the " - "queue %s:%s", - chunk_instance->stream->name, chunk_instance->chunk->name); - cio_chunk_close(chunk_instance->chunk, FLB_TRUE); - sb_remove_chunk_from_segregated_backlogs(chunk_instance->chunk, ctx); - /* This function will indirecly release chunk_instance so it has to be - * called last. - */ - continue; - } - - ch = chunk_instance->chunk; - - /* Associate this backlog chunk to this instance into the engine */ - ic = flb_input_chunk_map(in, event_type, ch); - if (!ic) { - flb_plg_error(ctx->ins, "removing chunk %s:%s from the queue", - chunk_instance->stream->name, chunk_instance->chunk->name); - cio_chunk_down(chunk_instance->chunk); - - /* - * If the file cannot be mapped, just drop it. Failures are all - * associated with data corruption. - */ - cio_chunk_close(chunk_instance->chunk, FLB_TRUE); - sb_remove_chunk_from_segregated_backlogs(chunk_instance->chunk, ctx); - /* This function will indirecly release chunk_instance so it has to be - * called last. - */ - continue; - } - - flb_plg_info(ctx->ins, "queueing %s:%s", - chunk_instance->stream->name, chunk_instance->chunk->name); - - /* We are removing this chunk reference from this specific backlog - * queue but we need to leave it in the remainder queues. - */ - sb_remove_chunk_from_segregated_backlogs(chunk_instance->chunk, ctx); - cio_chunk_down(ch); - - /* check our limits */ - total += size; - } - else { - empty_output_queue_count++; - } - } - } - - return 0; -} - -/* Initialize plugin */ -static int cb_sb_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - int ret; - char mem[32]; - struct flb_sb *ctx; - - ctx = flb_malloc(sizeof(struct flb_sb)); - if (!ctx) { - flb_errno(); - return -1; - } - - ctx->cio = data; - ctx->ins = in; - ctx->mem_limit = flb_utils_size_to_bytes(config->storage_bl_mem_limit); - - mk_list_init(&ctx->backlogs); - - flb_utils_bytes_to_human_readable_size(ctx->mem_limit, mem, sizeof(mem) - 1); - flb_plg_info(ctx->ins, "queue memory limit: %s", mem); - - /* export plugin context */ - flb_input_set_context(in, ctx); - - /* Set a collector to trigger the callback to queue data every second */ - ret = flb_input_set_collector_time(in, cb_queue_chunks, 1, 0, config); - if (ret < 0) { - flb_plg_error(ctx->ins, "could not create collector"); - flb_free(ctx); - return -1; - } - ctx->coll_fd = ret; - - return 0; -} - -static void cb_sb_pause(void *data, struct flb_config *config) -{ - struct flb_sb *ctx = data; - flb_input_collector_pause(ctx->coll_fd, ctx->ins); -} - -static void cb_sb_resume(void *data, struct flb_config *config) -{ - struct flb_sb *ctx = data; - flb_input_collector_resume(ctx->coll_fd, ctx->ins); -} - -static int cb_sb_exit(void *data, struct flb_config *config) -{ - struct flb_sb *ctx = data; - - flb_input_collector_pause(ctx->coll_fd, ctx->ins); - - sb_destroy_backlogs(ctx); - - flb_free(ctx); - - return 0; -} - -/* Plugin reference */ -struct flb_input_plugin in_storage_backlog_plugin = { - .name = "storage_backlog", - .description = "Storage Backlog", - .cb_init = cb_sb_init, - .cb_pre_run = NULL, - .cb_collect = NULL, - .cb_ingest = NULL, - .cb_flush_buf = NULL, - .cb_pause = cb_sb_pause, - .cb_resume = cb_sb_resume, - .cb_exit = cb_sb_exit, - - /* This plugin can only be configured and invoked by the Engine */ - .flags = FLB_INPUT_PRIVATE -}; diff --git a/fluent-bit/plugins/in_stream_processor/CMakeLists.txt b/fluent-bit/plugins/in_stream_processor/CMakeLists.txt deleted file mode 100644 index fa6a4b408..000000000 --- a/fluent-bit/plugins/in_stream_processor/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -set(src - sp.c - ) - -FLB_PLUGIN(in_stream_processor "${src}" "") diff --git a/fluent-bit/plugins/in_stream_processor/sp.c b/fluent-bit/plugins/in_stream_processor/sp.c deleted file mode 100644 index d4056a94c..000000000 --- a/fluent-bit/plugins/in_stream_processor/sp.c +++ /dev/null @@ -1,173 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -struct sp_chunk { - char *buf_data; - size_t buf_size; - struct mk_list _head; -}; - -struct sp_ctx { - int coll_fd; /* collector file descriptor to flush queue */ - flb_sds_t tag; /* outgoing Tag name */ - struct mk_list chunks; /* linked list with data chunks to ingest */ - struct flb_input_instance *ins; -}; - -/* - * This 'special' function is used by the Stream Processor engine to register - * data results of a query that needs to be ingested into the main pipeline. - * - * We usually don't do this in a plugin but for simplicity and to avoid - * extra memory-copies we just expose this function for direct use. - */ -int in_stream_processor_add_chunk(char *buf_data, size_t buf_size, - struct flb_input_instance *ins) -{ - struct sp_chunk *chunk; - struct sp_ctx *ctx = (struct sp_ctx *) ins->context; - - chunk = flb_malloc(sizeof(struct sp_chunk)); - if (!chunk) { - flb_errno(); - return -1; - } - - chunk->buf_data = buf_data; - chunk->buf_size = buf_size; - mk_list_add(&chunk->_head, &ctx->chunks); - - return 0; -} - -/* Callback used to queue pending data chunks */ -static int cb_chunks_append(struct flb_input_instance *in, - struct flb_config *config, void *in_context) -{ - struct mk_list *tmp; - struct mk_list *head; - struct sp_chunk *chunk; - struct sp_ctx *ctx = in_context; - (void) config; - - mk_list_foreach_safe(head, tmp, &ctx->chunks) { - chunk = mk_list_entry(head, struct sp_chunk, _head); - flb_input_log_append(in, - ctx->tag, flb_sds_len(ctx->tag), - chunk->buf_data, chunk->buf_size); - flb_free(chunk->buf_data); - mk_list_del(&chunk->_head); - flb_free(chunk); - } - return 0; -} - -/* Initialize plugin */ -static int cb_sp_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - int ret; - struct sp_ctx *ctx; - - /* Create plugin instance context */ - ctx = flb_malloc(sizeof(struct sp_ctx)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->ins = in; - mk_list_init(&ctx->chunks); - - /* Register context */ - flb_input_set_context(in, ctx); - - /* - * Configure the outgoing tag: when registering records into the Engine - * we need to specify a Tag, if we got the default name - * stream_processor.N, just override it using the Alias set by the - * Stream Processor interface. Otherwise if the Tag is different use - * that one. - */ - if (strncmp(in->tag, "stream_processor.", 17) == 0) { - ctx->tag = flb_sds_create(in->alias); - } - else { - ctx->tag = flb_sds_create(in->tag); - } - - /* Set our collector based on time, queue chunks every 0.5 sec */ - ret = flb_input_set_collector_time(in, - cb_chunks_append, - 0, - 500000000, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "Could not set collector"); - return -1; - } - ctx->coll_fd = ret; - - return 0; -} - -static void cb_sp_pause(void *data, struct flb_config *config) -{ - struct sp_ctx *ctx = data; - - flb_input_collector_pause(ctx->coll_fd, ctx->ins); -} - -static void cb_sp_resume(void *data, struct flb_config *config) -{ - struct sp_ctx *ctx = data; - - flb_input_collector_resume(ctx->coll_fd, ctx->ins); -} - -static int cb_sp_exit(void *data, struct flb_config *config) -{ - struct sp_ctx *ctx = data; - - /* Upon exit, put in the queue all pending chunks */ - cb_chunks_append(ctx->ins, config, ctx); - flb_sds_destroy(ctx->tag); - flb_free(ctx); - - return 0; -} - -/* Plugin reference */ -struct flb_input_plugin in_stream_processor_plugin = { - .name = "stream_processor", - .description = "Stream Processor", - .cb_init = cb_sp_init, - .cb_pre_run = NULL, - .cb_collect = NULL, - .cb_ingest = NULL, - .cb_flush_buf = NULL, - .cb_pause = cb_sp_pause, - .cb_resume = cb_sp_resume, - .cb_exit = cb_sp_exit, - - /* This plugin can only be configured and invoked by the Stream Processor */ - .flags = FLB_INPUT_PRIVATE | FLB_INPUT_NOTAG -}; diff --git a/fluent-bit/plugins/in_syslog/CMakeLists.txt b/fluent-bit/plugins/in_syslog/CMakeLists.txt deleted file mode 100644 index 88f698b12..000000000 --- a/fluent-bit/plugins/in_syslog/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -set(src - syslog_conf.c - syslog_server.c - syslog_conn.c - syslog_prot.c - syslog.c) - -FLB_PLUGIN(in_syslog "${src}" "") diff --git a/fluent-bit/plugins/in_syslog/syslog.c b/fluent-bit/plugins/in_syslog/syslog.c deleted file mode 100644 index d478dfc3f..000000000 --- a/fluent-bit/plugins/in_syslog/syslog.c +++ /dev/null @@ -1,263 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include "syslog.h" -#include "syslog_conf.h" -#include "syslog_server.h" -#include "syslog_conn.h" -#include "syslog_prot.h" - -/* cb_collect callback */ -static int in_syslog_collect_tcp(struct flb_input_instance *i_ins, - struct flb_config *config, void *in_context) -{ - struct flb_connection *connection; - struct syslog_conn *conn; - struct flb_syslog *ctx; - - (void) i_ins; - - ctx = in_context; - - connection = flb_downstream_conn_get(ctx->downstream); - - if (connection == NULL) { - flb_plg_error(ctx->ins, "could not accept new connection"); - - return -1; - } - - if (ctx->dgram_mode_flag) { - return syslog_dgram_conn_event(connection); - } - else { - flb_plg_trace(ctx->ins, "new Unix connection arrived FD=%i", connection->fd); - - conn = syslog_conn_add(connection, ctx); - - if (conn == NULL) { - flb_plg_error(ctx->ins, "could not accept new connection"); - - flb_downstream_conn_release(connection); - - return -1; - } - } - - return 0; -} - -/* - * Collect a datagram, per Syslog specification a datagram contains only - * one syslog message and it should not exceed 1KB. - */ -static int in_syslog_collect_udp(struct flb_input_instance *i_ins, - struct flb_config *config, - void *in_context) -{ - struct flb_syslog *ctx; - - (void) i_ins; - - ctx = in_context; - - return syslog_dgram_conn_event(ctx->dummy_conn->connection); -} - -/* Initialize plugin */ -static int in_syslog_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - int ret; - struct flb_syslog *ctx; - struct flb_connection *connection; - - /* Allocate space for the configuration */ - ctx = syslog_conf_create(in, config); - if (!ctx) { - flb_plg_error(in, "could not initialize plugin"); - return -1; - } - ctx->collector_id = -1; - - if ((ctx->mode == FLB_SYSLOG_UNIX_TCP || ctx->mode == FLB_SYSLOG_UNIX_UDP) - && !ctx->unix_path) { - flb_plg_error(ctx->ins, "Unix path not defined"); - syslog_conf_destroy(ctx); - return -1; - } - - /* Create Unix Socket */ - ret = syslog_server_create(ctx); - if (ret == -1) { - syslog_conf_destroy(ctx); - return -1; - } - - flb_input_downstream_set(ctx->downstream, ctx->ins); - - if (ctx->dgram_mode_flag) { - connection = flb_downstream_conn_get(ctx->downstream); - - if (connection == NULL) { - flb_plg_error(ctx->ins, "could not get DGRAM server dummy " - "connection"); - - syslog_conf_destroy(ctx); - - return -1; - } - - ctx->dummy_conn = syslog_conn_add(connection, ctx); - - if (ctx->dummy_conn == NULL) { - flb_plg_error(ctx->ins, "could not track DGRAM server dummy " - "connection"); - - syslog_conf_destroy(ctx); - - return -1; - } - } - - /* Set context */ - flb_input_set_context(in, ctx); - - /* Collect events for every opened connection to our socket */ - if (ctx->mode == FLB_SYSLOG_UNIX_TCP || - ctx->mode == FLB_SYSLOG_TCP) { - ret = flb_input_set_collector_socket(in, - in_syslog_collect_tcp, - ctx->downstream->server_fd, - config); - } - else { - ret = flb_input_set_collector_socket(in, - in_syslog_collect_udp, - ctx->downstream->server_fd, - config); - } - - if (ret == -1) { - flb_plg_error(ctx->ins, "Could not set collector"); - syslog_conf_destroy(ctx); - - return -1; - } - - ctx->collector_id = ret; - ctx->collector_event = flb_input_collector_get_event(ret, in); - - if (ret == -1) { - flb_plg_error(ctx->ins, "Could not get collector event"); - syslog_conf_destroy(ctx); - - return -1; - } - - return 0; -} - -static int in_syslog_exit(void *data, struct flb_config *config) -{ - struct flb_syslog *ctx = data; - (void) config; - - syslog_conn_exit(ctx); - syslog_conf_destroy(ctx); - - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "mode", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_syslog, mode_str), - "Set the socket mode: unix_tcp, unix_udp, tcp or udp" - }, - { - FLB_CONFIG_MAP_STR, "path", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_syslog, unix_path), - "Set the path for the UNIX socket" - }, - { - FLB_CONFIG_MAP_STR, "unix_perm", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_syslog, unix_perm_str), - "Set the permissions for the UNIX socket" - }, - { - FLB_CONFIG_MAP_SIZE, "buffer_chunk_size", FLB_SYSLOG_CHUNK, - 0, FLB_TRUE, offsetof(struct flb_syslog, buffer_chunk_size), - "Set the buffer chunk size" - }, - { - FLB_CONFIG_MAP_SIZE, "buffer_max_size", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_syslog, buffer_max_size), - "Set the buffer chunk size" - }, - { - FLB_CONFIG_MAP_STR, "parser", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_syslog, parser_name), - "Set the parser" - }, - { - FLB_CONFIG_MAP_SIZE, "receive_buffer_size", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_syslog, receive_buffer_size), - "Set the socket receiving buffer size" - }, - { - FLB_CONFIG_MAP_STR, "raw_message_key", (char *) NULL, - 0, FLB_TRUE, offsetof(struct flb_syslog, raw_message_key), - "Key where the raw message will be preserved" - }, - { - FLB_CONFIG_MAP_STR, "source_address_key", (char *) NULL, - 0, FLB_TRUE, offsetof(struct flb_syslog, source_address_key), - "Key where the source address will be injected" - }, - - - /* EOF */ - {0} -}; - -struct flb_input_plugin in_syslog_plugin = { - .name = "syslog", - .description = "Syslog", - .cb_init = in_syslog_init, - .cb_pre_run = NULL, - .cb_collect = NULL, - .cb_flush_buf = NULL, - .cb_exit = in_syslog_exit, - .config_map = config_map, - .flags = FLB_INPUT_NET_SERVER | FLB_IO_OPT_TLS -}; diff --git a/fluent-bit/plugins/in_syslog/syslog.h b/fluent-bit/plugins/in_syslog/syslog.h deleted file mode 100644 index 6da2fbd83..000000000 --- a/fluent-bit/plugins/in_syslog/syslog.h +++ /dev/null @@ -1,82 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_SYSLOG_H -#define FLB_IN_SYSLOG_H - -#include -#include -#include - -/* Syslog modes */ -#define FLB_SYSLOG_UNIX_TCP 1 -#define FLB_SYSLOG_UNIX_UDP 2 -#define FLB_SYSLOG_TCP 3 -#define FLB_SYSLOG_UDP 4 - -/* 32KB chunk size */ -#define FLB_SYSLOG_CHUNK "32768" - -struct syslog_conn; - -/* Context / Config*/ -struct flb_syslog { - /* Listening mode: unix udp, unix tcp or normal tcp */ - flb_sds_t mode_str; - int mode; - - /* Network mode */ - char *listen; - char *port; - - /* Unix socket (UDP/TCP)*/ - int server_fd; - flb_sds_t unix_path; - flb_sds_t unix_perm_str; - unsigned int unix_perm; - size_t receive_buffer_size; - - /* UDP buffer, data length and buffer size */ - // char *buffer_data; - // size_t buffer_len; - // size_t buffer_size; - - /* Buffers setup */ - size_t buffer_max_size; - size_t buffer_chunk_size; - - /* Configuration */ - flb_sds_t parser_name; - struct flb_parser *parser; - flb_sds_t raw_message_key; - flb_sds_t source_address_key; - - int dgram_mode_flag; - int collector_id; - struct mk_event *collector_event; - struct flb_downstream *downstream; - struct syslog_conn *dummy_conn; - - /* List for connections and event loop */ - struct mk_list connections; - struct flb_input_instance *ins; - struct flb_log_event_encoder *log_encoder; -}; - -#endif diff --git a/fluent-bit/plugins/in_syslog/syslog_conf.c b/fluent-bit/plugins/in_syslog/syslog_conf.c deleted file mode 100644 index 4db3f1626..000000000 --- a/fluent-bit/plugins/in_syslog/syslog_conf.c +++ /dev/null @@ -1,193 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "syslog.h" -#include "syslog_server.h" -#include "syslog_conf.h" - -struct flb_syslog *syslog_conf_create(struct flb_input_instance *ins, - struct flb_config *config) -{ - int ret; - char port[16]; - struct flb_syslog *ctx; - - ctx = flb_calloc(1, sizeof(struct flb_syslog)); - - if (ctx == NULL) { - flb_errno(); - - return NULL; - } - - ctx->ins = ins; - - ctx->log_encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ctx->log_encoder == NULL) { - flb_plg_error(ins, "could not initialize event encoder"); - syslog_conf_destroy(ctx); - - return NULL; - } - - mk_list_init(&ctx->connections); - - ret = flb_input_config_map_set(ins, (void *)ctx); - if (ret == -1) { - flb_log_event_encoder_destroy(ctx->log_encoder); - - flb_plg_error(ins, "unable to load configuration"); - flb_free(ctx); - - return NULL; - } - - /* Syslog mode: unix_udp, unix_tcp, tcp or udp */ - if (ctx->mode_str) { -#ifdef FLB_SYSTEM_WINDOWS - if (strcasestr(ctx->mode_str, "unix") != NULL) { - flb_log_event_encoder_destroy(ctx->log_encoder); - - flb_plg_error(ins, "unix sockets are note available in windows"); - flb_free(ctx); - - return NULL; - } - -#undef FLB_SYSLOG_UNIX_UDP -#define FLB_SYSLOG_UNIX_UDP FLB_SYSLOG_UDP -#endif - if (strcasecmp(ctx->mode_str, "unix_tcp") == 0) { - ctx->mode = FLB_SYSLOG_UNIX_TCP; - } - else if (strcasecmp(ctx->mode_str, "unix_udp") == 0) { - ctx->mode = FLB_SYSLOG_UNIX_UDP; - } - else if (strcasecmp(ctx->mode_str, "tcp") == 0) { - ctx->mode = FLB_SYSLOG_TCP; - } - else if (strcasecmp(ctx->mode_str, "udp") == 0) { - ctx->mode = FLB_SYSLOG_UDP; - } - else { - flb_log_event_encoder_destroy(ctx->log_encoder); - - flb_error("[in_syslog] Unknown syslog mode %s", ctx->mode_str); - flb_free(ctx); - return NULL; - } - } - else { - ctx->mode = FLB_SYSLOG_UNIX_UDP; - } - - /* Check if TCP mode was requested */ - if (ctx->mode == FLB_SYSLOG_TCP || ctx->mode == FLB_SYSLOG_UDP) { - /* Listen interface (if not set, defaults to 0.0.0.0:5140) */ - flb_input_net_default_listener("0.0.0.0", 5140, ins); - ctx->listen = ins->host.listen; - snprintf(port, sizeof(port) - 1, "%d", ins->host.port); - ctx->port = flb_strdup(port); - } - - /* Unix socket path and permission */ - if (ctx->mode == FLB_SYSLOG_UNIX_UDP || ctx->mode == FLB_SYSLOG_UNIX_TCP) { - if (ctx->unix_perm_str) { - ctx->unix_perm = strtol(ctx->unix_perm_str, NULL, 8) & 07777; - } else { - ctx->unix_perm = 0644; - } - } - - /* Buffer Chunk Size */ - if (ctx->buffer_chunk_size == -1) { - flb_log_event_encoder_destroy(ctx->log_encoder); - - flb_plg_error(ins, "invalid buffer_chunk_size"); - flb_free(ctx); - return NULL; - } - - /* Buffer Max Size */ - if (ctx->buffer_max_size == -1) { - flb_log_event_encoder_destroy(ctx->log_encoder); - - flb_plg_error(ins, "invalid buffer_max_size"); - flb_free(ctx); - return NULL; - } - else if (ctx->buffer_max_size == 0) { - ctx->buffer_max_size = ctx->buffer_chunk_size; - } - - /* Socket rcv buffer size */ - if (ctx->receive_buffer_size == -1 || ctx->receive_buffer_size>INT_MAX) { - flb_log_event_encoder_destroy(ctx->log_encoder); - - flb_plg_error(ins, "invalid receive_buffer_size"); - flb_free(ctx); - return NULL; - } - - /* Parser */ - if (ctx->parser_name) { - ctx->parser = flb_parser_get(ctx->parser_name, config); - } - else { - if (ctx->mode == FLB_SYSLOG_TCP || ctx->mode == FLB_SYSLOG_UDP) { - ctx->parser = flb_parser_get("syslog-rfc5424", config); - } - else { - ctx->parser = flb_parser_get("syslog-rfc3164-local", config); - } - } - - if (!ctx->parser) { - flb_log_event_encoder_destroy(ctx->log_encoder); - - flb_error("[in_syslog] parser not set"); - syslog_conf_destroy(ctx); - return NULL; - } - - return ctx; -} - -int syslog_conf_destroy(struct flb_syslog *ctx) -{ - if (ctx->log_encoder != NULL) { - flb_log_event_encoder_destroy(ctx->log_encoder); - } - - syslog_server_destroy(ctx); - - flb_free(ctx); - - return 0; -} diff --git a/fluent-bit/plugins/in_syslog/syslog_conf.h b/fluent-bit/plugins/in_syslog/syslog_conf.h deleted file mode 100644 index ac2304031..000000000 --- a/fluent-bit/plugins/in_syslog/syslog_conf.h +++ /dev/null @@ -1,32 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_SYSLOG_CONF_H -#define FLB_IN_SYSLOG_CONF_H - -#include -#include - -#include "syslog.h" - -struct flb_syslog *syslog_conf_create(struct flb_input_instance *i_ins, - struct flb_config *config); -int syslog_conf_destroy(struct flb_syslog *ctx); - -#endif diff --git a/fluent-bit/plugins/in_syslog/syslog_conn.c b/fluent-bit/plugins/in_syslog/syslog_conn.c deleted file mode 100644 index 8785c1e86..000000000 --- a/fluent-bit/plugins/in_syslog/syslog_conn.c +++ /dev/null @@ -1,247 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "syslog.h" -#include "syslog_conf.h" -#include "syslog_conn.h" -#include "syslog_prot.h" - -/* Callback invoked every time an event is triggered for a connection */ -int syslog_conn_event(void *data) -{ - struct flb_connection *connection; - struct syslog_conn *conn; - struct flb_syslog *ctx; - - connection = (struct flb_connection *) data; - - conn = connection->user_data; - - ctx = conn->ctx; - - if (ctx->dgram_mode_flag) { - return syslog_dgram_conn_event(data); - } - - return syslog_stream_conn_event(data); -} - -int syslog_stream_conn_event(void *data) -{ - int ret; - int bytes; - int available; - size_t size; - char *tmp; - struct mk_event *event; - struct syslog_conn *conn; - struct flb_syslog *ctx; - struct flb_connection *connection; - - connection = (struct flb_connection *) data; - - conn = connection->user_data; - - ctx = conn->ctx; - - event = &connection->event; - - if (event->mask & MK_EVENT_READ) { - available = (conn->buf_size - conn->buf_len) - 1; - if (available < 1) { - if (conn->buf_size + ctx->buffer_chunk_size > ctx->buffer_max_size) { - flb_plg_debug(ctx->ins, - "fd=%i incoming data exceed limit (%zd bytes)", - event->fd, (ctx->buffer_max_size)); - syslog_conn_del(conn); - return -1; - } - - size = conn->buf_size + ctx->buffer_chunk_size; - tmp = flb_realloc(conn->buf_data, size); - if (!tmp) { - flb_errno(); - return -1; - } - flb_plg_trace(ctx->ins, "fd=%i buffer realloc %zd -> %zd", - event->fd, conn->buf_size, size); - - conn->buf_data = tmp; - conn->buf_size = size; - available = (conn->buf_size - conn->buf_len) - 1; - } - - bytes = flb_io_net_read(connection, - (void *) &conn->buf_data[conn->buf_len], - available); - - if (bytes > 0) { - flb_plg_trace(ctx->ins, "read()=%i pre_len=%zu now_len=%zu", - bytes, conn->buf_len, conn->buf_len + bytes); - conn->buf_len += bytes; - conn->buf_data[conn->buf_len] = '\0'; - ret = syslog_prot_process(conn); - if (ret == -1) { - return -1; - } - return bytes; - } - else { - flb_plg_trace(ctx->ins, "fd=%i closed connection", event->fd); - syslog_conn_del(conn); - return -1; - } - } - - if (event->mask & MK_EVENT_CLOSE) { - flb_plg_trace(ctx->ins, "fd=%i hangup", event->fd); - syslog_conn_del(conn); - return -1; - } - return 0; -} - -int syslog_dgram_conn_event(void *data) -{ - struct flb_connection *connection; - int bytes; - struct syslog_conn *conn; - - connection = (struct flb_connection *) data; - - conn = connection->user_data; - - bytes = flb_io_net_read(connection, - (void *) &conn->buf_data[conn->buf_len], - conn->buf_size - 1); - - if (bytes > 0) { - conn->buf_data[bytes] = '\0'; - conn->buf_len = bytes; - - syslog_prot_process_udp(conn); - } - else { - flb_errno(); - } - - conn->buf_len = 0; - - return 0; -} - -/* Create a new mqtt request instance */ -struct syslog_conn *syslog_conn_add(struct flb_connection *connection, - struct flb_syslog *ctx) -{ - int ret; - struct syslog_conn *conn; - - conn = flb_malloc(sizeof(struct syslog_conn)); - if (!conn) { - return NULL; - } - - conn->connection = connection; - - /* Set data for the event-loop */ - MK_EVENT_NEW(&connection->event); - - connection->user_data = conn; - connection->event.type = FLB_ENGINE_EV_CUSTOM; - connection->event.handler = syslog_conn_event; - - /* Connection info */ - conn->ctx = ctx; - conn->ins = ctx->ins; - conn->buf_len = 0; - conn->buf_parsed = 0; - - /* Allocate read buffer */ - conn->buf_data = flb_malloc(ctx->buffer_chunk_size); - if (!conn->buf_data) { - flb_errno(); - - flb_free(conn); - - return NULL; - } - conn->buf_size = ctx->buffer_chunk_size; - - /* Register instance into the event loop if we're in - * stream mode (UDP events are received through the collector) - */ - if (!ctx->dgram_mode_flag) { - ret = mk_event_add(flb_engine_evl_get(), - connection->fd, - FLB_ENGINE_EV_CUSTOM, - MK_EVENT_READ, - &connection->event); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not register new connection"); - - flb_free(conn->buf_data); - flb_free(conn); - - return NULL; - } - } - - mk_list_add(&conn->_head, &ctx->connections); - - return conn; -} - -int syslog_conn_del(struct syslog_conn *conn) -{ - /* The downstream unregisters the file descriptor from the event-loop - * so there's nothing to be done by the plugin - */ - if (!conn->ctx->dgram_mode_flag) { - flb_downstream_conn_release(conn->connection); - } - - /* Release resources */ - mk_list_del(&conn->_head); - - flb_free(conn->buf_data); - flb_free(conn); - - return 0; -} - -int syslog_conn_exit(struct flb_syslog *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - struct syslog_conn *conn; - - mk_list_foreach_safe(head, tmp, &ctx->connections) { - conn = mk_list_entry(head, struct syslog_conn, _head); - syslog_conn_del(conn); - } - - return 0; -} diff --git a/fluent-bit/plugins/in_syslog/syslog_conn.h b/fluent-bit/plugins/in_syslog/syslog_conn.h deleted file mode 100644 index 684d3f9a4..000000000 --- a/fluent-bit/plugins/in_syslog/syslog_conn.h +++ /dev/null @@ -1,53 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_SYSLOG_CONN_H -#define FLB_IN_SYSLOG_CONN_H - -#include -#include -#include - -#include "syslog.h" - -/* Respresents a connection */ -struct syslog_conn { - int status; /* Connection status */ - - /* Buffer */ - char *buf_data; /* Buffer data */ - size_t buf_size; /* Buffer size */ - size_t buf_len; /* Buffer length */ - size_t buf_parsed; /* Parsed buffer (offset) */ - struct flb_input_instance *ins; /* Parent plugin instance */ - struct flb_syslog *ctx; /* Plugin configuration context */ - struct flb_connection *connection; - - struct mk_list _head; -}; - -int syslog_conn_event(void *data); -int syslog_stream_conn_event(void *data); -int syslog_dgram_conn_event(void *data); -struct syslog_conn *syslog_conn_add(struct flb_connection *connection, - struct flb_syslog *ctx); -int syslog_conn_del(struct syslog_conn *conn); -int syslog_conn_exit(struct flb_syslog *ctx); - -#endif diff --git a/fluent-bit/plugins/in_syslog/syslog_prot.c b/fluent-bit/plugins/in_syslog/syslog_prot.c deleted file mode 100644 index 1ec2c97cd..000000000 --- a/fluent-bit/plugins/in_syslog/syslog_prot.c +++ /dev/null @@ -1,324 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include "syslog.h" -#include "syslog_conn.h" -#include "syslog_prot.h" - -#include - -static inline void consume_bytes(char *buf, int bytes, int length) -{ - memmove(buf, buf + bytes, length - bytes); -} - -static int append_message_to_record_data(char **result_buffer, - size_t *result_size, - flb_sds_t message_key_name, - char *base_object_buffer, - size_t base_object_size, - char *message_buffer, - size_t message_size, - int message_type) -{ - int result = FLB_MAP_NOT_MODIFIED; - char *modified_data_buffer; - int modified_data_size; - msgpack_object_kv *new_map_entries[1]; - msgpack_object_kv message_entry; - *result_buffer = NULL; - *result_size = 0; - modified_data_buffer = NULL; - - if (message_key_name != NULL) { - new_map_entries[0] = &message_entry; - - message_entry.key.type = MSGPACK_OBJECT_STR; - message_entry.key.via.str.size = flb_sds_len(message_key_name); - message_entry.key.via.str.ptr = message_key_name; - - if (message_type == MSGPACK_OBJECT_BIN) { - message_entry.val.type = MSGPACK_OBJECT_BIN; - message_entry.val.via.bin.size = message_size; - message_entry.val.via.bin.ptr = message_buffer; - } - else if (message_type == MSGPACK_OBJECT_STR) { - message_entry.val.type = MSGPACK_OBJECT_STR; - message_entry.val.via.str.size = message_size; - message_entry.val.via.str.ptr = message_buffer; - } - else { - result = FLB_MAP_EXPANSION_INVALID_VALUE_TYPE; - } - - if (result == FLB_MAP_NOT_MODIFIED) { - result = flb_msgpack_expand_map(base_object_buffer, - base_object_size, - new_map_entries, 1, - &modified_data_buffer, - &modified_data_size); - if (result == 0) { - result = FLB_MAP_EXPAND_SUCCESS; - } - else { - result = FLB_MAP_EXPANSION_ERROR; - } - } - } - - if (result == FLB_MAP_EXPAND_SUCCESS) { - *result_buffer = modified_data_buffer; - *result_size = modified_data_size; - } - - return result; -} - -static inline int pack_line(struct flb_syslog *ctx, - struct flb_time *time, - struct flb_connection *connection, - char *data, size_t data_size, - char *raw_data, size_t raw_data_size) -{ - char *modified_data_buffer; - size_t modified_data_size; - char *appended_address_buffer; - size_t appended_address_size; - int result; - char *source_address; - - source_address = NULL; - modified_data_buffer = NULL; - appended_address_buffer = NULL; - - if (ctx->raw_message_key != NULL) { - result = append_message_to_record_data(&modified_data_buffer, - &modified_data_size, - ctx->raw_message_key, - data, - data_size, - raw_data, - raw_data_size, - MSGPACK_OBJECT_BIN); - - if (result == FLB_MAP_EXPANSION_ERROR) { - flb_plg_debug(ctx->ins, "error expanding raw message : %d", result); - } - } - - if (ctx->source_address_key != NULL) { - source_address = flb_connection_get_remote_address(connection); - if (source_address != NULL) { - if (modified_data_buffer != NULL) { - result = append_message_to_record_data(&appended_address_buffer, - &appended_address_size, - ctx->source_address_key, - modified_data_buffer, - modified_data_size, - source_address, - strlen(source_address), - MSGPACK_OBJECT_STR); - } - else { - result = append_message_to_record_data(&appended_address_buffer, - &appended_address_size, - ctx->source_address_key, - data, - data_size, - source_address, - strlen(source_address), - MSGPACK_OBJECT_STR); - } - - if (result == FLB_MAP_EXPANSION_ERROR) { - flb_plg_debug(ctx->ins, "error expanding source_address : %d", result); - } - } - } - - result = flb_log_event_encoder_begin_record(ctx->log_encoder); - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_set_timestamp(ctx->log_encoder, time); - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - if (appended_address_buffer != NULL) { - result = flb_log_event_encoder_set_body_from_raw_msgpack( - ctx->log_encoder, appended_address_buffer, appended_address_size); - } - else if (modified_data_buffer != NULL) { - result = flb_log_event_encoder_set_body_from_raw_msgpack( - ctx->log_encoder, modified_data_buffer, modified_data_size); - } - else { - result = flb_log_event_encoder_set_body_from_raw_msgpack( - ctx->log_encoder, data, data_size); - } - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_commit_record(ctx->log_encoder); - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(ctx->ins, NULL, 0, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); - result = 0; - } - else { - flb_plg_error(ctx->ins, "log event encoding error : %d", result); - - result = -1; - } - - flb_log_event_encoder_reset(ctx->log_encoder); - - if (modified_data_buffer != NULL) { - flb_free(modified_data_buffer); - } - if (appended_address_buffer != NULL) { - flb_free(appended_address_buffer); - } - - return result; -} - -int syslog_prot_process(struct syslog_conn *conn) -{ - int len; - int ret; - char *p; - char *eof; - char *end; - void *out_buf; - size_t out_size; - struct flb_time out_time; - struct flb_syslog *ctx = conn->ctx; - - eof = conn->buf_data; - end = conn->buf_data + conn->buf_len; - - /* Always parse while some remaining bytes exists */ - while (eof < end) { - /* Lookup the ending byte */ - eof = p = conn->buf_data + conn->buf_parsed; - while (*eof != '\n' && *eof != '\0' && eof < end) { - eof++; - } - - /* Incomplete message */ - if (eof == end || (*eof != '\n' && *eof != '\0')) { - break; - } - - /* No data ? */ - len = (eof - p); - if (len == 0) { - consume_bytes(conn->buf_data, 1, conn->buf_len); - conn->buf_len--; - conn->buf_parsed = 0; - conn->buf_data[conn->buf_len] = '\0'; - end = conn->buf_data + conn->buf_len; - - if (conn->buf_len == 0) { - break; - } - - continue; - } - - /* Process the string */ - ret = flb_parser_do(ctx->parser, p, len, - &out_buf, &out_size, &out_time); - if (ret >= 0) { - if (flb_time_to_nanosec(&out_time) == 0L) { - flb_time_get(&out_time); - } - pack_line(ctx, &out_time, - conn->connection, - out_buf, out_size, - p, len); - flb_free(out_buf); - } - else { - flb_plg_warn(ctx->ins, "error parsing log message with parser '%s'", - ctx->parser->name); - flb_plg_debug(ctx->ins, "unparsed log message: %.*s", len, p); - } - - conn->buf_parsed += len + 1; - end = conn->buf_data + conn->buf_len; - eof = conn->buf_data + conn->buf_parsed; - } - - if (conn->buf_parsed > 0) { - consume_bytes(conn->buf_data, conn->buf_parsed, conn->buf_len); - conn->buf_len -= conn->buf_parsed; - conn->buf_parsed = 0; - conn->buf_data[conn->buf_len] = '\0'; - } - - return 0; -} - -int syslog_prot_process_udp(struct syslog_conn *conn) -{ - int ret; - void *out_buf; - size_t out_size; - struct flb_time out_time = {0}; - char *buf; - size_t size; - struct flb_syslog *ctx; - struct flb_connection *connection; - - buf = conn->buf_data; - size = conn->buf_len; - ctx = conn->ctx; - connection = conn->connection; - - ret = flb_parser_do(ctx->parser, buf, size, - &out_buf, &out_size, &out_time); - if (ret >= 0) { - if (flb_time_to_double(&out_time) == 0) { - flb_time_get(&out_time); - } - pack_line(ctx, &out_time, - connection, - out_buf, out_size, - buf, size); - flb_free(out_buf); - } - else { - flb_plg_warn(ctx->ins, "error parsing log message with parser '%s'", - ctx->parser->name); - flb_plg_debug(ctx->ins, "unparsed log message: %.*s", - (int) size, buf); - return -1; - } - - return 0; -} diff --git a/fluent-bit/plugins/in_syslog/syslog_prot.h b/fluent-bit/plugins/in_syslog/syslog_prot.h deleted file mode 100644 index cb5976b7b..000000000 --- a/fluent-bit/plugins/in_syslog/syslog_prot.h +++ /dev/null @@ -1,35 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_SYSLOG_PROT_H -#define FLB_IN_SYSLOG_PROT_H - -#include - -#include "syslog.h" - -#define FLB_MAP_EXPAND_SUCCESS 0 -#define FLB_MAP_NOT_MODIFIED -1 -#define FLB_MAP_EXPANSION_ERROR -2 -#define FLB_MAP_EXPANSION_INVALID_VALUE_TYPE -3 - -int syslog_prot_process(struct syslog_conn *conn); -int syslog_prot_process_udp(struct syslog_conn *conn); - -#endif diff --git a/fluent-bit/plugins/in_syslog/syslog_server.c b/fluent-bit/plugins/in_syslog/syslog_server.c deleted file mode 100644 index 5317851db..000000000 --- a/fluent-bit/plugins/in_syslog/syslog_server.c +++ /dev/null @@ -1,235 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#if !defined(FLB_SYSTEM_WINDOWS) -#include -#include -#include -#endif -#include -#include - -#include "syslog.h" - -static int remove_existing_socket_file(char *socket_path) -{ - struct stat file_data; - int result; - - result = stat(socket_path, &file_data); - - if (result == -1) { - if (errno == ENOENT) { - return 0; - } - - flb_errno(); - - return -1; - } - - if (S_ISSOCK(file_data.st_mode) == 0) { - return -2; - } - - result = unlink(socket_path); - - if (result != 0) { - return -3; - } - - return 0; -} - -#if !defined(FLB_SYSTEM_WINDOWS) -static int syslog_server_unix_create(struct flb_syslog *ctx) -{ - int result; - int mode; - struct flb_tls *tls; - - if (ctx->mode == FLB_SYSLOG_UNIX_TCP) { - mode = FLB_TRANSPORT_UNIX_STREAM; - tls = ctx->ins->tls; - } - else if (ctx->mode == FLB_SYSLOG_UNIX_UDP) { - ctx->dgram_mode_flag = FLB_TRUE; - - mode = FLB_TRANSPORT_UNIX_DGRAM; - tls = NULL; - } - else { - return -1; - } - - result = remove_existing_socket_file(ctx->unix_path); - - if (result != 0) { - if (result == -2) { - flb_plg_error(ctx->ins, - "%s exists and it is not a unix socket. Aborting", - ctx->unix_path); - } - else { - flb_plg_error(ctx->ins, - "could not remove existing unix socket %s. Aborting", - ctx->unix_path); - } - - return -1; - } - - ctx->downstream = flb_downstream_create(mode, - ctx->ins->flags, - ctx->unix_path, - 0, - tls, - ctx->ins->config, - &ctx->ins->net_setup); - - if (ctx->downstream == NULL) { - return -1; - } - - if (chmod(ctx->unix_path, ctx->unix_perm)) { - flb_errno(); - flb_error("[in_syslog] cannot set permission on '%s' to %04o", - ctx->unix_path, ctx->unix_perm); - - return -1; - } - - return 0; -} -#else -static int syslog_server_unix_create(struct flb_syslog *ctx) -{ - return -1; -} -#endif - -static int syslog_server_net_create(struct flb_syslog *ctx) -{ - unsigned short int port; - int mode; - struct flb_tls *tls; - - port = (unsigned short int) strtoul(ctx->port, NULL, 10); - - if (ctx->mode == FLB_SYSLOG_TCP) { - mode = FLB_TRANSPORT_TCP; - tls = ctx->ins->tls; - } - else if (ctx->mode == FLB_SYSLOG_UDP) { - ctx->dgram_mode_flag = FLB_TRUE; - - mode = FLB_TRANSPORT_UDP; - tls = NULL; - } - else { - return -1; - } - - ctx->downstream = flb_downstream_create(mode, - ctx->ins->flags, - ctx->listen, - port, - tls, - ctx->ins->config, - &ctx->ins->net_setup); - - if (ctx->downstream != NULL) { - flb_info("[in_syslog] %s server binding %s:%s", - ((ctx->mode == FLB_SYSLOG_TCP) ? "TCP" : "UDP"), - ctx->listen, ctx->port); - } - else { - flb_error("[in_syslog] could not bind address %s:%s. Aborting", - ctx->listen, ctx->port); - - return -1; - } - - if (ctx->receive_buffer_size) { - if (flb_net_socket_rcv_buffer(ctx->downstream->server_fd, - ctx->receive_buffer_size)) { - flb_error("[in_syslog] could not set rcv buffer to %ld. Aborting", - ctx->receive_buffer_size); - return -1; - } - } - - flb_net_socket_nonblocking(ctx->downstream->server_fd); - - return 0; -} - -int syslog_server_create(struct flb_syslog *ctx) -{ - int ret; - - if (ctx->mode == FLB_SYSLOG_TCP || ctx->mode == FLB_SYSLOG_UDP) { - ret = syslog_server_net_create(ctx); - } - else { - /* Create unix socket end-point */ - ret = syslog_server_unix_create(ctx); - } - - if (ret != 0) { - return -1; - } - - return 0; -} - -int syslog_server_destroy(struct flb_syslog *ctx) -{ - if (ctx->collector_id != -1) { - flb_input_collector_delete(ctx->collector_id, ctx->ins); - - ctx->collector_id = -1; - } - - if (ctx->downstream != NULL) { - flb_downstream_destroy(ctx->downstream); - - ctx->downstream = NULL; - } - - if (ctx->mode == FLB_SYSLOG_UNIX_TCP || ctx->mode == FLB_SYSLOG_UNIX_UDP) { - if (ctx->unix_path) { - unlink(ctx->unix_path); - } - } - else { - flb_free(ctx->port); - } - - return 0; -} diff --git a/fluent-bit/plugins/in_syslog/syslog_server.h b/fluent-bit/plugins/in_syslog/syslog_server.h deleted file mode 100644 index d14feba7c..000000000 --- a/fluent-bit/plugins/in_syslog/syslog_server.h +++ /dev/null @@ -1,31 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_SYSLOG_SERVER_H -#define FLB_IN_SYSLOG_SERVER_H - -#include -#include - -#include "syslog.h" - -int syslog_server_create(struct flb_syslog *ctx); -int syslog_server_destroy(struct flb_syslog *ctx); - -#endif diff --git a/fluent-bit/plugins/in_systemd/CMakeLists.txt b/fluent-bit/plugins/in_systemd/CMakeLists.txt deleted file mode 100644 index 3e52d52b7..000000000 --- a/fluent-bit/plugins/in_systemd/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -set(src - systemd_config.c - systemd.c) - -if(FLB_SQLDB) -set(src - ${src} - systemd_db.c) -endif() - -FLB_PLUGIN(in_systemd "${src}" ${JOURNALD_LIBRARIES}) diff --git a/fluent-bit/plugins/in_systemd/systemd.c b/fluent-bit/plugins/in_systemd/systemd.c deleted file mode 100644 index 02f81144b..000000000 --- a/fluent-bit/plugins/in_systemd/systemd.c +++ /dev/null @@ -1,555 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include "systemd_config.h" -#include "systemd_db.h" - -#include - -/* msgpack helpers to pack unsigned ints (it takes care of endianness */ -#define pack_uint16(buf, d) _msgpack_store16(buf, (uint16_t) d) -#define pack_uint32(buf, d) _msgpack_store32(buf, (uint32_t) d) - -/* tag composer */ -static int tag_compose(const char *tag, const char *unit_name, - int unit_size, char **out_buf, size_t *out_size) -{ - int len; - const char *p; - char *buf = *out_buf; - size_t buf_s = 0; - - p = strchr(tag, '*'); - if (!p) { - return -1; - } - - /* Copy tag prefix if any */ - len = (p - tag); - if (len > 0) { - memcpy(buf, tag, len); - buf_s += len; - } - - /* Append file name */ - memcpy(buf + buf_s, unit_name, unit_size); - buf_s += unit_size; - - /* Tag suffix (if any) */ - p++; - if (*p) { - len = strlen(tag); - memcpy(buf + buf_s, p, (len - (p - tag))); - buf_s += (len - (p - tag)); - } - - buf[buf_s] = '\0'; - *out_size = buf_s; - - return 0; -} - -static int in_systemd_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - int ret; - int ret_j; - int i; - int len; - int entries = 0; - int skip_entries = 0; - int rows = 0; - time_t sec; - long nsec; - uint64_t usec; - size_t length; - size_t threshold; - const char *sep; - const char *key; - const char *val; - char *buf = NULL; -#ifdef FLB_HAVE_SQLDB - char *cursor = NULL; -#endif - char *tag = NULL; - char new_tag[PATH_MAX]; - char last_tag[PATH_MAX] = {0}; - size_t tag_len; - size_t last_tag_len = 0; - const void *data; - struct flb_systemd_config *ctx = in_context; - struct flb_time tm; - - /* Restricted by mem_buf_limit */ - if (flb_input_buf_paused(ins) == FLB_TRUE) { - return FLB_SYSTEMD_BUSY; - } - - /* - * if there are not pending records from a previous round, likely we got - * some changes in the journal, otherwise go ahead and continue reading - * the journal. - */ - if (ctx->pending_records == FLB_FALSE) { - ret = sd_journal_process(ctx->j); - if (ret == SD_JOURNAL_INVALIDATE) { - flb_plg_debug(ctx->ins, - "received event on added or removed journal file"); - } - if (ret != SD_JOURNAL_APPEND && ret != SD_JOURNAL_NOP) { - return FLB_SYSTEMD_NONE; - } - } - - if (ctx->lowercase == FLB_TRUE) { - ret = sd_journal_get_data_threshold(ctx->j, &threshold); - if (ret != 0) { - flb_plg_error(ctx->ins, - "error setting up systemd data. " - "sd_journal_get_data_threshold() return value '%i'", - ret); - return FLB_SYSTEMD_ERROR; - } - } - - while ((ret_j = sd_journal_next(ctx->j)) > 0) { - /* If the tag is composed dynamically, gather the Systemd Unit name */ - if (ctx->dynamic_tag) { - ret = sd_journal_get_data(ctx->j, "_SYSTEMD_UNIT", &data, &length); - if (ret == 0) { - tag = new_tag; - tag_compose(ctx->ins->tag, (const char *) data + 14, length - 14, - &tag, &tag_len); - } - else { - tag = new_tag; - tag_compose(ctx->ins->tag, - FLB_SYSTEMD_UNKNOWN, sizeof(FLB_SYSTEMD_UNKNOWN) - 1, - &tag, &tag_len); - } - } - else { - tag = ctx->ins->tag; - tag_len = ctx->ins->tag_len; - } - - if (last_tag_len == 0) { - strncpy(last_tag, tag, tag_len); - last_tag_len = tag_len; - } - - /* Set time */ - ret = sd_journal_get_realtime_usec(ctx->j, &usec); - if (ret != 0) { - flb_plg_error(ctx->ins, - "error reading from systemd journal. " - "sd_journal_get_realtime_usec() return value '%i'", - ret); - /* It seems the journal file was deleted (rotated). */ - ret_j = -1; - break; - } - sec = usec / 1000000; - nsec = (usec % 1000000) * 1000; - flb_time_set(&tm, sec, nsec); - - /* - * The new incoming record can have a different tag than previous one, - * so a new msgpack buffer is required. We ingest the data and prepare - * a new buffer. - */ - if (ctx->log_encoder->output_length > 0 && - ((last_tag_len != tag_len) || - (strncmp(last_tag, tag, tag_len) != 0))) { - flb_input_log_append(ctx->ins, - last_tag, last_tag_len, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); - - flb_log_event_encoder_reset(ctx->log_encoder); - - strncpy(last_tag, tag, tag_len); - last_tag_len = tag_len; - } - - - ret = flb_log_event_encoder_begin_record(ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_timestamp(ctx->log_encoder, &tm); - } - - /* Pack every field in the entry */ - entries = 0; - skip_entries = 0; - while (sd_journal_enumerate_data(ctx->j, &data, &length) > 0 && - entries < ctx->max_fields) { - key = (const char *) data; - if (ctx->strip_underscores == FLB_TRUE && key[0] == '_') { - key++; - length--; - } - - sep = strchr(key, '='); - if (sep == NULL) { - skip_entries++; - continue; - } - - len = (sep - key); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_string_length( - ctx->log_encoder, len); - } - - if (ctx->lowercase == FLB_TRUE) { - /* - * Ensure buf to have enough space for the key because the libsystemd - * might return larger data than the threshold. - */ - if (buf == NULL) { - buf = flb_sds_create_len(NULL, threshold); - } - if (flb_sds_alloc(buf) < len) { - buf = flb_sds_increase(buf, len - flb_sds_alloc(buf)); - } - for (i = 0; i < len; i++) { - buf[i] = tolower(key[i]); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_string_body( - ctx->log_encoder, buf, len); - } - } - else { - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_string_body( - ctx->log_encoder, (char *) key, len); - } - } - - val = sep + 1; - len = length - (sep - key) - 1; - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_string( - ctx->log_encoder, (char *) val, len); - } - - entries++; - } - rows++; - - if (skip_entries > 0) { - flb_plg_error(ctx->ins, "Skip %d broken entries", skip_entries); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(ctx->log_encoder); - } - - /* - * Some journals can have too much data, pause if we have processed - * more than 1MB. Journal will resume later. - */ - if (ctx->log_encoder->output_length > 1024000) { - flb_input_log_append(ctx->ins, - tag, tag_len, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); - - flb_log_event_encoder_reset(ctx->log_encoder); - - strncpy(last_tag, tag, tag_len); - last_tag_len = tag_len; - - break; - } - - if (rows >= ctx->max_entries) { - break; - } - } - - flb_sds_destroy(buf); - -#ifdef FLB_HAVE_SQLDB - /* Save cursor */ - if (ctx->db) { - sd_journal_get_cursor(ctx->j, &cursor); - if (cursor) { - flb_systemd_db_set_cursor(ctx, cursor); - flb_free(cursor); - } - } -#endif - - /* Write any pending data into the buffer */ - if (ctx->log_encoder->output_length > 0) { - flb_input_log_append(ctx->ins, - tag, tag_len, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); - - flb_log_event_encoder_reset(ctx->log_encoder); - } - - /* the journal is empty, no more records */ - if (ret_j == 0) { - ctx->pending_records = FLB_FALSE; - return FLB_SYSTEMD_OK; - } - else if (ret_j > 0) { - /* - * ret_j == 1, but the loop was broken due to some special condition like - * buffer size limit or it reach the max number of rows that it supposed to - * process on this call. Assume there are pending records. - */ - ctx->pending_records = FLB_TRUE; - return FLB_SYSTEMD_MORE; - } - else { - /* Supposedly, current cursor points to a deleted file. - * Re-seeking to the first journal entry. - * Other failures, such as disk read error, would still lead to infinite loop there, - * but at least FLB log will be full of errors. */ - ret = sd_journal_seek_head(ctx->j); - flb_plg_error(ctx->ins, - "sd_journal_next() returned error %i; " - "journal is re-opened, unread logs are lost; " - "sd_journal_seek_head() returned %i", ret_j, ret); - ctx->pending_records = FLB_TRUE; - return FLB_SYSTEMD_ERROR; - } -} - -static int in_systemd_collect_archive(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - int ret; - uint64_t val; - ssize_t bytes; - struct flb_systemd_config *ctx = in_context; - - bytes = read(ctx->ch_manager[0], &val, sizeof(uint64_t)); - if (bytes == -1) { - flb_errno(); - return -1; - } - - ret = in_systemd_collect(ins, config, in_context); - if (ret == FLB_SYSTEMD_OK) { - /* Events collector: journald events */ - ret = flb_input_set_collector_event(ins, - in_systemd_collect, - ctx->fd, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "error setting up collector events"); - flb_systemd_config_destroy(ctx); - return -1; - } - ctx->coll_fd_journal = ret; - flb_input_collector_start(ctx->coll_fd_journal, ins); - - /* Timer to collect pending events */ - ret = flb_input_set_collector_time(ins, - in_systemd_collect, - 1, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "error setting up collector for pending events"); - flb_systemd_config_destroy(ctx); - return -1; - } - ctx->coll_fd_pending = ret; - flb_input_collector_start(ctx->coll_fd_pending, ins); - - return 0; - } - - /* If FLB_SYSTEMD_NONE or FLB_SYSTEMD_MORE, keep trying */ - write(ctx->ch_manager[1], &val, sizeof(uint64_t)); - - return 0; -} - -static int in_systemd_init(struct flb_input_instance *ins, - struct flb_config *config, void *data) -{ - int ret; - struct flb_systemd_config *ctx; - - ctx = flb_systemd_config_create(ins, config); - if (!ctx) { - flb_plg_error(ins, "cannot initialize"); - return -1; - } - - /* Set the context */ - flb_input_set_context(ins, ctx); - - /* Events collector: archive */ - ret = flb_input_set_collector_event(ins, in_systemd_collect_archive, - ctx->ch_manager[0], config); - if (ret == -1) { - flb_systemd_config_destroy(ctx); - return -1; - } - ctx->coll_fd_archive = ret; - - return 0; -} - -static int in_systemd_pre_run(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - int n; - uint64_t val = 0xc002; - struct flb_systemd_config *ctx = in_context; - (void) ins; - (void) config; - - /* Insert a dummy event into the channel manager */ - n = write(ctx->ch_manager[1], &val, sizeof(val)); - if (n == -1) { - flb_errno(); - return -1; - } - - return n; -} - -static void in_systemd_pause(void *data, struct flb_config *config) -{ - int ret; - struct flb_systemd_config *ctx = data; - - flb_input_collector_pause(ctx->coll_fd_archive, ctx->ins); - - /* pause only if it's running */ - ret = flb_input_collector_running(ctx->coll_fd_journal, ctx->ins); - if (ret == FLB_TRUE) { - flb_input_collector_pause(ctx->coll_fd_journal, ctx->ins); - flb_input_collector_pause(ctx->coll_fd_pending, ctx->ins); - } -} - -static void in_systemd_resume(void *data, struct flb_config *config) -{ - int ret; - struct flb_systemd_config *ctx = data; - - flb_input_collector_resume(ctx->coll_fd_archive, ctx->ins); - - /* resume only if is not running */ - ret = flb_input_collector_running(ctx->coll_fd_journal, ctx->ins); - if (ret == FLB_FALSE) { - flb_input_collector_resume(ctx->coll_fd_journal, ctx->ins); - flb_input_collector_resume(ctx->coll_fd_pending, ctx->ins); - } -} - -static int in_systemd_exit(void *data, struct flb_config *config) -{ - (void) *config; - struct flb_systemd_config *ctx = data; - - flb_systemd_config_destroy(ctx); - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "path", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_systemd_config, path), - "Set the systemd journal path" - }, - { - FLB_CONFIG_MAP_INT, "max_fields", FLB_SYSTEMD_MAX_FIELDS, - 0, FLB_TRUE, offsetof(struct flb_systemd_config, max_fields), - "Set the maximum fields per notification" - }, - { - FLB_CONFIG_MAP_INT, "max_entries", FLB_SYSTEMD_MAX_ENTRIES, - 0, FLB_TRUE, offsetof(struct flb_systemd_config, max_entries), - "Set the maximum entries per notification" - }, - { - FLB_CONFIG_MAP_STR, "systemd_filter_type", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_systemd_config, filter_type), - "Set the systemd filter type to either 'and' or 'or'" - }, - { - FLB_CONFIG_MAP_STR, "systemd_filter", (char *)NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct flb_systemd_config, systemd_filters), - "Add a systemd filter, can be set multiple times" - }, - { - FLB_CONFIG_MAP_BOOL, "read_from_tail", "false", - 0, FLB_TRUE, offsetof(struct flb_systemd_config, read_from_tail), - "Read the journal from the end (tail)" - }, - { - FLB_CONFIG_MAP_BOOL, "lowercase", "false", - 0, FLB_TRUE, offsetof(struct flb_systemd_config, lowercase), - "Lowercase the fields" - }, - { - FLB_CONFIG_MAP_BOOL, "strip_underscores", "false", - 0, FLB_TRUE, offsetof(struct flb_systemd_config, strip_underscores), - "Strip undersecores from fields" - }, -#ifdef FLB_HAVE_SQLDB - { - FLB_CONFIG_MAP_STR, "db.sync", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_systemd_config, db_sync_mode), - "Set the database sync mode: extra, full, normal or off" - }, - { - FLB_CONFIG_MAP_STR, "db", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_systemd_config, db_path), - "Set the database path" - }, -#endif /* FLB_HAVE_SQLDB */ - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_input_plugin in_systemd_plugin = { - .name = "systemd", - .description = "Systemd (Journal) reader", - .cb_init = in_systemd_init, - .cb_pre_run = in_systemd_pre_run, - .cb_flush_buf = NULL, - .cb_pause = in_systemd_pause, - .cb_resume = in_systemd_resume, - .cb_exit = in_systemd_exit, - .config_map = config_map, - .flags = 0 -}; diff --git a/fluent-bit/plugins/in_systemd/systemd_config.c b/fluent-bit/plugins/in_systemd/systemd_config.c deleted file mode 100644 index 57c13a859..000000000 --- a/fluent-bit/plugins/in_systemd/systemd_config.c +++ /dev/null @@ -1,314 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include -#include -#include - -#ifdef FLB_HAVE_SQLDB -#include "systemd_db.h" -#endif - -#include "systemd_config.h" - -struct flb_systemd_config *flb_systemd_config_create(struct flb_input_instance *ins, - struct flb_config *config) -{ - int ret; - const char *tmp; - char *cursor = NULL; - struct stat st; - struct mk_list *head; - struct flb_systemd_config *ctx; - int journal_filter_is_and; - size_t size; - struct flb_config_map_val *mv; - - - /* Allocate space for the configuration */ - ctx = flb_calloc(1, sizeof(struct flb_systemd_config)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; -#ifdef FLB_HAVE_SQLDB - ctx->db_sync = -1; -#endif - - /* Load the config_map */ - ret = flb_input_config_map_set(ins, (void *)ctx); - if (ret == -1) { - flb_plg_error(ins, "unable to load configuration"); - flb_free(config); - return NULL; - } - - /* Create the channel manager */ - ret = pipe(ctx->ch_manager); - if (ret == -1) { - flb_errno(); - flb_free(ctx); - return NULL; - } - - /* Config: path */ - if (ctx->path) { - ret = stat(ctx->path, &st); - if (ret == -1) { - flb_errno(); - flb_plg_error(ctx->ins, "given path %s is invalid", ctx->path); - flb_free(ctx); - return NULL; - } - - if (!S_ISDIR(st.st_mode)) { - flb_errno(); - flb_plg_error(ctx->ins, "given path is not a directory: %s", ctx->path); - flb_free(ctx); - return NULL; - } - } - else { - ctx->path = NULL; - } - - /* Open the Journal */ - if (ctx->path) { - ret = sd_journal_open_directory(&ctx->j, ctx->path, 0); - } - else { - ret = sd_journal_open(&ctx->j, SD_JOURNAL_LOCAL_ONLY); - } - if (ret != 0) { - flb_plg_error(ctx->ins, "could not open the Journal"); - flb_free(ctx); - return NULL; - } - ctx->fd = sd_journal_get_fd(ctx->j); - - /* Tag settings */ - tmp = strchr(ins->tag, '*'); - if (tmp) { - ctx->dynamic_tag = FLB_TRUE; - } - else { - ctx->dynamic_tag = FLB_FALSE; - } - -#ifdef FLB_HAVE_SQLDB - /* Database options (needs to be set before the context) */ - if (ctx->db_sync_mode) { - if (strcasecmp(ctx->db_sync_mode, "extra") == 0) { - ctx->db_sync = 3; - } - else if (strcasecmp(ctx->db_sync_mode, "full") == 0) { - ctx->db_sync = 2; - } - else if (strcasecmp(ctx->db_sync_mode, "normal") == 0) { - ctx->db_sync = 1; - } - else if (strcasecmp(ctx->db_sync_mode, "off") == 0) { - ctx->db_sync = 0; - } - else { - flb_plg_error(ctx->ins, "invalid database 'db.sync' value: %s", ctx->db_sync_mode); - } - } - - /* Database file */ - if (ctx->db_path) { - ctx->db = flb_systemd_db_open(ctx->db_path, ins, ctx, config); - if (!ctx->db) { - flb_plg_error(ctx->ins, "could not open/create database '%s'", ctx->db_path); - } - } - -#endif - - if (ctx->filter_type) { - if (strcasecmp(ctx->filter_type, "and") == 0) { - journal_filter_is_and = FLB_TRUE; - } - else if (strcasecmp(ctx->filter_type, "or") == 0) { - journal_filter_is_and = FLB_FALSE; - } - else { - flb_plg_error(ctx->ins, - "systemd_filter_type must be 'and' or 'or'. Got %s", - ctx->filter_type); - flb_free(ctx); - return NULL; - } - } - else { - journal_filter_is_and = FLB_FALSE; - } - - /* Load Systemd filters */ - if (ctx->systemd_filters) { - flb_config_map_foreach(head, mv, ctx->systemd_filters) { - flb_plg_debug(ctx->ins, "add filter: %s (%s)", mv->val.str, - journal_filter_is_and ? "and" : "or"); - ret = sd_journal_add_match(ctx->j, mv->val.str, 0); - if (ret < 0) { - if (ret == -EINVAL) { - flb_plg_error(ctx->ins, - "systemd_filter error: invalid input '%s'", - mv->val.str); - } - else { - flb_plg_error(ctx->ins, - "systemd_filter error: status=%d input '%s'", - ret, mv->val.str); - } - flb_systemd_config_destroy(ctx); - return NULL; - } - if (journal_filter_is_and) { - ret = sd_journal_add_conjunction(ctx->j); - if (ret < 0) { - flb_plg_error(ctx->ins, - "sd_journal_add_conjunction failed. ret=%d", - ret); - flb_systemd_config_destroy(ctx); - return NULL; - } - } - else { - ret = sd_journal_add_disjunction(ctx->j); - if (ret < 0) { - flb_plg_error(ctx->ins, - "sd_journal_add_disjunction failed. ret=%d", - ret); - flb_systemd_config_destroy(ctx); - return NULL; - } - } - } - } - - if (ctx->read_from_tail == FLB_TRUE) { - sd_journal_seek_tail(ctx->j); - /* - * Skip up to 350 records until the end of journal is found. - * Workaround for bug https://github.com/systemd/systemd/issues/9934 - * Due to the bug, sd_journal_next() returns 2 last records of each journal file. - * 4 GB is the default journal limit, so with 25 MB/file we may get - * up to 4096/25*2 ~= 350 old log messages. See also fluent-bit PR #1565. - */ - ret = sd_journal_next_skip(ctx->j, 350); - flb_plg_debug(ctx->ins, - "jump to the end of journal and skip %d last entries", ret); - } - else { - ret = sd_journal_seek_head(ctx->j); - } - -#ifdef FLB_HAVE_SQLDB - /* Check if we have a cursor in our database */ - if (ctx->db) { - /* Initialize prepared statement */ - ret = sqlite3_prepare_v2(ctx->db->handler, - SQL_UPDATE_CURSOR, - -1, - &ctx->stmt_cursor, - 0); - if (ret != SQLITE_OK) { - flb_plg_error(ctx->ins, "error preparing database SQL statement"); - flb_systemd_config_destroy(ctx); - return NULL; - } - - /* Get current cursor */ - cursor = flb_systemd_db_get_cursor(ctx); - if (cursor) { - ret = sd_journal_seek_cursor(ctx->j, cursor); - if (ret == 0) { - flb_plg_info(ctx->ins, "seek_cursor=%.40s... OK", cursor); - - /* Skip the first entry, already processed */ - sd_journal_next_skip(ctx->j, 1); - } - else { - flb_plg_warn(ctx->ins, "seek_cursor failed"); - } - flb_free(cursor); - } - else { - /* Insert the first row */ - cursor = NULL; - flb_systemd_db_init_cursor(ctx, cursor); - if (cursor) { - flb_free(cursor); - } - } - } -#endif - - ctx->log_encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ctx->log_encoder == NULL) { - flb_plg_error(ctx->ins, "could not initialize event encoder"); - flb_systemd_config_destroy(ctx); - - return NULL; - } - - - sd_journal_get_data_threshold(ctx->j, &size); - flb_plg_debug(ctx->ins, - "sd_journal library may truncate values " - "to sd_journal_get_data_threshold() bytes: %zu", size); - - return ctx; -} - -int flb_systemd_config_destroy(struct flb_systemd_config *ctx) -{ - if (ctx->log_encoder != NULL) { - flb_log_event_encoder_destroy(ctx->log_encoder); - - ctx->log_encoder = NULL; - } - - /* Close context */ - if (ctx->j) { - sd_journal_close(ctx->j); - } - -#ifdef FLB_HAVE_SQLDB - if (ctx->db) { - sqlite3_finalize(ctx->stmt_cursor); - flb_systemd_db_close(ctx->db); - } -#endif - - close(ctx->ch_manager[0]); - close(ctx->ch_manager[1]); - - flb_free(ctx); - return 0; -} diff --git a/fluent-bit/plugins/in_systemd/systemd_config.h b/fluent-bit/plugins/in_systemd/systemd_config.h deleted file mode 100644 index db5c4cf53..000000000 --- a/fluent-bit/plugins/in_systemd/systemd_config.h +++ /dev/null @@ -1,82 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_SYSTEMD_CONFIG_H -#define FLB_SYSTEMD_CONFIG_H - -#include -#include -#include -#include -#include - -#include - -/* return values */ -#define FLB_SYSTEMD_ERROR -1 /* Systemd journal file read error. */ -#define FLB_SYSTEMD_NONE 0 -#define FLB_SYSTEMD_OK 1 -#define FLB_SYSTEMD_MORE 2 -#define FLB_SYSTEMD_BUSY 3 - -/* constants */ -#define FLB_SYSTEMD_UNIT "_SYSTEMD_UNIT" -#define FLB_SYSTEMD_UNKNOWN "unknown" -#define FLB_SYSTEMD_MAX_FIELDS "8000" -#define FLB_SYSTEMD_MAX_ENTRIES "5000" - -/* Input configuration & context */ -struct flb_systemd_config { - /* Journal */ - int fd; /* Journal file descriptor */ - sd_journal *j; /* Journal context */ - char *cursor; - flb_sds_t path; - flb_sds_t filter_type; /* sysytemd filter type: and|or */ - struct mk_list *systemd_filters; - int pending_records; - int read_from_tail; /* read_from_tail option */ - int lowercase; - int strip_underscores; - - /* Internal */ - int ch_manager[2]; /* pipe: channel manager */ - int coll_fd_archive; /* archive collector */ - int coll_fd_journal; /* journal, events mode */ - int coll_fd_pending; /* pending records */ - int dynamic_tag; - int max_fields; /* max number of fields per record */ - int max_entries; /* max number of records per iteration */ - -#ifdef FLB_HAVE_SQLDB - flb_sds_t db_path; - struct flb_sqldb *db; - flb_sds_t db_sync_mode; - int db_sync; - sqlite3_stmt *stmt_cursor; -#endif - struct flb_input_instance *ins; - struct flb_log_event_encoder *log_encoder; -}; - -struct flb_systemd_config *flb_systemd_config_create(struct flb_input_instance *i_ins, - struct flb_config *config); - -int flb_systemd_config_destroy(struct flb_systemd_config *ctx); -#endif diff --git a/fluent-bit/plugins/in_systemd/systemd_db.c b/fluent-bit/plugins/in_systemd/systemd_db.c deleted file mode 100644 index 0abc4d999..000000000 --- a/fluent-bit/plugins/in_systemd/systemd_db.c +++ /dev/null @@ -1,197 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include "systemd_config.h" -#include "systemd_db.h" - -struct query_status { - int rows; - char *cursor; - time_t updated; -}; - -static int cb_cursor_check(void *data, int argc, char **argv, char **cols) -{ - struct query_status *qs = data; - - qs->cursor = flb_strdup(argv[0]); /* cursor string */ - qs->updated = atoll(argv[1]); /* timestamp */ - qs->rows++; - - return 0; -} - -static int cb_count_check(void *data, int argc, char **argv, char **cols) -{ - struct query_status *qs = data; - - qs->rows = atoll(argv[0]); - return 0; -} - -/* sanitize database table if required */ -static void flb_systemd_db_sanitize(struct flb_sqldb *db, - struct flb_input_instance *ins) -{ - int ret; - struct query_status qs = {0}; - - memset(&qs, '\0', sizeof(qs)); - ret = flb_sqldb_query(db, - SQL_COUNT_CURSOR, cb_count_check, &qs); - if (ret != FLB_OK) { - flb_plg_error(ins, "db: failed counting number of rows"); - return; - } - - if (qs.rows > 1) { - flb_plg_warn(ins, - "db: table in_systemd_cursor looks corrupted, it has " - "more than one entry (rows=%i), the table content will be " - "fixed", qs.rows); - - /* Delete duplicates, we only preserve the last record based on it ROWID */ - ret = flb_sqldb_query(db, SQL_DELETE_DUPS, NULL, NULL); - if (ret != FLB_OK) { - flb_plg_error(ins, "could not delete in_systemd_cursor duplicates"); - return; - } - flb_plg_info(ins, "table in_systemd_cursor has been fixed"); - } - -} - -struct flb_sqldb *flb_systemd_db_open(const char *path, - struct flb_input_instance *ins, - struct flb_systemd_config *ctx, - struct flb_config *config) -{ - int ret; - char tmp[64]; - struct flb_sqldb *db; - - /* Open/create the database */ - db = flb_sqldb_open(path, ins->name, config); - if (!db) { - return NULL; - } - - /* Create table schema if it don't exists */ - ret = flb_sqldb_query(db, SQL_CREATE_CURSOR, NULL, NULL); - if (ret != FLB_OK) { - flb_plg_error(ins, "db: could not create 'cursor' table"); - flb_sqldb_close(db); - return NULL; - } - - if (ctx->db_sync >= 0) { - snprintf(tmp, sizeof(tmp) - 1, SQL_PRAGMA_SYNC, - ctx->db_sync); - ret = flb_sqldb_query(db, tmp, NULL, NULL); - if (ret != FLB_OK) { - flb_plg_error(ctx->ins, "db could not set pragma 'sync'"); - flb_sqldb_close(db); - return NULL; - } - } - - flb_systemd_db_sanitize(db, ins); - - return db; -} - -int flb_systemd_db_close(struct flb_sqldb *db) -{ - flb_sqldb_close(db); - return 0; -} - -int flb_systemd_db_init_cursor(struct flb_systemd_config *ctx, const char *cursor) -{ - int ret; - char query[PATH_MAX]; - struct query_status qs = {0}; - - /* Check if the file exists */ - memset(&qs, '\0', sizeof(qs)); - ret = flb_sqldb_query(ctx->db, - SQL_GET_CURSOR, cb_cursor_check, &qs); - - if (ret != FLB_OK) { - return -1; - } - - if (qs.rows == 0) { - /* Register the cursor */ - snprintf(query, sizeof(query) - 1, - SQL_INSERT_CURSOR, - cursor, time(NULL)); - ret = flb_sqldb_query(ctx->db, - query, NULL, NULL); - if (ret == FLB_ERROR) { - return -1; - } - return 0; - } - - return -1; -} - -int flb_systemd_db_set_cursor(struct flb_systemd_config *ctx, const char *cursor) -{ - int ret; - - /* Bind parameters */ - sqlite3_bind_text(ctx->stmt_cursor, 1, (char *) cursor, -1, 0); - sqlite3_bind_int64(ctx->stmt_cursor, 2, time(NULL)); - - ret = sqlite3_step(ctx->stmt_cursor); - - sqlite3_clear_bindings(ctx->stmt_cursor); - sqlite3_reset(ctx->stmt_cursor); - - if (ret != SQLITE_DONE) { - return -1; - } - return 0; -} - -char *flb_systemd_db_get_cursor(struct flb_systemd_config *ctx) -{ - int ret; - struct query_status qs = {0}; - - memset(&qs, '\0', sizeof(qs)); - ret = flb_sqldb_query(ctx->db, - SQL_GET_CURSOR, cb_cursor_check, &qs); - if (ret != FLB_OK) { - return NULL; - } - - if (qs.rows > 0) { - /* cursor must be freed by the caller */ - return qs.cursor; - } - - return NULL; -} diff --git a/fluent-bit/plugins/in_systemd/systemd_db.h b/fluent-bit/plugins/in_systemd/systemd_db.h deleted file mode 100644 index da8d6e195..000000000 --- a/fluent-bit/plugins/in_systemd/systemd_db.h +++ /dev/null @@ -1,64 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_SYSTEMD_DB_H -#define FLB_SYSTEMD_DB_H - -#include -#include -#include - -#include "systemd_config.h" - -#define SQL_CREATE_CURSOR \ - "CREATE TABLE IF NOT EXISTS in_systemd_cursor (" \ - " cursor TEXT NOT NULL," \ - " updated INTEGER" \ - ");" - -#define SQL_GET_CURSOR \ - "SELECT * FROM in_systemd_cursor LIMIT 1;" - -#define SQL_INSERT_CURSOR \ - "INSERT INTO in_systemd_cursor (cursor, updated)" \ - " VALUES ('%s', %lu);" - -#define SQL_COUNT_CURSOR \ - "SELECT COUNT(*) FROM in_systemd_cursor;" - -#define SQL_UPDATE_CURSOR \ - "UPDATE in_systemd_cursor SET cursor=@cursor, updated=@updated;" - -#define SQL_DELETE_DUPS \ - "DELETE FROM in_systemd_cursor WHERE ROWID < " \ - "(SELECT MAX(ROWID) FROM in_systemd_cursor);" - -#define SQL_PRAGMA_SYNC \ - "PRAGMA synchronous=%i;" - -struct flb_sqldb *flb_systemd_db_open(const char *path, - struct flb_input_instance *ins, - struct flb_systemd_config *ctx, - struct flb_config *config); -int flb_systemd_db_close(struct flb_sqldb *db); -int flb_systemd_db_init_cursor(struct flb_systemd_config *ctx, const char *cursor); -int flb_systemd_db_set_cursor(struct flb_systemd_config *ctx, const char *cursor); -char *flb_systemd_db_get_cursor(struct flb_systemd_config *ctx); - -#endif diff --git a/fluent-bit/plugins/in_tail/CMakeLists.txt b/fluent-bit/plugins/in_tail/CMakeLists.txt deleted file mode 100644 index 31d865218..000000000 --- a/fluent-bit/plugins/in_tail/CMakeLists.txt +++ /dev/null @@ -1,37 +0,0 @@ -set(src - tail_file.c - tail_dockermode.c - tail_scan.c - tail_config.c - tail_fs_stat.c - tail.c) - -if(FLB_HAVE_INOTIFY) -set(src - ${src} - tail_fs_inotify.c) -endif() - -if(FLB_SQLDB) -set(src - ${src} - tail_db.c) -endif() - -if(FLB_PARSER) - set(src - ${src} - tail_multiline.c - ) -endif() - -if(MSVC) - set(src - ${src} - win32/stat.c - win32/io.c - ) - FLB_PLUGIN(in_tail "${src}" "Shlwapi") -else() - FLB_PLUGIN(in_tail "${src}" "") -endif() diff --git a/fluent-bit/plugins/in_tail/tail.c b/fluent-bit/plugins/in_tail/tail.c deleted file mode 100644 index 34a0fec3d..000000000 --- a/fluent-bit/plugins/in_tail/tail.c +++ /dev/null @@ -1,783 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "tail.h" -#include "tail_fs.h" -#include "tail_db.h" -#include "tail_file.h" -#include "tail_scan.h" -#include "tail_signal.h" -#include "tail_config.h" -#include "tail_dockermode.h" -#include "tail_multiline.h" - -static inline int consume_byte(flb_pipefd_t fd) -{ - int ret; - uint64_t val; - - /* We need to consume the byte */ - ret = flb_pipe_r(fd, (char *) &val, sizeof(val)); - if (ret <= 0) { - flb_errno(); - return -1; - } - - return 0; -} - -/* cb_collect callback */ -static int in_tail_collect_pending(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - int ret; - int active = 0; - struct mk_list *tmp; - struct mk_list *head; - struct flb_tail_config *ctx = in_context; - struct flb_tail_file *file; - struct stat st; - uint64_t pre; - uint64_t total_processed = 0; - - /* Iterate promoted event files with pending bytes */ - mk_list_foreach_safe(head, tmp, &ctx->files_event) { - file = mk_list_entry(head, struct flb_tail_file, _head); - - if (file->watch_fd == -1 || - (file->offset >= file->size)) { - ret = fstat(file->fd, &st); - - if (ret == -1) { - flb_errno(); - flb_tail_file_remove(file); - continue; - } - - file->size = st.st_size; - file->pending_bytes = (file->size - file->offset); - } - else { - memset(&st, 0, sizeof(struct stat)); - } - - if (file->pending_bytes <= 0) { - file->pending_bytes = (file->size - file->offset); - } - - if (file->pending_bytes <= 0) { - continue; - } - - if (ctx->event_batch_size > 0 && - total_processed >= ctx->event_batch_size) { - break; - } - - /* get initial offset to calculate the number of processed bytes later */ - pre = file->offset; - - ret = flb_tail_file_chunk(file); - - /* Update the total number of bytes processed */ - if (file->offset > pre) { - total_processed += (file->offset - pre); - } - - switch (ret) { - case FLB_TAIL_ERROR: - /* Could not longer read the file */ - flb_tail_file_remove(file); - break; - case FLB_TAIL_OK: - case FLB_TAIL_BUSY: - /* - * Adjust counter to verify if we need a further read(2) later. - * For more details refer to tail_fs_inotify.c:96. - */ - if (file->offset < file->size) { - file->pending_bytes = (file->size - file->offset); - active++; - } - else { - file->pending_bytes = 0; - } - break; - } - } - - /* If no more active files, consume pending signal so we don't get called again. */ - if (active == 0) { - tail_consume_pending(ctx); - } - - return 0; -} - -/* cb_collect callback */ -static int in_tail_collect_static(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - int ret; - int active = 0; - int pre_size; - int pos_size; - int alter_size = 0; - int completed = FLB_FALSE; - char s_size[32]; - struct mk_list *tmp; - struct mk_list *head; - struct flb_tail_config *ctx = in_context; - struct flb_tail_file *file; - uint64_t pre; - uint64_t total_processed = 0; - - /* Do a data chunk collection for each file */ - mk_list_foreach_safe(head, tmp, &ctx->files_static) { - file = mk_list_entry(head, struct flb_tail_file, _head); - - /* - * The list 'files_static' represents all the files that were discovered - * on startup that already contains data: these are called 'static files'. - * - * When processing static files, we don't know what kind of content they - * have and what kind of 'latency' might add to process all of them in - * a row. Despite we always 'try' to do a full round and process a - * fraction of them on every invocation of this function if we have a - * huge number of files we will face latency and make the main pipeline - * to degrade performance. - * - * In order to avoid this situation, we added a new option to the plugin - * called 'static_batch_size' which basically defines how many bytes can - * be processed on every invocation to process the static files. - * - * When the limit is reached, we just break the loop and as a side effect - * we allow other events keep processing. - */ - if (ctx->static_batch_size > 0 && - total_processed >= ctx->static_batch_size) { - break; - } - - /* get initial offset to calculate the number of processed bytes later */ - pre = file->offset; - - /* Process the file */ - ret = flb_tail_file_chunk(file); - - /* Update the total number of bytes processed */ - if (file->offset > pre) { - total_processed += (file->offset - pre); - } - - switch (ret) { - case FLB_TAIL_ERROR: - /* Could not longer read the file */ - flb_plg_debug(ctx->ins, "inode=%"PRIu64" collect static ERROR", - file->inode); - flb_tail_file_remove(file); - break; - case FLB_TAIL_OK: - case FLB_TAIL_BUSY: - active++; - break; - case FLB_TAIL_WAIT: - if (file->config->exit_on_eof) { - flb_plg_info(ctx->ins, "inode=%"PRIu64" file=%s ended, stop", - file->inode, file->name); - if (ctx->files_static_count == 1) { - flb_engine_exit(config); - } - } - /* Promote file to 'events' type handler */ - flb_plg_debug(ctx->ins, "inode=%"PRIu64" file=%s promote to TAIL_EVENT", - file->inode, file->name); - - /* - * When promoting a file from 'static' to 'event' mode, the promoter - * will check if the file has been rotated while it was being - * processed on this function, if so, it will try to check for the - * following condition: - * - * "discover a new possible file created due to rotation" - * - * If the condition above is met, a new file entry will be added to - * the list that we are processing and a 'new signal' will be send - * to the signal manager. But the signal manager will trigger the - * message only if no pending messages exists (to avoid queue size - * exhaustion). - * - * All good, but there is a corner case where if no 'active' files - * exists, the signal will be read and this function will not be - * called again and since the signal did not triggered the - * message, the 'new file' enqueued by the nested function - * might stay in stale mode (note that altering the length of this - * list will not be reflected yet) - * - * To fix the corner case, we use a variable called 'alter_size' - * that determinate if the size of the list keeps the same after - * a rotation, so it means: a new file was added. - * - * We use 'alter_size' as a helper in the conditional below to know - * when to stop processing the static list. - */ - if (alter_size == 0) { - pre_size = ctx->files_static_count; - } - ret = flb_tail_file_to_event(file); - if (ret == -1) { - flb_plg_debug(ctx->ins, "file=%s cannot promote, unregistering", - file->name); - flb_tail_file_remove(file); - } - - if (alter_size == 0) { - pos_size = ctx->files_static_count; - if (pre_size == pos_size) { - alter_size++; - } - } - break; - } - } - - /* - * If there are no more active static handlers, we consume the 'byte' that - * triggered this event so this is not longer called again. - */ - if (active == 0 && alter_size == 0) { - consume_byte(ctx->ch_manager[0]); - ctx->ch_reads++; - completed = FLB_TRUE; - } - - /* Debugging number of processed bytes */ - if (flb_log_check_level(ctx->ins->log_level, FLB_LOG_DEBUG)) { - flb_utils_bytes_to_human_readable_size(total_processed, - s_size, sizeof(s_size)); - if (completed) { - flb_plg_debug(ctx->ins, "[static files] processed %s, done", s_size); - } - else { - flb_plg_debug(ctx->ins, "[static files] processed %s", s_size); - } - } - - return 0; -} - -static int in_tail_watcher_callback(struct flb_input_instance *ins, - struct flb_config *config, void *context) -{ - int ret = 0; - struct mk_list *tmp; - struct mk_list *head; - struct flb_tail_config *ctx = context; - struct flb_tail_file *file; - (void) config; - - mk_list_foreach_safe(head, tmp, &ctx->files_event) { - file = mk_list_entry(head, struct flb_tail_file, _head); - if (file->is_link == FLB_TRUE) { - ret = flb_tail_file_is_rotated(ctx, file); - if (ret == FLB_FALSE) { - continue; - } - - /* The symbolic link name has been rotated */ - flb_tail_file_rotated(file); - } - } - return ret; -} - -int in_tail_collect_event(void *file, struct flb_config *config) -{ - int ret; - struct stat st; - struct flb_tail_file *f = file; - - ret = fstat(f->fd, &st); - if (ret == -1) { - flb_tail_file_remove(f); - return 0; - } - - ret = flb_tail_file_chunk(f); - switch (ret) { - case FLB_TAIL_ERROR: - /* Could not longer read the file */ - flb_tail_file_remove(f); - break; - case FLB_TAIL_OK: - case FLB_TAIL_WAIT: - break; - } - - return 0; -} - -/* Initialize plugin */ -static int in_tail_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - int ret = -1; - struct flb_tail_config *ctx = NULL; - - /* Allocate space for the configuration */ - ctx = flb_tail_config_create(in, config); - if (!ctx) { - return -1; - } - ctx->ins = in; - - /* Initialize file-system watcher */ - ret = flb_tail_fs_init(in, ctx, config); - if (ret == -1) { - flb_tail_config_destroy(ctx); - return -1; - } - - /* Scan path */ - flb_tail_scan(ctx->path_list, ctx); - - /* - * After the first scan (on start time), all new files discovered needs to be - * read from head, so we switch the 'read_from_head' flag to true so any - * other file discovered after a scan or a rotation are read from the - * beginning. - */ - ctx->read_from_head = FLB_TRUE; - - /* Set plugin context */ - flb_input_set_context(in, ctx); - - /* Register an event collector */ - ret = flb_input_set_collector_event(in, in_tail_collect_static, - ctx->ch_manager[0], config); - if (ret == -1) { - flb_tail_config_destroy(ctx); - return -1; - } - ctx->coll_fd_static = ret; - - /* Register re-scan: time managed by 'refresh_interval' property */ - ret = flb_input_set_collector_time(in, flb_tail_scan_callback, - ctx->refresh_interval_sec, - ctx->refresh_interval_nsec, - config); - if (ret == -1) { - flb_tail_config_destroy(ctx); - return -1; - } - ctx->coll_fd_scan = ret; - - /* Register watcher, interval managed by 'watcher_interval' property */ - ret = flb_input_set_collector_time(in, in_tail_watcher_callback, - ctx->watcher_interval, 0, - config); - if (ret == -1) { - flb_tail_config_destroy(ctx); - return -1; - } - ctx->coll_fd_watcher = ret; - - /* Register callback to purge rotated files */ - ret = flb_input_set_collector_time(in, flb_tail_file_purge, - ctx->rotate_wait, 0, - config); - if (ret == -1) { - flb_tail_config_destroy(ctx); - return -1; - } - ctx->coll_fd_rotated = ret; - - /* Register callback to process pending bytes in promoted files */ - ret = flb_input_set_collector_event(in, in_tail_collect_pending, - ctx->ch_pending[0], config);//1, 0, config); - if (ret == -1) { - flb_tail_config_destroy(ctx); - return -1; - } - ctx->coll_fd_pending = ret; - - - if (ctx->multiline == FLB_TRUE && ctx->parser) { - ctx->parser = NULL; - flb_plg_warn(in, "on multiline mode 'Parser' is not allowed " - "(parser disabled)"); - } - - /* Register callback to process docker mode queued buffer */ - if (ctx->docker_mode == FLB_TRUE) { - ret = flb_input_set_collector_time(in, flb_tail_dmode_pending_flush, - ctx->docker_mode_flush, 0, - config); - if (ret == -1) { - ctx->docker_mode = FLB_FALSE; - flb_tail_config_destroy(ctx); - return -1; - } - ctx->coll_fd_dmode_flush = ret; - } - -#ifdef FLB_HAVE_PARSER - /* Register callback to process multiline queued buffer */ - if (ctx->multiline == FLB_TRUE) { - ret = flb_input_set_collector_time(in, flb_tail_mult_pending_flush, - ctx->multiline_flush, 0, - config); - if (ret == -1) { - ctx->multiline = FLB_FALSE; - flb_tail_config_destroy(ctx); - return -1; - } - ctx->coll_fd_mult_flush = ret; - } -#endif - - return 0; -} - -/* Pre-run callback / before the event loop */ -static int in_tail_pre_run(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_tail_config *ctx = in_context; - (void) ins; - - return tail_signal_manager(ctx); -} - -static int in_tail_exit(void *data, struct flb_config *config) -{ - (void) *config; - struct flb_tail_config *ctx = data; - - flb_tail_file_remove_all(ctx); - flb_tail_fs_exit(ctx); - flb_tail_config_destroy(ctx); - - return 0; -} - -static void in_tail_pause(void *data, struct flb_config *config) -{ - struct flb_tail_config *ctx = data; - - /* - * Pause general collectors: - * - * - static : static files lookup before promotion - */ - flb_input_collector_pause(ctx->coll_fd_static, ctx->ins); - flb_input_collector_pause(ctx->coll_fd_pending, ctx->ins); - - if (ctx->docker_mode == FLB_TRUE) { - flb_input_collector_pause(ctx->coll_fd_dmode_flush, ctx->ins); - if (config->is_ingestion_active == FLB_FALSE) { - flb_plg_info(ctx->ins, "flushing pending docker mode data..."); - flb_tail_dmode_pending_flush_all(ctx); - } - } - - if (ctx->multiline == FLB_TRUE) { - flb_input_collector_pause(ctx->coll_fd_mult_flush, ctx->ins); - if (config->is_ingestion_active == FLB_FALSE) { - flb_plg_info(ctx->ins, "flushing pending multiline data..."); - flb_tail_mult_pending_flush_all(ctx); - } - } - - /* Pause file system backend handlers */ - flb_tail_fs_pause(ctx); -} - -static void in_tail_resume(void *data, struct flb_config *config) -{ - struct flb_tail_config *ctx = data; - - flb_input_collector_resume(ctx->coll_fd_static, ctx->ins); - flb_input_collector_resume(ctx->coll_fd_pending, ctx->ins); - - if (ctx->docker_mode == FLB_TRUE) { - flb_input_collector_resume(ctx->coll_fd_dmode_flush, ctx->ins); - } - - if (ctx->multiline == FLB_TRUE) { - flb_input_collector_resume(ctx->coll_fd_mult_flush, ctx->ins); - } - - /* Pause file system backend handlers */ - flb_tail_fs_resume(ctx); -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_CLIST, "path", NULL, - 0, FLB_TRUE, offsetof(struct flb_tail_config, path_list), - "pattern specifying log files or multiple ones through " - "the use of common wildcards." - }, - { - FLB_CONFIG_MAP_CLIST, "exclude_path", NULL, - 0, FLB_TRUE, offsetof(struct flb_tail_config, exclude_list), - "Set one or multiple shell patterns separated by commas to exclude " - "files matching a certain criteria, e.g: 'exclude_path *.gz,*.zip'" - }, - { - FLB_CONFIG_MAP_STR, "key", "log", - 0, FLB_TRUE, offsetof(struct flb_tail_config, key), - "when a message is unstructured (no parser applied), it's appended " - "as a string under the key name log. This option allows to define an " - "alternative name for that key." - }, - { - FLB_CONFIG_MAP_BOOL, "read_from_head", "false", - 0, FLB_TRUE, offsetof(struct flb_tail_config, read_from_head), - "For new discovered files on start (without a database offset/position), read the " - "content from the head of the file, not tail." - }, - { - FLB_CONFIG_MAP_STR, "refresh_interval", "60", - 0, FLB_FALSE, 0, - "interval to refresh the list of watched files expressed in seconds." - }, - { - FLB_CONFIG_MAP_TIME, "watcher_interval", "2s", - 0, FLB_TRUE, offsetof(struct flb_tail_config, watcher_interval), - }, - { - FLB_CONFIG_MAP_TIME, "progress_check_interval", "2s", - 0, FLB_TRUE, offsetof(struct flb_tail_config, progress_check_interval), - }, - { - FLB_CONFIG_MAP_INT, "progress_check_interval_nsec", "0", - 0, FLB_TRUE, offsetof(struct flb_tail_config, progress_check_interval_nsec), - }, - { - FLB_CONFIG_MAP_TIME, "rotate_wait", FLB_TAIL_ROTATE_WAIT, - 0, FLB_TRUE, offsetof(struct flb_tail_config, rotate_wait), - "specify the number of extra time in seconds to monitor a file once is " - "rotated in case some pending data is flushed." - }, - { - FLB_CONFIG_MAP_BOOL, "docker_mode", "false", - 0, FLB_TRUE, offsetof(struct flb_tail_config, docker_mode), - "If enabled, the plugin will recombine split Docker log lines before " - "passing them to any parser as configured above. This mode cannot be " - "used at the same time as Multiline." - }, - { - FLB_CONFIG_MAP_INT, "docker_mode_flush", "4", - 0, FLB_TRUE, offsetof(struct flb_tail_config, docker_mode_flush), - "wait period time in seconds to flush queued unfinished split lines." - - }, -#ifdef FLB_HAVE_REGEX - { - FLB_CONFIG_MAP_STR, "docker_mode_parser", NULL, - 0, FLB_FALSE, 0, - "specify the parser name to fetch log first line for muliline log" - }, -#endif - { - FLB_CONFIG_MAP_STR, "path_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_tail_config, path_key), - "set the 'key' name where the name of monitored file will be appended." - }, - { - FLB_CONFIG_MAP_STR, "offset_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_tail_config, offset_key), - "set the 'key' name where the offset of monitored file will be appended." - }, - { - FLB_CONFIG_MAP_TIME, "ignore_older", "0", - 0, FLB_TRUE, offsetof(struct flb_tail_config, ignore_older), - "ignore files older than 'ignore_older'. Supports m,h,d (minutes, " - "hours, days) syntax. Default behavior is to read all the files." - }, - { - FLB_CONFIG_MAP_SIZE, "buffer_chunk_size", FLB_TAIL_CHUNK, - 0, FLB_TRUE, offsetof(struct flb_tail_config, buf_chunk_size), - "set the initial buffer size to read data from files. This value is " - "used too to increase buffer size." - }, - { - FLB_CONFIG_MAP_SIZE, "buffer_max_size", FLB_TAIL_CHUNK, - 0, FLB_TRUE, offsetof(struct flb_tail_config, buf_max_size), - "set the limit of the buffer size per monitored file. When a buffer " - "needs to be increased (e.g: very long lines), this value is used to " - "restrict how much the memory buffer can grow. If reading a file exceed " - "this limit, the file is removed from the monitored file list." - }, - { - FLB_CONFIG_MAP_SIZE, "static_batch_size", FLB_TAIL_STATIC_BATCH_SIZE, - 0, FLB_TRUE, offsetof(struct flb_tail_config, static_batch_size), - "On start, Fluent Bit might process files which already contains data, " - "these files are called 'static' files. The configuration property " - "in question set's the maximum number of bytes to process per iteration " - "for the static files monitored." - }, - { - FLB_CONFIG_MAP_SIZE, "event_batch_size", FLB_TAIL_EVENT_BATCH_SIZE, - 0, FLB_TRUE, offsetof(struct flb_tail_config, event_batch_size), - "When Fluent Bit is processing files in event based mode the amount of" - "data available for consumption could be too much and cause the input plugin " - "to over extend and smother other plugins" - "The configuration property sets the maximum number of bytes to process per iteration " - "for the files monitored (in event mode)." - }, - { - FLB_CONFIG_MAP_BOOL, "skip_long_lines", "false", - 0, FLB_TRUE, offsetof(struct flb_tail_config, skip_long_lines), - "if a monitored file reach it buffer capacity due to a very long line " - "(buffer_max_size), the default behavior is to stop monitoring that " - "file. This option alter that behavior and instruct Fluent Bit to skip " - "long lines and continue processing other lines that fits into the buffer." - }, - { - FLB_CONFIG_MAP_BOOL, "exit_on_eof", "false", - 0, FLB_TRUE, offsetof(struct flb_tail_config, exit_on_eof), - "exit Fluent Bit when reaching EOF on a monitored file." - }, - - { - FLB_CONFIG_MAP_BOOL, "skip_empty_lines", "false", - 0, FLB_TRUE, offsetof(struct flb_tail_config, skip_empty_lines), - "Allows to skip empty lines." - }, - -#ifdef FLB_HAVE_INOTIFY - { - FLB_CONFIG_MAP_BOOL, "inotify_watcher", "true", - 0, FLB_TRUE, offsetof(struct flb_tail_config, inotify_watcher), - "set to false to use file stat watcher instead of inotify." - }, -#endif -#ifdef FLB_HAVE_REGEX - { - FLB_CONFIG_MAP_STR, "parser", NULL, - 0, FLB_FALSE, 0, - "specify the parser name to process an unstructured message." - }, - { - FLB_CONFIG_MAP_STR, "tag_regex", NULL, - 0, FLB_FALSE, 0, - "set a regex to extract fields from the file name and use them later to " - "compose the Tag." - }, -#endif - -#ifdef FLB_HAVE_SQLDB - { - FLB_CONFIG_MAP_STR, "db", NULL, - 0, FLB_FALSE, 0, - "set a database file to keep track of monitored files and it offsets." - }, - { - FLB_CONFIG_MAP_STR, "db.sync", "normal", - 0, FLB_FALSE, 0, - "set a database sync method. values: extra, full, normal and off." - }, - { - FLB_CONFIG_MAP_BOOL, "db.locking", "false", - 0, FLB_TRUE, offsetof(struct flb_tail_config, db_locking), - "set exclusive locking mode, increase performance but don't allow " - "external connections to the database file." - }, - { - FLB_CONFIG_MAP_STR, "db.journal_mode", "WAL", - 0, FLB_TRUE, offsetof(struct flb_tail_config, db_journal_mode), - "Option to provide WAL configuration for Work Ahead Logging mechanism (WAL). Enabling WAL " - "provides higher performance. Note that WAL is not compatible with " - "shared network file systems." - }, -#endif - - /* Multiline Options */ -#ifdef FLB_HAVE_PARSER - { - FLB_CONFIG_MAP_BOOL, "multiline", "false", - 0, FLB_TRUE, offsetof(struct flb_tail_config, multiline), - "if enabled, the plugin will try to discover multiline messages and use " - "the proper parsers to compose the outgoing messages. Note that when this " - "option is enabled the Parser option is not used." - }, - { - FLB_CONFIG_MAP_TIME, "multiline_flush", FLB_TAIL_MULT_FLUSH, - 0, FLB_TRUE, offsetof(struct flb_tail_config, multiline_flush), - "wait period time in seconds to process queued multiline messages." - }, - { - FLB_CONFIG_MAP_STR, "parser_firstline", NULL, - 0, FLB_FALSE, 0, - "name of the parser that matches the beginning of a multiline message. " - "Note that the regular expression defined in the parser must include a " - "group name (named capture)." - }, - { - FLB_CONFIG_MAP_STR_PREFIX, "parser_", NULL, - 0, FLB_FALSE, 0, - "optional extra parser to interpret and structure multiline entries. This " - "option can be used to define multiple parsers, e.g: Parser_1 ab1, " - "Parser_2 ab2, Parser_N abN." - }, - - /* Multiline Core Engine based API */ - { - FLB_CONFIG_MAP_CLIST, "multiline.parser", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct flb_tail_config, multiline_parsers), - "specify one or multiple multiline parsers: docker, cri, go, java, etc." - }, -#endif - - /* EOF */ - {0} -}; - -struct flb_input_plugin in_tail_plugin = { - .name = "tail", - .description = "Tail files", - .cb_init = in_tail_init, - .cb_pre_run = in_tail_pre_run, - .cb_collect = NULL, - .cb_flush_buf = NULL, - .cb_pause = in_tail_pause, - .cb_resume = in_tail_resume, - .cb_exit = in_tail_exit, - .config_map = config_map, - .flags = 0 -}; diff --git a/fluent-bit/plugins/in_tail/tail.h b/fluent-bit/plugins/in_tail/tail.h deleted file mode 100644 index 074f7a49f..000000000 --- a/fluent-bit/plugins/in_tail/tail.h +++ /dev/null @@ -1,45 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_TAIL_H -#define FLB_TAIL_H - -#include -#include - -/* Internal return values */ -#define FLB_TAIL_ERROR -1 -#define FLB_TAIL_OK 0 -#define FLB_TAIL_WAIT 1 -#define FLB_TAIL_BUSY 2 - -/* Consuming mode */ -#define FLB_TAIL_STATIC 0 /* Data is being consumed through read(2) */ -#define FLB_TAIL_EVENT 1 /* Data is being consumed through inotify */ - -/* Config */ -#define FLB_TAIL_CHUNK "32768" /* buffer chunk = 32KB */ -#define FLB_TAIL_REFRESH 60 /* refresh every 60 seconds */ -#define FLB_TAIL_ROTATE_WAIT "5" /* time to monitor after rotation */ -#define FLB_TAIL_STATIC_BATCH_SIZE "50M" /* static batch size */ -#define FLB_TAIL_EVENT_BATCH_SIZE "50M" /* event batch size */ - -int in_tail_collect_event(void *file, struct flb_config *config); - -#endif diff --git a/fluent-bit/plugins/in_tail/tail_config.c b/fluent-bit/plugins/in_tail/tail_config.c deleted file mode 100644 index 360b3dbea..000000000 --- a/fluent-bit/plugins/in_tail/tail_config.c +++ /dev/null @@ -1,472 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include -#include - -#include "tail_fs.h" -#include "tail_db.h" -#include "tail_config.h" -#include "tail_scan.h" -#include "tail_sql.h" -#include "tail_dockermode.h" - -#ifdef FLB_HAVE_PARSER -#include "tail_multiline.h" -#endif - -static int multiline_load_parsers(struct flb_tail_config *ctx) -{ - struct mk_list *head; - struct mk_list *head_p; - struct flb_config_map_val *mv; - struct flb_slist_entry *val = NULL; - struct flb_ml_parser_ins *parser_i; - - if (!ctx->multiline_parsers) { - return 0; - } - - /* Create Multiline context using the plugin instance name */ - ctx->ml_ctx = flb_ml_create(ctx->config, ctx->ins->name); - if (!ctx->ml_ctx) { - return -1; - } - - /* - * Iterate all 'multiline.parser' entries. Every entry is considered - * a group which can have multiple multiline parser instances. - */ - flb_config_map_foreach(head, mv, ctx->multiline_parsers) { - mk_list_foreach(head_p, mv->val.list) { - val = mk_list_entry(head_p, struct flb_slist_entry, _head); - - /* Create an instance of the defined parser */ - parser_i = flb_ml_parser_instance_create(ctx->ml_ctx, val->str); - if (!parser_i) { - return -1; - } - } - } - - return 0; -} - -struct flb_tail_config *flb_tail_config_create(struct flb_input_instance *ins, - struct flb_config *config) -{ - int ret; - int sec; - int i; - long nsec; - const char *tmp; - struct flb_tail_config *ctx; - - ctx = flb_calloc(1, sizeof(struct flb_tail_config)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->config = config; - ctx->ins = ins; - ctx->ignore_older = 0; - ctx->skip_long_lines = FLB_FALSE; -#ifdef FLB_HAVE_SQLDB - ctx->db_sync = 1; /* sqlite sync 'normal' */ -#endif - - /* Load the config map */ - ret = flb_input_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return NULL; - } - - /* Create the channel manager */ - ret = flb_pipe_create(ctx->ch_manager); - if (ret == -1) { - flb_errno(); - flb_free(ctx); - return NULL; - } - ctx->ch_reads = 0; - ctx->ch_writes = 0; - - /* Create the pending channel */ - ret = flb_pipe_create(ctx->ch_pending); - if (ret == -1) { - flb_errno(); - flb_tail_config_destroy(ctx); - return NULL; - } - /* Make pending channel non-blocking */ - for (i = 0; i <= 1; i++) { - ret = flb_pipe_set_nonblocking(ctx->ch_pending[i]); - if (ret == -1) { - flb_errno(); - flb_tail_config_destroy(ctx); - return NULL; - } - } - - /* Config: path/pattern to read files */ - if (!ctx->path_list || mk_list_size(ctx->path_list) == 0) { - flb_plg_error(ctx->ins, "no input 'path' was given"); - flb_tail_config_destroy(ctx); - return NULL; - } - - /* Config: seconds interval before to re-scan the path */ - tmp = flb_input_get_property("refresh_interval", ins); - if (!tmp) { - ctx->refresh_interval_sec = FLB_TAIL_REFRESH; - ctx->refresh_interval_nsec = 0; - } - else { - ret = flb_utils_time_split(tmp, &sec, &nsec); - if (ret == 0) { - ctx->refresh_interval_sec = sec; - ctx->refresh_interval_nsec = nsec; - - if (sec == 0 && nsec == 0) { - flb_plg_error(ctx->ins, "invalid 'refresh_interval' config " - "value (%s)", tmp); - flb_free(ctx); - return NULL; - } - - if (sec == 0 && nsec <= 1000000) { - flb_plg_warn(ctx->ins, "very low refresh_interval " - "(%i.%lu nanoseconds) might cause high CPU usage", - sec, nsec); - } - } - else { - flb_plg_error(ctx->ins, - "invalid 'refresh_interval' config value (%s)", - tmp); - flb_tail_config_destroy(ctx); - return NULL; - } - } - - /* Config: seconds interval to monitor file after rotation */ - if (ctx->rotate_wait <= 0) { - flb_plg_error(ctx->ins, "invalid 'rotate_wait' config value"); - flb_free(ctx); - return NULL; - } - -#ifdef FLB_HAVE_PARSER - /* Config: multi-line support */ - if (ctx->multiline == FLB_TRUE) { - ret = flb_tail_mult_create(ctx, ins, config); - if (ret == -1) { - flb_tail_config_destroy(ctx); - return NULL; - } - } -#endif - - /* Config: Docker mode */ - if(ctx->docker_mode == FLB_TRUE) { - ret = flb_tail_dmode_create(ctx, ins, config); - if (ret == -1) { - flb_tail_config_destroy(ctx); - return NULL; - } - } - - /* Validate buffer limit */ - if (ctx->buf_chunk_size > ctx->buf_max_size) { - flb_plg_error(ctx->ins, "buffer_max_size must be >= buffer_chunk"); - flb_free(ctx); - return NULL; - } - -#ifdef FLB_HAVE_REGEX - /* Parser / Format */ - tmp = flb_input_get_property("parser", ins); - if (tmp) { - ctx->parser = flb_parser_get(tmp, config); - if (!ctx->parser) { - flb_plg_error(ctx->ins, "parser '%s' is not registered", tmp); - } - } -#endif - - mk_list_init(&ctx->files_static); - mk_list_init(&ctx->files_event); - mk_list_init(&ctx->files_rotated); - - /* hash table for files lookups */ - ctx->static_hash = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, 1000, 0); - if (!ctx->static_hash) { - flb_plg_error(ctx->ins, "could not create static hash"); - flb_tail_config_destroy(ctx); - return NULL; - } - - ctx->event_hash = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, 1000, 0); - if (!ctx->event_hash) { - flb_plg_error(ctx->ins, "could not create event hash"); - flb_tail_config_destroy(ctx); - return NULL; - } - -#ifdef FLB_HAVE_SQLDB - ctx->db = NULL; -#endif - -#ifdef FLB_HAVE_REGEX - tmp = flb_input_get_property("tag_regex", ins); - if (tmp) { - ctx->tag_regex = flb_regex_create(tmp); - if (ctx->tag_regex) { - ctx->dynamic_tag = FLB_TRUE; - } - else { - flb_plg_error(ctx->ins, "invalid 'tag_regex' config value"); - } - } - else { - ctx->tag_regex = NULL; - } -#endif - - /* Check if it should use dynamic tags */ - tmp = strchr(ins->tag, '*'); - if (tmp) { - ctx->dynamic_tag = FLB_TRUE; - } - -#ifdef FLB_HAVE_SQLDB - /* Database options (needs to be set before the context) */ - tmp = flb_input_get_property("db.sync", ins); - if (tmp) { - if (strcasecmp(tmp, "extra") == 0) { - ctx->db_sync = 3; - } - else if (strcasecmp(tmp, "full") == 0) { - ctx->db_sync = 2; - } - else if (strcasecmp(tmp, "normal") == 0) { - ctx->db_sync = 1; - } - else if (strcasecmp(tmp, "off") == 0) { - ctx->db_sync = 0; - } - else { - flb_plg_error(ctx->ins, "invalid database 'db.sync' value"); - } - } - - /* Initialize database */ - tmp = flb_input_get_property("db", ins); - if (tmp) { - ctx->db = flb_tail_db_open(tmp, ins, ctx, config); - if (!ctx->db) { - flb_plg_error(ctx->ins, "could not open/create database"); - flb_tail_config_destroy(ctx); - return NULL; - } - } - - /* Journal mode check */ - tmp = flb_input_get_property("db.journal_mode", ins); - if (tmp) { - if (strcasecmp(tmp, "DELETE") != 0 && - strcasecmp(tmp, "TRUNCATE") != 0 && - strcasecmp(tmp, "PERSIST") != 0 && - strcasecmp(tmp, "MEMORY") != 0 && - strcasecmp(tmp, "WAL") != 0 && - strcasecmp(tmp, "OFF") != 0) { - - flb_plg_error(ctx->ins, "invalid db.journal_mode=%s", tmp); - flb_tail_config_destroy(ctx); - return NULL; - } - } - - /* Prepare Statement */ - if (ctx->db) { - /* SQL_GET_FILE */ - ret = sqlite3_prepare_v2(ctx->db->handler, - SQL_GET_FILE, - -1, - &ctx->stmt_get_file, - 0); - if (ret != SQLITE_OK) { - flb_plg_error(ctx->ins, "error preparing database SQL statement"); - flb_tail_config_destroy(ctx); - return NULL; - } - - /* SQL_INSERT_FILE */ - ret = sqlite3_prepare_v2(ctx->db->handler, - SQL_INSERT_FILE, - -1, - &ctx->stmt_insert_file, - 0); - if (ret != SQLITE_OK) { - flb_plg_error(ctx->ins, "error preparing database SQL statement"); - flb_tail_config_destroy(ctx); - return NULL; - } - - /* SQL_ROTATE_FILE */ - ret = sqlite3_prepare_v2(ctx->db->handler, - SQL_ROTATE_FILE, - -1, - &ctx->stmt_rotate_file, - 0); - if (ret != SQLITE_OK) { - flb_plg_error(ctx->ins, "error preparing database SQL statement"); - flb_tail_config_destroy(ctx); - return NULL; - } - - /* SQL_UPDATE_OFFSET */ - ret = sqlite3_prepare_v2(ctx->db->handler, - SQL_UPDATE_OFFSET, - -1, - &ctx->stmt_offset, - 0); - if (ret != SQLITE_OK) { - flb_plg_error(ctx->ins, "error preparing database SQL statement"); - flb_tail_config_destroy(ctx); - return NULL; - } - - /* SQL_DELETE_FILE */ - ret = sqlite3_prepare_v2(ctx->db->handler, - SQL_DELETE_FILE, - -1, - &ctx->stmt_delete_file, - 0); - if (ret != SQLITE_OK) { - flb_plg_error(ctx->ins, "error preparing database SQL statement"); - flb_tail_config_destroy(ctx); - return NULL; - } - - } -#endif - -#ifdef FLB_HAVE_PARSER - /* Multiline core API */ - if (ctx->multiline_parsers && mk_list_size(ctx->multiline_parsers) > 0) { - ret = multiline_load_parsers(ctx); - if (ret != 0) { - flb_plg_error(ctx->ins, "could not load multiline parsers"); - flb_tail_config_destroy(ctx); - return NULL; - } - - /* Enable auto-flush routine */ - ret = flb_ml_auto_flush_init(ctx->ml_ctx); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not start multiline auto-flush"); - flb_tail_config_destroy(ctx); - return NULL; - } - flb_plg_info(ctx->ins, "multiline core started"); - } -#endif - -#ifdef FLB_HAVE_METRICS - ctx->cmt_files_opened = cmt_counter_create(ins->cmt, - "fluentbit", "input", - "files_opened_total", - "Total number of opened files", - 1, (char *[]) {"name"}); - - ctx->cmt_files_closed = cmt_counter_create(ins->cmt, - "fluentbit", "input", - "files_closed_total", - "Total number of closed files", - 1, (char *[]) {"name"}); - - ctx->cmt_files_rotated = cmt_counter_create(ins->cmt, - "fluentbit", "input", - "files_rotated_total", - "Total number of rotated files", - 1, (char *[]) {"name"}); - - /* OLD metrics */ - flb_metrics_add(FLB_TAIL_METRIC_F_OPENED, - "files_opened", ctx->ins->metrics); - flb_metrics_add(FLB_TAIL_METRIC_F_CLOSED, - "files_closed", ctx->ins->metrics); - flb_metrics_add(FLB_TAIL_METRIC_F_ROTATED, - "files_rotated", ctx->ins->metrics); -#endif - - return ctx; -} - -int flb_tail_config_destroy(struct flb_tail_config *config) -{ - -#ifdef FLB_HAVE_PARSER - flb_tail_mult_destroy(config); - - if (config->ml_ctx) { - flb_ml_destroy(config->ml_ctx); - } -#endif - - /* Close pipe ends */ - flb_pipe_close(config->ch_manager[0]); - flb_pipe_close(config->ch_manager[1]); - flb_pipe_close(config->ch_pending[0]); - flb_pipe_close(config->ch_pending[1]); - -#ifdef FLB_HAVE_REGEX - if (config->tag_regex) { - flb_regex_destroy(config->tag_regex); - } -#endif - -#ifdef FLB_HAVE_SQLDB - if (config->db != NULL) { - sqlite3_finalize(config->stmt_get_file); - sqlite3_finalize(config->stmt_insert_file); - sqlite3_finalize(config->stmt_delete_file); - sqlite3_finalize(config->stmt_rotate_file); - sqlite3_finalize(config->stmt_offset); - flb_tail_db_close(config->db); - } -#endif - - if (config->static_hash) { - flb_hash_table_destroy(config->static_hash); - } - if (config->event_hash) { - flb_hash_table_destroy(config->event_hash); - } - - flb_free(config); - return 0; -} diff --git a/fluent-bit/plugins/in_tail/tail_config.h b/fluent-bit/plugins/in_tail/tail_config.h deleted file mode 100644 index dcfa54e02..000000000 --- a/fluent-bit/plugins/in_tail/tail_config.h +++ /dev/null @@ -1,168 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_TAIL_CONFIG_H -#define FLB_TAIL_CONFIG_H - -#include -#include -#include -#include -#include -#include -#include -#ifdef FLB_HAVE_REGEX -#include -#endif -#ifdef FLB_HAVE_PARSER -#include -#endif - -#include - -/* Metrics */ -#ifdef FLB_HAVE_METRICS -#define FLB_TAIL_METRIC_F_OPENED 100 /* number of opened files */ -#define FLB_TAIL_METRIC_F_CLOSED 101 /* number of closed files */ -#define FLB_TAIL_METRIC_F_ROTATED 102 /* number of rotated files */ -#endif - -struct flb_tail_config { - int fd_notify; /* inotify fd */ - flb_pipefd_t ch_manager[2]; /* pipe: channel manager */ - flb_pipefd_t ch_pending[2]; /* pipe: pending events */ - int ch_reads; /* count number if signal reads */ - int ch_writes; /* count number of signal writes */ - - /* Buffer Config */ - size_t buf_chunk_size; /* allocation chunks */ - size_t buf_max_size; /* max size of a buffer */ - - /* Static files processor */ - size_t static_batch_size; - - /* Event files processor */ - size_t event_batch_size; - - /* Collectors */ - int coll_fd_static; - int coll_fd_scan; - int coll_fd_watcher; - int coll_fd_rotated; - int coll_fd_pending; - int coll_fd_inactive; - int coll_fd_dmode_flush; - int coll_fd_mult_flush; - int coll_fd_progress_check; - - /* Backend collectors */ - int coll_fd_fs1; /* used by fs_inotify & fs_stat */ - int coll_fd_fs2; /* only used by fs_stat */ - - /* Configuration */ - int dynamic_tag; /* dynamic tag ? e.g: abc.* */ -#ifdef FLB_HAVE_REGEX - struct flb_regex *tag_regex;/* path to tag regex */ -#endif - int refresh_interval_sec; /* seconds to re-scan */ - long refresh_interval_nsec;/* nanoseconds to re-scan */ - int read_from_head; /* read new files from head */ - int rotate_wait; /* sec to wait on rotated files */ - int watcher_interval; /* watcher interval */ - int ignore_older; /* ignore fields older than X seconds */ - time_t last_pending; /* last time a 'pending signal' was emitted' */ - struct mk_list *path_list; /* list of paths to scan (glob) */ - flb_sds_t path_key; /* key name of file path */ - flb_sds_t key; /* key for unstructured record */ - int skip_long_lines; /* skip long lines */ - int skip_empty_lines; /* skip empty lines (off) */ - int exit_on_eof; /* exit fluent-bit on EOF, test */ - - int progress_check_interval; /* watcher interval */ - int progress_check_interval_nsec; /* watcher interval */ - -#ifdef FLB_HAVE_INOTIFY - int inotify_watcher; /* enable/disable inotify monitor */ -#endif - flb_sds_t offset_key; /* key name of file offset */ - - /* Database */ -#ifdef FLB_HAVE_SQLDB - struct flb_sqldb *db; - int db_sync; - int db_locking; - flb_sds_t db_journal_mode; - sqlite3_stmt *stmt_get_file; - sqlite3_stmt *stmt_insert_file; - sqlite3_stmt *stmt_delete_file; - sqlite3_stmt *stmt_rotate_file; - sqlite3_stmt *stmt_offset; -#endif - - /* Parser / Format */ - struct flb_parser *parser; - - /* Multiline */ - int multiline; /* multiline enabled ? */ - int multiline_flush; /* multiline flush/wait */ - struct flb_parser *mult_parser_firstline; - struct mk_list mult_parsers; - - /* Docker mode */ - int docker_mode; /* Docker mode enabled ? */ - int docker_mode_flush; /* Docker mode flush/wait */ - struct flb_parser *docker_mode_parser; /* Parser for separate multiline logs */ - - /* Multiline core engine */ - struct flb_ml *ml_ctx; - struct mk_list *multiline_parsers; - - uint64_t files_static_count; /* number of items in the static file list */ - struct mk_list files_static; - struct mk_list files_event; - - /* List of rotated files that needs to be removed after 'rotate_wait' */ - struct mk_list files_rotated; - - /* List of shell patterns used to exclude certain file names */ - struct mk_list *exclude_list; - - /* Plugin input instance */ - struct flb_input_instance *ins; - - struct flb_log_event_encoder log_event_encoder; - struct flb_log_event_decoder log_event_decoder; - - /* Metrics */ - struct cmt_counter *cmt_files_opened; - struct cmt_counter *cmt_files_closed; - struct cmt_counter *cmt_files_rotated; - - /* Hash: hash tables for quick acess to registered files */ - struct flb_hash_table *static_hash; - struct flb_hash_table *event_hash; - - struct flb_config *config; -}; - -struct flb_tail_config *flb_tail_config_create(struct flb_input_instance *ins, - struct flb_config *config); -int flb_tail_config_destroy(struct flb_tail_config *config); - -#endif diff --git a/fluent-bit/plugins/in_tail/tail_db.c b/fluent-bit/plugins/in_tail/tail_db.c deleted file mode 100644 index 664963b6d..000000000 --- a/fluent-bit/plugins/in_tail/tail_db.c +++ /dev/null @@ -1,277 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include "tail_db.h" -#include "tail_sql.h" -#include "tail_file.h" - -struct query_status { - int id; - int rows; - int64_t offset; -}; - -/* Open or create database required by tail plugin */ -struct flb_sqldb *flb_tail_db_open(const char *path, - struct flb_input_instance *in, - struct flb_tail_config *ctx, - struct flb_config *config) -{ - int ret; - char tmp[64]; - struct flb_sqldb *db; - - /* Open/create the database */ - db = flb_sqldb_open(path, in->name, config); - if (!db) { - return NULL; - } - - /* Create table schema if it don't exists */ - ret = flb_sqldb_query(db, SQL_CREATE_FILES, NULL, NULL); - if (ret != FLB_OK) { - flb_plg_error(ctx->ins, "db: could not create 'in_tail_files' table"); - flb_sqldb_close(db); - return NULL; - } - - if (ctx->db_sync >= 0) { - snprintf(tmp, sizeof(tmp) - 1, SQL_PRAGMA_SYNC, - ctx->db_sync); - ret = flb_sqldb_query(db, tmp, NULL, NULL); - if (ret != FLB_OK) { - flb_plg_error(ctx->ins, "db could not set pragma 'sync'"); - flb_sqldb_close(db); - return NULL; - } - } - - if (ctx->db_locking == FLB_TRUE) { - ret = flb_sqldb_query(db, SQL_PRAGMA_LOCKING_MODE, NULL, NULL); - if (ret != FLB_OK) { - flb_plg_error(ctx->ins, "db: could not set pragma 'locking_mode'"); - flb_sqldb_close(db); - return NULL; - } - } - - if (ctx->db_journal_mode) { - snprintf(tmp, sizeof(tmp) - 1, SQL_PRAGMA_JOURNAL_MODE, - ctx->db_journal_mode); - ret = flb_sqldb_query(db, tmp, NULL, NULL); - if (ret != FLB_OK) { - flb_plg_error(ctx->ins, "db could not set pragma 'journal_mode'"); - flb_sqldb_close(db); - return NULL; - } - } - - return db; -} - -int flb_tail_db_close(struct flb_sqldb *db) -{ - flb_sqldb_close(db); - return 0; -} - -/* - * Check if an file inode exists in the database. Return FLB_TRUE or - * FLB_FALSE - */ -static int db_file_exists(struct flb_tail_file *file, - struct flb_tail_config *ctx, - uint64_t *id, uint64_t *inode, off_t *offset) -{ - int ret; - int exists = FLB_FALSE; - - /* Bind parameters */ - sqlite3_bind_int64(ctx->stmt_get_file, 1, file->inode); - ret = sqlite3_step(ctx->stmt_get_file); - - if (ret == SQLITE_ROW) { - exists = FLB_TRUE; - - /* id: column 0 */ - *id = sqlite3_column_int64(ctx->stmt_get_file, 0); - - /* offset: column 2 */ - *offset = sqlite3_column_int64(ctx->stmt_get_file, 2); - - /* inode: column 3 */ - *inode = sqlite3_column_int64(ctx->stmt_get_file, 3); - } - else if (ret == SQLITE_DONE) { - /* all good */ - } - else { - exists = -1; - } - - sqlite3_clear_bindings(ctx->stmt_get_file); - sqlite3_reset(ctx->stmt_get_file); - - return exists; - -} - -static int db_file_insert(struct flb_tail_file *file, struct flb_tail_config *ctx) - -{ - int ret; - time_t created; - - /* Register the file */ - created = time(NULL); - - /* Bind parameters */ - sqlite3_bind_text(ctx->stmt_insert_file, 1, file->name, -1, 0); - sqlite3_bind_int64(ctx->stmt_insert_file, 2, file->offset); - sqlite3_bind_int64(ctx->stmt_insert_file, 3, file->inode); - sqlite3_bind_int64(ctx->stmt_insert_file, 4, created); - - /* Run the insert */ - ret = sqlite3_step(ctx->stmt_insert_file); - if (ret != SQLITE_DONE) { - sqlite3_clear_bindings(ctx->stmt_insert_file); - sqlite3_reset(ctx->stmt_insert_file); - flb_plg_error(ctx->ins, "cannot execute insert file %s inode=%lu", - file->name, file->inode); - return -1; - } - - sqlite3_clear_bindings(ctx->stmt_insert_file); - sqlite3_reset(ctx->stmt_insert_file); - - /* Get the database ID for this file */ - return flb_sqldb_last_id(ctx->db); -} - -int flb_tail_db_file_set(struct flb_tail_file *file, - struct flb_tail_config *ctx) -{ - int ret; - uint64_t id = 0; - off_t offset = 0; - uint64_t inode = 0; - - /* Check if the file exists */ - ret = db_file_exists(file, ctx, &id, &inode, &offset); - if (ret == -1) { - flb_plg_error(ctx->ins, "cannot execute query to check inode: %lu", - file->inode); - return -1; - } - - if (ret == FLB_FALSE) { - /* Get the database ID for this file */ - file->db_id = db_file_insert(file, ctx); - } - else { - file->db_id = id; - file->offset = offset; - } - - return 0; -} - -/* Update Offset v2 */ -int flb_tail_db_file_offset(struct flb_tail_file *file, - struct flb_tail_config *ctx) -{ - int ret; - - /* Bind parameters */ - sqlite3_bind_int64(ctx->stmt_offset, 1, file->offset); - sqlite3_bind_int64(ctx->stmt_offset, 2, file->db_id); - - ret = sqlite3_step(ctx->stmt_offset); - - if (ret != SQLITE_DONE) { - sqlite3_clear_bindings(ctx->stmt_offset); - sqlite3_reset(ctx->stmt_offset); - return -1; - } - - /* Verify number of updated rows */ - ret = sqlite3_changes(ctx->db->handler); - if (ret == 0) { - /* - * 'someone' like you 'the reader' or another user has deleted the database - * entry, just restore it. - */ - file->db_id = db_file_insert(file, ctx); - } - - sqlite3_clear_bindings(ctx->stmt_offset); - sqlite3_reset(ctx->stmt_offset); - - return 0; -} - -/* Mark a file as rotated v2 */ -int flb_tail_db_file_rotate(const char *new_name, - struct flb_tail_file *file, - struct flb_tail_config *ctx) -{ - int ret; - - /* Bind parameters */ - sqlite3_bind_text(ctx->stmt_rotate_file, 1, new_name, -1, 0); - sqlite3_bind_int64(ctx->stmt_rotate_file, 2, file->db_id); - - ret = sqlite3_step(ctx->stmt_rotate_file); - - sqlite3_clear_bindings(ctx->stmt_rotate_file); - sqlite3_reset(ctx->stmt_rotate_file); - - if (ret != SQLITE_DONE) { - return -1; - } - - return 0; -} - -/* Delete file entry from the database */ -int flb_tail_db_file_delete(struct flb_tail_file *file, - struct flb_tail_config *ctx) -{ - int ret; - - /* Bind parameters */ - sqlite3_bind_int64(ctx->stmt_delete_file, 1, file->db_id); - ret = sqlite3_step(ctx->stmt_delete_file); - - sqlite3_clear_bindings(ctx->stmt_delete_file); - sqlite3_reset(ctx->stmt_delete_file); - - if (ret != SQLITE_DONE) { - flb_plg_error(ctx->ins, "db: error deleting entry from database: %s", - file->name); - return -1; - } - - flb_plg_debug(ctx->ins, "db: file deleted from database: %s", file->name); - return 0; -} diff --git a/fluent-bit/plugins/in_tail/tail_db.h b/fluent-bit/plugins/in_tail/tail_db.h deleted file mode 100644 index 7b5355d22..000000000 --- a/fluent-bit/plugins/in_tail/tail_db.h +++ /dev/null @@ -1,43 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_TAIL_DB_H -#define FLB_TAIL_DB_H - -#include -#include - -#include "tail_file.h" - -struct flb_sqldb *flb_tail_db_open(const char *path, - struct flb_input_instance *in, - struct flb_tail_config *ctx, - struct flb_config *config); - -int flb_tail_db_close(struct flb_sqldb *db); -int flb_tail_db_file_set(struct flb_tail_file *file, - struct flb_tail_config *ctx); -int flb_tail_db_file_offset(struct flb_tail_file *file, - struct flb_tail_config *ctx); -int flb_tail_db_file_rotate(const char *new_name, - struct flb_tail_file *file, - struct flb_tail_config *ctx); -int flb_tail_db_file_delete(struct flb_tail_file *file, - struct flb_tail_config *ctx); -#endif diff --git a/fluent-bit/plugins/in_tail/tail_dockermode.c b/fluent-bit/plugins/in_tail/tail_dockermode.c deleted file mode 100644 index a8f20f9cd..000000000 --- a/fluent-bit/plugins/in_tail/tail_dockermode.c +++ /dev/null @@ -1,459 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include "tail_config.h" -#include "tail_dockermode.h" -#include "tail_file_internal.h" - -int flb_tail_dmode_create(struct flb_tail_config *ctx, - struct flb_input_instance *ins, - struct flb_config *config) -{ - const char *tmp; - - if (ctx->multiline == FLB_TRUE) { - flb_plg_error(ctx->ins, "Docker mode cannot be enabled when multiline " - "is enabled"); - return -1; - } - -#ifdef FLB_HAVE_REGEX - /* First line Parser */ - tmp = flb_input_get_property("docker_mode_parser", ins); - if (tmp) { - ctx->docker_mode_parser = flb_parser_get(tmp, config); - if (!ctx->docker_mode_parser) { - flb_plg_error(ctx->ins, "parser '%s' is not registered", tmp); - } - } - else { - ctx->docker_mode_parser = NULL; - } -#endif - - tmp = flb_input_get_property("docker_mode_flush", ins); - if (!tmp) { - ctx->docker_mode_flush = FLB_TAIL_DMODE_FLUSH; - } - else { - ctx->docker_mode_flush = atoi(tmp); - if (ctx->docker_mode_flush <= 0) { - ctx->docker_mode_flush = 1; - } - } - - return 0; -} - -static int modify_json_cond(char *js, size_t js_len, - char **val, size_t *val_len, - char **out, size_t *out_len, - int cond(char*, size_t), - int mod(char*, size_t, char**, size_t*, void*), void *data) -{ - int ret; - struct flb_pack_state state; - jsmntok_t *t; - jsmntok_t *t_val = NULL; - int i; - int i_root = -1; - int i_key = -1; - char *old_val; - size_t old_val_len; - char *new_val = NULL; - size_t new_val_len = 0; - size_t mod_len; - - ret = flb_pack_state_init(&state); - if (ret != 0) { - ret = -1; - goto modify_json_cond_end; - } - - ret = flb_json_tokenise(js, js_len, &state); - if (ret != 0 || state.tokens_count == 0) { - ret = -1; - goto modify_json_cond_end; - } - - for (i = 0; i < state.tokens_count; i++) { - t = &state.tokens[i]; - - if (i_key >= 0) { - if (t->parent == i_key) { - if (t->type == JSMN_STRING) { - t_val = t; - } - break; - } - continue; - } - - if (t->start == 0 && t->parent == -1 && t->type == JSMN_OBJECT) { - i_root = i; - continue; - } - if (i_root == -1) { - continue; - } - - if (t->parent == i_root && t->type == JSMN_STRING && t->end - t->start == 3 && strncmp(js + t->start, "log", 3) == 0) { - i_key = i; - } - } - - if (!t_val) { - ret = -1; - goto modify_json_cond_end; - } - - *out = js; - *out_len = js_len; - - if (val) { - *val = js + t_val->start; - } - if (val_len) { - *val_len = t_val->end - t_val->start; - } - - if (!cond || cond(js + t_val->start, t_val->end - t_val->start)) { - old_val = js + t_val->start; - old_val_len = t_val->end - t_val->start; - ret = mod(old_val, old_val_len, &new_val, &new_val_len, data); - if (ret != 0) { - ret = -1; - goto modify_json_cond_end; - } - - ret = 1; - - if (new_val == old_val) { - goto modify_json_cond_end; - } - - mod_len = js_len + new_val_len - old_val_len; - *out = flb_malloc(mod_len); - if (!*out) { - flb_errno(); - flb_free(new_val); - ret = -1; - goto modify_json_cond_end; - } - *out_len = mod_len; - - memcpy(*out, js, t_val->start); - memcpy(*out + t_val->start, new_val, new_val_len); - memcpy(*out + t_val->start + new_val_len, js + t_val->end, js_len - t_val->end); - - flb_free(new_val); - } - - modify_json_cond_end: - flb_pack_state_reset(&state); - if (ret < 0) { - *out = NULL; - } - return ret; -} - -static int unesc_ends_with_nl(char *str, size_t len) -{ - char* unesc; - int unesc_len; - int nl; - - unesc = flb_malloc(len + 1); - if (!unesc) { - flb_errno(); - return FLB_FALSE; - } - unesc_len = flb_unescape_string(str, len, &unesc); - nl = unesc[unesc_len - 1] == '\n'; - flb_free(unesc); - return nl; -} - -static int prepend_sds_to_str(char *str, size_t len, char **out, size_t *out_len, void *data) -{ - flb_sds_t sds = data; - - if (flb_sds_len(sds) == 0) { - *out = str; - *out_len = len; - return 0; - } - - size_t mod_len = flb_sds_len(sds) + len; - *out = flb_malloc(mod_len); - if (!*out) { - flb_errno(); - return -1; - } - *out_len = mod_len; - - memcpy(*out, sds, flb_sds_len(sds)); - memcpy(*out + flb_sds_len(sds), str, len); - return 0; -} - -static int use_sds(char *str, size_t len, char **out, size_t *out_len, void *data) -{ - flb_sds_t sds = data; - size_t mod_len = flb_sds_len(sds); - *out = flb_malloc(mod_len); - if (!*out) { - flb_errno(); - return -1; - } - *out_len = mod_len; - - memcpy(*out, sds, flb_sds_len(sds)); - return 0; -} - -int flb_tail_dmode_process_content(time_t now, - char* line, size_t line_len, - char **repl_line, size_t *repl_line_len, - struct flb_tail_file *file, - struct flb_tail_config *ctx) -{ - char* val = NULL; - size_t val_len; - int ret; - void *out_buf = NULL; - size_t out_size; - struct flb_time out_time = {0}; - *repl_line = NULL; - *repl_line_len = 0; - flb_sds_t tmp; - flb_sds_t tmp_copy; - -#ifdef FLB_HAVE_REGEX - if (ctx->docker_mode_parser) { - ret = flb_parser_do(ctx->docker_mode_parser, line, line_len, - &out_buf, &out_size, &out_time); - flb_free(out_buf); - - /* - * Set dmode_firstline if the line meets the first-line requirement - */ - if(ret >= 0) { - file->dmode_firstline = true; - } - - /* - * Process if buffer contains full log line - */ - if (flb_sds_len(file->dmode_lastline) > 0 && file->dmode_complete) { - /* - * Buffered log should be flushed out - * as current line meets first-line requirement - */ - if(ret >= 0) { - flb_tail_dmode_flush(file, ctx); - } - - /* - * Flush the buffer if multiline has not been detected yet - */ - if (!file->dmode_firstline) { - flb_tail_dmode_flush(file, ctx); - } - } - } -#endif - - ret = modify_json_cond(line, line_len, - &val, &val_len, - repl_line, repl_line_len, - unesc_ends_with_nl, - prepend_sds_to_str, file->dmode_buf); - if (ret >= 0) { - /* line is a valid json */ - flb_sds_len_set(file->dmode_lastline, 0); - - /* concatenate current log line with buffered one */ - tmp = flb_sds_cat(file->dmode_buf, val, val_len); - if (!tmp) { - flb_errno(); - return -1; - } - file->dmode_buf = tmp; - - tmp_copy = flb_sds_copy(file->dmode_lastline, line, line_len); - if (!tmp_copy) { - flb_errno(); - return -1; - } - - file->dmode_lastline = tmp_copy; - file->dmode_flush_timeout = now + (ctx->docker_mode_flush - 1); - - if (ret == 0) { - /* Line not ended with newline */ - file->dmode_complete = false; - } - else { - /* Line ended with newline */ - file->dmode_complete = true; -#ifdef FLB_HAVE_REGEX - if (!ctx->docker_mode_parser) { - flb_tail_dmode_flush(file, ctx); - } -#else - flb_tail_dmode_flush(file, ctx); -#endif - } - } - return ret; -} - -void flb_tail_dmode_flush(struct flb_tail_file *file, struct flb_tail_config *ctx) -{ - int ret; - char *repl_line = NULL; - size_t repl_line_len = 0; - void *out_buf = NULL; - size_t out_size; - struct flb_time out_time = {0}; - time_t now = time(NULL); - - if (flb_sds_len(file->dmode_lastline) == 0) { - return; - } - - flb_time_zero(&out_time); - - ret = modify_json_cond(file->dmode_lastline, - flb_sds_len(file->dmode_lastline), - NULL, NULL, - &repl_line, &repl_line_len, - NULL, - use_sds, file->dmode_buf); - if (ret < 0) { - return; - } - - flb_sds_len_set(file->dmode_buf, 0); - flb_sds_len_set(file->dmode_lastline, 0); - file->dmode_flush_timeout = 0; - -#ifdef FLB_HAVE_REGEX - if (ctx->parser) { - ret = flb_parser_do(ctx->parser, repl_line, repl_line_len, - &out_buf, &out_size, &out_time); - if (ret >= 0) { - if (flb_time_to_double(&out_time) == 0) { - flb_time_get(&out_time); - } - if (ctx->ignore_older > 0 && (now - ctx->ignore_older) > out_time.tm.tv_sec) { - goto dmode_flush_end; - } - - flb_tail_pack_line_map(&out_time, (char**) &out_buf, &out_size, file, 0); - - goto dmode_flush_end; - } - } -#endif - - flb_tail_file_pack_line(NULL, repl_line, repl_line_len, file, 0); - - dmode_flush_end: - flb_free(repl_line); - flb_free(out_buf); -} - -static void file_pending_flush(struct flb_tail_config *ctx, - struct flb_tail_file *file, time_t now) -{ - if (file->dmode_flush_timeout > now) { - return; - } - - if (flb_sds_len(file->dmode_lastline) == 0) { - return; - } - - flb_tail_dmode_flush(file, ctx); - - if (file->sl_log_event_encoder->output_length > 0) { - flb_input_log_append(ctx->ins, - file->tag_buf, - file->tag_len, - file->sl_log_event_encoder->output_buffer, - file->sl_log_event_encoder->output_length); - - flb_log_event_encoder_reset(file->sl_log_event_encoder); - } -} - -int flb_tail_dmode_pending_flush_all(struct flb_tail_config *ctx) -{ - time_t expired; - struct mk_list *head; - struct flb_tail_file *file; - - expired = time(NULL) + 3600; - - /* Iterate promoted event files with pending bytes */ - mk_list_foreach(head, &ctx->files_static) { - file = mk_list_entry(head, struct flb_tail_file, _head); - file_pending_flush(ctx, file, expired); - } - - /* Iterate promoted event files with pending bytes */ - mk_list_foreach(head, &ctx->files_event) { - file = mk_list_entry(head, struct flb_tail_file, _head); - file_pending_flush(ctx, file, expired); - } - - return 0; -} - -int flb_tail_dmode_pending_flush(struct flb_input_instance *ins, - struct flb_config *config, void *context) -{ - time_t now; - struct mk_list *head; - struct flb_tail_file *file; - struct flb_tail_config *ctx = context; - - now = time(NULL); - - /* Iterate static event files with pending bytes */ - mk_list_foreach(head, &ctx->files_static) { - file = mk_list_entry(head, struct flb_tail_file, _head); - file_pending_flush(ctx, file, now); - } - - /* Iterate promoted event files with pending bytes */ - mk_list_foreach(head, &ctx->files_event) { - file = mk_list_entry(head, struct flb_tail_file, _head); - file_pending_flush(ctx, file, now); - } - - return 0; -} diff --git a/fluent-bit/plugins/in_tail/tail_dockermode.h b/fluent-bit/plugins/in_tail/tail_dockermode.h deleted file mode 100644 index 50869ff62..000000000 --- a/fluent-bit/plugins/in_tail/tail_dockermode.h +++ /dev/null @@ -1,38 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_TAIL_DOCKERMODE_H -#define FLB_TAIL_DOCKERMODE_H - -#include "tail_file.h" -#define FLB_TAIL_DMODE_FLUSH 4 - -int flb_tail_dmode_create(struct flb_tail_config *ctx, - struct flb_input_instance *ins, struct flb_config *config); -int flb_tail_dmode_process_content(time_t now, - char* line, size_t line_len, - char **repl_line, size_t *repl_line_len, - struct flb_tail_file *file, - struct flb_tail_config *ctx); -void flb_tail_dmode_flush(struct flb_tail_file *file, struct flb_tail_config *ctx); -int flb_tail_dmode_pending_flush(struct flb_input_instance *ins, - struct flb_config *config, void *context); -int flb_tail_dmode_pending_flush_all(struct flb_tail_config *ctx); - -#endif diff --git a/fluent-bit/plugins/in_tail/tail_file.c b/fluent-bit/plugins/in_tail/tail_file.c deleted file mode 100644 index 2385f0626..000000000 --- a/fluent-bit/plugins/in_tail/tail_file.c +++ /dev/null @@ -1,1860 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#ifdef FLB_SYSTEM_FREEBSD -#include -#include -#endif - -#include -#include -#include -#include -#ifdef FLB_HAVE_REGEX -#include -#include -#endif - -#include "tail.h" -#include "tail_file.h" -#include "tail_config.h" -#include "tail_db.h" -#include "tail_signal.h" -#include "tail_dockermode.h" -#include "tail_multiline.h" -#include "tail_scan.h" - -#ifdef FLB_SYSTEM_WINDOWS -#include "win32.h" -#endif - -#include - -static inline void consume_bytes(char *buf, int bytes, int length) -{ - memmove(buf, buf + bytes, length - bytes); -} - -static uint64_t stat_get_st_dev(struct stat *st) -{ -#ifdef FLB_SYSTEM_WINDOWS - /* do you want to contribute with a way to extract volume serial number ? */ - return 0; -#else - return st->st_dev; -#endif -} - -static int stat_to_hash_bits(struct flb_tail_config *ctx, struct stat *st, - uint64_t *out_hash) -{ - int len; - uint64_t st_dev; - char tmp[64]; - - st_dev = stat_get_st_dev(st); - - len = snprintf(tmp, sizeof(tmp) - 1, "%" PRIu64 ":%" PRIu64, - st_dev, (uint64_t)st->st_ino); - - *out_hash = cfl_hash_64bits(tmp, len); - return 0; -} - -static int stat_to_hash_key(struct flb_tail_config *ctx, struct stat *st, - flb_sds_t *key) -{ - uint64_t st_dev; - flb_sds_t tmp; - flb_sds_t buf; - - buf = flb_sds_create_size(64); - if (!buf) { - return -1; - } - - st_dev = stat_get_st_dev(st); - tmp = flb_sds_printf(&buf, "%" PRIu64 ":%" PRIu64, - st_dev, (uint64_t)st->st_ino); - if (!tmp) { - flb_sds_destroy(buf); - return -1; - } - - *key = buf; - return 0; -} - -/* Append custom keys and report the number of records processed */ -static int record_append_custom_keys(struct flb_tail_file *file, - char *in_data, size_t in_size, - char **out_data, size_t *out_size) -{ - int i; - int ret; - int records = 0; - msgpack_object k; - msgpack_object v; - struct flb_log_event event; - struct flb_tail_config *ctx; - struct flb_log_event_encoder encoder; - struct flb_log_event_decoder decoder; - - ctx = (struct flb_tail_config *) file->config; - - ret = flb_log_event_decoder_init(&decoder, in_data, in_size); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - return -1; - } - - ret = flb_log_event_encoder_init(&encoder, FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_log_event_decoder_destroy(&decoder); - - return -2; - } - - while (flb_log_event_decoder_next(&decoder, &event) == - FLB_EVENT_DECODER_SUCCESS) { - - ret = flb_log_event_encoder_begin_record(&encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_timestamp(&encoder, &event.timestamp); - } - - /* append previous map keys */ - for (i = 0; i < event.body->via.map.size; i++) { - k = event.body->via.map.ptr[i].key; - v = event.body->via.map.ptr[i].val; - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_msgpack_object( - &encoder, - &k); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_msgpack_object( - &encoder, - &v); - } - } - - /* path_key */ - if (ctx->path_key != NULL) { - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_cstring( - &encoder, - file->config->path_key); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_cstring( - &encoder, - file->orig_name); - } - } - - /* offset_key */ - if (ctx->offset_key != NULL) { - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_cstring( - &encoder, - file->config->offset_key); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_uint64( - &encoder, - file->offset + - file->last_processed_bytes); - } - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(&encoder); - } - else { - flb_plg_error(file->config->ins, "error packing event : %d", ret); - - flb_log_event_encoder_rollback_record(&encoder); - } - - /* counter */ - records++; - } - - *out_data = encoder.output_buffer; - *out_size = encoder.output_length; - - /* This function transfers ownership of the internal memory allocated by - * sbuffer using msgpack_sbuffer_release which means the caller is - * responsible for releasing the memory. - */ - flb_log_event_encoder_claim_internal_buffer_ownership(&encoder); - - flb_log_event_decoder_destroy(&decoder); - flb_log_event_encoder_destroy(&encoder); - - return records; -} - -static int flb_tail_repack_map(struct flb_log_event_encoder *encoder, - char *data, - size_t data_size) -{ - msgpack_unpacked source_map; - size_t offset; - int result; - size_t index; - msgpack_object value; - msgpack_object key; - - result = FLB_EVENT_ENCODER_SUCCESS; - - if (data_size > 0) { - msgpack_unpacked_init(&source_map); - - offset = 0; - result = msgpack_unpack_next(&source_map, - data, - data_size, - &offset); - - if (result == MSGPACK_UNPACK_SUCCESS) { - result = FLB_EVENT_ENCODER_SUCCESS; - } - else { - result = FLB_EVENT_DECODER_ERROR_DESERIALIZATION_FAILURE; - } - - for (index = 0; - index < source_map.data.via.map.size && - result == FLB_EVENT_ENCODER_SUCCESS; - index++) { - key = source_map.data.via.map.ptr[index].key; - value = source_map.data.via.map.ptr[index].val; - - result = flb_log_event_encoder_append_body_msgpack_object( - encoder, - &key); - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_append_body_msgpack_object( - encoder, - &value); - } - } - - msgpack_unpacked_destroy(&source_map); - } - - return result; -} - -int flb_tail_pack_line_map(struct flb_time *time, char **data, - size_t *data_size, struct flb_tail_file *file, - size_t processed_bytes) -{ - int result; - - result = flb_log_event_encoder_begin_record(file->sl_log_event_encoder); - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_set_timestamp( - file->sl_log_event_encoder, time); - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_tail_repack_map(file->sl_log_event_encoder, - *data, - *data_size); - } - - /* path_key */ - if (file->config->path_key != NULL) { - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_append_body_values( - file->sl_log_event_encoder, - FLB_LOG_EVENT_CSTRING_VALUE(file->config->path_key), - FLB_LOG_EVENT_STRING_VALUE(file->orig_name, - file->orig_name_len)); - } - } - - /* offset_key */ - if (file->config->offset_key != NULL) { - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_append_body_values( - file->sl_log_event_encoder, - FLB_LOG_EVENT_CSTRING_VALUE(file->config->offset_key), - FLB_LOG_EVENT_UINT64_VALUE(file->offset + processed_bytes)); - } - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_commit_record(file->sl_log_event_encoder); - } - else { - flb_log_event_encoder_rollback_record(file->sl_log_event_encoder); - } - - if (result != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(file->config->ins, "error packing event"); - - return -1; - } - - return 0; -} - -int flb_tail_file_pack_line(struct flb_time *time, char *data, size_t data_size, - struct flb_tail_file *file, size_t processed_bytes) -{ - int result; - - result = flb_log_event_encoder_begin_record(file->sl_log_event_encoder); - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_set_timestamp( - file->sl_log_event_encoder, time); - } - - /* path_key */ - if (file->config->path_key != NULL) { - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_append_body_values( - file->sl_log_event_encoder, - FLB_LOG_EVENT_CSTRING_VALUE(file->config->path_key), - FLB_LOG_EVENT_STRING_VALUE(file->orig_name, - file->orig_name_len)); - } - } - - /* offset_key */ - if (file->config->offset_key != NULL) { - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_append_body_values( - file->sl_log_event_encoder, - FLB_LOG_EVENT_CSTRING_VALUE(file->config->offset_key), - FLB_LOG_EVENT_UINT64_VALUE(file->offset + processed_bytes)); - } - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_append_body_values( - file->sl_log_event_encoder, - FLB_LOG_EVENT_CSTRING_VALUE(file->config->key), - FLB_LOG_EVENT_STRING_VALUE(data, - data_size)); - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_commit_record(file->sl_log_event_encoder); - } - - if (result != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(file->config->ins, "error packing event : %d", result); - - return -1; - } - - return 0; -} - -static int ml_stream_buffer_append(struct flb_tail_file *file, char *buf_data, size_t buf_size) -{ - int result; - - result = flb_log_event_encoder_emit_raw_record( - file->ml_log_event_encoder, - buf_data, buf_size); - - if (result != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(file->config->ins, - "log event raw append error : %d", - result); - - return -1; - } - - return 0; -} - -static int ml_stream_buffer_flush(struct flb_tail_config *ctx, struct flb_tail_file *file) -{ - if (file->ml_log_event_encoder->output_length > 0) { - flb_input_log_append(ctx->ins, - file->tag_buf, - file->tag_len, - file->ml_log_event_encoder->output_buffer, - file->ml_log_event_encoder->output_length); - - flb_log_event_encoder_reset(file->ml_log_event_encoder); - } - - return 0; -} - -static int process_content(struct flb_tail_file *file, size_t *bytes) -{ - size_t len; - int lines = 0; - int ret; - size_t processed_bytes = 0; - char *data; - char *end; - char *p; - void *out_buf; - size_t out_size; - int crlf; - char *line; - size_t line_len; - char *repl_line; - size_t repl_line_len; - time_t now = time(NULL); - struct flb_time out_time = {0}; - struct flb_tail_config *ctx; - - ctx = (struct flb_tail_config *) file->config; - - /* Parse the data content */ - data = file->buf_data; - end = data + file->buf_len; - - /* reset last processed bytes */ - file->last_processed_bytes = 0; - - /* Skip null characters from the head (sometimes introduced by copy-truncate log rotation) */ - while (data < end && *data == '\0') { - data++; - processed_bytes++; - } - - while (data < end && (p = memchr(data, '\n', end - data))) { - len = (p - data); - crlf = 0; - if (file->skip_next == FLB_TRUE) { - data += len + 1; - processed_bytes += len + 1; - file->skip_next = FLB_FALSE; - continue; - } - - /* - * Empty line (just breakline) - * --------------------------- - * [NOTE] with the new Multiline core feature and Multiline Filter on - * Fluent Bit v1.8.2, there are a couple of cases where stack traces - * or multi line patterns expects an empty line (meaning only the - * breakline), skipping empty lines on this plugin will break that - * functionality. - * - * We are introducing 'skip_empty_lines=off' configuration - * property to revert this behavior if some user is affected by - * this change. - */ - - if (len == 0 && ctx->skip_empty_lines) { - data++; - processed_bytes++; - continue; - } - - /* Process '\r\n' */ - if (len >= 2) { - crlf = (data[len-1] == '\r'); - if (len == 1 && crlf) { - data += 2; - processed_bytes += 2; - continue; - } - } - - /* Reset time for each line */ - flb_time_zero(&out_time); - - line = data; - line_len = len - crlf; - repl_line = NULL; - - if (ctx->ml_ctx) { - ret = flb_ml_append_text(ctx->ml_ctx, - file->ml_stream_id, - &out_time, - line, - line_len); - goto go_next; - } - else if (ctx->docker_mode) { - ret = flb_tail_dmode_process_content(now, line, line_len, - &repl_line, &repl_line_len, - file, ctx); - if (ret >= 0) { - if (repl_line == line) { - repl_line = NULL; - } - else { - line = repl_line; - line_len = repl_line_len; - } - /* Skip normal parsers flow */ - goto go_next; - } - else { - flb_tail_dmode_flush(file, ctx); - } - } - -#ifdef FLB_HAVE_PARSER - if (ctx->parser) { - /* Common parser (non-multiline) */ - ret = flb_parser_do(ctx->parser, line, line_len, - &out_buf, &out_size, &out_time); - if (ret >= 0) { - if (flb_time_to_nanosec(&out_time) == 0L) { - flb_time_get(&out_time); - } - - /* If multiline is enabled, flush any buffered data */ - if (ctx->multiline == FLB_TRUE) { - flb_tail_mult_flush(file, ctx); - } - - flb_tail_pack_line_map(&out_time, - (char**) &out_buf, &out_size, file, - processed_bytes); - - flb_free(out_buf); - } - else { - /* Parser failed, pack raw text */ - flb_tail_file_pack_line(NULL, data, len, file, processed_bytes); - } - } - else if (ctx->multiline == FLB_TRUE) { - ret = flb_tail_mult_process_content(now, - line, line_len, - file, ctx, processed_bytes); - - /* No multiline */ - if (ret == FLB_TAIL_MULT_NA) { - flb_tail_mult_flush(file, ctx); - - flb_tail_file_pack_line(NULL, - line, line_len, file, processed_bytes); - } - else if (ret == FLB_TAIL_MULT_MORE) { - /* we need more data, do nothing */ - goto go_next; - } - else if (ret == FLB_TAIL_MULT_DONE) { - /* Finalized */ - } - } - else { - flb_tail_file_pack_line(NULL, - line, line_len, file, processed_bytes); - } -#else - flb_tail_file_pack_line(NULL, - line, line_len, file, processed_bytes); -#endif - - go_next: - flb_free(repl_line); - repl_line = NULL; - /* Adjust counters */ - data += len + 1; - processed_bytes += len + 1; - lines++; - file->parsed = 0; - file->last_processed_bytes += processed_bytes; - } - file->parsed = file->buf_len; - - if (lines > 0) { - /* Append buffer content to a chunk */ - *bytes = processed_bytes; - - if (file->sl_log_event_encoder->output_length > 0) { - flb_input_log_append_records(ctx->ins, - lines, - file->tag_buf, - file->tag_len, - file->sl_log_event_encoder->output_buffer, - file->sl_log_event_encoder->output_length); - - flb_log_event_encoder_reset(file->sl_log_event_encoder); - } - } - else if (file->skip_next) { - *bytes = file->buf_len; - } - else { - *bytes = processed_bytes; - } - - if (ctx->ml_ctx) { - ml_stream_buffer_flush(ctx, file); - } - - return lines; -} - -static inline void drop_bytes(char *buf, size_t len, int pos, int bytes) -{ - memmove(buf + pos, - buf + pos + bytes, - len - pos - bytes); -} - -#ifdef FLB_HAVE_REGEX -static void cb_results(const char *name, const char *value, - size_t vlen, void *data) -{ - struct flb_hash_table *ht = data; - - if (vlen == 0) { - return; - } - - flb_hash_table_add(ht, name, strlen(name), (void *) value, vlen); -} -#endif - -#ifdef FLB_HAVE_REGEX -static int tag_compose(char *tag, struct flb_regex *tag_regex, char *fname, - char *out_buf, size_t *out_size, - struct flb_tail_config *ctx) -#else -static int tag_compose(char *tag, char *fname, char *out_buf, size_t *out_size, - struct flb_tail_config *ctx) -#endif -{ - int i; - size_t len; - char *p; - size_t buf_s = 0; -#ifdef FLB_HAVE_REGEX - ssize_t n; - struct flb_regex_search result; - struct flb_hash_table *ht; - char *beg; - char *end; - int ret; - const char *tmp; - size_t tmp_s; -#endif - -#ifdef FLB_HAVE_REGEX - if (tag_regex) { - n = flb_regex_do(tag_regex, fname, strlen(fname), &result); - if (n <= 0) { - flb_plg_error(ctx->ins, "invalid tag_regex pattern for file %s", - fname); - return -1; - } - else { - ht = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, - FLB_HASH_TABLE_SIZE, FLB_HASH_TABLE_SIZE); - flb_regex_parse(tag_regex, &result, cb_results, ht); - - for (p = tag, beg = p; (beg = strchr(p, '<')); p = end + 2) { - if (beg != p) { - len = (beg - p); - memcpy(out_buf + buf_s, p, len); - buf_s += len; - } - - beg++; - - end = strchr(beg, '>'); - if (end && !memchr(beg, '<', end - beg)) { - end--; - - len = end - beg + 1; - ret = flb_hash_table_get(ht, beg, len, (void *) &tmp, &tmp_s); - if (ret != -1) { - memcpy(out_buf + buf_s, tmp, tmp_s); - buf_s += tmp_s; - } - else { - memcpy(out_buf + buf_s, "_", 1); - buf_s++; - } - } - else { - flb_plg_error(ctx->ins, - "missing closing angle bracket in tag %s " - "at position %lu", tag, beg - tag); - flb_hash_table_destroy(ht); - return -1; - } - } - - flb_hash_table_destroy(ht); - if (*p) { - len = strlen(p); - memcpy(out_buf + buf_s, p, len); - buf_s += len; - } - } - } - else { -#endif - p = strchr(tag, '*'); - if (!p) { - return -1; - } - - /* Copy tag prefix if any */ - len = (p - tag); - if (len > 0) { - memcpy(out_buf, tag, len); - buf_s += len; - } - - /* Append file name */ - len = strlen(fname); - memcpy(out_buf + buf_s, fname, len); - buf_s += len; - - /* Tag suffix (if any) */ - p++; - if (*p) { - len = strlen(tag); - memcpy(out_buf + buf_s, p, (len - (p - tag))); - buf_s += (len - (p - tag)); - } - - /* Sanitize buffer */ - for (i = 0; i < buf_s; i++) { - if (out_buf[i] == '/' || out_buf[i] == '\\' || out_buf[i] == ':') { - if (i > 0) { - out_buf[i] = '.'; - } - else { - drop_bytes(out_buf, buf_s, i, 1); - buf_s--; - i--; - } - } - - if (i > 0 && out_buf[i] == '.') { - if (out_buf[i - 1] == '.') { - drop_bytes(out_buf, buf_s, i, 1); - buf_s--; - i--; - } - } - else if (out_buf[i] == '*') { - drop_bytes(out_buf, buf_s, i, 1); - buf_s--; - i--; - } - } - - /* Check for an ending '.' */ - if (out_buf[buf_s - 1] == '.') { - drop_bytes(out_buf, buf_s, buf_s - 1, 1); - buf_s--; - } -#ifdef FLB_HAVE_REGEX - } -#endif - - out_buf[buf_s] = '\0'; - *out_size = buf_s; - - return 0; -} - -static inline int flb_tail_file_exists(struct stat *st, - struct flb_tail_config *ctx) -{ - int ret; - uint64_t hash; - - ret = stat_to_hash_bits(ctx, st, &hash); - if (ret != 0) { - return -1; - } - - /* static hash */ - if (flb_hash_table_exists(ctx->static_hash, hash)) { - return FLB_TRUE; - } - - /* event hash */ - if (flb_hash_table_exists(ctx->event_hash, hash)) { - return FLB_TRUE; - } - - return FLB_FALSE; -} - -/* - * Based in the configuration or database offset, set the proper 'offset' for the - * file in question. - */ -static int set_file_position(struct flb_tail_config *ctx, - struct flb_tail_file *file) -{ - int64_t ret; - -#ifdef FLB_HAVE_SQLDB - /* - * If the database option is enabled, try to gather the file position. The - * database function updates the file->offset entry. - */ - if (ctx->db) { - ret = flb_tail_db_file_set(file, ctx); - if (ret == 0) { - if (file->offset > 0) { - ret = lseek(file->fd, file->offset, SEEK_SET); - if (ret == -1) { - flb_errno(); - return -1; - } - } - else if (ctx->read_from_head == FLB_FALSE) { - ret = lseek(file->fd, 0, SEEK_END); - if (ret == -1) { - flb_errno(); - return -1; - } - file->offset = ret; - flb_tail_db_file_offset(file, ctx); - } - return 0; - } - } -#endif - - if (ctx->read_from_head == FLB_TRUE) { - /* no need to seek, offset position is already zero */ - return 0; - } - - /* tail... */ - ret = lseek(file->fd, 0, SEEK_END); - if (ret == -1) { - flb_errno(); - return -1; - } - file->offset = ret; - - return 0; -} - -/* Multiline flush callback: invoked every time some content is complete */ -static int ml_flush_callback(struct flb_ml_parser *parser, - struct flb_ml_stream *mst, - void *data, char *buf_data, size_t buf_size) -{ - int result; - size_t mult_size = 0; - char *mult_buf = NULL; - struct flb_tail_file *file = data; - struct flb_tail_config *ctx = file->config; - - if (ctx->path_key == NULL && ctx->offset_key == NULL) { - ml_stream_buffer_append(file, buf_data, buf_size); - } - else { - /* adjust the records in a new buffer */ - result = record_append_custom_keys(file, - buf_data, - buf_size, - &mult_buf, - &mult_size); - - if (result < 0) { - ml_stream_buffer_append(file, buf_data, buf_size); - } - else { - ml_stream_buffer_append(file, mult_buf, mult_size); - - flb_free(mult_buf); - } - } - - if (mst->forced_flush) { - ml_stream_buffer_flush(ctx, file); - } - - return 0; -} - -int flb_tail_file_append(char *path, struct stat *st, int mode, - struct flb_tail_config *ctx) -{ - int fd; - int ret; - uint64_t stream_id; - uint64_t ts; - uint64_t hash_bits; - flb_sds_t hash_key; - size_t len; - char *tag; - char *name; - size_t tag_len; - struct flb_tail_file *file; - struct stat lst; - flb_sds_t inode_str; - - if (!S_ISREG(st->st_mode)) { - return -1; - } - - if (flb_tail_file_exists(st, ctx) == FLB_TRUE) { - return -1; - } - - fd = open(path, O_RDONLY); - if (fd == -1) { - flb_errno(); - flb_plg_error(ctx->ins, "cannot open %s", path); - return -1; - } - - file = flb_calloc(1, sizeof(struct flb_tail_file)); - if (!file) { - flb_errno(); - goto error; - } - - /* Initialize */ - file->watch_fd = -1; - file->fd = fd; - - /* On non-windows environments check if the original path is a link */ - ret = lstat(path, &lst); - if (ret == 0) { - if (S_ISLNK(lst.st_mode)) { - file->is_link = FLB_TRUE; - file->link_inode = lst.st_ino; - } - } - - /* get unique hash for this file */ - ret = stat_to_hash_bits(ctx, st, &hash_bits); - if (ret != 0) { - flb_plg_error(ctx->ins, "error procesisng hash bits for file %s", path); - goto error; - } - file->hash_bits = hash_bits; - - /* store the hash key used for hash_bits */ - ret = stat_to_hash_key(ctx, st, &hash_key); - if (ret != 0) { - flb_plg_error(ctx->ins, "error procesisng hash key for file %s", path); - goto error; - } - file->hash_key = hash_key; - - file->inode = st->st_ino; - file->offset = 0; - file->size = st->st_size; - file->buf_len = 0; - file->parsed = 0; - file->config = ctx; - file->tail_mode = mode; - file->tag_len = 0; - file->tag_buf = NULL; - file->rotated = 0; - file->pending_bytes = 0; - file->mult_firstline = FLB_FALSE; - file->mult_keys = 0; - file->mult_flush_timeout = 0; - file->mult_skipping = FLB_FALSE; - - /* - * Duplicate string into 'file' structure, the called function - * take cares to resolve real-name of the file in case we are - * running in a non-Linux system. - * - * Depending of the operating system, the way to obtain the file - * name associated to it file descriptor can have different behaviors - * specifically if it root path it's under a symbolic link. On Linux - * we can trust the file name but in others it's better to solve it - * with some extra calls. - */ - ret = flb_tail_file_name_dup(path, file); - if (!file->name) { - flb_errno(); - goto error; - } - - /* We keep a copy of the initial filename in orig_name. This is required - * for path_key to continue working after rotation. */ - file->orig_name = flb_strdup(file->name); - if (!file->orig_name) { - flb_free(file->name); - flb_errno(); - goto error; - } - file->orig_name_len = file->name_len; - - /* multiline msgpack buffers */ - file->mult_records = 0; - msgpack_sbuffer_init(&file->mult_sbuf); - msgpack_packer_init(&file->mult_pck, &file->mult_sbuf, - msgpack_sbuffer_write); - - /* docker mode */ - file->dmode_flush_timeout = 0; - file->dmode_complete = true; - file->dmode_buf = flb_sds_create_size(ctx->docker_mode == FLB_TRUE ? 65536 : 0); - file->dmode_lastline = flb_sds_create_size(ctx->docker_mode == FLB_TRUE ? 20000 : 0); - file->dmode_firstline = false; -#ifdef FLB_HAVE_SQLDB - file->db_id = 0; -#endif - file->skip_next = FLB_FALSE; - file->skip_warn = FLB_FALSE; - - /* Multiline core mode */ - if (ctx->ml_ctx) { - /* - * Create inode str to get stream_id. - * - * If stream_id is created by filename, - * it will be same after file rotation and it causes invalid destruction: - * - * - https://github.com/fluent/fluent-bit/issues/4190 - */ - inode_str = flb_sds_create_size(64); - flb_sds_printf(&inode_str, "%"PRIu64, file->inode); - - /* Create a stream for this file */ - ret = flb_ml_stream_create(ctx->ml_ctx, - inode_str, flb_sds_len(inode_str), - ml_flush_callback, file, - &stream_id); - if (ret != 0) { - flb_plg_error(ctx->ins, - "could not create multiline stream for file: %s", - inode_str); - flb_sds_destroy(inode_str); - goto error; - } - file->ml_stream_id = stream_id; - flb_sds_destroy(inode_str); - - /* - * Multiline core file buffer: the multiline core functionality invokes a callback everytime a message is ready - * to be processed by the caller, this can be a multiline message or a message that is considered 'complete'. In - * the previous version of Tail, when it received a message this message was automatically ingested into the pipeline - * without any previous buffering which leads to performance degradation. - * - * The msgpack buffer 'ml_sbuf' keeps all ML provided records and it's flushed just when the file processor finish - * processing the "read() bytes". - */ - } - - /* Local buffer */ - file->buf_size = ctx->buf_chunk_size; - file->buf_data = flb_malloc(file->buf_size); - if (!file->buf_data) { - flb_errno(); - goto error; - } - - /* Initialize (optional) dynamic tag */ - if (ctx->dynamic_tag == FLB_TRUE) { - len = ctx->ins->tag_len + strlen(path) + 1; - tag = flb_malloc(len); - if (!tag) { - flb_errno(); - flb_plg_error(ctx->ins, "failed to allocate tag buffer"); - goto error; - } -#ifdef FLB_HAVE_REGEX - ret = tag_compose(ctx->ins->tag, ctx->tag_regex, path, tag, &tag_len, ctx); -#else - ret = tag_compose(ctx->ins->tag, path, tag, &tag_len, ctx); -#endif - if (ret == 0) { - file->tag_len = tag_len; - file->tag_buf = flb_strdup(tag); - } - flb_free(tag); - if (ret != 0) { - flb_plg_error(ctx->ins, "failed to compose tag for file: %s", path); - goto error; - } - } - else { - file->tag_len = strlen(ctx->ins->tag); - file->tag_buf = flb_strdup(ctx->ins->tag); - } - if (!file->tag_buf) { - flb_plg_error(ctx->ins, "failed to set tag for file: %s", path); - flb_errno(); - goto error; - } - - if (mode == FLB_TAIL_STATIC) { - mk_list_add(&file->_head, &ctx->files_static); - ctx->files_static_count++; - flb_hash_table_add(ctx->static_hash, file->hash_key, flb_sds_len(file->hash_key), - file, sizeof(file)); - tail_signal_manager(file->config); - } - else if (mode == FLB_TAIL_EVENT) { - mk_list_add(&file->_head, &ctx->files_event); - flb_hash_table_add(ctx->event_hash, file->hash_key, flb_sds_len(file->hash_key), - file, sizeof(file)); - - /* Register this file into the fs_event monitoring */ - ret = flb_tail_fs_add(ctx, file); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not register file into fs_events"); - goto error; - } - } - - /* Set the file position (database offset, head or tail) */ - ret = set_file_position(ctx, file); - if (ret == -1) { - flb_tail_file_remove(file); - goto error; - } - - /* Remaining bytes to read */ - file->pending_bytes = file->size - file->offset; - -#ifdef FLB_HAVE_METRICS - name = (char *) flb_input_name(ctx->ins); - ts = cfl_time_now(); - cmt_counter_inc(ctx->cmt_files_opened, ts, 1, (char *[]) {name}); - - /* Old api */ - flb_metrics_sum(FLB_TAIL_METRIC_F_OPENED, 1, ctx->ins->metrics); -#endif - - file->sl_log_event_encoder = flb_log_event_encoder_create( - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (file->sl_log_event_encoder == NULL) { - flb_tail_file_remove(file); - - goto error; - } - - file->ml_log_event_encoder = flb_log_event_encoder_create( - FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (file->ml_log_event_encoder == NULL) { - flb_tail_file_remove(file); - - goto error; - } - - flb_plg_debug(ctx->ins, - "inode=%"PRIu64" with offset=%"PRId64" appended as %s", - file->inode, file->offset, path); - return 0; - -error: - if (file) { - if (file->buf_data) { - flb_free(file->buf_data); - } - if (file->name) { - flb_free(file->name); - } - flb_free(file); - } - close(fd); - - return -1; -} - -void flb_tail_file_remove(struct flb_tail_file *file) -{ - uint64_t ts; - char *name; - struct flb_tail_config *ctx; - - ctx = file->config; - - flb_plg_debug(ctx->ins, "inode=%"PRIu64" removing file name %s", - file->inode, file->name); - - /* remove the multiline.core stream */ - if (ctx->ml_ctx && file->ml_stream_id > 0) { - /* destroy ml stream */ - flb_ml_stream_id_destroy_all(ctx->ml_ctx, file->ml_stream_id); - } - - if (file->rotated > 0) { -#ifdef FLB_HAVE_SQLDB - /* - * Make sure to remove a the file entry from the database if the file - * was rotated and it's not longer being monitored. - */ - if (ctx->db) { - flb_tail_db_file_delete(file, file->config); - } -#endif - mk_list_del(&file->_rotate_head); - } - - msgpack_sbuffer_destroy(&file->mult_sbuf); - - if (file->sl_log_event_encoder != NULL) { - flb_log_event_encoder_destroy(file->sl_log_event_encoder); - } - - if (file->ml_log_event_encoder != NULL) { - flb_log_event_encoder_destroy(file->ml_log_event_encoder); - } - - flb_sds_destroy(file->dmode_buf); - flb_sds_destroy(file->dmode_lastline); - mk_list_del(&file->_head); - flb_tail_fs_remove(ctx, file); - - /* avoid deleting file with -1 fd */ - if (file->fd != -1) { - close(file->fd); - } - if (file->tag_buf) { - flb_free(file->tag_buf); - } - - /* remove any potential entry from the hash tables */ - flb_hash_table_del(ctx->static_hash, file->hash_key); - flb_hash_table_del(ctx->event_hash, file->hash_key); - - flb_free(file->buf_data); - flb_free(file->name); - flb_free(file->orig_name); - flb_free(file->real_name); - flb_sds_destroy(file->hash_key); - -#ifdef FLB_HAVE_METRICS - name = (char *) flb_input_name(ctx->ins); - ts = cfl_time_now(); - cmt_counter_inc(ctx->cmt_files_closed, ts, 1, (char *[]) {name}); - - /* old api */ - flb_metrics_sum(FLB_TAIL_METRIC_F_CLOSED, 1, ctx->ins->metrics); -#endif - - flb_free(file); -} - -int flb_tail_file_remove_all(struct flb_tail_config *ctx) -{ - int count = 0; - struct mk_list *head; - struct mk_list *tmp; - struct flb_tail_file *file; - - mk_list_foreach_safe(head, tmp, &ctx->files_static) { - file = mk_list_entry(head, struct flb_tail_file, _head); - flb_tail_file_remove(file); - count++; - } - - mk_list_foreach_safe(head, tmp, &ctx->files_event) { - file = mk_list_entry(head, struct flb_tail_file, _head); - flb_tail_file_remove(file); - count++; - } - - return count; -} - -static int adjust_counters(struct flb_tail_config *ctx, struct flb_tail_file *file) -{ - int ret; - int64_t offset; - struct stat st; - - ret = fstat(file->fd, &st); - if (ret == -1) { - flb_errno(); - return FLB_TAIL_ERROR; - } - - /* Check if the file was truncated */ - if (file->offset > st.st_size) { - offset = lseek(file->fd, 0, SEEK_SET); - if (offset == -1) { - flb_errno(); - return FLB_TAIL_ERROR; - } - - flb_plg_debug(ctx->ins, "inode=%"PRIu64" file truncated %s", - file->inode, file->name); - file->offset = offset; - file->buf_len = 0; - - /* Update offset in the database file */ -#ifdef FLB_HAVE_SQLDB - if (ctx->db) { - flb_tail_db_file_offset(file, ctx); - } -#endif - } - else { - file->size = st.st_size; - file->pending_bytes = (st.st_size - file->offset); - } - - return FLB_TAIL_OK; -} - -int flb_tail_file_chunk(struct flb_tail_file *file) -{ - int ret; - char *tmp; - size_t size; - size_t capacity; - size_t processed_bytes; - ssize_t bytes; - struct flb_tail_config *ctx; - - /* Check if we the engine issued a pause */ - ctx = file->config; - if (flb_input_buf_paused(ctx->ins) == FLB_TRUE) { - return FLB_TAIL_BUSY; - } - - capacity = (file->buf_size - file->buf_len) - 1; - if (capacity < 1) { - /* - * If there is no more room for more data, try to increase the - * buffer under the limit of buffer_max_size. - */ - if (file->buf_size >= ctx->buf_max_size) { - if (ctx->skip_long_lines == FLB_FALSE) { - flb_plg_error(ctx->ins, "file=%s requires a larger buffer size, " - "lines are too long. Skipping file.", file->name); - return FLB_TAIL_ERROR; - } - - /* Warn the user */ - if (file->skip_warn == FLB_FALSE) { - flb_plg_warn(ctx->ins, "file=%s have long lines. " - "Skipping long lines.", file->name); - file->skip_warn = FLB_TRUE; - } - - /* Do buffer adjustments */ - file->offset += file->buf_len; - file->buf_len = 0; - file->skip_next = FLB_TRUE; - } - else { - size = file->buf_size + ctx->buf_chunk_size; - if (size > ctx->buf_max_size) { - size = ctx->buf_max_size; - } - - /* Increase the buffer size */ - tmp = flb_realloc(file->buf_data, size); - if (tmp) { - flb_plg_trace(ctx->ins, "file=%s increase buffer size " - "%lu => %lu bytes", - file->name, file->buf_size, size); - file->buf_data = tmp; - file->buf_size = size; - } - else { - flb_errno(); - flb_plg_error(ctx->ins, "cannot increase buffer size for %s, " - "skipping file.", file->name); - return FLB_TAIL_ERROR; - } - } - capacity = (file->buf_size - file->buf_len) - 1; - } - - bytes = read(file->fd, file->buf_data + file->buf_len, capacity); - if (bytes > 0) { - /* we read some data, let the content processor take care of it */ - file->buf_len += bytes; - file->buf_data[file->buf_len] = '\0'; - - /* Now that we have some data in the buffer, call the data processor - * which aims to cut lines and register the entries into the engine. - * - * The returned value is the absolute offset the file must be seek - * now. It may need to get back a few bytes at the beginning of a new - * line. - */ - ret = process_content(file, &processed_bytes); - if (ret < 0) { - flb_plg_debug(ctx->ins, "inode=%"PRIu64" file=%s process content ERROR", - file->inode, file->name); - return FLB_TAIL_ERROR; - } - - /* Adjust the file offset and buffer */ - file->offset += processed_bytes; - consume_bytes(file->buf_data, processed_bytes, file->buf_len); - file->buf_len -= processed_bytes; - file->buf_data[file->buf_len] = '\0'; - -#ifdef FLB_HAVE_SQLDB - if (file->config->db) { - flb_tail_db_file_offset(file, file->config); - } -#endif - - /* adjust file counters, returns FLB_TAIL_OK or FLB_TAIL_ERROR */ - ret = adjust_counters(ctx, file); - - /* Data was consumed but likely some bytes still remain */ - return ret; - } - else if (bytes == 0) { - /* We reached the end of file, let's wait for some incoming data */ - ret = adjust_counters(ctx, file); - if (ret == FLB_TAIL_OK) { - return FLB_TAIL_WAIT; - } - else { - return FLB_TAIL_ERROR; - } - } - else { - /* error */ - flb_errno(); - flb_plg_error(ctx->ins, "error reading %s", file->name); - return FLB_TAIL_ERROR; - } - - return FLB_TAIL_ERROR; -} - -/* Returns FLB_TRUE if a file has been rotated, otherwise FLB_FALSE */ -int flb_tail_file_is_rotated(struct flb_tail_config *ctx, - struct flb_tail_file *file) -{ - int ret; - char *name; - struct stat st; - - /* - * Do not double-check already rotated files since the caller of this - * function will trigger a rotation. - */ - if (file->rotated != 0) { - return FLB_FALSE; - } - - /* Check if the 'original monitored file' is a link and rotated */ - if (file->is_link == FLB_TRUE) { - ret = lstat(file->name, &st); - if (ret == -1) { - /* Broken link or missing file */ - if (errno == ENOENT) { - flb_plg_info(ctx->ins, "inode=%"PRIu64" link_rotated: %s", - file->link_inode, file->name); - return FLB_TRUE; - } - else { - flb_errno(); - flb_plg_error(ctx->ins, - "link_inode=%"PRIu64" cannot detect if file: %s", - file->link_inode, file->name); - return -1; - } - } - else { - /* The file name is there, check if the same that we have */ - if (st.st_ino != file->link_inode) { - return FLB_TRUE; - } - } - } - - /* Retrieve the real file name, operating system lookup */ - name = flb_tail_file_name(file); - if (!name) { - flb_plg_error(ctx->ins, - "inode=%"PRIu64" cannot detect if file was rotated: %s", - file->inode, file->name); - return -1; - } - - - /* Get stats from the file name */ - ret = stat(name, &st); - if (ret == -1) { - flb_errno(); - flb_free(name); - return -1; - } - - /* Compare inodes and names */ - if (file->inode == st.st_ino && - flb_tail_target_file_name_cmp(name, file) == 0) { - flb_free(name); - return FLB_FALSE; - } - - flb_plg_debug(ctx->ins, "inode=%"PRIu64" rotated: %s => %s", - file->inode, file->name, name); - - flb_free(name); - return FLB_TRUE; -} - -/* Promote a event in the static list to the dynamic 'events' interface */ -int flb_tail_file_to_event(struct flb_tail_file *file) -{ - int ret; - struct stat st; - struct flb_tail_config *ctx = file->config; - - /* Check if the file promoted have pending bytes */ - ret = fstat(file->fd, &st); - if (ret != 0) { - flb_errno(); - return -1; - } - - if (file->offset < st.st_size) { - file->pending_bytes = (st.st_size - file->offset); - tail_signal_pending(file->config); - } - else { - file->pending_bytes = 0; - } - - /* Check if the file has been rotated */ - ret = flb_tail_file_is_rotated(ctx, file); - if (ret == FLB_TRUE) { - flb_tail_file_rotated(file); - } - - /* Notify the fs-event handler that we will start monitoring this 'file' */ - ret = flb_tail_fs_add(ctx, file); - if (ret == -1) { - return -1; - } - - /* List swap: change from 'static' to 'event' list */ - mk_list_del(&file->_head); - ctx->files_static_count--; - flb_hash_table_del(ctx->static_hash, file->hash_key); - - mk_list_add(&file->_head, &file->config->files_event); - flb_hash_table_add(ctx->event_hash, file->hash_key, flb_sds_len(file->hash_key), - file, sizeof(file)); - - file->tail_mode = FLB_TAIL_EVENT; - - return 0; -} - -/* - * Given an open file descriptor, return the filename. This function is a - * bit slow and it aims to be used only when a file is rotated. - */ -char *flb_tail_file_name(struct flb_tail_file *file) -{ - int ret; - char *buf; -#ifdef __linux__ - ssize_t s; - char tmp[128]; -#elif defined(__APPLE__) - char path[PATH_MAX]; -#elif defined(FLB_SYSTEM_WINDOWS) - HANDLE h; -#elif defined(FLB_SYSTEM_FREEBSD) - struct kinfo_file *file_entries; - int file_count; - int file_index; -#endif - - buf = flb_malloc(PATH_MAX); - if (!buf) { - flb_errno(); - return NULL; - } - -#ifdef __linux__ - ret = snprintf(tmp, sizeof(tmp) - 1, "/proc/%i/fd/%i", getpid(), file->fd); - if (ret == -1) { - flb_errno(); - flb_free(buf); - return NULL; - } - - s = readlink(tmp, buf, PATH_MAX); - if (s == -1) { - flb_free(buf); - flb_errno(); - return NULL; - } - buf[s] = '\0'; - -#elif __APPLE__ - int len; - - ret = fcntl(file->fd, F_GETPATH, path); - if (ret == -1) { - flb_errno(); - flb_free(buf); - return NULL; - } - - len = strlen(path); - memcpy(buf, path, len); - buf[len] = '\0'; - -#elif defined(FLB_SYSTEM_WINDOWS) - int len; - - h = (HANDLE) _get_osfhandle(file->fd); - if (h == INVALID_HANDLE_VALUE) { - flb_errno(); - flb_free(buf); - return NULL; - } - - /* This function returns the length of the string excluding "\0" - * and the resulting path has a "\\?\" prefix. - */ - len = GetFinalPathNameByHandleA(h, buf, PATH_MAX, FILE_NAME_NORMALIZED); - if (len == 0 || len >= PATH_MAX) { - flb_free(buf); - return NULL; - } - - if (strstr(buf, "\\\\?\\")) { - memmove(buf, buf + 4, len + 1); - } -#elif defined(FLB_SYSTEM_FREEBSD) - if ((file_entries = kinfo_getfile(getpid(), &file_count)) == NULL) { - flb_free(buf); - return NULL; - } - - for (file_index=0; file_index < file_count; file_index++) { - if (file_entries[file_index].kf_fd == file->fd) { - strncpy(buf, file_entries[file_index].kf_path, PATH_MAX - 1); - buf[PATH_MAX - 1] = 0; - break; - } - } - free(file_entries); -#endif - return buf; -} - -int flb_tail_file_name_dup(char *path, struct flb_tail_file *file) -{ - file->name = flb_strdup(path); - if (!file->name) { - flb_errno(); - return -1; - } - file->name_len = strlen(file->name); - - if (file->real_name) { - flb_free(file->real_name); - } - - file->real_name = flb_tail_file_name(file); - if (!file->real_name) { - flb_errno(); - flb_free(file->name); - file->name = NULL; - return -1; - } - - return 0; -} - -/* Invoked every time a file was rotated */ -int flb_tail_file_rotated(struct flb_tail_file *file) -{ - int ret; - uint64_t ts; - char *name; - char *i_name; - char *tmp; - struct stat st; - struct flb_tail_config *ctx = file->config; - - /* Get the new file name */ - name = flb_tail_file_name(file); - if (!name) { - return -1; - } - - flb_plg_debug(ctx->ins, "inode=%"PRIu64" rotated %s -> %s", - file->inode, file->name, name); - - /* Update local file entry */ - tmp = file->name; - flb_tail_file_name_dup(name, file); - flb_plg_info(ctx->ins, "inode=%"PRIu64" handle rotation(): %s => %s", - file->inode, tmp, file->name); - if (file->rotated == 0) { - file->rotated = time(NULL); - mk_list_add(&file->_rotate_head, &file->config->files_rotated); - - /* Rotate the file in the database */ -#ifdef FLB_HAVE_SQLDB - if (file->config->db) { - ret = flb_tail_db_file_rotate(name, file, file->config); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not rotate file %s->%s in database", - file->name, name); - } - } -#endif - -#ifdef FLB_HAVE_METRICS - i_name = (char *) flb_input_name(ctx->ins); - ts = cfl_time_now(); - cmt_counter_inc(ctx->cmt_files_rotated, ts, 1, (char *[]) {i_name}); - - /* OLD api */ - flb_metrics_sum(FLB_TAIL_METRIC_F_ROTATED, - 1, file->config->ins->metrics); -#endif - - /* Check if a new file has been created */ - ret = stat(tmp, &st); - if (ret == 0 && st.st_ino != file->inode) { - if (flb_tail_file_exists(&st, ctx) == FLB_FALSE) { - ret = flb_tail_file_append(tmp, &st, FLB_TAIL_STATIC, ctx); - if (ret == -1) { - flb_tail_scan(ctx->path_list, ctx); - } - else { - tail_signal_manager(file->config); - } - } - } - } - flb_free(tmp); - flb_free(name); - - return 0; -} - -static int check_purge_deleted_file(struct flb_tail_config *ctx, - struct flb_tail_file *file, time_t ts) -{ - int ret; - struct stat st; - - ret = fstat(file->fd, &st); - if (ret == -1) { - flb_plg_debug(ctx->ins, "error stat(2) %s, removing", file->name); - flb_tail_file_remove(file); - return FLB_TRUE; - } - - if (st.st_nlink == 0) { - flb_plg_debug(ctx->ins, "purge: monitored file has been deleted: %s", - file->name); -#ifdef FLB_HAVE_SQLDB - if (ctx->db) { - /* Remove file entry from the database */ - flb_tail_db_file_delete(file, file->config); - } -#endif - /* Remove file from the monitored list */ - flb_tail_file_remove(file); - return FLB_TRUE; - } - - return FLB_FALSE; -} - -/* Purge rotated and deleted files */ -int flb_tail_file_purge(struct flb_input_instance *ins, - struct flb_config *config, void *context) -{ - int ret; - int count = 0; - struct mk_list *tmp; - struct mk_list *head; - struct flb_tail_file *file; - struct flb_tail_config *ctx = context; - time_t now; - struct stat st; - - /* Rotated files */ - now = time(NULL); - mk_list_foreach_safe(head, tmp, &ctx->files_rotated) { - file = mk_list_entry(head, struct flb_tail_file, _rotate_head); - if ((file->rotated + ctx->rotate_wait) <= now) { - ret = fstat(file->fd, &st); - if (ret == 0) { - flb_plg_debug(ctx->ins, - "inode=%"PRIu64" purge rotated file %s " \ - "(offset=%"PRId64" / size = %"PRIu64")", - file->inode, file->name, file->offset, (uint64_t)st.st_size); - if (file->pending_bytes > 0 && flb_input_buf_paused(ins)) { - flb_plg_warn(ctx->ins, "purged rotated file while data " - "ingestion is paused, consider increasing " - "rotate_wait"); - } - } - else { - flb_plg_debug(ctx->ins, - "inode=%"PRIu64" purge rotated file %s (offset=%"PRId64")", - file->inode, file->name, file->offset); - } - - flb_tail_file_remove(file); - count++; - } - } - - /* - * Deleted files: under high load scenarios, exists the chances that in - * our event loop we miss some notifications about a file. In order to - * sanitize our list of monitored files we will iterate all of them and check - * if they have been deleted or not. - */ - mk_list_foreach_safe(head, tmp, &ctx->files_static) { - file = mk_list_entry(head, struct flb_tail_file, _head); - check_purge_deleted_file(ctx, file, now); - } - mk_list_foreach_safe(head, tmp, &ctx->files_event) { - file = mk_list_entry(head, struct flb_tail_file, _head); - check_purge_deleted_file(ctx, file, now); - } - - return count; -} diff --git a/fluent-bit/plugins/in_tail/tail_file.h b/fluent-bit/plugins/in_tail/tail_file.h deleted file mode 100644 index 796224c37..000000000 --- a/fluent-bit/plugins/in_tail/tail_file.h +++ /dev/null @@ -1,137 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_TAIL_FILE_H -#define FLB_TAIL_FILE_H - -#include -#include - -#include -#include - -#include "tail.h" -#include "tail_fs.h" -#include "tail_config.h" -#include "tail_file_internal.h" - -#ifdef FLB_SYSTEM_WINDOWS -#include "win32.h" -#endif - -#ifdef FLB_HAVE_REGEX -#define FLB_HASH_TABLE_SIZE 50 -#endif - -/* return the file modification time in seconds since epoch */ -static inline int64_t flb_tail_stat_mtime(struct stat *st) -{ -#if defined(FLB_HAVE_WINDOWS) - return (int64_t) st->st_mtime; -#elif defined(__APPLE__) && !defined(_POSIX_C_SOURCE) - return (int64_t) st->st_mtimespec.tv_sec; -#elif (_POSIX_C_SOURCE >= 200809L || \ - defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || \ - defined(__BIONIC__) || (defined (__SVR4) && defined (__sun)) || \ - defined(__FreeBSD__) || defined (__linux__)) - return (int64_t) st->st_mtim.tv_sec; -#elif defined(_AIX) - return (int64_t) st->st_mtime; -#else - return (int64_t) st->st_mtime; -#endif - - /* backend unsupported: submit a PR :) */ - return -1; -} - -static inline int flb_tail_target_file_name_cmp(char *name, - struct flb_tail_file *file) -{ - int ret; - char *name_a = NULL; - char *name_b = NULL; - char *base_a = NULL; - char *base_b = NULL; - - name_a = flb_strdup(name); - if (!name_a) { - flb_errno(); - ret = -1; - goto out; - } - - base_a = flb_strdup(basename(name_a)); - if (!base_a) { - flb_errno(); - ret = -1; - goto out; - } - -#if defined(FLB_SYSTEM_WINDOWS) - name_b = flb_strdup(file->real_name); - if (!name_b) { - flb_errno(); - ret = -1; - goto out; - } - - base_b = basename(name_b); - ret = _stricmp(base_a, base_b); -#else - name_b = flb_strdup(file->real_name); - if (!name_b) { - flb_errno(); - ret = -1; - goto out; - } - base_b = basename(name_b); - ret = strcmp(base_a, base_b); -#endif - - out: - flb_free(name_a); - flb_free(name_b); - flb_free(base_a); - - /* FYI: 'base_b' never points to a new allocation, no flb_free is needed */ - - return ret; -} - -int flb_tail_file_name_dup(char *path, struct flb_tail_file *file); -int flb_tail_file_to_event(struct flb_tail_file *file); -int flb_tail_file_chunk(struct flb_tail_file *file); -int flb_tail_file_append(char *path, struct stat *st, int mode, - struct flb_tail_config *ctx); -void flb_tail_file_remove(struct flb_tail_file *file); -int flb_tail_file_remove_all(struct flb_tail_config *ctx); -char *flb_tail_file_name(struct flb_tail_file *file); -int flb_tail_file_is_rotated(struct flb_tail_config *ctx, - struct flb_tail_file *file); -int flb_tail_file_rotated(struct flb_tail_file *file); -int flb_tail_file_purge(struct flb_input_instance *ins, - struct flb_config *config, void *context); -int flb_tail_pack_line_map(struct flb_time *time, char **data, - size_t *data_size, struct flb_tail_file *file, - size_t processed_bytes); -int flb_tail_file_pack_line(struct flb_time *time, char *data, size_t data_size, - struct flb_tail_file *file, size_t processed_bytes); - -#endif diff --git a/fluent-bit/plugins/in_tail/tail_file_internal.h b/fluent-bit/plugins/in_tail/tail_file_internal.h deleted file mode 100644 index 6d95c87c1..000000000 --- a/fluent-bit/plugins/in_tail/tail_file_internal.h +++ /dev/null @@ -1,130 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_TAIL_INTERNAL_H -#define FLB_TAIL_INTERNAL_H - -#include -#include -#include -#include -#include - -#ifdef FLB_HAVE_PARSER -#include -#endif - -#include "tail.h" -#include "tail_config.h" - -struct flb_tail_file { - /* Inotify */ - int watch_fd; - /* file lookup info */ - int fd; - int64_t size; - int64_t offset; - int64_t last_line; - uint64_t dev_id; - uint64_t inode; - uint64_t link_inode; - int is_link; - char *name; /* target file name given by scan routine */ - char *real_name; /* real file name in the file system */ - char *orig_name; /* original file name (before rotation) */ - size_t name_len; - size_t orig_name_len; - time_t rotated; - int64_t pending_bytes; - - /* dynamic tag for this file */ - int tag_len; - char *tag_buf; - - /* OLD multiline */ - time_t mult_flush_timeout; /* time when multiline started */ - int mult_firstline; /* bool: mult firstline found ? */ - int mult_firstline_append; /* bool: mult firstline appendable ? */ - int mult_skipping; /* skipping because ignode_older than ? */ - int mult_keys; /* total number of buffered keys */ - - int mult_records; /* multiline records counter mult_sbuf */ - msgpack_sbuffer mult_sbuf; /* temporary msgpack buffer */ - msgpack_packer mult_pck; /* temporary msgpack packer */ - struct flb_time mult_time; /* multiline time parsed from first line */ - - /* OLD docker mode */ - time_t dmode_flush_timeout; /* time when docker mode started */ - flb_sds_t dmode_buf; /* buffer for docker mode */ - flb_sds_t dmode_lastline; /* last incomplete line */ - bool dmode_complete; /* buffer contains completed log */ - bool dmode_firstline; /* dmode mult firstline found ? */ - - /* multiline engine: file stream_id and local buffers */ - uint64_t ml_stream_id; - - /* content parsing, positions and buffer */ - size_t parsed; - size_t buf_len; - size_t buf_size; - char *buf_data; - - /* - * This value represent the number of bytes procesed by process_content() - * in the last iteration. - */ - size_t last_processed_bytes; - - /* - * Long-lines handling: this flag is enabled when a previous line was - * too long and the buffer did not contain a \n, so when reaching the - * missing \n, skip that content and move forward. - * - * This flag is only set when Skip_Long_Lines is On. - */ - int skip_next; - - /* Did the plugin already warn the user about long lines ? */ - int skip_warn; - - /* Opaque data type for specific fs-event backend data */ - void *fs_backend; - - /* database reference */ - uint64_t db_id; - - uint64_t hash_bits; - flb_sds_t hash_key; - - /* There are dedicated log event encoders for - * single and multi line events because I am respecting - * the old behavior which resulted in grouping both types - * of logs in tail_file.c but I don't know if this is - * strictly necessary. - */ - struct flb_log_event_encoder *ml_log_event_encoder; - struct flb_log_event_encoder *sl_log_event_encoder; - - /* reference */ - int tail_mode; - struct flb_tail_config *config; - struct mk_list _head; - struct mk_list _rotate_head; -}; -#endif diff --git a/fluent-bit/plugins/in_tail/tail_fs.h b/fluent-bit/plugins/in_tail/tail_fs.h deleted file mode 100644 index 948954333..000000000 --- a/fluent-bit/plugins/in_tail/tail_fs.h +++ /dev/null @@ -1,96 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_TAIL_FS_H -#define FLB_TAIL_FS_H - -#include -#include - -#include "tail_config.h" -#include "tail_file_internal.h" - -#include "tail_fs_stat.h" -#ifdef FLB_HAVE_INOTIFY -#include "tail_fs_inotify.h" -#endif - -static inline int flb_tail_fs_init(struct flb_input_instance *in, - struct flb_tail_config *ctx, struct flb_config *config) -{ -#ifdef FLB_HAVE_INOTIFY - if (ctx->inotify_watcher) { - return flb_tail_fs_inotify_init(in, ctx, config); - } -#endif - return flb_tail_fs_stat_init(in, ctx, config); -} - -static inline void flb_tail_fs_pause(struct flb_tail_config *ctx) -{ -#ifdef FLB_HAVE_INOTIFY - if (ctx->inotify_watcher) { - return flb_tail_fs_inotify_pause(ctx); - } -#endif - return flb_tail_fs_stat_pause(ctx); -} - -static inline void flb_tail_fs_resume(struct flb_tail_config *ctx) -{ -#ifdef FLB_HAVE_INOTIFY - if (ctx->inotify_watcher) { - return flb_tail_fs_inotify_resume(ctx); - } -#endif - return flb_tail_fs_stat_resume(ctx); -} - -static inline int flb_tail_fs_add(struct flb_tail_config *ctx, struct flb_tail_file *file) -{ -#ifdef FLB_HAVE_INOTIFY - if (ctx->inotify_watcher) { - return flb_tail_fs_inotify_add(file); - } -#endif - return flb_tail_fs_stat_add(file); -} - -static inline int flb_tail_fs_remove(struct flb_tail_config *ctx, struct flb_tail_file *file) -{ -#ifdef FLB_HAVE_INOTIFY - if (ctx->inotify_watcher) { - return flb_tail_fs_inotify_remove(file); - } -#endif - return flb_tail_fs_stat_remove(file); -} - -static inline int flb_tail_fs_exit(struct flb_tail_config *ctx) -{ -#ifdef FLB_HAVE_INOTIFY - if (ctx->inotify_watcher) { - return flb_tail_fs_inotify_exit(ctx); - } -#endif - return flb_tail_fs_stat_exit(ctx); -} - - -#endif diff --git a/fluent-bit/plugins/in_tail/tail_fs_inotify.c b/fluent-bit/plugins/in_tail/tail_fs_inotify.c deleted file mode 100644 index 59d10ca08..000000000 --- a/fluent-bit/plugins/in_tail/tail_fs_inotify.c +++ /dev/null @@ -1,433 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define _DEFAULT_SOURCE - -#include -#include -#include - -#include -#include -#include -#include - -#include "tail_config.h" -#include "tail_file.h" -#include "tail_db.h" -#include "tail_signal.h" - -#include -#include - -#include - -static int debug_event_mask(struct flb_tail_config *ctx, - struct flb_tail_file *file, - uint32_t mask) -{ - flb_sds_t buf; - int buf_size = 256; - - /* Only enter this function if debug mode is allowed */ - if (flb_log_check(FLB_LOG_DEBUG) == 0) { - return 0; - } - - if (file) { - buf_size = file->name_len + 128; - } - - if (buf_size < 256) { - buf_size = 256; - } - - /* Create buffer */ - buf = flb_sds_create_size(buf_size); - if (!buf) { - return -1; - } - - /* Print info into sds */ - if (file) { - flb_sds_printf(&buf, "inode=%"PRIu64", %s, events: ", file->inode, file->name); - } - else { - flb_sds_printf(&buf, "events: "); - } - - if (mask & IN_ATTRIB) { - flb_sds_printf(&buf, "IN_ATTRIB "); - } - if (mask & IN_IGNORED) { - flb_sds_printf(&buf, "IN_IGNORED "); - } - if (mask & IN_MODIFY) { - flb_sds_printf(&buf, "IN_MODIFY "); - } - if (mask & IN_MOVE_SELF) { - flb_sds_printf(&buf, "IN_MOVE_SELF "); - } - if (mask & IN_Q_OVERFLOW) { - flb_sds_printf(&buf, "IN_Q_OVERFLOW "); - } - - flb_plg_debug(ctx->ins, "%s", buf); - flb_sds_destroy(buf); - - return 0; -} - -static int tail_fs_add(struct flb_tail_file *file, int check_rotated) -{ - int flags; - int watch_fd; - char *name; - struct flb_tail_config *ctx = file->config; - - /* - * If there is no watcher associated, we only want to monitor events if - * this file is rotated to somewhere. Note at this point we are polling - * lines from the file and once we reach EOF (and a watch_fd exists), - * we update the flags to receive notifications. - */ - flags = IN_ATTRIB | IN_IGNORED | IN_MODIFY | IN_Q_OVERFLOW; - - if (check_rotated == FLB_TRUE) { - flags |= IN_MOVE_SELF; - } - - /* - * Double check real name of the file associated to the inode: - * - * Inotify interface in the Kernel uses the inode number as a real reference - * for the file we have opened. If for some reason the file we are pointing - * out in file->name has been rotated and not been updated, we might not add - * the watch to the real file we aim to. - * - * A case like this can generate the issue: - * - * 1. inode=1 : file a.log is being watched - * 2. inode=1 : file a.log is rotated to a.log.1, but notification not - * delivered yet. - * 3. inode=2 : new file 'a.log' is created - * 4. inode=2 : the scan_path routine discover the new 'a.log' file - * 5. inode=2 : add an inotify watcher for 'a.log' - * 6. conflict: inotify_add_watch() receives the path 'a.log', - */ - - name = flb_tail_file_name(file); - if (!name) { - flb_plg_error(ctx->ins, "inode=%"PRIu64" cannot get real filename for inotify", - file->inode); - return -1; - } - - /* Register or update the flags */ - watch_fd = inotify_add_watch(ctx->fd_notify, name, flags); - flb_free(name); - - if (watch_fd == -1) { - flb_errno(); - if (errno == ENOSPC) { - flb_plg_error(ctx->ins, "inotify: The user limit on the total " - "number of inotify watches was reached or the kernel " - "failed to allocate a needed resource (ENOSPC)"); - } - return -1; - } - file->watch_fd = watch_fd; - flb_plg_info(ctx->ins, "inotify_fs_add(): inode=%"PRIu64" watch_fd=%i name=%s", - file->inode, watch_fd, file->name); - return 0; -} - -static int flb_tail_fs_add_rotated(struct flb_tail_file *file) -{ - return tail_fs_add(file, FLB_FALSE); -} - -static int tail_fs_event(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - int ret; - int64_t offset; - struct mk_list *head; - struct mk_list *tmp; - struct flb_tail_config *ctx = in_context; - struct flb_tail_file *file = NULL; - struct inotify_event ev; - struct stat st; - - /* Read the event */ - ret = read(ctx->fd_notify, &ev, sizeof(struct inotify_event)); - if (ret < 1) { - return -1; - } - - /* Lookup watched file */ - mk_list_foreach_safe(head, tmp, &ctx->files_event) { - file = mk_list_entry(head, struct flb_tail_file, _head); - if (file->watch_fd != ev.wd) { - file = NULL; - continue; - } - break; - } - - if (!file) { - return -1; - } - - /* Debug event */ - debug_event_mask(ctx, file, ev.mask); - - if (ev.mask & IN_IGNORED) { - flb_plg_debug(ctx->ins, "inode=%"PRIu64" watch_fd=%i IN_IGNORED", - file->inode, ev.wd); - return -1; - } - - /* Check file rotation (only if it has not been rotated before) */ - if (ev.mask & IN_MOVE_SELF && file->rotated == 0) { - flb_plg_debug(ins, "inode=%"PRIu64" rotated IN_MOVE SELF '%s'", - file->inode, file->name); - - /* A rotated file must be re-registered */ - flb_tail_file_rotated(file); - flb_tail_fs_remove(ctx, file); - flb_tail_fs_add_rotated(file); - } - - ret = fstat(file->fd, &st); - if (ret == -1) { - flb_plg_debug(ins, "inode=%"PRIu64" error stat(2) %s, removing", - file->inode, file->name); - flb_tail_file_remove(file); - return 0; - } - file->size = st.st_size; - file->pending_bytes = (file->size - file->offset); - - /* File was removed ? */ - if (ev.mask & IN_ATTRIB) { - /* Check if the file have been deleted */ - if (st.st_nlink == 0) { - flb_plg_debug(ins, "inode=%"PRIu64" file has been deleted: %s", - file->inode, file->name); - -#ifdef FLB_HAVE_SQLDB - if (ctx->db) { - /* Remove file entry from the database */ - flb_tail_db_file_delete(file, ctx); - } -#endif - /* Remove file from the monitored list */ - flb_tail_file_remove(file); - return 0; - } - } - - if (ev.mask & IN_MODIFY) { - /* - * The file was modified, check how many new bytes do - * we have. - */ - - /* Check if the file was truncated */ - if (file->offset > st.st_size) { - offset = lseek(file->fd, 0, SEEK_SET); - if (offset == -1) { - flb_errno(); - return -1; - } - - flb_plg_debug(ctx->ins, "inode=%"PRIu64" file truncated %s", - file->inode, file->name); - file->offset = offset; - file->buf_len = 0; - - /* Update offset in the database file */ -#ifdef FLB_HAVE_SQLDB - if (ctx->db) { - flb_tail_db_file_offset(file, ctx); - } -#endif - } - } - - /* Collect the data */ - ret = in_tail_collect_event(file, config); - if (ret != FLB_TAIL_ERROR) { - /* - * Due to read buffer size capacity, there are some cases where the - * read operation cannot consume all new data available on one - * round; upon successfull read(2) some data can still remain. - * - * If that is the case, we set in the structure how - * many bytes are available 'now', so then the further - * routine that check pending bytes and then the inotified-file - * can process them properly after an internal signal. - * - * The goal to defer this routine is to avoid a blocking - * read(2) operation, that might kill performance. Just let's - * wait a second and do a good job. - */ - tail_signal_pending(ctx); - } - else { - return ret; - } - - return 0; -} - -static int in_tail_progress_check_callback(struct flb_input_instance *ins, - struct flb_config *config, void *context) -{ - int ret = 0; - struct mk_list *tmp; - struct mk_list *head; - struct flb_tail_config *ctx = context; - struct flb_tail_file *file; - int pending_data_detected; - struct stat st; - - (void) config; - - pending_data_detected = FLB_FALSE; - - mk_list_foreach_safe(head, tmp, &ctx->files_event) { - file = mk_list_entry(head, struct flb_tail_file, _head); - - if (file->offset < file->size) { - pending_data_detected = FLB_TRUE; - - continue; - } - - ret = fstat(file->fd, &st); - if (ret == -1) { - flb_errno(); - flb_plg_error(ins, "fstat error"); - - continue; - } - - if (file->offset < st.st_size) { - file->size = st.st_size; - file->pending_bytes = (file->size - file->offset); - - pending_data_detected = FLB_TRUE; - } - } - - if (pending_data_detected) { - tail_signal_pending(ctx); - } - - return 0; -} - -/* File System events based on Inotify(2). Linux >= 2.6.32 is suggested */ -int flb_tail_fs_inotify_init(struct flb_input_instance *in, - struct flb_tail_config *ctx, struct flb_config *config) -{ - int fd; - int ret; - - flb_plg_debug(ctx->ins, "flb_tail_fs_inotify_init() initializing inotify tail input"); - - /* Create inotify instance */ - fd = inotify_init1(IN_NONBLOCK | IN_CLOEXEC); - if (fd == -1) { - flb_errno(); - return -1; - } - flb_plg_debug(ctx->ins, "inotify watch fd=%i", fd); - ctx->fd_notify = fd; - - /* This backend use Fluent Bit event-loop to trigger notifications */ - ret = flb_input_set_collector_event(in, tail_fs_event, - ctx->fd_notify, config); - if (ret < 0) { - close(fd); - return -1; - } - ctx->coll_fd_fs1 = ret; - - /* Register callback to check current tail offsets */ - ret = flb_input_set_collector_time(in, in_tail_progress_check_callback, - ctx->progress_check_interval, - ctx->progress_check_interval_nsec, - config); - if (ret == -1) { - flb_tail_config_destroy(ctx); - return -1; - } - ctx->coll_fd_progress_check = ret; - - return 0; -} - -void flb_tail_fs_inotify_pause(struct flb_tail_config *ctx) -{ - flb_input_collector_pause(ctx->coll_fd_fs1, ctx->ins); -} - -void flb_tail_fs_inotify_resume(struct flb_tail_config *ctx) -{ - flb_input_collector_resume(ctx->coll_fd_fs1, ctx->ins); -} - -int flb_tail_fs_inotify_add(struct flb_tail_file *file) -{ - int ret; - struct flb_tail_config *ctx = file->config; - - ret = tail_fs_add(file, FLB_TRUE); - if (ret == -1) { - flb_plg_error(ctx->ins, "inode=%"PRIu64" cannot register file %s", - file->inode, file->name); - return -1; - } - - return 0; -} - -int flb_tail_fs_inotify_remove(struct flb_tail_file *file) -{ - struct flb_tail_config *ctx = file->config; - - if (file->watch_fd == -1) { - return 0; - } - - flb_plg_info(ctx->ins, "inotify_fs_remove(): inode=%"PRIu64" watch_fd=%i", - file->inode, file->watch_fd); - - inotify_rm_watch(file->config->fd_notify, file->watch_fd); - file->watch_fd = -1; - return 0; -} - -int flb_tail_fs_inotify_exit(struct flb_tail_config *ctx) -{ - return close(ctx->fd_notify); -} diff --git a/fluent-bit/plugins/in_tail/tail_fs_inotify.h b/fluent-bit/plugins/in_tail/tail_fs_inotify.h deleted file mode 100644 index 128ab0624..000000000 --- a/fluent-bit/plugins/in_tail/tail_fs_inotify.h +++ /dev/null @@ -1,37 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_TAIL_FS_INOTIFY_H -#define FLB_TAIL_FS_INOTIFY_H - -#include -#include - -#include "tail_config.h" -#include "tail_file_internal.h" - -int flb_tail_fs_inotify_init(struct flb_input_instance *in, - struct flb_tail_config *ctx, struct flb_config *config); -int flb_tail_fs_inotify_add(struct flb_tail_file *file); -int flb_tail_fs_inotify_remove(struct flb_tail_file *file); -int flb_tail_fs_inotify_exit(struct flb_tail_config *ctx); -void flb_tail_fs_inotify_pause(struct flb_tail_config *ctx); -void flb_tail_fs_inotify_resume(struct flb_tail_config *ctx); - -#endif diff --git a/fluent-bit/plugins/in_tail/tail_fs_stat.c b/fluent-bit/plugins/in_tail/tail_fs_stat.c deleted file mode 100644 index 6b312c9bd..000000000 --- a/fluent-bit/plugins/in_tail/tail_fs_stat.c +++ /dev/null @@ -1,253 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define _DEFAULT_SOURCE - -#include -#include - -#include -#include - -#include "tail_file.h" -#include "tail_db.h" -#include "tail_config.h" -#include "tail_signal.h" - -#ifdef FLB_SYSTEM_WINDOWS -#include "win32.h" -#endif - -struct fs_stat { - /* last time check */ - time_t checked; - - /* previous status */ - struct stat st; -}; - -static int tail_fs_event(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - int ret; - struct mk_list *head; - struct mk_list *tmp; - struct flb_tail_config *ctx = in_context; - struct flb_tail_file *file = NULL; - struct fs_stat *fst; - struct stat st; - time_t t; - - t = time(NULL); - - /* Lookup watched file */ - mk_list_foreach_safe(head, tmp, &ctx->files_event) { - file = mk_list_entry(head, struct flb_tail_file, _head); - fst = file->fs_backend; - - /* Check current status of the file */ - ret = fstat(file->fd, &st); - if (ret == -1) { - flb_errno(); - continue; - } - - /* Check if the file was modified */ - if ((fst->st.st_mtime != st.st_mtime) || - (fst->st.st_size != st.st_size)) { - /* Update stat info and trigger the notification */ - memcpy(&fst->st, &st, sizeof(struct stat)); - fst->checked = t; - in_tail_collect_event(file, config); - } - } - - return 0; -} - -static int tail_fs_check(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - int ret; - int64_t offset; - char *name; - struct mk_list *tmp; - struct mk_list *head; - struct flb_tail_config *ctx = in_context; - struct flb_tail_file *file = NULL; - struct fs_stat *fst; - struct stat st; - - /* Lookup watched file */ - mk_list_foreach_safe(head, tmp, &ctx->files_event) { - file = mk_list_entry(head, struct flb_tail_file, _head); - fst = file->fs_backend; - - ret = fstat(file->fd, &st); - if (ret == -1) { - flb_plg_debug(ctx->ins, "error stat(2) %s, removing", file->name); - flb_tail_file_remove(file); - continue; - } - - /* Check if the file have been deleted */ - if (st.st_nlink == 0) { - flb_plg_debug(ctx->ins, "file has been deleted: %s", file->name); -#ifdef FLB_HAVE_SQLDB - if (ctx->db) { - /* Remove file entry from the database */ - flb_tail_db_file_delete(file, ctx); - } -#endif - flb_tail_file_remove(file); - continue; - } - - /* Check if the file was truncated */ - if (file->offset > st.st_size) { - offset = lseek(file->fd, 0, SEEK_SET); - if (offset == -1) { - flb_errno(); - return -1; - } - - flb_plg_debug(ctx->ins, "file truncated %s", file->name); - file->offset = offset; - file->buf_len = 0; - memcpy(&fst->st, &st, sizeof(struct stat)); - -#ifdef FLB_HAVE_SQLDB - /* Update offset in database file */ - if (ctx->db) { - flb_tail_db_file_offset(file, ctx); - } -#endif - } - - if (file->offset < st.st_size) { - file->pending_bytes = (st.st_size - file->offset); - tail_signal_pending(ctx); - } - else { - file->pending_bytes = 0; - } - - - /* Discover the current file name for the open file descriptor */ - name = flb_tail_file_name(file); - if (!name) { - flb_plg_debug(ctx->ins, "could not resolve %s, removing", file->name); - flb_tail_file_remove(file); - continue; - } - - /* - * Check if file still exists. This method requires explicity that the - * user is using an absolute path, otherwise we will be rotating the - * wrong file. - * - * flb_tail_target_file_name_cmp is a deeper compare than - * flb_tail_file_name_cmp. If applicable, it compares to the underlying - * real_name of the file. - */ - if (flb_tail_file_is_rotated(ctx, file) == FLB_TRUE) { - flb_tail_file_rotated(file); - } - flb_free(name); - - } - - return 0; -} - -/* File System events based on stat(2) */ -int flb_tail_fs_stat_init(struct flb_input_instance *in, - struct flb_tail_config *ctx, struct flb_config *config) -{ - int ret; - - flb_plg_debug(ctx->ins, "flb_tail_fs_stat_init() initializing stat tail input"); - - /* Set a manual timer to collect events every 0.250 seconds */ - ret = flb_input_set_collector_time(in, tail_fs_event, - 0, 250000000, config); - if (ret < 0) { - return -1; - } - ctx->coll_fd_fs1 = ret; - - /* Set a manual timer to check deleted/rotated files every 2.5 seconds */ - ret = flb_input_set_collector_time(in, tail_fs_check, - 2, 500000000, config); - if (ret < 0) { - return -1; - } - ctx->coll_fd_fs2 = ret; - - return 0; -} - -void flb_tail_fs_stat_pause(struct flb_tail_config *ctx) -{ - flb_input_collector_pause(ctx->coll_fd_fs1, ctx->ins); - flb_input_collector_pause(ctx->coll_fd_fs2, ctx->ins); -} - -void flb_tail_fs_stat_resume(struct flb_tail_config *ctx) -{ - flb_input_collector_resume(ctx->coll_fd_fs1, ctx->ins); - flb_input_collector_resume(ctx->coll_fd_fs2, ctx->ins); -} - -int flb_tail_fs_stat_add(struct flb_tail_file *file) -{ - int ret; - struct fs_stat *fst; - - fst = flb_malloc(sizeof(struct fs_stat)); - if (!fst) { - flb_errno(); - return -1; - } - - fst->checked = time(NULL); - ret = stat(file->name, &fst->st); - if (ret == -1) { - flb_errno(); - flb_free(fst); - return -1; - } - file->fs_backend = fst; - - return 0; -} - -int flb_tail_fs_stat_remove(struct flb_tail_file *file) -{ - if (file->tail_mode == FLB_TAIL_EVENT) { - flb_free(file->fs_backend); - } - return 0; -} - -int flb_tail_fs_stat_exit(struct flb_tail_config *ctx) -{ - (void) ctx; - return 0; -} diff --git a/fluent-bit/plugins/in_tail/tail_fs_stat.h b/fluent-bit/plugins/in_tail/tail_fs_stat.h deleted file mode 100644 index 21a0704cb..000000000 --- a/fluent-bit/plugins/in_tail/tail_fs_stat.h +++ /dev/null @@ -1,37 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_TAIL_FS_STAT_H -#define FLB_TAIL_FS_STAT_H - -#include -#include - -#include "tail_config.h" -#include "tail_file_internal.h" - -int flb_tail_fs_stat_init(struct flb_input_instance *in, - struct flb_tail_config *ctx, struct flb_config *config); -int flb_tail_fs_stat_add(struct flb_tail_file *file); -int flb_tail_fs_stat_remove(struct flb_tail_file *file); -int flb_tail_fs_stat_exit(struct flb_tail_config *ctx); -void flb_tail_fs_stat_pause(struct flb_tail_config *ctx); -void flb_tail_fs_stat_resume(struct flb_tail_config *ctx); - -#endif diff --git a/fluent-bit/plugins/in_tail/tail_multiline.c b/fluent-bit/plugins/in_tail/tail_multiline.c deleted file mode 100644 index 71c031014..000000000 --- a/fluent-bit/plugins/in_tail/tail_multiline.c +++ /dev/null @@ -1,606 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include "tail_config.h" -#include "tail_multiline.h" - -static int tail_mult_append(struct flb_parser *parser, - struct flb_tail_config *ctx) -{ - struct flb_tail_mult *mp; - - mp = flb_malloc(sizeof(struct flb_tail_mult)); - if (!mp) { - flb_errno(); - return -1; - } - - mp->parser = parser; - mk_list_add(&mp->_head, &ctx->mult_parsers); - - return 0; -} - -int flb_tail_mult_create(struct flb_tail_config *ctx, - struct flb_input_instance *ins, - struct flb_config *config) -{ - int ret; - const char *tmp; - struct mk_list *head; - struct flb_parser *parser; - struct flb_kv *kv; - - if (ctx->multiline_flush <= 0) { - ctx->multiline_flush = 1; - } - - mk_list_init(&ctx->mult_parsers); - - /* Get firstline parser */ - tmp = flb_input_get_property("parser_firstline", ins); - if (!tmp) { - flb_plg_error(ctx->ins, "multiline: no parser defined for firstline"); - return -1; - } - parser = flb_parser_get(tmp, config); - if (!parser) { - flb_plg_error(ctx->ins, "multiline: invalid parser '%s'", tmp); - return -1; - } - - ctx->mult_parser_firstline = parser; - - /* Read all multiline rules */ - mk_list_foreach(head, &ins->properties) { - kv = mk_list_entry(head, struct flb_kv, _head); - if (strcasecmp("parser_firstline", kv->key) == 0) { - continue; - } - - if (strncasecmp("parser_", kv->key, 7) == 0) { - parser = flb_parser_get(kv->val, config); - if (!parser) { - flb_plg_error(ctx->ins, "multiline: invalid parser '%s'", kv->val); - return -1; - } - - ret = tail_mult_append(parser, ctx); - if (ret == -1) { - return -1; - } - } - } - - return 0; -} - -int flb_tail_mult_destroy(struct flb_tail_config *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_tail_mult *mp; - - if (ctx->multiline == FLB_FALSE) { - return 0; - } - - mk_list_foreach_safe(head, tmp, &ctx->mult_parsers) { - mp = mk_list_entry(head, struct flb_tail_mult, _head); - mk_list_del(&mp->_head); - flb_free(mp); - } - - return 0; -} - -/* Process the result of a firstline match */ -int flb_tail_mult_process_first(time_t now, - char *buf, size_t size, - struct flb_time *out_time, - struct flb_tail_file *file, - struct flb_tail_config *ctx) -{ - int ret; - size_t off; - msgpack_object map; - msgpack_unpacked result; - - /* If a previous multiline context already exists, flush first */ - if (file->mult_firstline && !file->mult_skipping) { - flb_tail_mult_flush(file, ctx); - } - - /* Remark as first multiline message */ - file->mult_firstline = FLB_TRUE; - - /* Validate obtained time, if not set, set the current time */ - if (flb_time_to_nanosec(out_time) == 0L) { - flb_time_get(out_time); - } - - /* Should we skip this multiline record ? */ - if (ctx->ignore_older > 0) { - if ((now - ctx->ignore_older) > out_time->tm.tv_sec) { - flb_free(buf); - file->mult_skipping = FLB_TRUE; - file->mult_firstline = FLB_TRUE; - - /* we expect more data to skip */ - return FLB_TAIL_MULT_MORE; - } - } - - /* Re-initiate buffers */ - msgpack_sbuffer_init(&file->mult_sbuf); - msgpack_packer_init(&file->mult_pck, &file->mult_sbuf, msgpack_sbuffer_write); - - /* - * flb_parser_do() always return a msgpack buffer, so we tweak our - * local msgpack reference to avoid an extra allocation. The only - * concern is that we don't know what's the real size of the memory - * allocated, so we assume it's just 'out_size'. - */ - file->mult_flush_timeout = now + (ctx->multiline_flush - 1); - file->mult_sbuf.data = buf; - file->mult_sbuf.size = size; - file->mult_sbuf.alloc = size; - - /* Set multiline status */ - file->mult_firstline = FLB_TRUE; - file->mult_skipping = FLB_FALSE; - flb_time_copy(&file->mult_time, out_time); - - off = 0; - msgpack_unpacked_init(&result); - ret = msgpack_unpack_next(&result, buf, size, &off); - if (ret != MSGPACK_UNPACK_SUCCESS) { - msgpack_sbuffer_destroy(&file->mult_sbuf); - msgpack_unpacked_destroy(&result); - return FLB_TAIL_MULT_NA; - } - - map = result.data; - file->mult_keys = map.via.map.size; - msgpack_unpacked_destroy(&result); - - /* We expect more data */ - return FLB_TAIL_MULT_MORE; -} - -/* Append a raw log entry to the last structured field in the mult buffer */ -static inline void flb_tail_mult_append_raw(char *buf, int size, - struct flb_tail_file *file, - struct flb_tail_config *config) -{ - /* Append the raw string */ - msgpack_pack_str(&file->mult_pck, size); - msgpack_pack_str_body(&file->mult_pck, buf, size); -} - -/* Check if the last key value type of a map is string or not */ -static inline int is_last_key_val_string(char *buf, size_t size) -{ - int ret = FLB_FALSE; - size_t off; - msgpack_unpacked result; - msgpack_object v; - msgpack_object root; - - off = 0; - msgpack_unpacked_init(&result); - ret = msgpack_unpack_next(&result, buf, size, &off); - if (ret != MSGPACK_UNPACK_SUCCESS) { - return ret; - } - - root = result.data; - if (root.type != MSGPACK_OBJECT_MAP) { - ret = FLB_FALSE; - } - else { - if (root.via.map.size == 0) { - ret = FLB_FALSE; - } - else { - v = root.via.map.ptr[root.via.map.size - 1].val; - if (v.type == MSGPACK_OBJECT_STR) { - ret = FLB_TRUE; - } - } - } - - msgpack_unpacked_destroy(&result); - return ret; -} - -int flb_tail_mult_process_content(time_t now, - char *buf, size_t len, - struct flb_tail_file *file, - struct flb_tail_config *ctx, - size_t processed_bytes) -{ - int ret; - size_t off; - void *out_buf; - size_t out_size = 0; - struct mk_list *head; - struct flb_tail_mult *mult_parser = NULL; - struct flb_time out_time = {0}; - msgpack_object map; - msgpack_unpacked result; - - /* Always check if this line is the beginning of a new multiline message */ - ret = flb_parser_do(ctx->mult_parser_firstline, - buf, len, - &out_buf, &out_size, &out_time); - if (ret >= 0) { - /* - * The content is a candidate for a firstline, but we need to perform - * the extra-mandatory check where the last key value type must be - * a string, otherwise no string concatenation with continuation lines - * will be possible. - */ - ret = is_last_key_val_string(out_buf, out_size); - if (ret == FLB_TRUE) - file->mult_firstline_append = FLB_TRUE; - else - file->mult_firstline_append = FLB_FALSE; - - flb_tail_mult_process_first(now, out_buf, out_size, &out_time, - file, ctx); - return FLB_TAIL_MULT_MORE; - } - - if (file->mult_skipping == FLB_TRUE) { - return FLB_TAIL_MULT_MORE; - } - - /* - * Once here means we have some data that is a continuation, iterate - * parsers trying to find a match - */ - out_buf = NULL; - mk_list_foreach(head, &ctx->mult_parsers) { - mult_parser = mk_list_entry(head, struct flb_tail_mult, _head); - - /* Process line text with current parser */ - out_buf = NULL; - out_size = 0; - ret = flb_parser_do(mult_parser->parser, - buf, len, - &out_buf, &out_size, &out_time); - if (ret < 0) { - mult_parser = NULL; - continue; - } - - /* The line was processed, break the loop and buffer the data */ - break; - } - - if (!mult_parser) { - /* - * If no parser was found means the string log must be appended - * to the last structured field. - */ - if (file->mult_firstline && file->mult_firstline_append) { - flb_tail_mult_append_raw(buf, len, file, ctx); - } - else { - flb_tail_file_pack_line(NULL, buf, len, file, processed_bytes); - } - - return FLB_TAIL_MULT_MORE; - } - - off = 0; - msgpack_unpacked_init(&result); - msgpack_unpack_next(&result, out_buf, out_size, &off); - map = result.data; - - /* Append new map to our local msgpack buffer */ - file->mult_keys += map.via.map.size; - msgpack_unpacked_destroy(&result); - msgpack_sbuffer_write(&file->mult_sbuf, out_buf, out_size); - flb_free(out_buf); - - return FLB_TAIL_MULT_MORE; -} - -static int flb_tail_mult_pack_line_body( - struct flb_log_event_encoder *context, - struct flb_tail_file *file) -{ - size_t adjacent_object_offset; - size_t continuation_length; - msgpack_unpacked adjacent_object; - msgpack_unpacked current_object; - size_t entry_index; - msgpack_object entry_value; - msgpack_object entry_key; - msgpack_object_map *data_map; - int map_size; - size_t offset; - struct flb_tail_config *config; - int result; - - result = FLB_EVENT_ENCODER_SUCCESS; - config = (struct flb_tail_config *) file->config; - - /* New Map size */ - map_size = file->mult_keys; - - if (file->config->path_key != NULL) { - map_size++; - - result = flb_log_event_encoder_append_body_values( - context, - FLB_LOG_EVENT_CSTRING_VALUE(config->path_key), - FLB_LOG_EVENT_CSTRING_VALUE(file->name)); - } - - - msgpack_unpacked_init(¤t_object); - msgpack_unpacked_init(&adjacent_object); - - offset = 0; - - while (result == FLB_EVENT_ENCODER_SUCCESS && - msgpack_unpack_next(¤t_object, - file->mult_sbuf.data, - file->mult_sbuf.size, - &offset) == MSGPACK_UNPACK_SUCCESS) { - if (current_object.data.type != MSGPACK_OBJECT_MAP) { - continue; - } - - data_map = ¤t_object.data.via.map; - - continuation_length = 0; - - for (entry_index = 0; entry_index < data_map->size; entry_index++) { - entry_key = data_map->ptr[entry_index].key; - entry_value = data_map->ptr[entry_index].val; - - result = flb_log_event_encoder_append_body_msgpack_object(context, - &entry_key); - - if (result != FLB_EVENT_ENCODER_SUCCESS) { - break; - } - - /* Check if this is the last entry in the map and if that is - * the case then add the lengths of all the trailing string - * objects after the map in order to append them to the value - * but only if the value object is a string - */ - if (entry_index + 1 == data_map->size && - entry_value.type == MSGPACK_OBJECT_STR) { - adjacent_object_offset = offset; - - while (msgpack_unpack_next( - &adjacent_object, - file->mult_sbuf.data, - file->mult_sbuf.size, - &adjacent_object_offset) == MSGPACK_UNPACK_SUCCESS) { - if (adjacent_object.data.type != MSGPACK_OBJECT_STR) { - break; - } - - /* Sum total bytes to append */ - continuation_length += adjacent_object.data.via.str.size + 1; - } - - result = flb_log_event_encoder_append_body_string_length( - context, - entry_value.via.str.size + - continuation_length); - - if (result != FLB_EVENT_ENCODER_SUCCESS) { - break; - } - - result = flb_log_event_encoder_append_body_string_body( - context, - (char *) entry_value.via.str.ptr, - entry_value.via.str.size); - - if (result != FLB_EVENT_ENCODER_SUCCESS) { - break; - } - - if (continuation_length > 0) { - adjacent_object_offset = offset; - - while (msgpack_unpack_next( - &adjacent_object, - file->mult_sbuf.data, - file->mult_sbuf.size, - &adjacent_object_offset) == MSGPACK_UNPACK_SUCCESS) { - if (adjacent_object.data.type != MSGPACK_OBJECT_STR) { - break; - } - - result = flb_log_event_encoder_append_body_string_body( - context, - "\n", - 1); - - if (result != FLB_EVENT_ENCODER_SUCCESS) { - break; - } - - result = flb_log_event_encoder_append_body_string_body( - context, - (char *) adjacent_object.data.via.str.ptr, - adjacent_object.data.via.str.size); - - if (result != FLB_EVENT_ENCODER_SUCCESS) { - break; - } - } - } - } - else { - result = flb_log_event_encoder_append_body_msgpack_object(context, - &entry_value); - } - } - } - - msgpack_unpacked_destroy(¤t_object); - msgpack_unpacked_destroy(&adjacent_object); - - /* Reset status */ - file->mult_firstline = FLB_FALSE; - file->mult_skipping = FLB_FALSE; - file->mult_keys = 0; - file->mult_flush_timeout = 0; - - msgpack_sbuffer_destroy(&file->mult_sbuf); - - file->mult_sbuf.data = NULL; - - flb_time_zero(&file->mult_time); - - return result; -} - -/* Flush any multiline context data into outgoing buffers */ -int flb_tail_mult_flush(struct flb_tail_file *file, struct flb_tail_config *ctx) -{ - int result; - - /* nothing to flush */ - if (file->mult_firstline == FLB_FALSE) { - return -1; - } - - if (file->mult_keys == 0) { - return -1; - } - - result = flb_log_event_encoder_begin_record(file->ml_log_event_encoder); - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_set_timestamp( - file->ml_log_event_encoder, &file->mult_time); - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_tail_mult_pack_line_body( - file->ml_log_event_encoder, - file); - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_commit_record( - file->ml_log_event_encoder); - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(ctx->ins, - file->tag_buf, - file->tag_len, - file->ml_log_event_encoder->output_buffer, - file->ml_log_event_encoder->output_length); - result = 0; - } - else { - flb_plg_error(file->config->ins, "error packing event : %d", result); - - result = -1; - } - - flb_log_event_encoder_reset(file->ml_log_event_encoder); - - return result; -} - -static void file_pending_flush(struct flb_tail_config *ctx, - struct flb_tail_file *file, time_t now) -{ - if (file->mult_flush_timeout > now) { - return; - } - - if (file->mult_firstline == FLB_FALSE) { - if (file->mult_sbuf.data == NULL || file->mult_sbuf.size <= 0) { - return; - } - } - - flb_tail_mult_flush(file, ctx); -} - -int flb_tail_mult_pending_flush_all(struct flb_tail_config *ctx) -{ - time_t expired; - struct mk_list *head; - struct flb_tail_file *file; - - expired = time(NULL) + 3600; - - /* Iterate promoted event files with pending bytes */ - mk_list_foreach(head, &ctx->files_static) { - file = mk_list_entry(head, struct flb_tail_file, _head); - file_pending_flush(ctx, file, expired); - } - - /* Iterate promoted event files with pending bytes */ - mk_list_foreach(head, &ctx->files_event) { - file = mk_list_entry(head, struct flb_tail_file, _head); - file_pending_flush(ctx, file, expired); - } - - return 0; -} - -int flb_tail_mult_pending_flush(struct flb_input_instance *ins, - struct flb_config *config, void *context) -{ - time_t now; - struct mk_list *head; - struct flb_tail_file *file; - struct flb_tail_config *ctx = context; - - now = time(NULL); - - /* Iterate promoted event files with pending bytes */ - mk_list_foreach(head, &ctx->files_static) { - file = mk_list_entry(head, struct flb_tail_file, _head); - - file_pending_flush(ctx, file, now); - } - - /* Iterate promoted event files with pending bytes */ - mk_list_foreach(head, &ctx->files_event) { - file = mk_list_entry(head, struct flb_tail_file, _head); - - file_pending_flush(ctx, file, now); - } - - return 0; -} diff --git a/fluent-bit/plugins/in_tail/tail_multiline.h b/fluent-bit/plugins/in_tail/tail_multiline.h deleted file mode 100644 index d7f7539b1..000000000 --- a/fluent-bit/plugins/in_tail/tail_multiline.h +++ /dev/null @@ -1,57 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_TAIL_TAIL_MULT_H -#define FLB_TAIL_TAIL_MULT_H - -#include -#include - -#include "tail_config.h" -#include "tail_file.h" - -#define FLB_TAIL_MULT_NA -1 /* not applicable as a multiline stream */ -#define FLB_TAIL_MULT_DONE 0 /* finished a multiline stream */ -#define FLB_TAIL_MULT_MORE 1 /* expect more lines to come */ -#define FLB_TAIL_MULT_FLUSH "4" /* max flush time for multiline: 4 seconds */ - -struct flb_tail_mult { - struct flb_parser *parser; - struct mk_list _head; -}; - -int flb_tail_mult_create(struct flb_tail_config *ctx, - struct flb_input_instance *ins, - struct flb_config *config); - -int flb_tail_mult_destroy(struct flb_tail_config *ctx); - -int flb_tail_mult_process_content(time_t now, - char *buf, size_t len, - struct flb_tail_file *file, - struct flb_tail_config *ctx, - size_t processed_bytes); -int flb_tail_mult_flush(struct flb_tail_file *file, - struct flb_tail_config *ctx); - -int flb_tail_mult_pending_flush(struct flb_input_instance *ins, - struct flb_config *config, void *context); -int flb_tail_mult_pending_flush_all(struct flb_tail_config *ctx); - -#endif diff --git a/fluent-bit/plugins/in_tail/tail_scan.c b/fluent-bit/plugins/in_tail/tail_scan.c deleted file mode 100644 index ccb8e070a..000000000 --- a/fluent-bit/plugins/in_tail/tail_scan.c +++ /dev/null @@ -1,71 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include "tail.h" -#include "tail_config.h" - -/* - * Include proper scan backend - */ -#ifdef FLB_SYSTEM_WINDOWS -#include "tail_scan_win32.c" -#else -#include "tail_scan_glob.c" -#endif - -int flb_tail_scan(struct mk_list *path_list, struct flb_tail_config *ctx) -{ - int ret; - struct mk_list *head; - struct flb_slist_entry *pattern; - - mk_list_foreach(head, path_list) { - pattern = mk_list_entry(head, struct flb_slist_entry, _head); - ret = tail_scan_path(pattern->str, ctx); - if (ret == -1) { - flb_plg_warn(ctx->ins, "error scanning path: %s", pattern->str); - } - else { - flb_plg_debug(ctx->ins, "%i new files found on path '%s'", - ret, pattern->str); - } - } - - return 0; -} - -/* - * Triggered by refresh_interval, it re-scan the path looking for new files - * that match the original path pattern. - */ -int flb_tail_scan_callback(struct flb_input_instance *ins, - struct flb_config *config, void *context) -{ - int ret; - struct flb_tail_config *ctx = context; - (void) config; - - ret = flb_tail_scan(ctx->path_list, ctx); - if (ret > 0) { - flb_plg_debug(ins, "%i new files found", ret); - } - - return ret; -} diff --git a/fluent-bit/plugins/in_tail/tail_scan.h b/fluent-bit/plugins/in_tail/tail_scan.h deleted file mode 100644 index ec3c96a2a..000000000 --- a/fluent-bit/plugins/in_tail/tail_scan.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_TAIL_SCAN_H -#define FLB_TAIL_SCAN_H - -#include "tail_config.h" - -int flb_tail_scan(struct mk_list *path, struct flb_tail_config *ctx); -int flb_tail_scan_callback(struct flb_input_instance *ins, - struct flb_config *config, void *context); - -#endif diff --git a/fluent-bit/plugins/in_tail/tail_scan_glob.c b/fluent-bit/plugins/in_tail/tail_scan_glob.c deleted file mode 100644 index b330b7c3b..000000000 --- a/fluent-bit/plugins/in_tail/tail_scan_glob.c +++ /dev/null @@ -1,278 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include -#include -#include - -#include "tail.h" -#include "tail_file.h" -#include "tail_signal.h" -#include "tail_scan.h" -#include "tail_config.h" - -/* Define missing GLOB_TILDE if not exists */ -#ifndef GLOB_TILDE -#define GLOB_TILDE 1<<2 /* use GNU Libc value */ -#define UNSUP_TILDE 1 - -/* we need these extra headers for path resolution */ -#include -#include -#include - -static char *expand_tilde(const char *path) -{ - int len; - char user[256]; - char *p = NULL; - char *dir = NULL; - char *tmp = NULL; - struct passwd *uinfo = NULL; - - if (path[0] == '~') { - p = strchr(path, '/'); - - if (p) { - /* check case '~/' */ - if ((p - path) == 1) { - dir = getenv("HOME"); - if (!dir) { - return path; - } - } - else { - /* - * it refers to a different user: ~user/abc, first step grab - * the user name. - */ - len = (p - path) - 1; - memcpy(user, path + 1, len); - user[len] = '\0'; - - /* use getpwnam() to resolve user information */ - uinfo = getpwnam(user); - if (!uinfo) { - return path; - } - - dir = uinfo->pw_dir; - } - } - else { - dir = getenv("HOME"); - if (!dir) { - return path; - } - } - - if (p) { - tmp = flb_malloc(PATH_MAX); - if (!tmp) { - flb_errno(); - return NULL; - } - snprintf(tmp, PATH_MAX - 1, "%s%s", dir, p); - } - else { - dir = getenv("HOME"); - if (!dir) { - return path; - } - - tmp = flb_strdup(dir); - if (!tmp) { - return path; - } - } - - return tmp; - } - - return path; -} -#endif - -static int tail_is_excluded(char *path, struct flb_tail_config *ctx) -{ - struct mk_list *head; - struct flb_slist_entry *pattern; - - if (!ctx->exclude_list) { - return FLB_FALSE; - } - - mk_list_foreach(head, ctx->exclude_list) { - pattern = mk_list_entry(head, struct flb_slist_entry, _head); - if (fnmatch(pattern->str, path, 0) == 0) { - return FLB_TRUE; - } - } - - return FLB_FALSE; -} - -static inline int do_glob(const char *pattern, int flags, - void *not_used, glob_t *pglob) -{ - int ret; - int new_flags; - char *tmp = NULL; - int tmp_needs_free = FLB_FALSE; - (void) not_used; - - /* Save current values */ - new_flags = flags; - - if (flags & GLOB_TILDE) { -#ifdef UNSUP_TILDE - /* - * Some libc libraries like Musl do not support GLOB_TILDE for tilde - * expansion. A workaround is to use wordexp(3) but looking at it - * implementation in Musl it looks quite expensive: - * - * http://git.musl-libc.org/cgit/musl/tree/src/misc/wordexp.c - * - * the workaround is to do our own tilde expansion in a temporary buffer. - */ - - /* Look for a tilde */ - tmp = expand_tilde(pattern); - if (tmp != pattern) { - /* the path was expanded */ - pattern = tmp; - tmp_needs_free = FLB_TRUE; - } - - /* remove unused flag */ - new_flags &= ~GLOB_TILDE; -#endif - } - - /* invoke glob with new parameters */ - ret = glob(pattern, new_flags, NULL, pglob); - - /* remove temporary buffer, if allocated by expand_tilde above. - * Note that this buffer is only used for libc implementations - * that do not support the GLOB_TILDE flag, like musl. */ - if ((tmp != NULL) && (tmp_needs_free == FLB_TRUE)) { - flb_free(tmp); - } - - return ret; -} - -/* Scan a path, register the entries and return how many */ -static int tail_scan_path(const char *path, struct flb_tail_config *ctx) -{ - int i; - int ret; - int count = 0; - glob_t globbuf; - time_t now; - int64_t mtime; - struct stat st; - - flb_plg_debug(ctx->ins, "scanning path %s", path); - - /* Safe reset for globfree() */ - globbuf.gl_pathv = NULL; - - /* Scan the given path */ - ret = do_glob(path, GLOB_TILDE | GLOB_ERR, NULL, &globbuf); - if (ret != 0) { - switch (ret) { - case GLOB_NOSPACE: - flb_plg_error(ctx->ins, "no memory space available"); - return -1; - case GLOB_ABORTED: - flb_plg_error(ctx->ins, "read error, check permissions: %s", path); - return -1; - case GLOB_NOMATCH: - ret = stat(path, &st); - if (ret == -1) { - flb_plg_debug(ctx->ins, "cannot read info from: %s", path); - } - else { - ret = access(path, R_OK); - if (ret == -1 && errno == EACCES) { - flb_plg_error(ctx->ins, "NO read access for path: %s", path); - } - else { - flb_plg_debug(ctx->ins, "NO matches for path: %s", path); - } - } - return 0; - } - } - - - /* For every entry found, generate an output list */ - now = time(NULL); - for (i = 0; i < globbuf.gl_pathc; i++) { - ret = stat(globbuf.gl_pathv[i], &st); - if (ret == 0 && S_ISREG(st.st_mode)) { - /* Check if this file is blacklisted */ - if (tail_is_excluded(globbuf.gl_pathv[i], ctx) == FLB_TRUE) { - flb_plg_debug(ctx->ins, "excluded=%s", globbuf.gl_pathv[i]); - continue; - } - - if (ctx->ignore_older > 0) { - mtime = flb_tail_stat_mtime(&st); - if (mtime > 0) { - if ((now - ctx->ignore_older) > mtime) { - flb_plg_debug(ctx->ins, "excluded=%s (ignore_older)", - globbuf.gl_pathv[i]); - continue; - } - } - } - - /* Append file to list */ - ret = flb_tail_file_append(globbuf.gl_pathv[i], &st, - FLB_TAIL_STATIC, ctx); - if (ret == 0) { - flb_plg_debug(ctx->ins, "scan_glob add(): %s, inode %li", - globbuf.gl_pathv[i], st.st_ino); - count++; - } - else { - flb_plg_debug(ctx->ins, "scan_blog add(): dismissed: %s, inode %li", - globbuf.gl_pathv[i], st.st_ino); - } - } - else { - flb_plg_debug(ctx->ins, "skip (invalid) entry=%s", - globbuf.gl_pathv[i]); - } - } - - if (count > 0) { - tail_signal_manager(ctx); - } - - globfree(&globbuf); - return count; -} diff --git a/fluent-bit/plugins/in_tail/tail_scan_win32.c b/fluent-bit/plugins/in_tail/tail_scan_win32.c deleted file mode 100644 index 94733f065..000000000 --- a/fluent-bit/plugins/in_tail/tail_scan_win32.c +++ /dev/null @@ -1,245 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * This file implements glob-like patch matching feature for Windows - * based on Win32 API. - */ - -#include -#include -#include -#include - -#include - -#include "tail.h" -#include "tail_file.h" -#include "tail_signal.h" -#include "tail_config.h" - -#include "win32.h" - -static int tail_is_excluded(char *path, struct flb_tail_config *ctx) -{ - struct mk_list *head; - struct flb_slist_entry *pattern; - - if (!ctx->exclude_list) { - return FLB_FALSE; - } - - mk_list_foreach(head, ctx->exclude_list) { - pattern = mk_list_entry(head, struct flb_slist_entry, _head); - if (PathMatchSpecA(path, pattern->str)) { - return FLB_TRUE; - } - } - - return FLB_FALSE; -} - -/* - * This function is a thin wrapper over flb_tail_file_append(), - * adding normalization and sanity checks on top of it. - */ -static int tail_register_file(const char *target, struct flb_tail_config *ctx, - time_t ts) -{ - int64_t mtime; - struct stat st; - char path[MAX_PATH]; - - if (_fullpath(path, target, MAX_PATH) == NULL) { - flb_plg_error(ctx->ins, "cannot get absolute path of %s", target); - return -1; - } - - if (stat(path, &st) != 0 || !S_ISREG(st.st_mode)) { - return -1; - } - - if (ctx->ignore_older > 0) { - mtime = flb_tail_stat_mtime(&st); - if (mtime > 0) { - if ((ts - ctx->ignore_older) > mtime) { - flb_plg_debug(ctx->ins, "excluded=%s (ignore_older)", - target); - return -1; - } - } - } - - if (tail_is_excluded(path, ctx) == FLB_TRUE) { - flb_plg_trace(ctx->ins, "skip '%s' (excluded)", path); - return -1; - } - - return flb_tail_file_append(path, &st, FLB_TAIL_STATIC, ctx); -} - -/* - * Perform patern match on the given path string. This function - * supports patterns with "nested" wildcards like below. - * - * tail_scan_pattern("C:\fluent-bit\*\*.txt", ctx); - * - * On success, the number of files found is returned (zero indicates - * "no file found"). On error, -1 is returned. - */ -static int tail_scan_pattern(const char *path, struct flb_tail_config *ctx) -{ - char *star, *p0, *p1; - char pattern[MAX_PATH]; - char buf[MAX_PATH]; - int ret; - int n_added = 0; - time_t now; - int64_t mtime; - HANDLE h; - WIN32_FIND_DATA data; - - if (strlen(path) > MAX_PATH - 1) { - flb_plg_error(ctx->ins, "path too long '%s'"); - return -1; - } - - star = strchr(path, '*'); - if (star == NULL) { - return -1; - } - - /* - * C:\data\tmp\input_*.conf - * 0<-----| - */ - p0 = star; - while (path <= p0 && *p0 != '\\') { - p0--; - } - - /* - * C:\data\tmp\input_*.conf - * |---->1 - */ - p1 = star; - while (*p1 && *p1 != '\\') { - p1++; - } - - memcpy(pattern, path, (p1 - path)); - pattern[p1 - path] = '\0'; - - h = FindFirstFileA(pattern, &data); - if (h == INVALID_HANDLE_VALUE) { - return 0; /* none matched */ - } - - now = time(NULL); - do { - /* Ignore the current and parent dirs */ - if (!strcmp(".", data.cFileName) || !strcmp("..", data.cFileName)) { - continue; - } - - /* Avoid an infinite loop */ - if (strchr(data.cFileName, '*')) { - continue; - } - - /* Create a path (prefix + filename + suffix) */ - memcpy(buf, path, p0 - path + 1); - buf[p0 - path + 1] = '\0'; - - if (strlen(buf) + strlen(data.cFileName) + strlen(p1) > MAX_PATH - 1) { - flb_plg_warn(ctx->ins, "'%s%s%s' is too long", buf, data.cFileName, p1); - continue; - } - strcat(buf, data.cFileName); - strcat(buf, p1); - - if (strchr(p1, '*')) { - ret = tail_scan_pattern(buf, ctx); /* recursive */ - if (ret >= 0) { - n_added += ret; - } - continue; - } - - /* Try to register the target file */ - ret = tail_register_file(buf, ctx, now); - if (ret == 0) { - n_added++; - } - } while (FindNextFileA(h, &data) != 0); - - FindClose(h); - return n_added; -} - -static int tail_filepath(char *buf, int len, const char *basedir, const char *filename) -{ - char drive[_MAX_DRIVE]; - char dir[_MAX_DIR]; - char fname[_MAX_FNAME]; - char ext[_MAX_EXT]; - char tmp[MAX_PATH]; - int ret; - - ret = _splitpath_s(basedir, drive, _MAX_DRIVE, dir, _MAX_DIR, NULL, 0, NULL, 0); - if (ret) { - return -1; - } - - ret = _splitpath_s(filename, NULL, 0, NULL, 0, fname, _MAX_FNAME, ext, _MAX_EXT); - if (ret) { - return -1; - } - - ret = _makepath_s(tmp, MAX_PATH, drive, dir, fname, ext); - if (ret) { - return -1; - } - - if (_fullpath(buf, tmp, len) == NULL) { - return -1; - } - - return 0; -} - -static int tail_scan_path(const char *path, struct flb_tail_config *ctx) -{ - int ret; - int n_added = 0; - time_t now; - - if (strchr(path, '*')) { - return tail_scan_pattern(path, ctx); - } - - /* No wildcard involved. Let's just handle the file... */ - now = time(NULL); - ret = tail_register_file(path, ctx, now); - if (ret == 0) { - n_added++; - } - - return n_added; -} diff --git a/fluent-bit/plugins/in_tail/tail_signal.h b/fluent-bit/plugins/in_tail/tail_signal.h deleted file mode 100644 index 1a81fec64..000000000 --- a/fluent-bit/plugins/in_tail/tail_signal.h +++ /dev/null @@ -1,98 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_TAIL_SIGNAL_H -#define FLB_TAIL_SIGNAL_H - -#include "tail_config.h" - -static inline int tail_signal_manager(struct flb_tail_config *ctx) -{ - int n; - uint64_t val = 0xc001; - - /* - * The number of signal reads might be less than the written signals, this - * means that some event is still pending in the queue. On that case we - * don't need to signal it again. - */ - if (ctx->ch_reads < ctx->ch_writes) { - return 1; - } - - /* Reset counters: prevent an overflow, unlikely..but let's keep safe */ - if (ctx->ch_reads == ctx->ch_writes) { - ctx->ch_reads = 0; - ctx->ch_writes = 0; - } - - /* Insert a dummy event into the channel manager */ - n = flb_pipe_w(ctx->ch_manager[1], (const char *) &val, sizeof(val)); - if (n == -1) { - flb_errno(); - return -1; - } - else { - ctx->ch_writes++; - } - - return n; -} - -static inline int tail_signal_pending(struct flb_tail_config *ctx) -{ - int n; - uint64_t val = 0xc002; - - /* Insert a dummy event into the 'pending' channel */ - n = flb_pipe_w(ctx->ch_pending[1], (const char *) &val, sizeof(val)); - - /* - * If we get EAGAIN, it simply means pending channel is full. As - * notification is already pending, it's safe to ignore. - */ - if (n == -1 && !FLB_PIPE_WOULDBLOCK()) { - flb_errno(); - return -1; - } - - return n; -} - -static inline int tail_consume_pending(struct flb_tail_config *ctx) -{ - int ret; - uint64_t val; - - /* - * We need to consume the pending bytes. Loop until we would have - * blocked (pipe is empty). - */ - do { - ret = flb_pipe_r(ctx->ch_pending[0], (char *) &val, sizeof(val)); - if (ret <= 0 && !FLB_PIPE_WOULDBLOCK()) { - flb_errno(); - return -1; - } - } while (!FLB_PIPE_WOULDBLOCK()); - - return 0; -} - -#endif diff --git a/fluent-bit/plugins/in_tail/tail_sql.h b/fluent-bit/plugins/in_tail/tail_sql.h deleted file mode 100644 index 855933a01..000000000 --- a/fluent-bit/plugins/in_tail/tail_sql.h +++ /dev/null @@ -1,65 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_TAIL_SQL_H -#define FLB_TAIL_SQL_H - -/* - * In Fluent Bit we try to have a common convention for table names, - * if the table belong to an input/output plugin, use plugin name - * plus to what it's about, e.g: - * - * in_tail plugin table to track files: in_tail_files - */ -#define SQL_CREATE_FILES \ - "CREATE TABLE IF NOT EXISTS in_tail_files (" \ - " id INTEGER PRIMARY KEY," \ - " name TEXT NOT NULL," \ - " offset INTEGER," \ - " inode INTEGER," \ - " created INTEGER," \ - " rotated INTEGER DEFAULT 0" \ - ");" - -#define SQL_GET_FILE \ - "SELECT * from in_tail_files WHERE inode=@inode order by id desc;" - -#define SQL_INSERT_FILE \ - "INSERT INTO in_tail_files (name, offset, inode, created)" \ - " VALUES (@name, @offset, @inode, @created);" - -#define SQL_ROTATE_FILE \ - "UPDATE in_tail_files set name=@name,rotated=1 WHERE id=@id;" - -#define SQL_UPDATE_OFFSET \ - "UPDATE in_tail_files set offset=@offset WHERE id=@id;" - -#define SQL_DELETE_FILE \ - "DELETE FROM in_tail_files WHERE id=@id;" - -#define SQL_PRAGMA_SYNC \ - "PRAGMA synchronous=%i;" - -#define SQL_PRAGMA_JOURNAL_MODE \ - "PRAGMA journal_mode=%s;" - -#define SQL_PRAGMA_LOCKING_MODE \ - "PRAGMA locking_mode=EXCLUSIVE;" - -#endif diff --git a/fluent-bit/plugins/in_tail/win32.h b/fluent-bit/plugins/in_tail/win32.h deleted file mode 100644 index a9414f892..000000000 --- a/fluent-bit/plugins/in_tail/win32.h +++ /dev/null @@ -1,67 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * This is the interface file that replaces POSIX functions - * with our own custom implementation. - */ - -#ifndef FLB_TAIL_WIN32_H -#define FLB_TAIL_WIN32_H - -#include "win32/interface.h" - -#undef open -#undef stat -#undef lstat -#undef fstat -#undef lseek - -#undef S_IFDIR -#undef S_IFCHR -#undef S_IFIFO -#undef S_IFREG -#undef S_IFLNK -#undef S_IFMT -#undef S_ISDIR -#undef S_ISCHR -#undef S_ISFIFO -#undef S_ISREG -#undef S_ISLNK - -#define open win32_open -#define stat win32_stat -#define lstat win32_lstat -#define fstat win32_fstat - -#define lseek _lseeki64 - -#define S_IFDIR WIN32_S_IFDIR -#define S_IFCHR WIN32_S_IFCHR -#define S_IFIFO WIN32_S_IFIFO -#define S_IFREG WIN32_S_IFREG -#define S_IFLNK WIN32_S_IFLNK -#define S_IFMT WIN32_S_IFMT - -#define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR) -#define S_ISCHR(m) (((m) & S_IFMT) == S_IFCHR) -#define S_ISIFO(m) (((m) & S_IFMT) == S_IFIFO) -#define S_ISREG(m) (((m) & S_IFMT) == S_IFREG) -#define S_ISLNK(m) (((m) & S_IFMT) == S_IFLNK) -#endif diff --git a/fluent-bit/plugins/in_tail/win32/interface.h b/fluent-bit/plugins/in_tail/win32/interface.h deleted file mode 100644 index 73b2ef233..000000000 --- a/fluent-bit/plugins/in_tail/win32/interface.h +++ /dev/null @@ -1,44 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_TAIL_WIN32_INTERFACE_H -#define FLB_TAIL_WIN32_INTERFACE_H - -struct win32_stat { - uint64_t st_ino; - uint16_t st_mode; - int64_t st_mtime; - int16_t st_nlink; - int64_t st_size; -}; - -int win32_stat(const char *path, struct win32_stat *wst); -int win32_lstat(const char *path, struct win32_stat *wst); -int win32_fstat(int fd, struct win32_stat *wst); - -int win32_open(const char *path, int flags); - -#define WIN32_S_IFDIR 0x1000 -#define WIN32_S_IFCHR 0x2000 -#define WIN32_S_IFIFO 0x4000 -#define WIN32_S_IFREG 0x8000 -#define WIN32_S_IFLNK 0xc000 -#define WIN32_S_IFMT 0xf000 - -#endif diff --git a/fluent-bit/plugins/in_tail/win32/io.c b/fluent-bit/plugins/in_tail/win32/io.c deleted file mode 100644 index 45928b04a..000000000 --- a/fluent-bit/plugins/in_tail/win32/io.c +++ /dev/null @@ -1,47 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include "interface.h" - -/* - * POSIX IO emulation tailored for in_tail's usage. - * - * open(2) that does not acquire an exclusive lock. - */ - -int win32_open(const char *path, int flags) -{ - HANDLE h; - h = CreateFileA(path, - GENERIC_READ, - FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, - NULL, /* lpSecurityAttributes */ - OPEN_EXISTING, /* dwCreationDisposition */ - 0, /* dwFlagsAndAttributes */ - NULL); /* hTemplateFile */ - if (h == INVALID_HANDLE_VALUE) { - return -1; - } - return _open_osfhandle((intptr_t) h, _O_RDONLY); -} diff --git a/fluent-bit/plugins/in_tail/win32/stat.c b/fluent-bit/plugins/in_tail/win32/stat.c deleted file mode 100644 index bce802749..000000000 --- a/fluent-bit/plugins/in_tail/win32/stat.c +++ /dev/null @@ -1,332 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include "interface.h" - -/* - * NTFS stat(2) emulation tailored for in_tail's usage. - * - * (1) Support st_ino (inode) for Windows NTFS. - * (2) Support NTFS symlinks. - * (3) Support large files >= 2GB. - * - * To use it, include "win32.h" and it will transparently - * replace stat(), lstat() and fstat(). - */ - -#define UINT64(high, low) ((uint64_t) (high) << 32 | (low)) -#define WINDOWS_TICKS_TO_SECONDS_RATIO 10000000 -#define WINDOWS_EPOCH_TO_UNIX_EPOCH_DELTA 11644473600 - -/* - * FILETIME timestamps are represented in 100-nanosecond intervals, - * because of this, that's why we need to divide the number by 10000000 - * in order to convert it to seconds. - * - * While UNIX timestamps use January 1, 1970 as epoch Windows FILETIME - * timestamps use January 1, 1601. Because of this we need to subtract - * 11644473600 seconds to account for it. - * - * Note: Even though this does not account for leap seconds it should be - * accurate enough. - */ - -static uint64_t filetime_to_epoch(FILETIME *ft) -{ - ULARGE_INTEGER timestamp; - - if (ft == NULL) { - return 0; - } - - timestamp.HighPart = ft->dwHighDateTime; - timestamp.LowPart = ft->dwLowDateTime; - - timestamp.QuadPart /= WINDOWS_TICKS_TO_SECONDS_RATIO; - timestamp.QuadPart -= WINDOWS_EPOCH_TO_UNIX_EPOCH_DELTA; - - return timestamp.QuadPart; -} - -static void reset_errno() -{ - errno = 0; -} - -static void propagate_last_error_to_errno() -{ - DWORD error_code; - - error_code = GetLastError(); - - switch (error_code) { - case ERROR_INVALID_TARGET_HANDLE: - case ERROR_INVALID_HANDLE: - errno = EBADF; - break; - - case ERROR_TOO_MANY_OPEN_FILES: - errno = EMFILE; - break; - - case ERROR_INVALID_FLAG_NUMBER: - case ERROR_INVALID_PARAMETER: - errno = EINVAL; - break; - - case ERROR_NOT_ENOUGH_MEMORY: - case ERROR_OUTOFMEMORY: - errno = ENOMEM; - break; - - case ERROR_SHARING_VIOLATION: - case ERROR_LOCK_VIOLATION: - case ERROR_PATH_BUSY: - case ERROR_BUSY: - errno = EBUSY; - break; - - case ERROR_HANDLE_DISK_FULL: - case ERROR_DISK_FULL: - errno = ENOSPC; - break; - - case ERROR_INVALID_ADDRESS: - errno = EFAULT; - break; - - case ERROR_FILE_TOO_LARGE: - errno = EFBIG; - break; - - case ERROR_ALREADY_EXISTS: - case ERROR_FILE_EXISTS: - errno = EEXIST; - break; - - case ERROR_FILE_NOT_FOUND: - case ERROR_PATH_NOT_FOUND: - case ERROR_INVALID_DRIVE: - case ERROR_BAD_PATHNAME: - case ERROR_INVALID_NAME: - case ERROR_BAD_UNIT: - errno = ENOENT; - break; - - case ERROR_SEEK_ON_DEVICE: - case ERROR_NEGATIVE_SEEK: - errno = ESPIPE; - break; - - case ERROR_ACCESS_DENIED: - errno = EACCES; - break; - - case ERROR_DIR_NOT_EMPTY: - errno = ENOTEMPTY; - break; - - case ERROR_BROKEN_PIPE: - errno = EPIPE; - break; - - case ERROR_GEN_FAILURE: - errno = EIO; - break; - - case ERROR_OPEN_FAILED: - errno = EIO; - break; - - case ERROR_SUCCESS: - errno = 0; - break; - - default: - /* This is just a canary, if you find this - * error then it means we need to expand the - * translation list. - */ - - errno = EOWNERDEAD; - break; - } -} - -static int get_mode(unsigned int attr) -{ - if (attr & FILE_ATTRIBUTE_DIRECTORY) { - return WIN32_S_IFDIR; - } - return WIN32_S_IFREG; -} - - - -static int is_symlink(const char *path) -{ - WIN32_FIND_DATA data; - HANDLE h; - - SetLastError(0); - reset_errno(); - - h = FindFirstFileA(path, &data); - - if (h == INVALID_HANDLE_VALUE) { - propagate_last_error_to_errno(); - - return 0; - } - - FindClose(h); - - /* - * A NTFS symlink is a file with a bit of metadata ("reparse point"), - * So (1) check if the file has metadata and then (2) confirm that - * it is indeed a symlink. - */ - if (data.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { - if (data.dwReserved0 == IO_REPARSE_TAG_SYMLINK) { - return 1; - } - } - - return 0; -} - -static int hstat(HANDLE h, struct win32_stat *wst) -{ - BY_HANDLE_FILE_INFORMATION info; - FILE_STANDARD_INFO std; - - SetLastError(0); - reset_errno(); - - if (!GetFileInformationByHandle(h, &info)) { - propagate_last_error_to_errno(); - - return -1; - } - - if (!GetFileInformationByHandleEx(h, FileStandardInfo, - &std, sizeof(std))) { - propagate_last_error_to_errno(); - - return -1; - } - - wst->st_nlink = std.NumberOfLinks; - if (std.DeletePending) { - wst->st_nlink = 0; - } - - wst->st_mode = get_mode(info.dwFileAttributes); - wst->st_size = UINT64(info.nFileSizeHigh, info.nFileSizeLow); - wst->st_ino = UINT64(info.nFileIndexHigh, info.nFileIndexLow); - wst->st_mtime = filetime_to_epoch(&info.ftLastWriteTime); - - return 0; -} - -int win32_stat(const char *path, struct win32_stat *wst) -{ - HANDLE h; - - SetLastError(0); - reset_errno(); - - h = CreateFileA(path, - GENERIC_READ, - FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, - NULL, /* lpSecurityAttributes */ - OPEN_EXISTING, /* dwCreationDisposition */ - 0, /* dwFlagsAndAttributes */ - NULL); /* hTemplateFile */ - - if (h == INVALID_HANDLE_VALUE) { - propagate_last_error_to_errno(); - - return -1; - } - - if (hstat(h, wst)) { - CloseHandle(h); - return -1; - } - - CloseHandle(h); - return 0; -} - -int win32_lstat(const char *path, struct win32_stat *wst) -{ - HANDLE h; - - SetLastError(0); - reset_errno(); - - h = CreateFileA(path, - GENERIC_READ, - FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, - NULL, /* lpSecurityAttributes */ - OPEN_EXISTING, /* dwCreationDisposition */ - FILE_FLAG_OPEN_REPARSE_POINT, - NULL); /* hTemplateFile */ - - if (h == INVALID_HANDLE_VALUE) { - propagate_last_error_to_errno(); - - return -1; - } - - if (hstat(h, wst)) { - CloseHandle(h); - return -1; - } - - if (is_symlink(path)) { - wst->st_mode = WIN32_S_IFLNK; - } - - CloseHandle(h); - return 0; -} - -int win32_fstat(int fd, struct win32_stat *wst) -{ - HANDLE h; - - SetLastError(0); - reset_errno(); - - h = (HANDLE) _get_osfhandle(fd); - - if (h == INVALID_HANDLE_VALUE) { - propagate_last_error_to_errno(); - - return -1; - } - - return hstat(h, wst); -} diff --git a/fluent-bit/plugins/in_tcp/CMakeLists.txt b/fluent-bit/plugins/in_tcp/CMakeLists.txt deleted file mode 100644 index df6763cd6..000000000 --- a/fluent-bit/plugins/in_tcp/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(src - tcp.c - tcp_conn.c - tcp_config.c) - -FLB_PLUGIN(in_tcp "${src}" "") diff --git a/fluent-bit/plugins/in_tcp/tcp.c b/fluent-bit/plugins/in_tcp/tcp.c deleted file mode 100644 index 084ea6887..000000000 --- a/fluent-bit/plugins/in_tcp/tcp.c +++ /dev/null @@ -1,184 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include "tcp.h" -#include "tcp_conn.h" -#include "tcp_config.h" - -/* - * For a server event, the collection event means a new client have arrived, we - * accept the connection and create a new TCP instance which will wait for - * JSON map messages. - */ -static int in_tcp_collect(struct flb_input_instance *in, - struct flb_config *config, void *in_context) -{ - struct flb_connection *connection; - struct tcp_conn *conn; - struct flb_in_tcp_config *ctx; - - ctx = in_context; - - connection = flb_downstream_conn_get(ctx->downstream); - - if (connection == NULL) { - flb_plg_error(ctx->ins, "could not accept new connection"); - - return -1; - } - - flb_plg_trace(ctx->ins, "new TCP connection arrived FD=%i", connection->fd); - - conn = tcp_conn_add(connection, ctx); - - if (conn == NULL) { - flb_plg_error(ctx->ins, "could not accept new connection"); - - flb_downstream_conn_release(connection); - - return -1; - } - - return 0; -} - -/* Initialize plugin */ -static int in_tcp_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - unsigned short int port; - int ret; - struct flb_in_tcp_config *ctx; - - (void) data; - - /* Allocate space for the configuration */ - ctx = tcp_config_init(in); - if (!ctx) { - return -1; - } - ctx->collector_id = -1; - ctx->ins = in; - mk_list_init(&ctx->connections); - - /* Set the context */ - flb_input_set_context(in, ctx); - - port = (unsigned short int) strtoul(ctx->tcp_port, NULL, 10); - - ctx->downstream = flb_downstream_create(FLB_TRANSPORT_TCP, - in->flags, - ctx->listen, - port, - in->tls, - config, - &in->net_setup); - - if (ctx->downstream == NULL) { - flb_plg_error(ctx->ins, - "could not initialize downstream on %s:%s. Aborting", - ctx->listen, ctx->tcp_port); - - tcp_config_destroy(ctx); - - return -1; - } - - flb_input_downstream_set(ctx->downstream, ctx->ins); - - /* Collect upon data available on the standard input */ - ret = flb_input_set_collector_socket(in, - in_tcp_collect, - ctx->downstream->server_fd, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "Could not set collector for IN_TCP input plugin"); - tcp_config_destroy(ctx); - - return -1; - } - - ctx->collector_id = ret; - - return 0; -} - -static int in_tcp_exit(void *data, struct flb_config *config) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_in_tcp_config *ctx; - struct tcp_conn *conn; - - (void) *config; - - ctx = data; - - mk_list_foreach_safe(head, tmp, &ctx->connections) { - conn = mk_list_entry(head, struct tcp_conn, _head); - - tcp_conn_del(conn); - } - - tcp_config_destroy(ctx); - - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "format", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_in_tcp_config, format_name), - "Set the format: json or none" - }, - { - FLB_CONFIG_MAP_STR, "separator", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_in_tcp_config, raw_separator), - "Set separator" - }, - { - FLB_CONFIG_MAP_STR, "chunk_size", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_in_tcp_config, chunk_size_str), - "Set the chunk size" - }, - { - FLB_CONFIG_MAP_STR, "buffer_size", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_in_tcp_config, buffer_size_str), - "Set the buffer size" - }, - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_input_plugin in_tcp_plugin = { - .name = "tcp", - .description = "TCP", - .cb_init = in_tcp_init, - .cb_pre_run = NULL, - .cb_collect = in_tcp_collect, - .cb_flush_buf = NULL, - .cb_exit = in_tcp_exit, - .config_map = config_map, - .flags = FLB_INPUT_NET_SERVER | FLB_IO_OPT_TLS -}; diff --git a/fluent-bit/plugins/in_tcp/tcp.h b/fluent-bit/plugins/in_tcp/tcp.h deleted file mode 100644 index 3ddcbed06..000000000 --- a/fluent-bit/plugins/in_tcp/tcp.h +++ /dev/null @@ -1,50 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_TCP_H -#define FLB_IN_TCP_H - -#define FLB_TCP_FMT_JSON 0 /* default */ -#define FLB_TCP_FMT_NONE 1 /* no format, use delimiters */ - -#include -#include -#include -#include -#include - -struct flb_in_tcp_config { - flb_sds_t format_name; /* Data format name */ - int format; /* Data format */ - size_t buffer_size; /* Buffer size for each reader */ - flb_sds_t buffer_size_str; /* Buffer size in string form */ - size_t chunk_size; /* Chunk allocation size */ - flb_sds_t chunk_size_str; /* Chunk size in string form */ - char *listen; /* Listen interface */ - char *tcp_port; /* TCP Port */ - flb_sds_t raw_separator; /* Unescaped string delimiterr */ - flb_sds_t separator; /* String delimiter */ - int collector_id; /* Listener collector id */ - struct flb_downstream *downstream; /* Client manager */ - struct mk_list connections; /* List of active connections */ - struct flb_input_instance *ins; /* Input plugin instace */ - struct flb_log_event_encoder *log_encoder; -}; - -#endif diff --git a/fluent-bit/plugins/in_tcp/tcp_config.c b/fluent-bit/plugins/in_tcp/tcp_config.c deleted file mode 100644 index db9a36a01..000000000 --- a/fluent-bit/plugins/in_tcp/tcp_config.c +++ /dev/null @@ -1,155 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include "tcp.h" -#include "tcp_conn.h" -#include "tcp_config.h" - -#include - -struct flb_in_tcp_config *tcp_config_init(struct flb_input_instance *ins) -{ - int ret; - int len; - char port[16]; - char *out; - struct flb_in_tcp_config *ctx; - - /* Allocate plugin context */ - ctx = flb_calloc(1, sizeof(struct flb_in_tcp_config)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - ctx->format = FLB_TCP_FMT_JSON; - - /* Load the config map */ - ret = flb_input_config_map_set(ins, (void *)ctx); - if (ret == -1) { - flb_plg_error(ins, "unable to load configuration"); - flb_free(ctx); - return NULL; - } - - /* Data format (expected payload) */ - if (ctx->format_name) { - if (strcasecmp(ctx->format_name, "json") == 0) { - ctx->format = FLB_TCP_FMT_JSON; - } - else if (strcasecmp(ctx->format_name, "none") == 0) { - ctx->format = FLB_TCP_FMT_NONE; - } - else { - flb_plg_error(ctx->ins, "unrecognized format value '%s'", ctx->format_name); - flb_free(ctx); - return NULL; - } - } - - /* String separator used to split records when using 'format none' */ - if (ctx->raw_separator) { - len = strlen(ctx->raw_separator); - out = flb_malloc(len + 1); - if (!out) { - flb_errno(); - flb_free(ctx); - return NULL; - } - ret = flb_unescape_string(ctx->raw_separator, len, &out); - if (ret <= 0) { - flb_plg_error(ctx->ins, "invalid separator"); - flb_free(out); - flb_free(ctx); - return NULL; - } - - ctx->separator = flb_sds_create_len(out, ret); - if (!ctx->separator) { - flb_free(out); - flb_free(ctx); - return NULL; - } - flb_free(out); - } - if (!ctx->separator) { - ctx->separator = flb_sds_create_len("\n", 1); - } - - /* Listen interface (if not set, defaults to 0.0.0.0:5170) */ - flb_input_net_default_listener("0.0.0.0", 5170, ins); - ctx->listen = ins->host.listen; - snprintf(port, sizeof(port) - 1, "%d", ins->host.port); - ctx->tcp_port = flb_strdup(port); - - /* Chunk size */ - if (ctx->chunk_size_str) { - /* Convert KB unit to Bytes */ - ctx->chunk_size = (atoi(ctx->chunk_size_str) * 1024); - } else { - ctx->chunk_size = atoi(FLB_IN_TCP_CHUNK); - } - - /* Buffer size */ - if (!ctx->buffer_size_str) { - ctx->buffer_size = ctx->chunk_size; - } - else { - /* Convert KB unit to Bytes */ - ctx->buffer_size = (atoi(ctx->buffer_size_str) * 1024); - } - - ctx->log_encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ctx->log_encoder == NULL) { - flb_plg_error(ctx->ins, "could not initialize event encoder"); - tcp_config_destroy(ctx); - - ctx = NULL; - } - - return ctx; -} - -int tcp_config_destroy(struct flb_in_tcp_config *ctx) -{ - if (ctx->log_encoder != NULL) { - flb_log_event_encoder_destroy(ctx->log_encoder); - } - - if (ctx->collector_id != -1) { - flb_input_collector_delete(ctx->collector_id, ctx->ins); - - ctx->collector_id = -1; - } - - if (ctx->downstream != NULL) { - flb_downstream_destroy(ctx->downstream); - } - - flb_sds_destroy(ctx->separator); - flb_free(ctx->tcp_port); - flb_free(ctx); - - return 0; -} diff --git a/fluent-bit/plugins/in_tcp/tcp_config.h b/fluent-bit/plugins/in_tcp/tcp_config.h deleted file mode 100644 index 36df27873..000000000 --- a/fluent-bit/plugins/in_tcp/tcp_config.h +++ /dev/null @@ -1,28 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_TCP_CONFIG_H -#define FLB_IN_TCP_CONFIG_H - -#include "tcp.h" - -struct flb_in_tcp_config *tcp_config_init(struct flb_input_instance *i_ins); -int tcp_config_destroy(struct flb_in_tcp_config *config); - -#endif diff --git a/fluent-bit/plugins/in_tcp/tcp_conn.c b/fluent-bit/plugins/in_tcp/tcp_conn.c deleted file mode 100644 index 28b4b3222..000000000 --- a/fluent-bit/plugins/in_tcp/tcp_conn.c +++ /dev/null @@ -1,412 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include - -#include "tcp.h" -#include "tcp_conn.h" - -static inline void consume_bytes(char *buf, int bytes, int length) -{ - memmove(buf, buf + bytes, length - bytes); -} - -static inline int process_pack(struct tcp_conn *conn, - char *pack, size_t size) -{ - int ret; - size_t off = 0; - msgpack_unpacked result; - msgpack_object entry; - struct flb_in_tcp_config *ctx; - - ctx = conn->ctx; - - flb_log_event_encoder_reset(ctx->log_encoder); - - /* First pack the results, iterate concatenated messages */ - msgpack_unpacked_init(&result); - while (msgpack_unpack_next(&result, pack, size, &off) == MSGPACK_UNPACK_SUCCESS) { - entry = result.data; - - ret = flb_log_event_encoder_begin_record(ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp(ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - if (entry.type == MSGPACK_OBJECT_MAP) { - ret = flb_log_event_encoder_set_body_from_msgpack_object( - ctx->log_encoder, &entry); - } - else if (entry.type == MSGPACK_OBJECT_ARRAY) { - ret = flb_log_event_encoder_append_body_values( - ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE("msg"), - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&entry)); - } - else { - ret = FLB_EVENT_ENCODER_ERROR_INVALID_VALUE_TYPE; - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(ctx->log_encoder); - } - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - break; - } - } - } - - msgpack_unpacked_destroy(&result); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(conn->ins, NULL, 0, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); - ret = 0; - } - else { - flb_plg_error(ctx->ins, "log event encoding error : %d", ret); - - ret = -1; - } - - return ret; -} - -/* Process a JSON payload, return the number of processed bytes */ -static ssize_t parse_payload_json(struct tcp_conn *conn) -{ - int ret; - int out_size; - char *pack; - - ret = flb_pack_json_state(conn->buf_data, conn->buf_len, - &pack, &out_size, &conn->pack_state); - if (ret == FLB_ERR_JSON_PART) { - flb_plg_debug(conn->ins, "JSON incomplete, waiting for more data..."); - return 0; - } - else if (ret == FLB_ERR_JSON_INVAL) { - flb_plg_warn(conn->ins, "invalid JSON message, skipping"); - conn->buf_len = 0; - conn->pack_state.multiple = FLB_TRUE; - return -1; - } - else if (ret == -1) { - return -1; - } - - /* Process the packaged JSON and return the last byte used */ - process_pack(conn, pack, out_size); - flb_free(pack); - - return conn->pack_state.last_byte; -} - -/* - * Process a raw text payload, uses the delimited character to split records, - * return the number of processed bytes - */ -static ssize_t parse_payload_none(struct tcp_conn *conn) -{ - int ret; - int len; - int sep_len; - size_t consumed = 0; - char *buf; - char *s; - char *separator; - struct flb_in_tcp_config *ctx; - - ctx = conn->ctx; - - separator = conn->ctx->separator; - sep_len = flb_sds_len(conn->ctx->separator); - - buf = conn->buf_data; - ret = FLB_EVENT_ENCODER_SUCCESS; - - flb_log_event_encoder_reset(ctx->log_encoder); - - while ((s = strstr(buf, separator))) { - len = (s - buf); - if (len == 0) { - break; - } - else if (len > 0) { - ret = flb_log_event_encoder_begin_record(ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp(ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE("log"), - FLB_LOG_EVENT_STRING_VALUE(buf, len)); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(ctx->log_encoder); - } - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - break; - } - - consumed += len + 1; - buf += len + sep_len; - } - else { - break; - } - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(conn->ins, NULL, 0, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); - } - else { - flb_plg_error(ctx->ins, "log event encoding error : %d", ret); - } - - return consumed; -} - -/* Callback invoked every time an event is triggered for a connection */ -int tcp_conn_event(void *data) -{ - int bytes; - int available; - int size; - ssize_t ret_payload = -1; - char *tmp; - struct mk_event *event; - struct tcp_conn *conn; - struct flb_connection *connection; - struct flb_in_tcp_config *ctx; - - connection = (struct flb_connection *) data; - - conn = connection->user_data; - - ctx = conn->ctx; - - event = &connection->event; - - if (event->mask & MK_EVENT_READ) { - available = (conn->buf_size - conn->buf_len) - 1; - if (available < 1) { - if (conn->buf_size + ctx->chunk_size > ctx->buffer_size) { - flb_plg_warn(ctx->ins, - "fd=%i incoming data exceeds 'Buffer_Size' (%zu KB)", - event->fd, (ctx->buffer_size / 1024)); - tcp_conn_del(conn); - return -1; - } - - size = conn->buf_size + ctx->chunk_size; - tmp = flb_realloc(conn->buf_data, size); - if (!tmp) { - flb_errno(); - return -1; - } - flb_plg_trace(ctx->ins, "fd=%i buffer realloc %i -> %i", - event->fd, conn->buf_size, size); - - conn->buf_data = tmp; - conn->buf_size = size; - available = (conn->buf_size - conn->buf_len) - 1; - } - - /* Read data */ - bytes = flb_io_net_read(connection, - (void *) &conn->buf_data[conn->buf_len], - available); - - if (bytes <= 0) { - flb_plg_trace(ctx->ins, "fd=%i closed connection", event->fd); - tcp_conn_del(conn); - return -1; - } - - flb_plg_trace(ctx->ins, "read()=%i pre_len=%i now_len=%i", - bytes, conn->buf_len, conn->buf_len + bytes); - conn->buf_len += bytes; - conn->buf_data[conn->buf_len] = '\0'; - - /* Strip CR or LF if found at first byte */ - if (conn->buf_data[0] == '\r' || conn->buf_data[0] == '\n') { - /* Skip message with one byte with CR or LF */ - flb_plg_trace(ctx->ins, "skip one byte message with ASCII code=%i", - conn->buf_data[0]); - consume_bytes(conn->buf_data, 1, conn->buf_len); - conn->buf_len--; - conn->buf_data[conn->buf_len] = '\0'; - } - - /* JSON Format handler */ - if (ctx->format == FLB_TCP_FMT_JSON) { - ret_payload = parse_payload_json(conn); - if (ret_payload == 0) { - /* Incomplete JSON message, we need more data */ - return -1; - } - else if (ret_payload == -1) { - flb_pack_state_reset(&conn->pack_state); - flb_pack_state_init(&conn->pack_state); - conn->pack_state.multiple = FLB_TRUE; - return -1; - } - } - else if (ctx->format == FLB_TCP_FMT_NONE) { - ret_payload = parse_payload_none(conn); - if (ret_payload == 0) { - return -1; - } - else if (ret_payload == -1) { - conn->buf_len = 0; - return -1; - } - } - - - consume_bytes(conn->buf_data, ret_payload, conn->buf_len); - conn->buf_len -= ret_payload; - conn->buf_data[conn->buf_len] = '\0'; - - if (ctx->format == FLB_TCP_FMT_JSON) { - jsmn_init(&conn->pack_state.parser); - conn->pack_state.tokens_count = 0; - conn->pack_state.last_byte = 0; - conn->pack_state.buf_len = 0; - } - - return bytes; - } - - if (event->mask & MK_EVENT_CLOSE) { - flb_plg_trace(ctx->ins, "fd=%i hangup", event->fd); - tcp_conn_del(conn); - return -1; - } - - return 0; -} - -/* Create a new mqtt request instance */ -struct tcp_conn *tcp_conn_add(struct flb_connection *connection, - struct flb_in_tcp_config *ctx) -{ - struct tcp_conn *conn; - int ret; - - conn = flb_malloc(sizeof(struct tcp_conn)); - if (!conn) { - flb_errno(); - return NULL; - } - - conn->connection = connection; - - /* Set data for the event-loop */ - MK_EVENT_NEW(&connection->event); - - connection->user_data = conn; - connection->event.type = FLB_ENGINE_EV_CUSTOM; - connection->event.handler = tcp_conn_event; - - /* Connection info */ - conn->ctx = ctx; - conn->buf_len = 0; - conn->rest = 0; - conn->status = TCP_NEW; - - conn->buf_data = flb_malloc(ctx->chunk_size); - if (!conn->buf_data) { - flb_errno(); - - flb_plg_error(ctx->ins, "could not allocate new connection"); - flb_free(conn); - - return NULL; - } - conn->buf_size = ctx->chunk_size; - conn->ins = ctx->ins; - - /* Initialize JSON parser */ - if (ctx->format == FLB_TCP_FMT_JSON) { - flb_pack_state_init(&conn->pack_state); - conn->pack_state.multiple = FLB_TRUE; - } - - /* Register instance into the event loop */ - ret = mk_event_add(flb_engine_evl_get(), - connection->fd, - FLB_ENGINE_EV_CUSTOM, - MK_EVENT_READ, - &connection->event); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not register new connection"); - - flb_free(conn->buf_data); - flb_free(conn); - - return NULL; - } - - mk_list_add(&conn->_head, &ctx->connections); - - return conn; -} - -int tcp_conn_del(struct tcp_conn *conn) -{ - struct flb_in_tcp_config *ctx; - - ctx = conn->ctx; - - if (ctx->format == FLB_TCP_FMT_JSON) { - flb_pack_state_reset(&conn->pack_state); - } - - /* The downstream unregisters the file descriptor from the event-loop - * so there's nothing to be done by the plugin - */ - flb_downstream_conn_release(conn->connection); - - /* Release resources */ - mk_list_del(&conn->_head); - - flb_free(conn->buf_data); - flb_free(conn); - - return 0; -} diff --git a/fluent-bit/plugins/in_tcp/tcp_conn.h b/fluent-bit/plugins/in_tcp/tcp_conn.h deleted file mode 100644 index f9af869f2..000000000 --- a/fluent-bit/plugins/in_tcp/tcp_conn.h +++ /dev/null @@ -1,59 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_TCP_CONN_H -#define FLB_IN_TCP_CONN_H - -#include -#include - -#define FLB_IN_TCP_CHUNK "32768" - -enum { - TCP_NEW = 1, /* it's a new connection */ - TCP_CONNECTED = 2, /* MQTT connection per protocol spec OK */ -}; - -struct tcp_conn_stream { - char *tag; - size_t tag_len; -}; - -/* Respresents a connection */ -struct tcp_conn { - int status; /* Connection status */ - - /* Buffer */ - char *buf_data; /* Buffer data */ - int buf_len; /* Data length */ - int buf_size; /* Buffer size */ - size_t rest; /* Unpacking offset */ - - struct flb_input_instance *ins; /* Parent plugin instance */ - struct flb_in_tcp_config *ctx; /* Plugin configuration context */ - struct flb_pack_state pack_state; /* Internal JSON parser */ - struct flb_connection *connection; - - struct mk_list _head; -}; - -struct tcp_conn *tcp_conn_add(struct flb_connection *connection, struct flb_in_tcp_config *ctx); -int tcp_conn_del(struct tcp_conn *conn); - -#endif diff --git a/fluent-bit/plugins/in_thermal/CMakeLists.txt b/fluent-bit/plugins/in_thermal/CMakeLists.txt deleted file mode 100644 index 693d0ed41..000000000 --- a/fluent-bit/plugins/in_thermal/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - in_thermal.c) - -FLB_PLUGIN(in_thermal "${src}" "") diff --git a/fluent-bit/plugins/in_thermal/in_thermal.c b/fluent-bit/plugins/in_thermal/in_thermal.c deleted file mode 100644 index 2eb9267de..000000000 --- a/fluent-bit/plugins/in_thermal/in_thermal.c +++ /dev/null @@ -1,372 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include -#include - -#include -#include -#include - -#include - -#include "in_thermal.h" - -struct flb_input_plugin in_thermal_plugin; - -/* Default collection time: every 1 second (0 nanoseconds) */ -#define DEFAULT_INTERVAL_SEC "1" -#define DEFAULT_INTERVAL_NSEC "0" - -#define IN_THERMAL_N_MAX 32 -#define IN_THERMAL_FILENAME_LEN 1024 -#define IN_THERMAL_TYPE_LEN 256 - -struct temp_info -{ - char name[IN_THERMAL_FILENAME_LEN]; /* .../thermal_zoneX/... */ - char type[IN_THERMAL_TYPE_LEN]; /* from /sys/class/thermal/thermal_zoneX/type */ - double temp; /* from /sys/class/thermal/thermal_zoneX/temp */ -}; - -/* Retrieve temperature(s) from the system (via /sys/class/thermal) */ -static inline int proc_temperature(struct flb_in_thermal_config *ctx, - struct temp_info *info, int n) -{ - int i, j; - DIR *d; - struct dirent *e; - char filename[IN_THERMAL_FILENAME_LEN]; - FILE *f; - int temp; - - d = opendir("/sys/class/thermal"); - if (d == NULL) { - return -1; - } - - i = 0; - while (id_name, ".") || !strcmp(e->d_name, "..")) { - continue; - } - - if (e->d_type == DT_REG) { - continue; - } - -#ifdef FLB_HAVE_REGEX - if (ctx->name_regex && !flb_regex_match(ctx->name_regex, - (unsigned char *) e->d_name, - strlen(e->d_name))) { - continue; - } -#endif - - if (!strncmp(e->d_name, "thermal_zone", 12)) { - strncpy(info[i].name, e->d_name, IN_THERMAL_FILENAME_LEN); - if (snprintf(filename, IN_THERMAL_FILENAME_LEN, - "/sys/class/thermal/%s/type", e->d_name) <=0 ) { - continue; - } - - f = fopen(filename, "r"); - if (!f) { - flb_errno(); - flb_error("[in_thermal] cannot read %s", filename); - continue; - } - - if (f && fgets(info[i].type, IN_THERMAL_TYPE_LEN, f) && - strlen(info[i].type) > 1) { - /* Remove trailing \n */ - for (j = 0; info[i].type[j]; ++j) { - if (info[i].type[j] == '\n') { - info[i].type[j] = 0; - break; - } - } - fclose(f); - -#ifdef FLB_HAVE_REGEX - if (ctx->type_regex && - !flb_regex_match(ctx->type_regex, - (unsigned char *) info[i].type, - strlen(info[i].type))) { - continue; - } -#endif - - if (snprintf(filename, IN_THERMAL_FILENAME_LEN, - "/sys/class/thermal/%s/temp", e->d_name) <= 0) { - continue; - } - f = fopen(filename, "r"); - if (f && fscanf(f, "%d", &temp) == 1) { - info[i].temp = temp/1000.0; - ++i; - } - } - - if (f) { - fclose(f); - } - } - } - - closedir(d); - return i; -} - -/* Init temperature input */ -static int in_thermal_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - int ret; - struct flb_in_thermal_config *ctx; - struct temp_info info[IN_THERMAL_N_MAX]; - (void) data; - - /* Allocate space for the configuration */ - ctx = flb_calloc(1, sizeof(struct flb_in_thermal_config)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->ins = in; - - ctx->log_encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ctx->log_encoder == NULL) { - flb_plg_error(in, "could not initialize event encoder"); - flb_free(ctx); - - return -1; - } - - /* Load the config map */ - ret = flb_input_config_map_set(in, (void *)ctx); - if (ret == -1) { - flb_log_event_encoder_destroy(ctx->log_encoder); - flb_free(ctx); - flb_plg_error(in, "unable to load configuration"); - return -1; - } - - /* Collection time setting */ - if (ctx->interval_sec <= 0 && ctx->interval_nsec <= 0) { - /* Illegal settings. Override them. */ - ctx->interval_sec = atoi(DEFAULT_INTERVAL_SEC); - ctx->interval_nsec = atoi(DEFAULT_INTERVAL_NSEC); - } - -#ifdef FLB_HAVE_REGEX - if (ctx->name_rgx && strcmp(ctx->name_rgx, "") != 0) { - ctx->name_regex = flb_regex_create(ctx->name_rgx); - if (!ctx->name_regex) { - flb_plg_error(ctx->ins, "invalid 'name_regex' config value"); - } - } - - if (ctx->type_rgx && strcmp(ctx->type_rgx, "") != 0) { - ctx->type_regex = flb_regex_create(ctx->type_rgx); - if (!ctx->type_regex) { - flb_plg_error(ctx->ins, "invalid 'type_regex' config value"); - } - } -#endif - - ctx->prev_device_num = proc_temperature(ctx, info, IN_THERMAL_N_MAX); - if (!ctx->prev_device_num) { - flb_plg_warn(ctx->ins, "thermal device file not found"); - } - - /* Set the context */ - flb_input_set_context(in, ctx); - - /* Set our collector based on time, temperature every 1 second */ - ret = flb_input_set_collector_time(in, - in_thermal_collect, - ctx->interval_sec, - ctx->interval_nsec, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "Could not set collector for temperature input plugin"); - - flb_log_event_encoder_destroy(ctx->log_encoder); - flb_free(ctx); - - return -1; - } - ctx->coll_fd = ret; - - return 0; -} - -/* Callback to gather temperature */ -int in_thermal_collect(struct flb_input_instance *i_ins, - struct flb_config *config, void *in_context) -{ - int n; - int i; - int ret; - struct temp_info info[IN_THERMAL_N_MAX]; - struct flb_in_thermal_config *ctx = in_context; - - (void) config; - - /* Get the current temperature(s) */ - n = proc_temperature(ctx, info, IN_THERMAL_N_MAX); - if (n != ctx->prev_device_num) { - flb_plg_info(ctx->ins, "the number of thermal devices changed %d -> %d", - ctx->prev_device_num, n); - } - ctx->prev_device_num = n; - if (!n) { - return 0; - } - - /* - * Store the new data into the MessagePack buffer - */ - - for (i = 0; i < n; ++i) { - ret = flb_log_event_encoder_begin_record(ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp(ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE("name"), - FLB_LOG_EVENT_CSTRING_VALUE(info[i].name), - - FLB_LOG_EVENT_CSTRING_VALUE("type"), - FLB_LOG_EVENT_CSTRING_VALUE(info[i].type), - - FLB_LOG_EVENT_CSTRING_VALUE("temp"), - FLB_LOG_EVENT_DOUBLE_VALUE(info[i].temp)); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(ctx->log_encoder); - } - - flb_plg_trace(ctx->ins, "%s temperature %0.2f", info[i].name, info[i].temp); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(ctx->ins, NULL, 0, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); - ret = 0; - } - else { - flb_plg_error(ctx->ins, "log event encoding error : %d", ret); - - ret = -1; - } - - flb_log_event_encoder_reset(ctx->log_encoder); - - return 0; -} - -static void in_thermal_pause(void *data, struct flb_config *config) -{ - struct flb_in_thermal_config *ctx = data; - flb_input_collector_pause(ctx->coll_fd, ctx->ins); -} - -static void in_thermal_resume(void *data, struct flb_config *config) -{ - struct flb_in_thermal_config *ctx = data; - flb_input_collector_resume(ctx->coll_fd, ctx->ins); -} - -static int in_thermal_exit(void *data, struct flb_config *config) -{ - (void) *config; - struct flb_in_thermal_config *ctx = data; - - if (ctx->log_encoder != NULL) { - flb_log_event_encoder_destroy(ctx->log_encoder); - } - -#ifdef FLB_HAVE_REGEX - if (ctx && ctx->name_regex) { - flb_regex_destroy(ctx->name_regex); - } - if (ctx && ctx->type_regex) { - flb_regex_destroy(ctx->type_regex); - } -#endif - - flb_free(ctx); - - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_INT, "interval_sec", DEFAULT_INTERVAL_SEC, - 0, FLB_TRUE, offsetof(struct flb_in_thermal_config, interval_sec), - "Set the collector interval" - }, - { - FLB_CONFIG_MAP_INT, "interval_nsec", DEFAULT_INTERVAL_NSEC, - 0, FLB_TRUE, offsetof(struct flb_in_thermal_config, interval_nsec), - "Set the collector interval (nanoseconds)" - }, -#ifdef FLB_HAVE_REGEX - { - FLB_CONFIG_MAP_STR, "name_regex", NULL, - 0, FLB_TRUE, offsetof(struct flb_in_thermal_config, name_rgx), - "Set thermal name regular expression filter" - }, - { - FLB_CONFIG_MAP_STR, "type_regex", NULL, - 0, FLB_TRUE, offsetof(struct flb_in_thermal_config, type_rgx), - "Set thermal type regular expression filter" - }, -#endif /* FLB_HAVE_REGEX */ - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_input_plugin in_thermal_plugin = { - .name = "thermal", - .description = "Thermal", - .cb_init = in_thermal_init, - .cb_pre_run = NULL, - .cb_collect = in_thermal_collect, - .cb_flush_buf = NULL, - .cb_pause = in_thermal_pause, - .cb_resume = in_thermal_resume, - .cb_exit = in_thermal_exit, - .config_map = config_map -}; diff --git a/fluent-bit/plugins/in_thermal/in_thermal.h b/fluent-bit/plugins/in_thermal/in_thermal.h deleted file mode 100644 index 17f14ca34..000000000 --- a/fluent-bit/plugins/in_thermal/in_thermal.h +++ /dev/null @@ -1,55 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_THERMAL_H -#define FLB_IN_THERMAL_H - -#include -#include -#include - -#ifdef FLB_HAVE_REGEX -#include -#endif - -/* Temperature Input configuration & context */ -struct flb_in_thermal_config { - /* setup */ - int coll_fd; /* collector id/fd */ - int interval_sec; /* interval collection time (Second) */ - int interval_nsec; /* interval collection time (Nanosecond) */ - int prev_device_num; /* number of thermal devices */ -#ifdef FLB_HAVE_REGEX - struct flb_regex *name_regex; /* compiled filter by name */ - struct flb_regex *type_regex; /* compiled filter by type */ - flb_sds_t name_rgx; /* optional filter by name */ - flb_sds_t type_rgx; /* optional filter by type */ -#endif - struct flb_input_instance *ins; - struct flb_log_event_encoder *log_encoder; -}; - -int in_thermal_pre_run(void *in_context, struct flb_config *config); -int in_thermal_collect(struct flb_input_instance *i_ins, - struct flb_config *config, void *in_context); -void *in_thermal_flush(void *in_context, size_t *size); - -extern struct flb_input_plugin in_thermal_plugin; - -#endif diff --git a/fluent-bit/plugins/in_udp/CMakeLists.txt b/fluent-bit/plugins/in_udp/CMakeLists.txt deleted file mode 100644 index 0b623f169..000000000 --- a/fluent-bit/plugins/in_udp/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(src - udp.c - udp_conn.c - udp_config.c) - -FLB_PLUGIN(in_udp "${src}" "") diff --git a/fluent-bit/plugins/in_udp/udp.c b/fluent-bit/plugins/in_udp/udp.c deleted file mode 100644 index ad5a28497..000000000 --- a/fluent-bit/plugins/in_udp/udp.c +++ /dev/null @@ -1,197 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include "udp.h" -#include "udp_conn.h" -#include "udp_config.h" - -static int in_udp_collect(struct flb_input_instance *in, - struct flb_config *config, - void *in_context) -{ - struct flb_connection *connection; - struct flb_in_udp_config *ctx; - - ctx = in_context; - - connection = flb_downstream_conn_get(ctx->downstream); - - if (connection == NULL) { - flb_plg_error(ctx->ins, "could get UDP server dummy connection"); - - return -1; - } - - return udp_conn_event(connection); -} - -/* Initialize plugin */ -static int in_udp_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - struct flb_connection *connection; - unsigned short int port; - int ret; - struct flb_in_udp_config *ctx; - - (void) data; - - /* Allocate space for the configuration */ - ctx = udp_config_init(in); - - if (ctx == NULL) { - return -1; - } - - ctx->collector_id = -1; - ctx->ins = in; - - /* Set the context */ - flb_input_set_context(in, ctx); - - port = (unsigned short int) strtoul(ctx->port, NULL, 10); - - ctx->downstream = flb_downstream_create(FLB_TRANSPORT_UDP, - in->flags, - ctx->listen, - port, - in->tls, - config, - &in->net_setup); - - if (ctx->downstream == NULL) { - flb_plg_error(ctx->ins, - "could not initialize downstream on %s:%s. Aborting", - ctx->listen, ctx->port); - - udp_config_destroy(ctx); - - return -1; - } - - flb_input_downstream_set(ctx->downstream, ctx->ins); - - connection = flb_downstream_conn_get(ctx->downstream); - - if (connection == NULL) { - flb_plg_error(ctx->ins, "could not get UDP server dummy connection"); - - udp_config_destroy(ctx); - - return -1; - } - - ctx->dummy_conn = udp_conn_add(connection, ctx); - - if (ctx->dummy_conn == NULL) { - flb_plg_error(ctx->ins, "could not track UDP server dummy connection"); - - udp_config_destroy(ctx); - - return -1; - } - - /* Collect upon data available on the standard input */ - ret = flb_input_set_collector_socket(in, - in_udp_collect, - ctx->downstream->server_fd, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "Could not set collector for IN_UDP input plugin"); - udp_config_destroy(ctx); - - return -1; - } - - ctx->collector_id = ret; - ctx->collector_event = flb_input_collector_get_event(ret, in); - - if (ret == -1) { - flb_plg_error(ctx->ins, "Could not get collector event"); - udp_config_destroy(ctx); - - return -1; - } - - return 0; -} - -static int in_udp_exit(void *data, struct flb_config *config) -{ - struct flb_in_udp_config *ctx; - - (void) *config; - - ctx = data; - - if (ctx->dummy_conn != NULL) { - udp_conn_del(ctx->dummy_conn); - } - - udp_config_destroy(ctx); - - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "format", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_in_udp_config, format_name), - "Set the format: json or none" - }, - { - FLB_CONFIG_MAP_STR, "separator", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_in_udp_config, raw_separator), - "Set separator" - }, - { - FLB_CONFIG_MAP_STR, "chunk_size", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_in_udp_config, chunk_size_str), - "Set the chunk size" - }, - { - FLB_CONFIG_MAP_STR, "buffer_size", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_in_udp_config, buffer_size_str), - "Set the buffer size" - }, - { - FLB_CONFIG_MAP_STR, "source_address_key", (char *) NULL, - 0, FLB_TRUE, offsetof(struct flb_in_udp_config, source_address_key), - "Key where the source address will be injected" - }, - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_input_plugin in_udp_plugin = { - .name = "udp", - .description = "UDP", - .cb_init = in_udp_init, - .cb_pre_run = NULL, - .cb_collect = in_udp_collect, - .cb_flush_buf = NULL, - .cb_exit = in_udp_exit, - .config_map = config_map, - .flags = FLB_INPUT_NET_SERVER, -}; diff --git a/fluent-bit/plugins/in_udp/udp.h b/fluent-bit/plugins/in_udp/udp.h deleted file mode 100644 index 1a7bfce30..000000000 --- a/fluent-bit/plugins/in_udp/udp.h +++ /dev/null @@ -1,54 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_UDP_H -#define FLB_IN_UDP_H - -#define FLB_UDP_FMT_JSON 0 /* default */ -#define FLB_UDP_FMT_NONE 1 /* no format, use delimiters */ - -#include -#include -#include -#include -#include - -struct udp_conn; - -struct flb_in_udp_config { - struct mk_event *collector_event; - flb_sds_t format_name; /* Data format name */ - int format; /* Data format */ - size_t buffer_size; /* Buffer size for each reader */ - flb_sds_t buffer_size_str; /* Buffer size in string form */ - size_t chunk_size; /* Chunk allocation size */ - flb_sds_t chunk_size_str; /* Chunk size in string form */ - char *listen; /* Listen interface */ - char *port; /* Port */ - flb_sds_t raw_separator; /* Unescaped string delimiterr */ - flb_sds_t separator; /* String delimiter */ - flb_sds_t source_address_key; /* Source IP address */ - int collector_id; /* Listener collector id */ - struct flb_downstream *downstream; /* Client manager */ - struct udp_conn *dummy_conn; /* Datagram dummy connection */ - struct flb_input_instance *ins; /* Input plugin instace */ - struct flb_log_event_encoder *log_encoder; -}; - -#endif diff --git a/fluent-bit/plugins/in_udp/udp_config.c b/fluent-bit/plugins/in_udp/udp_config.c deleted file mode 100644 index ad2995490..000000000 --- a/fluent-bit/plugins/in_udp/udp_config.c +++ /dev/null @@ -1,155 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include "udp.h" -#include "udp_conn.h" -#include "udp_config.h" - -#include - -struct flb_in_udp_config *udp_config_init(struct flb_input_instance *ins) -{ - int ret; - int len; - char port[16]; - char *out; - struct flb_in_udp_config *ctx; - - /* Allocate plugin context */ - ctx = flb_calloc(1, sizeof(struct flb_in_udp_config)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - ctx->format = FLB_UDP_FMT_JSON; - - /* Load the config map */ - ret = flb_input_config_map_set(ins, (void *)ctx); - if (ret == -1) { - flb_plg_error(ins, "unable to load configuration"); - flb_free(ctx); - return NULL; - } - - /* Data format (expected payload) */ - if (ctx->format_name) { - if (strcasecmp(ctx->format_name, "json") == 0) { - ctx->format = FLB_UDP_FMT_JSON; - } - else if (strcasecmp(ctx->format_name, "none") == 0) { - ctx->format = FLB_UDP_FMT_NONE; - } - else { - flb_plg_error(ctx->ins, "unrecognized format value '%s'", ctx->format_name); - flb_free(ctx); - return NULL; - } - } - - /* String separator used to split records when using 'format none' */ - if (ctx->raw_separator) { - len = strlen(ctx->raw_separator); - out = flb_malloc(len + 1); - if (!out) { - flb_errno(); - flb_free(ctx); - return NULL; - } - ret = flb_unescape_string(ctx->raw_separator, len, &out); - if (ret <= 0) { - flb_plg_error(ctx->ins, "invalid separator"); - flb_free(out); - flb_free(ctx); - return NULL; - } - - ctx->separator = flb_sds_create_len(out, ret); - if (!ctx->separator) { - flb_free(out); - flb_free(ctx); - return NULL; - } - flb_free(out); - } - if (!ctx->separator) { - ctx->separator = flb_sds_create_len("\n", 1); - } - - /* Listen interface (if not set, defaults to 0.0.0.0:5170) */ - flb_input_net_default_listener("0.0.0.0", 5170, ins); - ctx->listen = ins->host.listen; - snprintf(port, sizeof(port) - 1, "%d", ins->host.port); - ctx->port = flb_strdup(port); - - /* Chunk size */ - if (ctx->chunk_size_str) { - /* Convert KB unit to Bytes */ - ctx->chunk_size = (atoi(ctx->chunk_size_str) * 1024); - } else { - ctx->chunk_size = atoi(FLB_IN_UDP_CHUNK); - } - - /* Buffer size */ - if (!ctx->buffer_size_str) { - ctx->buffer_size = ctx->chunk_size; - } - else { - /* Convert KB unit to Bytes */ - ctx->buffer_size = (atoi(ctx->buffer_size_str) * 1024); - } - - ctx->log_encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ctx->log_encoder == NULL) { - flb_plg_error(ctx->ins, "could not initialize event encoder"); - udp_config_destroy(ctx); - - ctx = NULL; - } - - return ctx; -} - -int udp_config_destroy(struct flb_in_udp_config *ctx) -{ - if (ctx->log_encoder != NULL) { - flb_log_event_encoder_destroy(ctx->log_encoder); - } - - if (ctx->collector_id != -1) { - flb_input_collector_delete(ctx->collector_id, ctx->ins); - - ctx->collector_id = -1; - } - - if (ctx->downstream != NULL) { - flb_downstream_destroy(ctx->downstream); - } - - flb_sds_destroy(ctx->separator); - flb_free(ctx->port); - flb_free(ctx); - - return 0; -} diff --git a/fluent-bit/plugins/in_udp/udp_config.h b/fluent-bit/plugins/in_udp/udp_config.h deleted file mode 100644 index dcddb74a7..000000000 --- a/fluent-bit/plugins/in_udp/udp_config.h +++ /dev/null @@ -1,28 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_UDP_CONFIG_H -#define FLB_IN_UDP_CONFIG_H - -#include "udp.h" - -struct flb_in_udp_config *udp_config_init(struct flb_input_instance *i_ins); -int udp_config_destroy(struct flb_in_udp_config *config); - -#endif diff --git a/fluent-bit/plugins/in_udp/udp_conn.c b/fluent-bit/plugins/in_udp/udp_conn.c deleted file mode 100644 index d8cc4d5e6..000000000 --- a/fluent-bit/plugins/in_udp/udp_conn.c +++ /dev/null @@ -1,500 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include - -#include "udp.h" -#include "udp_conn.h" - -static inline void consume_bytes(char *buf, int bytes, int length) -{ - memmove(buf, buf + bytes, length - bytes); -} - -static int append_message_to_record_data(char **result_buffer, - size_t *result_size, - flb_sds_t message_key_name, - char *base_object_buffer, - size_t base_object_size, - char *message_buffer, - size_t message_size, - int message_type) -{ - int result = FLB_MAP_NOT_MODIFIED; - char *modified_data_buffer; - int modified_data_size; - msgpack_object_kv *new_map_entries[1]; - msgpack_object_kv message_entry; - *result_buffer = NULL; - *result_size = 0; - modified_data_buffer = NULL; - - if (message_key_name != NULL) { - new_map_entries[0] = &message_entry; - - message_entry.key.type = MSGPACK_OBJECT_STR; - message_entry.key.via.str.size = flb_sds_len(message_key_name); - message_entry.key.via.str.ptr = message_key_name; - - if (message_type == MSGPACK_OBJECT_BIN) { - message_entry.val.type = MSGPACK_OBJECT_BIN; - message_entry.val.via.bin.size = message_size; - message_entry.val.via.bin.ptr = message_buffer; - } - else if (message_type == MSGPACK_OBJECT_STR) { - message_entry.val.type = MSGPACK_OBJECT_STR; - message_entry.val.via.str.size = message_size; - message_entry.val.via.str.ptr = message_buffer; - } - else { - result = FLB_MAP_EXPANSION_INVALID_VALUE_TYPE; - } - - if (result == FLB_MAP_NOT_MODIFIED) { - result = flb_msgpack_expand_map(base_object_buffer, - base_object_size, - new_map_entries, 1, - &modified_data_buffer, - &modified_data_size); - if (result == 0) { - result = FLB_MAP_EXPAND_SUCCESS; - } - else { - result = FLB_MAP_EXPANSION_ERROR; - } - } - } - - if (result == FLB_MAP_EXPAND_SUCCESS) { - *result_buffer = modified_data_buffer; - *result_size = modified_data_size; - } - - return result; -} - -static inline int process_pack(struct udp_conn *conn, - char *pack, size_t size) -{ - int ret; - size_t off = 0; - msgpack_unpacked result; - msgpack_object entry; - msgpack_sbuffer sbuf; - msgpack_packer pck; - struct flb_in_udp_config *ctx; - char *appended_address_buffer; - size_t appended_address_size; - char *source_address; - int i; - int len; - - ctx = conn->ctx; - - flb_log_event_encoder_reset(ctx->log_encoder); - - /* First pack the results, iterate concatenated messages */ - msgpack_unpacked_init(&result); - while (msgpack_unpack_next(&result, pack, size, &off) == MSGPACK_UNPACK_SUCCESS) { - entry = result.data; - - appended_address_buffer = NULL; - source_address = NULL; - - ret = flb_log_event_encoder_begin_record(ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp(ctx->log_encoder); - } - - if (ctx->source_address_key != NULL) { - source_address = flb_connection_get_remote_address(conn->connection); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - if (entry.type == MSGPACK_OBJECT_MAP) { - if (source_address != NULL) { - msgpack_sbuffer_init(&sbuf); - msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); - - len = entry.via.map.size; - msgpack_pack_map(&pck, len); - - for (i=0; isource_address_key, - sbuf.data, - sbuf.size, - source_address, - strlen(source_address), - MSGPACK_OBJECT_STR); - msgpack_sbuffer_destroy(&sbuf); - } - - if (ret == FLB_MAP_EXPANSION_ERROR) { - flb_plg_debug(ctx->ins, "error expanding source_address : %d", ret); - } - - if (appended_address_buffer != NULL) { - ret = flb_log_event_encoder_set_body_from_raw_msgpack( - ctx->log_encoder, appended_address_buffer, appended_address_size); - } - else { - ret = flb_log_event_encoder_set_body_from_msgpack_object( - ctx->log_encoder, &entry); - } - } - else if (entry.type == MSGPACK_OBJECT_ARRAY) { - if (source_address != NULL) { - ret = flb_log_event_encoder_append_body_values( - ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE("msg"), - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&entry), - FLB_LOG_EVENT_CSTRING_VALUE(ctx->source_address_key), - FLB_LOG_EVENT_CSTRING_VALUE(source_address)); - } - else { - ret = flb_log_event_encoder_append_body_values( - ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE("msg"), - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&entry)); - } - } - else { - ret = FLB_EVENT_ENCODER_ERROR_INVALID_VALUE_TYPE; - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(ctx->log_encoder); - } - - if (appended_address_buffer != NULL) { - flb_free(appended_address_buffer); - } - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - break; - } - } - } - - msgpack_unpacked_destroy(&result); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(conn->ins, NULL, 0, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); - ret = 0; - } - else { - flb_plg_error(ctx->ins, "log event encoding error : %d", ret); - - ret = -1; - } - - return ret; -} - -/* Process a JSON payload, return the number of processed bytes */ -static ssize_t parse_payload_json(struct udp_conn *conn) -{ - int ret; - int out_size; - char *pack; - - ret = flb_pack_json_state(conn->buf_data, conn->buf_len, - &pack, &out_size, &conn->pack_state); - if (ret == FLB_ERR_JSON_PART) { - flb_plg_debug(conn->ins, "JSON incomplete, waiting for more data..."); - return 0; - } - else if (ret == FLB_ERR_JSON_INVAL) { - flb_plg_warn(conn->ins, "invalid JSON message, skipping"); - conn->buf_len = 0; - conn->pack_state.multiple = FLB_TRUE; - return -1; - } - else if (ret == -1) { - return -1; - } - - /* Process the packaged JSON and return the last byte used */ - process_pack(conn, pack, out_size); - flb_free(pack); - - return conn->pack_state.last_byte; -} - -/* - * Process a raw text payload, uses the delimited character to split records, - * return the number of processed bytes - */ -static ssize_t parse_payload_none(struct udp_conn *conn) -{ - int ret; - int len; - int sep_len; - size_t consumed = 0; - char *buf; - char *s; - char *separator; - struct flb_in_udp_config *ctx; - - ctx = conn->ctx; - - separator = conn->ctx->separator; - sep_len = flb_sds_len(conn->ctx->separator); - - buf = conn->buf_data; - ret = FLB_EVENT_ENCODER_SUCCESS; - - flb_log_event_encoder_reset(ctx->log_encoder); - - while ((s = strstr(buf, separator))) { - len = (s - buf); - if (len == 0) { - break; - } - else if (len > 0) { - ret = flb_log_event_encoder_begin_record(ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp(ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE("log"), - FLB_LOG_EVENT_STRING_VALUE(buf, len)); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(ctx->log_encoder); - } - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - break; - } - - consumed += len + 1; - buf += len + sep_len; - } - else { - break; - } - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(conn->ins, NULL, 0, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); - } - else { - flb_plg_error(ctx->ins, "log event encoding error : %d", ret); - } - - return consumed; -} - -/* Callback invoked every time an event is triggered for a connection */ -int udp_conn_event(void *data) -{ - int bytes; - int available; - int size; - ssize_t ret_payload = -1; - char *tmp; - struct udp_conn *conn; - struct flb_connection *connection; - struct flb_in_udp_config *ctx; - - connection = (struct flb_connection *) data; - - conn = connection->user_data; - - ctx = conn->ctx; - - if (ctx->format == FLB_UDP_FMT_JSON && - conn->buf_len > 0) { - flb_pack_state_reset(&conn->pack_state); - flb_pack_state_init(&conn->pack_state); - - conn->pack_state.multiple = FLB_TRUE; - } - - conn->buf_len = 0; - - available = (conn->buf_size - conn->buf_len) - 1; - if (available < 1) { - if (conn->buf_size + ctx->chunk_size > ctx->buffer_size) { - flb_plg_trace(ctx->ins, - "fd=%i incoming data exceed limit (%zu KB)", - connection->fd, (ctx->buffer_size / 1024)); - return -1; - } - - size = conn->buf_size + ctx->chunk_size; - tmp = flb_realloc(conn->buf_data, size); - if (!tmp) { - flb_errno(); - return -1; - } - flb_plg_trace(ctx->ins, "fd=%i buffer realloc %i -> %i", - connection->fd, conn->buf_size, size); - - conn->buf_data = tmp; - conn->buf_size = size; - available = (conn->buf_size - conn->buf_len) - 1; - } - - /* Read data */ - bytes = flb_io_net_read(connection, - (void *) &conn->buf_data[conn->buf_len], - available); - - if (bytes <= 0) { - return -1; - } - - flb_plg_trace(ctx->ins, "read()=%i pre_len=%i now_len=%i", - bytes, conn->buf_len, conn->buf_len + bytes); - conn->buf_len += bytes; - conn->buf_data[conn->buf_len] = '\0'; - - /* Strip CR or LF if found at first byte */ - if (conn->buf_data[0] == '\r' || conn->buf_data[0] == '\n') { - /* Skip message with one byte with CR or LF */ - flb_plg_trace(ctx->ins, "skip one byte message with ASCII code=%i", - conn->buf_data[0]); - consume_bytes(conn->buf_data, 1, conn->buf_len); - conn->buf_len--; - conn->buf_data[conn->buf_len] = '\0'; - } - - /* JSON Format handler */ - if (ctx->format == FLB_UDP_FMT_JSON) { - ret_payload = parse_payload_json(conn); - if (ret_payload == 0) { - /* Incomplete JSON message, we need more data */ - return -1; - } - else if (ret_payload == -1) { - flb_pack_state_reset(&conn->pack_state); - flb_pack_state_init(&conn->pack_state); - conn->pack_state.multiple = FLB_TRUE; - return -1; - } - } - else if (ctx->format == FLB_UDP_FMT_NONE) { - ret_payload = parse_payload_none(conn); - if (ret_payload == 0) { - return -1; - } - else if (ret_payload == -1) { - conn->buf_len = 0; - return -1; - } - } - - consume_bytes(conn->buf_data, ret_payload, conn->buf_len); - conn->buf_len -= ret_payload; - conn->buf_data[conn->buf_len] = '\0'; - - if (ctx->format == FLB_UDP_FMT_JSON) { - jsmn_init(&conn->pack_state.parser); - conn->pack_state.tokens_count = 0; - conn->pack_state.last_byte = 0; - conn->pack_state.buf_len = 0; - } - - return bytes; -} - -struct udp_conn *udp_conn_add(struct flb_connection *connection, - struct flb_in_udp_config *ctx) -{ - struct udp_conn *conn; - - conn = flb_malloc(sizeof(struct udp_conn)); - if (!conn) { - flb_errno(); - return NULL; - } - - conn->connection = connection; - - /* Set data for the event-loop */ - - MK_EVENT_NEW(&connection->event); - - connection->user_data = conn; - connection->event.type = FLB_ENGINE_EV_CUSTOM; - connection->event.handler = udp_conn_event; - - /* Connection info */ - conn->ctx = ctx; - conn->buf_len = 0; - - conn->buf_data = flb_malloc(ctx->chunk_size); - if (!conn->buf_data) { - flb_errno(); - - flb_plg_error(ctx->ins, "could not allocate new connection"); - flb_free(conn); - - return NULL; - } - conn->buf_size = ctx->chunk_size; - conn->ins = ctx->ins; - - /* Initialize JSON parser */ - if (ctx->format == FLB_UDP_FMT_JSON) { - flb_pack_state_init(&conn->pack_state); - conn->pack_state.multiple = FLB_TRUE; - } - - return conn; -} - -int udp_conn_del(struct udp_conn *conn) -{ - struct flb_in_udp_config *ctx; - - ctx = conn->ctx; - - if (ctx->format == FLB_UDP_FMT_JSON) { - flb_pack_state_reset(&conn->pack_state); - } - - flb_free(conn->buf_data); - flb_free(conn); - - return 0; -} diff --git a/fluent-bit/plugins/in_udp/udp_conn.h b/fluent-bit/plugins/in_udp/udp_conn.h deleted file mode 100644 index 25b8ef3ef..000000000 --- a/fluent-bit/plugins/in_udp/udp_conn.h +++ /dev/null @@ -1,57 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_UDP_CONN_H -#define FLB_IN_UDP_CONN_H - -#include -#include - -#define FLB_IN_UDP_CHUNK "32768" - -#define FLB_MAP_EXPAND_SUCCESS 0 -#define FLB_MAP_NOT_MODIFIED -1 -#define FLB_MAP_EXPANSION_ERROR -2 -#define FLB_MAP_EXPANSION_INVALID_VALUE_TYPE -3 - -struct udp_conn_stream { - char *tag; - size_t tag_len; -}; - -/* Respresents a connection */ -struct udp_conn { - /* Buffer */ - char *buf_data; /* Buffer data */ - int buf_len; /* Data length */ - int buf_size; /* Buffer size */ - - struct flb_input_instance *ins; /* Parent plugin instance */ - struct flb_in_udp_config *ctx; /* Plugin configuration context */ - struct flb_pack_state pack_state; /* Internal JSON parser */ - struct flb_connection *connection; - - struct mk_list _head; -}; - -struct udp_conn *udp_conn_add(struct flb_connection *connection, struct flb_in_udp_config *ctx); -int udp_conn_del(struct udp_conn *conn); -int udp_conn_event(void *data); - -#endif diff --git a/fluent-bit/plugins/in_unix_socket/CMakeLists.txt b/fluent-bit/plugins/in_unix_socket/CMakeLists.txt deleted file mode 100644 index f07027449..000000000 --- a/fluent-bit/plugins/in_unix_socket/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(src - unix_socket.c - unix_socket_conn.c - unix_socket_config.c) - -FLB_PLUGIN(in_unix_socket "${src}" "") diff --git a/fluent-bit/plugins/in_unix_socket/unix_socket.c b/fluent-bit/plugins/in_unix_socket/unix_socket.c deleted file mode 100644 index 9e7b19110..000000000 --- a/fluent-bit/plugins/in_unix_socket/unix_socket.c +++ /dev/null @@ -1,320 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include - -#include "unix_socket.h" -#include "unix_socket_conn.h" -#include "unix_socket_config.h" - -/* - * For a server event, the collection event means a new client have arrived, we - * accept the connection and create a new UNIX SOCKET instance which will wait for - * JSON map messages. - */ -static int in_unix_socket_collect(struct flb_input_instance *in, - struct flb_config *config, void *in_context) -{ - struct flb_connection *connection; - struct unix_socket_conn *conn; - struct flb_in_unix_socket_config *ctx; - - ctx = in_context; - - connection = flb_downstream_conn_get(ctx->downstream); - - if (connection == NULL) { - flb_plg_error(ctx->ins, "could not accept new connection"); - - return -1; - } - - if (ctx->dgram_mode_flag) { - return unix_socket_conn_event(connection); - } - else { - flb_plg_trace(ctx->ins, "new UNIX SOCKET connection arrived FD=%i", connection->fd); - - conn = unix_socket_conn_add(connection, ctx); - - if (conn == NULL) { - flb_plg_error(ctx->ins, "could not accept new connection"); - - flb_downstream_conn_release(connection); - - return -1; - } - } - - return 0; -} - -static int remove_existing_socket_file(char *socket_path) -{ - struct stat file_data; - int result; - - result = stat(socket_path, &file_data); - - if (result == -1) { - if (errno == ENOENT) { - return 0; - } - - flb_errno(); - - return -1; - } - - if (S_ISSOCK(file_data.st_mode) == 0) { - return -2; - } - - result = unlink(socket_path); - - if (result != 0) { - return -3; - } - - return 0; -} - -/* Initialize plugin */ -static int in_unix_socket_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - struct flb_connection *connection; - int mode; - struct flb_in_unix_socket_config *ctx; - int ret; - struct flb_tls *tls; - - (void) data; - - ctx = unix_socket_config_init(in); - - if (ctx == NULL) { - return -1; - } - - ctx->collector_id = -1; - ctx->ins = in; - - mk_list_init(&ctx->connections); - - /* Set the context */ - flb_input_set_context(in, ctx); - - ret = remove_existing_socket_file(ctx->listen); - - if (ret != 0) { - if (ret == -2) { - flb_plg_error(ctx->ins, - "%s exists and it is not a unix socket. Aborting", - ctx->listen); - } - else { - flb_plg_error(ctx->ins, - "could not remove existing unix socket %s. Aborting", - ctx->listen); - } - - unix_socket_config_destroy(ctx); - - return -1; - } - - mode = FLB_TRANSPORT_UNIX_STREAM; - - if (ctx->socket_mode != NULL && - strcasecmp(ctx->socket_mode, "DGRAM") == 0) { - mode = FLB_TRANSPORT_UNIX_DGRAM; - ctx->dgram_mode_flag = FLB_TRUE; - tls = NULL; - } - else { - tls = in->tls; - } - - ctx->downstream = flb_downstream_create(mode, - in->flags, - ctx->listen, - 0, - tls, - config, - &in->net_setup); - - if (ctx->downstream == NULL) { - flb_plg_error(ctx->ins, - "could not initialize downstream on unix://%s. Aborting", - ctx->listen); - - unix_socket_config_destroy(ctx); - - return -1; - } - - flb_input_downstream_set(ctx->downstream, ctx->ins); - - if (ctx->socket_permissions != NULL) { - ret = chmod(ctx->listen, ctx->socket_acl); - - if (ret != 0) { - flb_errno(); - - flb_plg_error(ctx->ins, "cannot set permission on '%s' to %04o", - ctx->listen, ctx->socket_acl); - - unix_socket_config_destroy(ctx); - - return -1; - } - } - - if (ctx->dgram_mode_flag) { - connection = flb_downstream_conn_get(ctx->downstream); - - if (connection == NULL) { - flb_plg_error(ctx->ins, "could not get DGRAM server dummy " - "connection"); - - unix_socket_config_destroy(ctx); - - return -1; - } - - ctx->dummy_conn = unix_socket_conn_add(connection, ctx); - - if (ctx->dummy_conn == NULL) { - flb_plg_error(ctx->ins, "could not track DGRAM server dummy " - "connection"); - - unix_socket_config_destroy(ctx); - - return -1; - } - } - - /* Collect upon data available on the standard input */ - ret = flb_input_set_collector_socket(in, - in_unix_socket_collect, - ctx->downstream->server_fd, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "Could not set collector for IN_UNIX_SOCKET " - "input plugin"); - - unix_socket_config_destroy(ctx); - - return -1; - } - - ctx->collector_id = ret; - ctx->collector_event = flb_input_collector_get_event(ret, in); - - if (ret == -1) { - flb_plg_error(ctx->ins, "Could not get collector event"); - - unix_socket_config_destroy(ctx); - - return -1; - } - - return 0; -} - -static int in_unix_socket_exit(void *data, struct flb_config *config) -{ - struct mk_list *head; - struct unix_socket_conn *conn; - struct flb_in_unix_socket_config *ctx; - struct mk_list *tmp; - - (void) *config; - - ctx = data; - - mk_list_foreach_safe(head, tmp, &ctx->connections) { - conn = mk_list_entry(head, struct unix_socket_conn, _head); - - unix_socket_conn_del(conn); - } - - unix_socket_config_destroy(ctx); - - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "socket_mode", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_in_unix_socket_config, socket_mode), - "Unix socket mode : STREAM or DGRAM" - }, - { - FLB_CONFIG_MAP_STR, "socket_path", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_in_unix_socket_config, listen), - "Unix socket path" - }, - { - FLB_CONFIG_MAP_STR, "socket_permissions", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_in_unix_socket_config, socket_permissions), - "Set the permissions for the UNIX socket" - }, - { - FLB_CONFIG_MAP_STR, "format", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_in_unix_socket_config, format_name), - "Set the format: json or none" - }, - { - FLB_CONFIG_MAP_STR, "separator", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_in_unix_socket_config, raw_separator), - "Set separator" - }, - { - FLB_CONFIG_MAP_STR, "chunk_size", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_in_unix_socket_config, chunk_size_str), - "Set the chunk size" - }, - { - FLB_CONFIG_MAP_STR, "buffer_size", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_in_unix_socket_config, buffer_size_str), - "Set the buffer size" - }, - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_input_plugin in_unix_socket_plugin = { - .name = "unix_socket", - .description = "UNIX_SOCKET", - .cb_init = in_unix_socket_init, - .cb_pre_run = NULL, - .cb_collect = in_unix_socket_collect, - .cb_flush_buf = NULL, - .cb_exit = in_unix_socket_exit, - .config_map = config_map, - .flags = FLB_INPUT_NET_SERVER | FLB_IO_OPT_TLS -}; diff --git a/fluent-bit/plugins/in_unix_socket/unix_socket.h b/fluent-bit/plugins/in_unix_socket/unix_socket.h deleted file mode 100644 index 08642f4fa..000000000 --- a/fluent-bit/plugins/in_unix_socket/unix_socket.h +++ /dev/null @@ -1,55 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_UNIX_SOCKET_H -#define FLB_IN_UNIX_SOCKET_H - -#define FLB_UNIX_SOCKET_FMT_JSON 0 /* default */ -#define FLB_UNIX_SOCKET_FMT_NONE 1 /* no format, use delimiters */ - -#include -#include -#include -#include -#include - -struct flb_in_unix_socket_config { - int dgram_mode_flag; /* Stateless mode flag (UDP alike) */ - struct mk_event *collector_event; - flb_sds_t format_name; /* Data format name */ - int format; /* Data format */ - size_t buffer_size; /* Buffer size for each reader */ - flb_sds_t buffer_size_str; /* Buffer size in string form */ - size_t chunk_size; /* Chunk allocation size */ - flb_sds_t chunk_size_str; /* Chunk size in string form */ - char *listen; /* Unix socket path */ - char *socket_permissions; /* Unix socket ACL as string */ - flb_sds_t socket_mode; /* Unix socket mode (STREAM or DGRAM) */ - int socket_acl; /* Unix socket ACL */ - flb_sds_t raw_separator; /* Unescaped string delimiterr */ - flb_sds_t separator; /* String delimiter */ - int collector_id; /* Listener collector id */ - struct flb_downstream *downstream; /* Client manager */ - struct unix_socket_conn *dummy_conn;/* Datagram dummy connection */ - struct mk_list connections; /* List of active connections */ - struct flb_input_instance *ins; /* Input plugin instace */ - struct flb_log_event_encoder *log_encoder; -}; - -#endif diff --git a/fluent-bit/plugins/in_unix_socket/unix_socket_config.c b/fluent-bit/plugins/in_unix_socket/unix_socket_config.c deleted file mode 100644 index e14d8aa1f..000000000 --- a/fluent-bit/plugins/in_unix_socket/unix_socket_config.c +++ /dev/null @@ -1,153 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include "unix_socket.h" -#include "unix_socket_conn.h" -#include "unix_socket_config.h" - -#include - -struct flb_in_unix_socket_config *unix_socket_config_init(struct flb_input_instance *ins) -{ - int ret; - int len; - char *out; - struct flb_in_unix_socket_config *ctx; - - /* Allocate plugin context */ - ctx = flb_calloc(1, sizeof(struct flb_in_unix_socket_config)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - ctx->format = FLB_UNIX_SOCKET_FMT_JSON; - - /* Load the config map */ - ret = flb_input_config_map_set(ins, (void *)ctx); - if (ret == -1) { - flb_plg_error(ins, "unable to load configuration"); - flb_free(ctx); - return NULL; - } - - if (ctx->socket_permissions != NULL) { - ctx->socket_acl = strtol(ctx->socket_permissions, NULL, 8); - ctx->socket_acl &= 07777; - } - - - /* Data format (expected payload) */ - if (ctx->format_name) { - if (strcasecmp(ctx->format_name, "json") == 0) { - ctx->format = FLB_UNIX_SOCKET_FMT_JSON; - } - else if (strcasecmp(ctx->format_name, "none") == 0) { - ctx->format = FLB_UNIX_SOCKET_FMT_NONE; - } - else { - flb_plg_error(ctx->ins, "unrecognized format value '%s'", ctx->format_name); - flb_free(ctx); - return NULL; - } - } - - /* String separator used to split records when using 'format none' */ - if (ctx->raw_separator) { - len = strlen(ctx->raw_separator); - out = flb_malloc(len + 1); - if (!out) { - flb_errno(); - flb_free(ctx); - return NULL; - } - ret = flb_unescape_string(ctx->raw_separator, len, &out); - if (ret <= 0) { - flb_plg_error(ctx->ins, "invalid separator"); - flb_free(out); - flb_free(ctx); - return NULL; - } - - ctx->separator = flb_sds_create_len(out, ret); - if (!ctx->separator) { - flb_free(out); - flb_free(ctx); - return NULL; - } - flb_free(out); - } - if (!ctx->separator) { - ctx->separator = flb_sds_create_len("\n", 1); - } - - /* Chunk size */ - if (ctx->chunk_size_str) { - /* Convert KB unit to Bytes */ - ctx->chunk_size = (atoi(ctx->chunk_size_str) * 1024); - } else { - ctx->chunk_size = atoi(FLB_IN_UNIX_SOCKET_CHUNK); - } - - /* Buffer size */ - if (!ctx->buffer_size_str) { - ctx->buffer_size = ctx->chunk_size; - } - else { - /* Convert KB unit to Bytes */ - ctx->buffer_size = (atoi(ctx->buffer_size_str) * 1024); - } - - ctx->log_encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ctx->log_encoder == NULL) { - flb_plg_error(ctx->ins, "could not initialize event encoder"); - unix_socket_config_destroy(ctx); - - ctx = NULL; - } - - return ctx; -} - -int unix_socket_config_destroy(struct flb_in_unix_socket_config *ctx) -{ - if (ctx->log_encoder != NULL) { - flb_log_event_encoder_destroy(ctx->log_encoder); - } - - if (ctx->collector_id != -1) { - flb_input_collector_delete(ctx->collector_id, ctx->ins); - - ctx->collector_id = -1; - } - - if (ctx->downstream != NULL) { - flb_downstream_destroy(ctx->downstream); - } - - flb_sds_destroy(ctx->separator); - flb_free(ctx); - - return 0; -} diff --git a/fluent-bit/plugins/in_unix_socket/unix_socket_config.h b/fluent-bit/plugins/in_unix_socket/unix_socket_config.h deleted file mode 100644 index 93e07075f..000000000 --- a/fluent-bit/plugins/in_unix_socket/unix_socket_config.h +++ /dev/null @@ -1,28 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_UNIX_SOCKET_CONFIG_H -#define FLB_IN_UNIX_SOCKET_CONFIG_H - -#include "unix_socket.h" - -struct flb_in_unix_socket_config *unix_socket_config_init(struct flb_input_instance *i_ins); -int unix_socket_config_destroy(struct flb_in_unix_socket_config *config); - -#endif diff --git a/fluent-bit/plugins/in_unix_socket/unix_socket_conn.c b/fluent-bit/plugins/in_unix_socket/unix_socket_conn.c deleted file mode 100644 index 472cbbeb2..000000000 --- a/fluent-bit/plugins/in_unix_socket/unix_socket_conn.c +++ /dev/null @@ -1,433 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include - -#include "unix_socket.h" -#include "unix_socket_conn.h" - -static inline void consume_bytes(char *buf, int bytes, int length) -{ - memmove(buf, buf + bytes, length - bytes); -} - -static inline int process_pack(struct unix_socket_conn *conn, - char *pack, size_t size) -{ - int ret; - size_t off = 0; - msgpack_unpacked result; - msgpack_object entry; - struct flb_in_unix_socket_config *ctx; - - ctx = (struct flb_in_unix_socket_config *) conn->ctx; - - flb_log_event_encoder_reset(ctx->log_encoder); - - /* First pack the results, iterate concatenated messages */ - msgpack_unpacked_init(&result); - while (msgpack_unpack_next(&result, pack, size, &off) == MSGPACK_UNPACK_SUCCESS) { - entry = result.data; - - entry = result.data; - - ret = flb_log_event_encoder_begin_record(ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp(ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - if (entry.type == MSGPACK_OBJECT_MAP) { - ret = flb_log_event_encoder_set_body_from_msgpack_object( - ctx->log_encoder, &entry); - } - else if (entry.type == MSGPACK_OBJECT_ARRAY) { - ret = flb_log_event_encoder_append_body_values( - ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE("msg"), - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&entry)); - } - else { - ret = FLB_EVENT_ENCODER_ERROR_INVALID_VALUE_TYPE; - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(ctx->log_encoder); - } - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - break; - } - } - } - - msgpack_unpacked_destroy(&result); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(conn->ins, NULL, 0, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); - ret = 0; - } - else { - flb_plg_error(ctx->ins, "log event encoding error : %d", ret); - - ret = -1; - } - - return ret; -} - -/* Process a JSON payload, return the number of processed bytes */ -static ssize_t parse_payload_json(struct unix_socket_conn *conn) -{ - int ret; - int out_size; - char *pack; - - ret = flb_pack_json_state(conn->buf_data, conn->buf_len, - &pack, &out_size, &conn->pack_state); - if (ret == FLB_ERR_JSON_PART) { - flb_plg_debug(conn->ins, "JSON incomplete, waiting for more data..."); - return 0; - } - else if (ret == FLB_ERR_JSON_INVAL) { - flb_plg_warn(conn->ins, "invalid JSON message, skipping"); - conn->buf_len = 0; - conn->pack_state.multiple = FLB_TRUE; - return -1; - } - else if (ret == -1) { - return -1; - } - - /* Process the packaged JSON and return the last byte used */ - process_pack(conn, pack, out_size); - flb_free(pack); - - return conn->pack_state.last_byte; -} - -/* - * Process a raw text payload, uses the delimited character to split records, - * return the number of processed bytes - */ -static ssize_t parse_payload_none(struct unix_socket_conn *conn) -{ - int ret; - int len; - int sep_len; - size_t consumed = 0; - char *buf; - char *s; - char *separator; - struct flb_in_unix_socket_config *ctx; - - ctx = (struct flb_in_unix_socket_config *) conn->ctx; - - separator = conn->ctx->separator; - sep_len = flb_sds_len(conn->ctx->separator); - - buf = conn->buf_data; - - flb_log_event_encoder_reset(ctx->log_encoder); - - while ((s = strstr(buf, separator))) { - len = (s - buf); - if (len == 0) { - break; - } - else if (len > 0) { - ret = flb_log_event_encoder_begin_record(ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp(ctx->log_encoder); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_body_values( - ctx->log_encoder, - FLB_LOG_EVENT_CSTRING_VALUE("log"), - FLB_LOG_EVENT_STRING_VALUE(buf, len)); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(ctx->log_encoder); - } - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - break; - } - - consumed += len + 1; - buf += len + sep_len; - } - else { - break; - } - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(conn->ins, NULL, 0, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); - } - else { - flb_plg_error(ctx->ins, "log event encoding error : %d", ret); - } - - return consumed; -} - -/* Callback invoked every time an event is triggered for a connection */ -int unix_socket_conn_event(void *data) -{ - int bytes; - int available; - int size; - ssize_t ret_payload = -1; - char *tmp; - struct mk_event *event; - struct unix_socket_conn *conn; - struct flb_connection *connection; - struct flb_in_unix_socket_config *ctx; - - connection = (struct flb_connection *) data; - - conn = connection->user_data; - - ctx = conn->ctx; - - if (ctx->dgram_mode_flag) { - if (ctx->format == FLB_UNIX_SOCKET_FMT_JSON && - conn->buf_len > 0) { - flb_pack_state_reset(&conn->pack_state); - flb_pack_state_init(&conn->pack_state); - - conn->pack_state.multiple = FLB_TRUE; - } - - event = ctx->collector_event; - } - else { - event = &connection->event; - } - - if (event->mask & MK_EVENT_READ) { - available = (conn->buf_size - conn->buf_len) - 1; - if (available < 1) { - if (conn->buf_size + ctx->chunk_size > ctx->buffer_size) { - flb_plg_trace(ctx->ins, - "fd=%i incoming data exceed limit (%zu KB)", - event->fd, (ctx->buffer_size / 1024)); - - if (!ctx->dgram_mode_flag) { - unix_socket_conn_del(conn); - } - - return -1; - } - - size = conn->buf_size + ctx->chunk_size; - tmp = flb_realloc(conn->buf_data, size); - if (!tmp) { - flb_errno(); - return -1; - } - flb_plg_trace(ctx->ins, "fd=%i buffer realloc %i -> %i", - event->fd, conn->buf_size, size); - - conn->buf_data = tmp; - conn->buf_size = size; - available = (conn->buf_size - conn->buf_len) - 1; - } - - /* Read data */ - bytes = flb_io_net_read(connection, - (void *) &conn->buf_data[conn->buf_len], - available); - - if (bytes <= 0) { - if (!ctx->dgram_mode_flag) { - flb_plg_trace(ctx->ins, "fd=%i closed connection", event->fd); - unix_socket_conn_del(conn); - } - - return -1; - } - - flb_plg_trace(ctx->ins, "read()=%i pre_len=%i now_len=%i", - bytes, conn->buf_len, conn->buf_len + bytes); - conn->buf_len += bytes; - conn->buf_data[conn->buf_len] = '\0'; - - /* Strip CR or LF if found at first byte */ - if (conn->buf_data[0] == '\r' || conn->buf_data[0] == '\n') { - /* Skip message with one byte with CR or LF */ - flb_plg_trace(ctx->ins, "skip one byte message with ASCII code=%i", - conn->buf_data[0]); - consume_bytes(conn->buf_data, 1, conn->buf_len); - conn->buf_len--; - conn->buf_data[conn->buf_len] = '\0'; - } - - /* JSON Format handler */ - if (ctx->format == FLB_UNIX_SOCKET_FMT_JSON) { - ret_payload = parse_payload_json(conn); - if (ret_payload == 0) { - /* Incomplete JSON message, we need more data */ - return -1; - } - else if (ret_payload == -1) { - flb_pack_state_reset(&conn->pack_state); - flb_pack_state_init(&conn->pack_state); - conn->pack_state.multiple = FLB_TRUE; - return -1; - } - } - else if (ctx->format == FLB_UNIX_SOCKET_FMT_NONE) { - ret_payload = parse_payload_none(conn); - if (ret_payload == 0) { - return -1; - } - else if (ret_payload == -1) { - conn->buf_len = 0; - return -1; - } - } - - - consume_bytes(conn->buf_data, ret_payload, conn->buf_len); - conn->buf_len -= ret_payload; - conn->buf_data[conn->buf_len] = '\0'; - - if (ctx->format == FLB_UNIX_SOCKET_FMT_JSON) { - jsmn_init(&conn->pack_state.parser); - conn->pack_state.tokens_count = 0; - conn->pack_state.last_byte = 0; - conn->pack_state.buf_len = 0; - } - - return bytes; - } - - if (event->mask & MK_EVENT_CLOSE) { - flb_plg_trace(ctx->ins, "fd=%i hangup", event->fd); - unix_socket_conn_del(conn); - return -1; - } - - return 0; -} - -/* Create a new mqtt request instance */ -struct unix_socket_conn *unix_socket_conn_add(struct flb_connection *connection, - struct flb_in_unix_socket_config *ctx) -{ - struct unix_socket_conn *conn; - int ret; - - conn = flb_malloc(sizeof(struct unix_socket_conn)); - if (!conn) { - flb_errno(); - return NULL; - } - - conn->connection = connection; - - /* Set data for the event-loop */ - MK_EVENT_NEW(&connection->event); - - connection->user_data = conn; - connection->event.type = FLB_ENGINE_EV_CUSTOM; - connection->event.handler = unix_socket_conn_event; - - /* Connection info */ - conn->ctx = ctx; - conn->buf_len = 0; - conn->rest = 0; - conn->status = UNIX_SOCKET_NEW; - - conn->buf_data = flb_malloc(ctx->chunk_size); - if (!conn->buf_data) { - flb_errno(); - - flb_plg_error(ctx->ins, "could not allocate new connection"); - flb_free(conn); - - return NULL; - } - conn->buf_size = ctx->chunk_size; - conn->ins = ctx->ins; - - /* Initialize JSON parser */ - if (ctx->format == FLB_UNIX_SOCKET_FMT_JSON) { - flb_pack_state_init(&conn->pack_state); - conn->pack_state.multiple = FLB_TRUE; - } - - /* Register instance into the event loop */ - ret = mk_event_add(flb_engine_evl_get(), - connection->fd, - FLB_ENGINE_EV_CUSTOM, - MK_EVENT_READ, - &connection->event); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not register new connection"); - - flb_free(conn->buf_data); - flb_free(conn); - - return NULL; - } - - mk_list_add(&conn->_head, &ctx->connections); - - return conn; -} - -int unix_socket_conn_del(struct unix_socket_conn *conn) -{ - struct flb_in_unix_socket_config *ctx; - - ctx = conn->ctx; - - if (ctx->format == FLB_UNIX_SOCKET_FMT_JSON) { - flb_pack_state_reset(&conn->pack_state); - } - - /* The downstream unregisters the file descriptor from the event-loop - * so there's nothing to be done by the plugin - */ - flb_downstream_conn_release(conn->connection); - - /* Release resources */ - mk_list_del(&conn->_head); - - flb_free(conn->buf_data); - flb_free(conn); - - return 0; -} diff --git a/fluent-bit/plugins/in_unix_socket/unix_socket_conn.h b/fluent-bit/plugins/in_unix_socket/unix_socket_conn.h deleted file mode 100644 index d46694b46..000000000 --- a/fluent-bit/plugins/in_unix_socket/unix_socket_conn.h +++ /dev/null @@ -1,60 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_UNIX_SOCKET_CONN_H -#define FLB_IN_UNIX_SOCKET_CONN_H - -#include -#include - -#define FLB_IN_UNIX_SOCKET_CHUNK "32768" - -enum { - UNIX_SOCKET_NEW = 1, /* it's a new connection */ - UNIX_SOCKET_CONNECTED = 2, /* MQTT connection per protocol spec OK */ -}; - -struct unix_socket_conn_stream { - char *tag; - size_t tag_len; -}; - -/* Respresents a connection */ -struct unix_socket_conn { - int status; /* Connection status */ - - /* Buffer */ - char *buf_data; /* Buffer data */ - int buf_len; /* Data length */ - int buf_size; /* Buffer size */ - size_t rest; /* Unpacking offset */ - - struct flb_input_instance *ins; /* Parent plugin instance */ - struct flb_in_unix_socket_config *ctx; /* Plugin configuration context */ - struct flb_pack_state pack_state; /* Internal JSON parser */ - struct flb_connection *connection; - - struct mk_list _head; -}; - -struct unix_socket_conn *unix_socket_conn_add(struct flb_connection *connection, struct flb_in_unix_socket_config *ctx); -int unix_socket_conn_del(struct unix_socket_conn *conn); -int unix_socket_conn_event(void *data); - -#endif diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/CMakeLists.txt b/fluent-bit/plugins/in_windows_exporter_metrics/CMakeLists.txt deleted file mode 100644 index 8cc7fe70a..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/CMakeLists.txt +++ /dev/null @@ -1,28 +0,0 @@ -set(src - we_config.c - we.c - we_cpu.c - we_os.c - we_net.c - we_logical_disk.c - we_cs.c - we_wmi.c - we_util.c - we_metric.c - we_perflib.c - we_wmi_thermalzone.c - we_wmi_cpu_info.c - we_wmi_logon.c - we_wmi_system.c - we_wmi_service.c - we_wmi_memory.c - we_wmi_paging_file.c - we_wmi_process.c - ) - -set(libs - wbemuuid - netapi32 -) - -FLB_PLUGIN(in_windows_exporter_metrics "${src}" "${libs}") diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we.c b/fluent-bit/plugins/in_windows_exporter_metrics/we.c deleted file mode 100644 index 0f99fb83f..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we.c +++ /dev/null @@ -1,1144 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2021 The Fluent Bit Authors - * Copyright (C) 2015-2018 Treasure Data Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "we.h" -#include "we_wmi.h" -#include "we_config.h" - -/* collectors */ -#include "we_cpu.h" -#include "we_os.h" -#include "we_net.h" -#include "we_logical_disk.h" -#include "we_cs.h" - -/* wmi collectors */ -#include "we_wmi_cpu_info.h" -#include "we_wmi_logon.h" -#include "we_wmi_system.h" -#include "we_wmi_thermalzone.h" -#include "we_wmi_service.h" -#include "we_wmi_memory.h" -#include "we_wmi_paging_file.h" -#include "we_wmi_process.h" - -static int we_timer_cpu_metrics_cb(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_ne *ctx = in_context; - - we_cpu_update(ctx); - - return 0; -} - -static int we_timer_os_metrics_cb(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_ne *ctx = in_context; - - we_os_update(ctx); - - return 0; -} - -static int we_timer_net_metrics_cb(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_ne *ctx = in_context; - - we_net_update(ctx); - - return 0; -} - -static int we_timer_logical_disk_metrics_cb(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_ne *ctx = in_context; - - we_logical_disk_update(ctx); - - return 0; -} - -static int we_timer_cs_metrics_cb(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_ne *ctx = in_context; - - we_cs_update(ctx); - - return 0; -} - -static int we_timer_wmi_thermalzone_metrics_cb(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_ne *ctx = in_context; - - we_wmi_thermalzone_update(ctx); - - return 0; -} - -static int we_timer_wmi_cpu_info_metrics_cb(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_ne *ctx = in_context; - - we_wmi_cpu_info_update(ctx); - - return 0; -} - -static int we_timer_wmi_logon_metrics_cb(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_ne *ctx = in_context; - - we_wmi_logon_update(ctx); - - return 0; -} - -static int we_timer_wmi_system_metrics_cb(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_ne *ctx = in_context; - - we_wmi_system_update(ctx); - - return 0; -} - -static int we_timer_wmi_service_metrics_cb(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_ne *ctx = in_context; - - we_wmi_service_update(ctx); - - return 0; -} - -static int we_timer_wmi_memory_metrics_cb(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_ne *ctx = in_context; - - we_wmi_memory_update(ctx); - - return 0; -} - -static int we_timer_wmi_paging_file_metrics_cb(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_ne *ctx = in_context; - - we_wmi_paging_file_update(ctx); - - return 0; -} - -static int we_timer_wmi_process_metrics_cb(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_ne *ctx = in_context; - - we_wmi_process_update(ctx); - - return 0; -} - -struct flb_we_callback { - char *name; - void (*func)(char *, void *, void *); -}; - -static int we_update_cb(struct flb_we *ctx, char *name); - -static void update_metrics(struct flb_input_instance *ins, struct flb_we *ctx) -{ - int ret; - struct mk_list *head; - struct flb_slist_entry *entry; - - /* Update our metrics */ - if (ctx->metrics) { - mk_list_foreach(head, ctx->metrics) { - entry = mk_list_entry(head, struct flb_slist_entry, _head); - ret = flb_callback_exists(ctx->callback, entry->str); - if (ret == FLB_TRUE) { - we_update_cb(ctx, entry->str); - } - else { - flb_plg_debug(ctx->ins, "Callback for metrics '%s' is not registered", entry->str); - } - } - } -} - -/* - * Update the metrics, this function is invoked every time 'scrape_interval' - * expires. - */ -static int cb_we_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - int ret; - struct flb_we *ctx; - - ctx = in_context; - - update_metrics(ins, ctx); - - /* Append the updated metrics */ - ret = flb_input_metrics_append(ins, NULL, 0, ctx->cmt); - - if (ret) { - flb_plg_error(ins, "could not append metrics"); - } - - return 0; -} - -static void we_cpu_update_cb(char *name, void *p1, void *p2) -{ - struct flb_we *ctx = p1; - - we_cpu_update(ctx); -} - -static void we_os_update_cb(char *name, void *p1, void *p2) -{ - struct flb_we *ctx = p1; - - we_os_update(ctx); -} - -static void we_net_update_cb(char *name, void *p1, void *p2) -{ - struct flb_we *ctx = p1; - - we_net_update(ctx); -} - -static void we_logical_disk_update_cb(char *name, void *p1, void *p2) -{ - struct flb_we *ctx = p1; - - we_logical_disk_update(ctx); -} - -static void we_cs_update_cb(char *name, void *p1, void *p2) -{ - struct flb_we *ctx = p1; - - we_cs_update(ctx); -} - -static void we_wmi_thermalzone_update_cb(char *name, void *p1, void *p2) -{ - struct flb_we *ctx = p1; - - we_wmi_thermalzone_update(ctx); -} - -static void we_wmi_cpu_info_update_cb(char *name, void *p1, void *p2) -{ - struct flb_we *ctx = p1; - - we_wmi_cpu_info_update(ctx); -} - -static void we_wmi_logon_update_cb(char *name, void *p1, void *p2) -{ - struct flb_we *ctx = p1; - - we_wmi_logon_update(ctx); -} - -static void we_wmi_system_update_cb(char *name, void *p1, void *p2) -{ - struct flb_we *ctx = p1; - - we_wmi_system_update(ctx); -} - -static void we_wmi_service_update_cb(char *name, void *p1, void *p2) -{ - struct flb_we *ctx = p1; - - we_wmi_service_update(ctx); -} - -static void we_wmi_memory_update_cb(char *name, void *p1, void *p2) -{ - struct flb_we *ctx = p1; - - we_wmi_memory_update(ctx); -} - -static void we_wmi_paging_file_update_cb(char *name, void *p1, void *p2) -{ - struct flb_we *ctx = p1; - - we_wmi_paging_file_update(ctx); -} - -static void we_wmi_process_update_cb(char *name, void *p1, void *p2) -{ - struct flb_we *ctx = p1; - - we_wmi_process_update(ctx); -} - -static int we_update_cb(struct flb_we *ctx, char *name) -{ - int ret; - - ret = flb_callback_do(ctx->callback, name, ctx, NULL); - return ret; -} - -/* - * Callbacks Table - */ -struct flb_we_callback ne_callbacks[] = { - /* metrics */ - { "cpu_info", we_wmi_cpu_info_update_cb }, - { "cpu", we_cpu_update_cb }, - { "os", we_os_update_cb }, - { "net", we_net_update_cb }, - { "logical_disk", we_logical_disk_update_cb }, - { "cs", we_cs_update_cb }, - { "thermalzone", we_wmi_thermalzone_update_cb }, - { "logon", we_wmi_logon_update_cb }, - { "system", we_wmi_system_update_cb }, - { "service", we_wmi_service_update_cb }, - { "memory", we_wmi_memory_update_cb }, - { "paging_file", we_wmi_paging_file_update_cb }, - { "process", we_wmi_process_update_cb }, - { 0 } -}; - -static int in_we_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - int ret; - int metric_idx = -1; - struct flb_we *ctx; - double windows_version = 0.0; - struct mk_list *head; - struct flb_slist_entry *entry; - struct flb_we_callback *cb; - - /* Create plugin context */ - ctx = flb_we_config_create(in, config); - - if (ctx == NULL) { - flb_errno(); - - return -1; - } - - /* Initialize fds */ - ctx->coll_cpu_fd = -1; - ctx->coll_net_fd = -1; - ctx->coll_logical_disk_fd = -1; - ctx->coll_cs_fd = -1; - ctx->coll_os_fd = -1; - ctx->coll_wmi_thermalzone_fd = -1; - ctx->coll_wmi_cpu_info_fd = -1; - ctx->coll_wmi_logon_fd = -1; - ctx->coll_wmi_system_fd = -1; - ctx->coll_wmi_service_fd = -1; - ctx->coll_wmi_memory_fd = -1; - ctx->coll_wmi_paging_file_fd = -1; - ctx->coll_wmi_process_fd = -1; - - ctx->callback = flb_callback_create(in->name); - if (!ctx->callback) { - flb_plg_error(ctx->ins, "Create callback failed"); - return -1; - } - - /* Associate context with the instance */ - flb_input_set_context(in, ctx); - - ret = we_get_windows_version(&windows_version); - - if (ret == FLB_FALSE) { - flb_plg_error(in, "could not get windows version"); - - return -1; - } - ctx->windows_version = windows_version; - - ret = we_perflib_init(ctx); - - if (ret) { - flb_plg_error(in, "could not initialize PERFLIB"); - return -1; - } - - ret = we_wmi_init(ctx); - - if (ret) { - flb_plg_error(in, "could not initialize WMI"); - - return -1; - } - - /* Create the collector */ - ret = flb_input_set_collector_time(in, - cb_we_collect, - ctx->scrape_interval, 0, - config); - - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set collector for " - "Windows Exporter Metrics plugin"); - return -1; - } - - ctx->coll_fd = ret; - - /* Check and initialize enabled metrics */ - if (ctx->metrics) { - mk_list_foreach(head, ctx->metrics) { - entry = mk_list_entry(head, struct flb_slist_entry, _head); - ret = flb_callback_exists(ctx->callback, entry->str); - - if (ret == FLB_FALSE) { - if (strncmp(entry->str, "cpu_info", 8) == 0) { - if (ctx->wmi_cpu_info_scrape_interval == 0) { - flb_plg_debug(ctx->ins, "enabled metrics %s", entry->str); - metric_idx = 0; - } - else { - /* Create the cpu_info collector */ - ret = flb_input_set_collector_time(in, - we_timer_wmi_cpu_info_metrics_cb, - ctx->wmi_cpu_info_scrape_interval, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set cpu_info collector for Windows Exporter Metrics plugin"); - return -1; - } - ctx->coll_wmi_cpu_info_fd = ret; - } - - /* Initialize cpu info metric collectors */ - ret = we_wmi_cpu_info_init(ctx); - if (ret == -1) { - return -1; - } - } - else if (strncmp(entry->str, "cpu", 3) == 0) { - if (ctx->cpu_scrape_interval == 0) { - flb_plg_debug(ctx->ins, "enabled metrics %s", entry->str); - metric_idx = 1; - } - else { - /* Create the cpu collector */ - ret = flb_input_set_collector_time(in, - we_timer_cpu_metrics_cb, - ctx->cpu_scrape_interval, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set cpu collector for Windows Exporter Metrics plugin"); - return -1; - } - ctx->coll_cpu_fd = ret; - } - - /* Initialize cpu metric collectors */ - ret = we_cpu_init(ctx); - if (ret < 0) { - return -1; - } - } - else if (strncmp(entry->str, "os", 2) == 0) { - if (ctx->os_scrape_interval == 0) { - flb_plg_debug(ctx->ins, "enabled metrics %s", entry->str); - metric_idx = 2; - } else { - /* Create the os collector */ - ret = flb_input_set_collector_time(in, - we_timer_os_metrics_cb, - ctx->os_scrape_interval, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set os collector for Windows Exporter Metrics plugin"); - return -1; - } - ctx->coll_os_fd = ret; - } - - /* Initialize os metric collectors */ - ret = we_os_init(ctx); - if (ret) { - return -1; - } - } - else if (strncmp(entry->str, "net", 3) == 0) { - if (ctx->net_scrape_interval == 0) { - flb_plg_debug(ctx->ins, "enabled metrics %s", entry->str); - metric_idx = 3; - } - else { - /* Create the net collector */ - ret = flb_input_set_collector_time(in, - we_timer_net_metrics_cb, - ctx->net_scrape_interval, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set net collector for Windows Exporter Metrics plugin"); - return -1; - } - ctx->coll_net_fd = ret; - } - - /* Initialize net metric collectors */ - ret = we_net_init(ctx); - if (ret) { - return -1; - } - } - else if (strncmp(entry->str, "logical_disk", 12) == 0) { - if (ctx->logical_disk_scrape_interval == 0) { - flb_plg_debug(ctx->ins, "enabled metrics %s", entry->str); - metric_idx = 4; - } - else { - /* Create the logical_disk collector */ - ret = flb_input_set_collector_time(in, - we_timer_logical_disk_metrics_cb, - ctx->logical_disk_scrape_interval, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set logical_disk collector for Windows Exporter Metrics plugin"); - return -1; - } - ctx->coll_logical_disk_fd = ret; - } - - /* Initialize logical_disk metric collectors */ - ret = we_logical_disk_init(ctx); - if (ret) { - return -1; - } - } - else if (strncmp(entry->str, "cs", 2) == 0) { - if (ctx->cs_scrape_interval == 0) { - flb_plg_debug(ctx->ins, "enabled metrics %s", entry->str); - metric_idx = 5; - } - else { - /* Create the logical_disk collector */ - ret = flb_input_set_collector_time(in, - we_timer_cs_metrics_cb, - ctx->cs_scrape_interval, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set cs collector for Windows Exporter Metrics plugin"); - return -1; - } - ctx->coll_cs_fd = ret; - } - - /* Initialize cs metric collectors */ - ret = we_cs_init(ctx); - if (ret) { - return -1; - } - } - else if (strncmp(entry->str, "thermalzone", 11) == 0) { - if (ctx->wmi_thermalzone_scrape_interval == 0) { - flb_plg_debug(ctx->ins, "enabled metrics %s", entry->str); - metric_idx = 6; - } - else { - /* Create the thermalzone collector */ - ret = flb_input_set_collector_time(in, - we_timer_wmi_thermalzone_metrics_cb, - ctx->wmi_thermalzone_scrape_interval, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set thermalzone collector for Windows Exporter Metrics plugin"); - return -1; - } - ctx->coll_wmi_thermalzone_fd = ret; - } - - /* Initialize thermalzone metric collectors */ - ret = we_wmi_thermalzone_init(ctx); - if (ret) { - return -1; - } - } - else if (strncmp(entry->str, "logon", 5) == 0) { - if (ctx->wmi_logon_scrape_interval == 0) { - flb_plg_debug(ctx->ins, "enabled metrics %s", entry->str); - metric_idx = 7; - } - else { - /* Create the logon collector */ - ret = flb_input_set_collector_time(in, - we_timer_wmi_logon_metrics_cb, - ctx->wmi_logon_scrape_interval, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set thermalzone collector for Windows Exporter Metrics plugin"); - return -1; - } - ctx->coll_wmi_logon_fd = ret; - } - - /* Initialize logon metric collectors */ - ret = we_wmi_logon_init(ctx); - if (ret) { - return -1; - } - } - else if (strncmp(entry->str, "system", 6) == 0) { - if (ctx->wmi_logon_scrape_interval == 0) { - flb_plg_debug(ctx->ins, "enabled metrics %s", entry->str); - metric_idx = 8; - } - else { - /* Create the logon collector */ - ret = flb_input_set_collector_time(in, - we_timer_wmi_system_metrics_cb, - ctx->wmi_system_scrape_interval, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set system collector for Windows Exporter Metrics plugin"); - return -1; - } - ctx->coll_wmi_system_fd = ret; - } - - /* Initialize system metric collectors */ - ret = we_wmi_system_init(ctx); - if (ret) { - return -1; - } - } - else if (strncmp(entry->str, "service", 7) == 0) { - if (ctx->wmi_service_scrape_interval == 0) { - flb_plg_debug(ctx->ins, "enabled metrics %s", entry->str); - metric_idx = 9; - } - else { - /* Create the service collector */ - ret = flb_input_set_collector_time(in, - we_timer_wmi_service_metrics_cb, - ctx->wmi_service_scrape_interval, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set service collector for Windows Exporter Metrics plugin"); - return -1; - } - ctx->coll_wmi_service_fd = ret; - } - - /* Initialize service metric collectors */ - ret = we_wmi_service_init(ctx); - if (ret) { - return -1; - } - } - else if (strncmp(entry->str, "memory", 6) == 0) { - if (ctx->wmi_memory_scrape_interval == 0) { - flb_plg_debug(ctx->ins, "enabled metrics %s", entry->str); - metric_idx = 10; - } - else { - /* Create the memory collector */ - ret = flb_input_set_collector_time(in, - we_timer_wmi_memory_metrics_cb, - ctx->wmi_memory_scrape_interval, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set memory collector for Windows Exporter Metrics plugin"); - return -1; - } - ctx->coll_wmi_memory_fd = ret; - } - - /* Initialize memory metric collectors */ - ret = we_wmi_memory_init(ctx); - if (ret) { - return -1; - } - } - else if (strncmp(entry->str, "paging_file", 11) == 0) { - if (ctx->wmi_paging_file_scrape_interval == 0) { - flb_plg_debug(ctx->ins, "enabled metrics %s", entry->str); - metric_idx = 11; - } - else { - /* Create the paging_file collector */ - ret = flb_input_set_collector_time(in, - we_timer_wmi_paging_file_metrics_cb, - ctx->wmi_paging_file_scrape_interval, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set paging_file collector for Windows Exporter Metrics plugin"); - return -1; - } - ctx->coll_wmi_paging_file_fd = ret; - } - - /* Initialize paging_file metric collectors */ - ret = we_wmi_paging_file_init(ctx); - if (ret) { - return -1; - } - } - else if (strncmp(entry->str, "process", 7) == 0) { - if (ctx->wmi_process_scrape_interval == 0) { - flb_plg_debug(ctx->ins, "enabled metrics %s", entry->str); - metric_idx = 12; - } - else { - /* Create the process collector */ - ret = flb_input_set_collector_time(in, - we_timer_wmi_process_metrics_cb, - ctx->wmi_process_scrape_interval, 0, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, - "could not set process collector for Windows Exporter Metrics plugin"); - return -1; - } - ctx->coll_wmi_process_fd = ret; - } - - /* Initialize paging_file metric collectors */ - ret = we_wmi_process_init(ctx); - if (ret) { - return -1; - } - } - else { - flb_plg_warn(ctx->ins, "Unknown metrics: %s", entry->str); - metric_idx = -1; - } - - if (metric_idx >= 0) { - cb = &ne_callbacks[metric_idx]; - ret = flb_callback_set(ctx->callback, cb->name, cb->func); - if (ret == -1) { - flb_plg_error(ctx->ins, "error setting up default " - "callback '%s'", cb->name); - } - } - } - } - } - else { - flb_plg_error(ctx->ins, "No metrics is specified"); - - return -1; - } - - return 0; -} - -static int in_we_exit(void *data, struct flb_config *config) -{ - int ret; - struct flb_we* ctx = data; - struct mk_list *head; - struct flb_slist_entry *entry; - - if (data == NULL) { - return 0; - } - - /* Teardown for callback tied up resources */ - if (ctx->metrics) { - mk_list_foreach(head, ctx->metrics) { - entry = mk_list_entry(head, struct flb_slist_entry, _head); - ret = flb_callback_exists(ctx->callback, entry->str); - - if (ret == FLB_TRUE) { - if (strncmp(entry->str, "cpu_info", 8) == 0) { - we_wmi_cpu_info_exit(ctx); - } - else if (strncmp(entry->str, "cpu", 3) == 0) { - /* nop */ - } - else if (strncmp(entry->str, "os", 2) == 0) { - we_os_exit(ctx); - } - else if (strncmp(entry->str, "net", 3) == 0) { - we_net_exit(ctx); - } - else if (strncmp(entry->str, "logical_disk", 12) == 0) { - we_logical_disk_exit(ctx); - } - else if (strncmp(entry->str, "cs", 2) == 0) { - we_cs_exit(ctx); - } - else if (strncmp(entry->str, "thermalzone", 11) == 0) { - we_wmi_thermalzone_exit(ctx); - } - else if (strncmp(entry->str, "logon", 5) == 0) { - we_wmi_logon_exit(ctx); - } - else if (strncmp(entry->str, "system", 6) == 0) { - we_wmi_system_exit(ctx); - } - else if (strncmp(entry->str, "service", 7) == 0) { - we_wmi_service_exit(ctx); - } - else if (strncmp(entry->str, "memory", 6) == 0) { - we_wmi_memory_exit(ctx); - } - else if (strncmp(entry->str, "paging_file", 11) == 0) { - we_wmi_paging_file_exit(ctx); - } - else if (strncmp(entry->str, "process", 7) == 0) { - we_wmi_process_exit(ctx); - } - else { - flb_plg_warn(ctx->ins, "Unknown metrics: %s", entry->str); - } - } - } - } - - /* destroy callback context */ - if (ctx->callback) { - flb_callback_destroy(ctx->callback); - } - - /* Teardown for timer tied up resources */ - if (ctx->coll_net_fd != -1) { - we_net_exit(ctx); - } - if (ctx->coll_logical_disk_fd != -1) { - we_logical_disk_exit(ctx); - } - if (ctx->coll_cs_fd != -1) { - we_cs_exit(ctx); - } - if (ctx->coll_os_fd != -1) { - we_os_exit(ctx); - } - if (ctx->coll_wmi_thermalzone_fd != -1) { - we_wmi_thermalzone_exit(ctx); - } - if (ctx->coll_wmi_cpu_info_fd != -1) { - we_wmi_cpu_info_exit(ctx); - } - if (ctx->coll_wmi_logon_fd != -1) { - we_wmi_logon_exit(ctx); - } - if (ctx->coll_wmi_system_fd != -1) { - we_wmi_system_exit(ctx); - } - if (ctx->coll_wmi_service_fd != -1) { - we_wmi_service_exit(ctx); - } - if (ctx->coll_wmi_memory_fd != -1) { - we_wmi_memory_exit(ctx); - } - if (ctx->coll_wmi_paging_file_fd != -1) { - we_wmi_paging_file_exit(ctx); - } - if (ctx->coll_wmi_process_fd != -1) { - we_wmi_process_exit(ctx); - } - - flb_we_config_destroy(ctx); - - return 0; -} - -static void in_we_pause(void *data, struct flb_config *config) -{ - struct flb_we *ctx; - - ctx = (struct flb_we *) data; - - flb_input_collector_pause(ctx->coll_fd, ctx->ins); - if (ctx->coll_cpu_fd != -1) { - flb_input_collector_pause(ctx->coll_cpu_fd, ctx->ins); - } - if (ctx->coll_net_fd != -1) { - flb_input_collector_pause(ctx->coll_net_fd, ctx->ins); - } - if (ctx->coll_logical_disk_fd != -1) { - flb_input_collector_pause(ctx->coll_logical_disk_fd, ctx->ins); - } - if (ctx->coll_cs_fd != -1) { - flb_input_collector_pause(ctx->coll_cs_fd, ctx->ins); - } - if (ctx->coll_os_fd != -1) { - flb_input_collector_pause(ctx->coll_os_fd, ctx->ins); - } - if (ctx->coll_wmi_thermalzone_fd != -1) { - flb_input_collector_pause(ctx->coll_wmi_thermalzone_fd, ctx->ins); - } - if (ctx->coll_wmi_cpu_info_fd != -1) { - flb_input_collector_pause(ctx->coll_wmi_cpu_info_fd, ctx->ins); - } - if (ctx->coll_wmi_logon_fd != -1) { - flb_input_collector_pause(ctx->coll_wmi_logon_fd, ctx->ins); - } - if (ctx->coll_wmi_system_fd != -1) { - flb_input_collector_pause(ctx->coll_wmi_system_fd, ctx->ins); - } - if (ctx->coll_wmi_service_fd != -1) { - flb_input_collector_pause(ctx->coll_wmi_service_fd, ctx->ins); - } - if (ctx->coll_wmi_memory_fd != -1) { - flb_input_collector_pause(ctx->coll_wmi_memory_fd, ctx->ins); - } - if (ctx->coll_wmi_paging_file_fd != -1) { - flb_input_collector_pause(ctx->coll_wmi_paging_file_fd, ctx->ins); - } - if (ctx->coll_wmi_process_fd != -1) { - flb_input_collector_pause(ctx->coll_wmi_process_fd, ctx->ins); - } -} - -static void in_we_resume(void *data, struct flb_config *config) -{ - struct flb_we *ctx; - - ctx = (struct flb_we *) data; - - flb_input_collector_resume(ctx->coll_fd, ctx->ins); - if (ctx->coll_cpu_fd != -1) { - flb_input_collector_resume(ctx->coll_cpu_fd, ctx->ins); - } - if (ctx->coll_net_fd != -1) { - flb_input_collector_resume(ctx->coll_net_fd, ctx->ins); - } - if (ctx->coll_logical_disk_fd != -1) { - flb_input_collector_resume(ctx->coll_logical_disk_fd, ctx->ins); - } - if (ctx->coll_wmi_process_fd != -1) { - flb_input_collector_resume(ctx->coll_wmi_process_fd, ctx->ins); - } - if (ctx->coll_cs_fd != -1) { - flb_input_collector_resume(ctx->coll_cs_fd, ctx->ins); - } - if (ctx->coll_os_fd != -1) { - flb_input_collector_resume(ctx->coll_os_fd, ctx->ins); - } - if (ctx->coll_wmi_thermalzone_fd != -1) { - flb_input_collector_resume(ctx->coll_wmi_thermalzone_fd, ctx->ins); - } - if (ctx->coll_wmi_cpu_info_fd != -1) { - flb_input_collector_resume(ctx->coll_wmi_cpu_info_fd, ctx->ins); - } - if (ctx->coll_wmi_logon_fd != -1) { - flb_input_collector_resume(ctx->coll_wmi_logon_fd, ctx->ins); - } - if (ctx->coll_wmi_system_fd != -1) { - flb_input_collector_resume(ctx->coll_wmi_system_fd, ctx->ins); - } - if (ctx->coll_wmi_service_fd != -1) { - flb_input_collector_resume(ctx->coll_wmi_service_fd, ctx->ins); - } - if (ctx->coll_wmi_memory_fd != -1) { - flb_input_collector_resume(ctx->coll_wmi_memory_fd, ctx->ins); - } - if (ctx->coll_wmi_paging_file_fd != -1) { - flb_input_collector_resume(ctx->coll_wmi_paging_file_fd, ctx->ins); - } -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_TIME, "scrape_interval", "1", - 0, FLB_TRUE, offsetof(struct flb_we, scrape_interval), - "scrape interval to collect metrics from the node." - }, - { - FLB_CONFIG_MAP_STR, "enable_collector", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct flb_we, collectors), - "Collector to enable." - }, - { - FLB_CONFIG_MAP_TIME, "collector.cpu.scrape_interval", "0", - 0, FLB_TRUE, offsetof(struct flb_we, cpu_scrape_interval), - "scrape interval to collect cpu metrics from the node." - }, - - { - FLB_CONFIG_MAP_TIME, "collector.net.scrape_interval", "0", - 0, FLB_TRUE, offsetof(struct flb_we, net_scrape_interval), - "scrape interval to collect net metrics from the node." - }, - { - FLB_CONFIG_MAP_TIME, "collector.logical_disk.scrape_interval", "0", - 0, FLB_TRUE, offsetof(struct flb_we, logical_disk_scrape_interval), - "scrape interval to collect logical_disk metrics from the node." - }, - - { - FLB_CONFIG_MAP_TIME, "collector.cs.scrape_interval", "0", - 0, FLB_TRUE, offsetof(struct flb_we, cs_scrape_interval), - "scrape interval to collect cs metrics from the node." - }, - - { - FLB_CONFIG_MAP_TIME, "collector.os.scrape_interval", "0", - 0, FLB_TRUE, offsetof(struct flb_we, os_scrape_interval), - "scrape interval to collect os metrics from the node." - }, - - { - FLB_CONFIG_MAP_TIME, "collector.thermalzone.scrape_interval", "0", - 0, FLB_TRUE, offsetof(struct flb_we, wmi_thermalzone_scrape_interval), - "scrape interval to collect thermalzone metrics from the node." - }, - - { - FLB_CONFIG_MAP_TIME, "collector.cpu_info.scrape_interval", "0", - 0, FLB_TRUE, offsetof(struct flb_we, wmi_cpu_info_scrape_interval), - "scrape interval to collect cpu_info metrics from the node." - }, - - { - FLB_CONFIG_MAP_TIME, "collector.logon.scrape_interval", "0", - 0, FLB_TRUE, offsetof(struct flb_we, wmi_logon_scrape_interval), - "scrape interval to collect logon metrics from the node." - }, - { - FLB_CONFIG_MAP_TIME, "collector.system.scrape_interval", "0", - 0, FLB_TRUE, offsetof(struct flb_we, wmi_system_scrape_interval), - "scrape interval to collect system metrics from the node." - }, - { - FLB_CONFIG_MAP_TIME, "collector.service.scrape_interval", "0", - 0, FLB_TRUE, offsetof(struct flb_we, wmi_service_scrape_interval), - "scrape interval to collect service metrics from the node." - }, - - { - FLB_CONFIG_MAP_TIME, "collector.memory.scrape_interval", "0", - 0, FLB_TRUE, offsetof(struct flb_we, wmi_memory_scrape_interval), - "scrape interval to collect memory metrics from the node." - }, - { - FLB_CONFIG_MAP_TIME, "collector.paging_file.scrape_interval", "0", - 0, FLB_TRUE, offsetof(struct flb_we, wmi_paging_file_scrape_interval), - "scrape interval to collect paging_file metrics from the node." - }, - - { - FLB_CONFIG_MAP_TIME, "collector.process.scrape_interval", "0", - 0, FLB_TRUE, offsetof(struct flb_we, wmi_process_scrape_interval), - "scrape interval to collect process metrics from the node." - }, - - { - FLB_CONFIG_MAP_CLIST, "metrics", - "cpu,cpu_info,os,net,logical_disk,cs,thermalzone,logon,system,service", - 0, FLB_TRUE, offsetof(struct flb_we, metrics), - "Comma separated list of keys to enable metrics." - }, - { - FLB_CONFIG_MAP_STR, "we.logical_disk.allow_disk_regex", "/.+/", - 0, FLB_TRUE, offsetof(struct flb_we, raw_allowing_disk), - "Specify to be scribable regex for logical disk metrics." - }, - { - FLB_CONFIG_MAP_STR, "we.logical_disk.deny_disk_regex", NULL, - 0, FLB_TRUE, offsetof(struct flb_we, raw_denying_disk), - "Specify to be denied regex for logical disk metrics." - }, - { - FLB_CONFIG_MAP_STR, "we.net.allow_nic_regex", "/.+/", - 0, FLB_TRUE, offsetof(struct flb_we, raw_allowing_nic), - "Specify to be scribable regex for net metrics by name of NIC." - }, - { - FLB_CONFIG_MAP_STR, "we.service.where", NULL, - 0, FLB_TRUE, offsetof(struct flb_we, raw_where_clause), - "Specify the where clause for retrieving service metrics." - }, - { - FLB_CONFIG_MAP_STR, "we.service.include", NULL, - 0, FLB_TRUE, offsetof(struct flb_we, raw_service_include), - "Specify the key value condition pairs for includeing condition to construct where clause of service metrics." - }, - { - FLB_CONFIG_MAP_STR, "we.service.exclude", NULL, - 0, FLB_TRUE, offsetof(struct flb_we, raw_service_exclude), - "Specify the key value condition pairs for excludeing condition to construct where clause of service metrics." - }, - { - FLB_CONFIG_MAP_STR, "we.process.allow_process_regex", "/.+/", - 0, FLB_TRUE, offsetof(struct flb_we, raw_allowing_process), - "Specify the regex covering the process metrics to collect." - }, - { - FLB_CONFIG_MAP_STR, "we.process.deny_process_regex", NULL, - 0, FLB_TRUE, offsetof(struct flb_we, raw_denying_process), - "Specify the regex for process metrics to prevent collection of/ignore." - }, - /* EOF */ - {0} -}; - -struct flb_input_plugin in_windows_exporter_metrics_plugin = { - .name = "windows_exporter_metrics", - .description = "Windows Exporter Metrics (Prometheus Compatible)", - .cb_init = in_we_init, - .cb_pre_run = NULL, - .cb_collect = cb_we_collect, - .cb_flush_buf = NULL, - .config_map = config_map, - .cb_pause = in_we_pause, - .cb_resume = in_we_resume, - .cb_exit = in_we_exit, -}; diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we.h b/fluent-bit/plugins/in_windows_exporter_metrics/we.h deleted file mode 100644 index 1ec848893..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we.h +++ /dev/null @@ -1,332 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2021 The Fluent Bit Authors - * Copyright (C) 2015-2018 Treasure Data Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_WINDOWS_EXPORTER_H -#define FLB_WINDOWS_EXPORTER_H - -#include -#include -#include -#include - -#include -#include - -#include -#include - -#include "we_metric.h" - -#define PERFLIB_COUNTER_TYPE_COUNTER 0x400 -#define PERFLIB_COUNTER_FLAG_BASE_VALUE 0x00030000 -#define PERFLIB_COUNTER_FLAG_BASE_NANOSECONDS 0x00100000 - -struct we_perflib_counter_definition { - char *name_index_str; - uint32_t name_index; - char *name; - uint32_t help_index; - char *help; - - uint32_t type; - uint32_t size; - uint32_t offset; - uint32_t detail_level; - - struct mk_list _head; -}; - -union we_perflib_value { - uint64_t as_qword; - double as_double; - uint32_t as_dword; - float as_float; -}; - -struct we_perflib_counter { - struct we_perflib_instance *parent; - struct we_perflib_counter_definition *definition; - union we_perflib_value primary_value; - union we_perflib_value secondary_value; - struct mk_list _head; -}; - -struct we_perflib_instance { - char *name; - struct we_perflib_object *parent; - struct flb_hash_table *counters; - struct mk_list _head; -}; - -struct we_perflib_object { - char *name; - int64_t time; - int64_t frequency; - int64_t hundred_ns_time; - size_t counter_count; - size_t instance_count; - struct flb_hash_table *instances; - struct mk_list counter_definitions; -}; - - -struct we_perflib_context { - struct flb_hash_table *counter_indexes; -}; - -struct we_cpu_counters { - struct we_perflib_metric_source *metric_sources; - struct we_perflib_metric_spec *metric_specs; - int operational; - struct flb_hash_table *metrics; - char *query; -}; - -struct we_net_counters { - struct we_perflib_metric_source *metric_sources; - struct we_perflib_metric_spec *metric_specs; - int operational; - struct flb_hash_table *metrics; - char *query; -}; - -struct we_logical_disk_counters { - struct we_perflib_metric_source *metric_sources; - struct we_perflib_metric_spec *metric_specs; - int operational; - struct flb_hash_table *metrics; - char *query; -}; - -struct wmi_query_spec; - -struct we_wmi_thermal_counters { - struct wmi_query_spec *temperature_celsius; - struct wmi_query_spec *percent_passive_limit; - struct wmi_query_spec *throttle_reasons; - int operational; -}; - -struct we_wmi_cpu_info_counters { - struct wmi_query_spec *info; - int operational; -}; - -struct we_wmi_logon_counters { - struct wmi_query_spec *info; - int operational; -}; - -struct we_wmi_system_counters { - struct wmi_query_spec *info; - struct cmt_gauge *context_switches; - struct cmt_gauge *exception_dispatches; - struct cmt_gauge *processor_queue; - struct cmt_gauge *system_calls; - struct cmt_gauge *system_up_time; - struct cmt_gauge *threads; - int operational; -}; - -struct we_wmi_service_counters { - struct wmi_query_spec *info; - struct cmt_gauge *information; - struct cmt_gauge *state; - struct cmt_gauge *start_mode; - struct cmt_gauge *status; - int operational; -}; - -struct we_wmi_memory_counters { - struct wmi_query_spec *info; - struct cmt_gauge *available_bytes; - struct cmt_gauge *cache_bytes; - struct cmt_gauge *cache_bytes_peak; - struct cmt_gauge *cache_faults_total; - struct cmt_gauge *commit_limit; - struct cmt_gauge *committed_bytes; - struct cmt_gauge *demand_zero_faults_total; - struct cmt_gauge *free_and_zero_page_list_bytes; - struct cmt_gauge *free_system_page_table_entries; - struct cmt_gauge *modified_page_list_bytes; - struct cmt_gauge *page_faults_total; - struct cmt_gauge *swap_page_reads_total; - struct cmt_gauge *swap_pages_read_total; - struct cmt_gauge *swap_pages_written_total; - struct cmt_gauge *swap_page_operations_total; - struct cmt_gauge *swap_page_writes_total; - struct cmt_gauge *pool_nonpaged_allocs_total; - struct cmt_gauge *pool_nonpaged_bytes; - struct cmt_gauge *pool_paged_allocs_total; - struct cmt_gauge *pool_paged_bytes; - struct cmt_gauge *pool_paged_resident_bytes; - struct cmt_gauge *standby_cache_core_bytes; - struct cmt_gauge *standby_cache_normal_priority_bytes; - struct cmt_gauge *standby_cache_reserve_bytes; - struct cmt_gauge *system_cache_resident_bytes; - struct cmt_gauge *system_code_resident_bytes; - struct cmt_gauge *system_code_total_bytes; - struct cmt_gauge *system_driver_resident_bytes; - struct cmt_gauge *system_driver_total_bytes; - struct cmt_gauge *transition_faults_total; - struct cmt_gauge *transition_pages_repurposed_total; - struct cmt_gauge *write_copies_total; - int operational; -}; - -struct we_wmi_paging_file_counters { - struct wmi_query_spec *info; - struct cmt_gauge *allocated_base_size_megabytes; - struct cmt_gauge *current_usage_megabytes; - struct cmt_gauge *peak_usage_megabytes; - int operational; -}; - -struct we_wmi_process_counters { - struct wmi_query_spec *info; - struct cmt_gauge *start_time; - struct cmt_gauge *handles; - struct cmt_gauge *cpu_time_total; - struct cmt_gauge *io_bytes_total; - struct cmt_gauge *io_operations_total; - struct cmt_gauge *page_faults_total; - struct cmt_gauge *page_file_bytes; - struct cmt_gauge *pool_bytes; - struct cmt_gauge *priority_base; - struct cmt_gauge *thread_count; - struct cmt_gauge *private_bytes; - struct cmt_gauge *virtual_bytes; - struct cmt_gauge *working_set_private_bytes; - struct cmt_gauge *working_set_peak_bytes; - struct cmt_gauge *working_set_bytes; - int operational; -}; - -struct we_os_counters { - struct cmt_gauge *info; - struct cmt_gauge *users; - struct cmt_gauge *physical_memory_free_bytes; - struct cmt_gauge *time; - struct cmt_gauge *tz; - struct cmt_gauge *virtual_memory_free_bytes; - struct cmt_gauge *processes_limit; - struct cmt_gauge *process_memory_limit_bytes; - struct cmt_gauge *processes; - struct cmt_gauge *virtual_memory_bytes; - struct cmt_gauge *visible_memory_bytes; - int operational; -}; - -struct we_cs_counters { - struct cmt_gauge *logical_processors; - struct cmt_gauge *physical_memory_bytes; - struct cmt_gauge *hostname; - int operational; -}; - -struct flb_we { - /* configuration */ - int scrape_interval; - - int coll_fd; /* collector fd */ - struct cmt *cmt; /* cmetrics context */ - struct flb_input_instance *ins; /* input instance */ - struct mk_list *collectors; - char *raw_allowing_disk; - char *raw_denying_disk; - char *raw_allowing_nic; - char *raw_where_clause; - char *raw_service_include; - char *raw_service_exclude; - char *raw_allowing_process; - char *raw_denying_process; - char *service_include_buffer; - int service_include_buffer_size; - char *service_exclude_buffer; - int service_exclude_buffer_size; - - struct flb_regex *allowing_disk_regex; - struct flb_regex *denying_disk_regex; - struct flb_regex *allowing_nic_regex; - struct flb_regex *allowing_process_regex; - struct flb_regex *denying_process_regex; - - struct we_perflib_context perflib_context; - /* WMI locator and service contexts */ - IWbemLocator *locator; - IWbemServices *service; - - float windows_version; - - struct flb_callback *callback; /* metric callback */ - struct mk_list *metrics; /* enabled metrics */ - - /* Individual intervals for metrics */ - int cpu_scrape_interval; - int net_scrape_interval; - int logical_disk_scrape_interval; - int cs_scrape_interval; - int os_scrape_interval; - int wmi_thermalzone_scrape_interval; - int wmi_cpu_info_scrape_interval; - int wmi_logon_scrape_interval; - int wmi_system_scrape_interval; - int wmi_service_scrape_interval; - int wmi_memory_scrape_interval; - int wmi_paging_file_scrape_interval; - int wmi_process_scrape_interval; - - int coll_cpu_fd; /* collector fd (cpu) */ - int coll_net_fd; /* collector fd (net) */ - int coll_logical_disk_fd; /* collector fd (logical_disk) */ - int coll_cs_fd; /* collector fd (cs) */ - int coll_os_fd; /* collector fd (os) */ - int coll_wmi_thermalzone_fd; /* collector fd (wmi_thermalzone) */ - int coll_wmi_cpu_info_fd; /* collector fd (wmi_cpu_info) */ - int coll_wmi_logon_fd; /* collector fd (wmi_logon) */ - int coll_wmi_system_fd; /* collector fd (wmi_system) */ - int coll_wmi_service_fd; /* collector fd (wmi_service) */ - int coll_wmi_memory_fd; /* collector fd (wmi_memory) */ - int coll_wmi_paging_file_fd; /* collector fd (wmi_paging_file) */ - int coll_wmi_process_fd; /* collector fd (wmi_process) */ - - /* - * Metrics Contexts - * ---------------- - */ - - struct we_cpu_counters cpu; - struct we_net_counters net; - struct we_logical_disk_counters logical_disk; - struct we_cs_counters cs; - struct we_os_counters *os; - struct we_wmi_thermal_counters *wmi_thermals; - struct we_wmi_cpu_info_counters *wmi_cpu_info; - struct we_wmi_logon_counters *wmi_logon; - struct we_wmi_system_counters *wmi_system; - struct we_wmi_service_counters *wmi_service; - struct we_wmi_memory_counters *wmi_memory; - struct we_wmi_paging_file_counters *wmi_paging_file; - struct we_wmi_process_counters *wmi_process; -}; - -typedef int (*collector_cb)(struct flb_we *); - -#endif diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_config.c b/fluent-bit/plugins/in_windows_exporter_metrics/we_config.c deleted file mode 100644 index 14913b433..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_config.c +++ /dev/null @@ -1,154 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2021 The Fluent Bit Authors - * Copyright (C) 2015-2018 Treasure Data Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include "we.h" - -struct flb_we *flb_we_config_create(struct flb_input_instance *ins, - struct flb_config *config) -{ - int ret; - struct flb_we *ctx; - int root_type; - - ctx = flb_calloc(1, sizeof(struct flb_we)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - ctx->allowing_disk_regex = NULL; - ctx->denying_disk_regex = NULL; - ctx->allowing_nic_regex = NULL; - ctx->service_include_buffer = NULL; - ctx->service_include_buffer_size = 0; - ctx->service_exclude_buffer = NULL; - ctx->service_exclude_buffer_size = 0; - ctx->allowing_process_regex = NULL; - ctx->denying_process_regex = NULL; - - /* Load the config map */ - ret = flb_input_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return NULL; - } - - /* Process allow/deny regex rules */ - if (ctx->raw_allowing_disk != NULL) { - ctx->allowing_disk_regex = flb_regex_create(ctx->raw_allowing_disk); - } - - if (ctx->raw_denying_disk != NULL) { - ctx->denying_disk_regex = flb_regex_create(ctx->raw_denying_disk); - } - - if (ctx->raw_allowing_nic != NULL) { - ctx->allowing_nic_regex = flb_regex_create(ctx->raw_allowing_nic); - } - - if (ctx->raw_service_include != NULL) { - ret = flb_pack_json(ctx->raw_service_include, - strlen(ctx->raw_service_include), - &ctx->service_include_buffer, - &ctx->service_include_buffer_size, - &root_type, - NULL); - if (ret != 0) { - flb_plg_warn(ctx->ins, "we.service.include is incomplete. Ignored."); - ctx->service_include_buffer = NULL; - ctx->service_include_buffer_size = 0; - } - } - - if (ctx->raw_service_exclude != NULL) { - ret = flb_pack_json(ctx->raw_service_exclude, - strlen(ctx->raw_service_exclude), - &ctx->service_exclude_buffer, - &ctx->service_exclude_buffer_size, - &root_type, - NULL); - if (ret != 0) { - flb_plg_warn(ctx->ins, "we.service.exclude is incomplete. Ignored."); - ctx->service_exclude_buffer = NULL; - ctx->service_exclude_buffer_size = 0; - } - } - - /* Process allow/deny regex rules for process metrics */ - if (ctx->raw_allowing_process != NULL) { - ctx->allowing_process_regex = flb_regex_create(ctx->raw_allowing_process); - } - - if (ctx->raw_denying_process != NULL) { - ctx->denying_process_regex = flb_regex_create(ctx->raw_denying_process); - } - - ctx->cmt = cmt_create(); - if (!ctx->cmt) { - flb_plg_error(ins, "could not initialize CMetrics"); - flb_free(ctx); - return NULL; - } - - return ctx; -} - -void flb_we_config_destroy(struct flb_we *ctx) -{ - if (!ctx) { - return; - } - - if (ctx->allowing_disk_regex != NULL) { - flb_regex_destroy(ctx->allowing_disk_regex); - } - - if (ctx->denying_disk_regex != NULL) { - flb_regex_destroy(ctx->denying_disk_regex); - } - - if (ctx->allowing_nic_regex != NULL) { - flb_regex_destroy(ctx->allowing_nic_regex); - } - - if (ctx->service_include_buffer != NULL) { - flb_free(ctx->service_include_buffer); - } - - if (ctx->service_exclude_buffer != NULL) { - flb_free(ctx->service_exclude_buffer); - } - - if (ctx->allowing_process_regex != NULL) { - flb_regex_destroy(ctx->allowing_process_regex); - } - - if (ctx->denying_disk_regex != NULL) { - flb_regex_destroy(ctx->denying_disk_regex); - } - - if (ctx->cmt) { - cmt_destroy(ctx->cmt); - } - - flb_free(ctx); -} diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_config.h b/fluent-bit/plugins/in_windows_exporter_metrics/we_config.h deleted file mode 100644 index 00eea3ebe..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_config.h +++ /dev/null @@ -1,32 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2021 The Fluent Bit Authors - * Copyright (C) 2015-2018 Treasure Data Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_WE_CONFIG_H -#define FLB_WE_CONFIG_H - -#include -#include "we.h" - -struct flb_we *flb_we_config_create(struct flb_input_instance *ins, - struct flb_config *config); - -void flb_we_config_destroy(struct flb_we *ctx); - -#endif diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_cpu.c b/fluent-bit/plugins/in_windows_exporter_metrics/we_cpu.c deleted file mode 100644 index d6013d797..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_cpu.c +++ /dev/null @@ -1,304 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2021 The Fluent Bit Authors - * Copyright (C) 2015-2018 Treasure Data Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include - -#include "we.h" -#include "we_cpu.h" -#include "we_util.h" -#include "we_metric.h" -#include "we_perflib.h" - - -struct we_perflib_metric_source basic_metric_sources[] = { - WE_PERFLIB_METRIC_SOURCE("cstate_seconds_total", - "% C1 Time", - "c1"), - - WE_PERFLIB_METRIC_SOURCE("cstate_seconds_total", - "% C2 Time", - "c2"), - - WE_PERFLIB_METRIC_SOURCE("cstate_seconds_total", - "% C3 Time", - "c3"), - - WE_PERFLIB_METRIC_SOURCE("time_total", - "% Idle Time", - "idle"), - - WE_PERFLIB_METRIC_SOURCE("time_total", - "% Interrupt Time", - "interrupt"), - - WE_PERFLIB_METRIC_SOURCE("time_total", - "% DPC Time", - "dpc"), - - WE_PERFLIB_METRIC_SOURCE("time_total", - "% Privileged Time", - "privileged"), - - WE_PERFLIB_METRIC_SOURCE("time_total", - "% User Time", - "user"), - - WE_PERFLIB_METRIC_SOURCE("interrupts_total", - "Interrupts/sec", - NULL), - - WE_PERFLIB_METRIC_SOURCE("dpcs_total", - "DPCs Queued/sec", - NULL), - - WE_PERFLIB_TERMINATOR_SOURCE() - }; - - -struct we_perflib_metric_source full_metric_sources[] = { - WE_PERFLIB_METRIC_SOURCE("cstate_seconds_total", - "% C1 Time", - "c1"), - - WE_PERFLIB_METRIC_SOURCE("cstate_seconds_total", - "% C2 Time", - "c2"), - - WE_PERFLIB_METRIC_SOURCE("cstate_seconds_total", - "% C3 Time", - "c3"), - - WE_PERFLIB_METRIC_SOURCE("time_total", - "% Idle Time", - "idle"), - - WE_PERFLIB_METRIC_SOURCE("time_total", - "% Interrupt Time", - "interrupt"), - - WE_PERFLIB_METRIC_SOURCE("time_total", - "% DPC Time", - "dpc"), - - WE_PERFLIB_METRIC_SOURCE("time_total", - "% Privileged Time", - "privileged"), - - WE_PERFLIB_METRIC_SOURCE("time_total", - "% User Time", - "user"), - - WE_PERFLIB_METRIC_SOURCE("interrupts_total", - "Interrupts/sec", - NULL), - - WE_PERFLIB_METRIC_SOURCE("dpcs_total", - "DPCs Queued/sec", - NULL), - - WE_PERFLIB_METRIC_SOURCE("clock_interrupts_total", - "Clock Interrupts/sec", - NULL), - - WE_PERFLIB_METRIC_SOURCE("idle_break_events_total", - "Idle Break Events/sec", - NULL), - - WE_PERFLIB_METRIC_SOURCE("parkings_status", - "Parking Status", - NULL), - - WE_PERFLIB_METRIC_SOURCE("core_frequency_mhz", - "Processor Frequency", - NULL), - - WE_PERFLIB_METRIC_SOURCE("processor_performance", - "% Processor Performance", - NULL), - - WE_PERFLIB_TERMINATOR_SOURCE() - }; - -struct we_perflib_metric_spec full_metric_specs[] = - { - WE_PERFLIB_COUNTER_SPEC("cstate_seconds_total", - "Time spent in low-power idle state.", - "core,state"), - - WE_PERFLIB_COUNTER_SPEC("time_total", - "Time that processor spent in different " \ - "modes (idle, user, system, ...)", - "core,mode"), - - WE_PERFLIB_COUNTER_SPEC("interrupts_total", - "Total number of received and serviced " \ - "hardware interrupts", - "core"), - - WE_PERFLIB_COUNTER_SPEC("dpcs_total", - "Total number of received and serviced " \ - "deferred procedure calls (DPCs)", - "core"), - - WE_PERFLIB_COUNTER_SPEC("clock_interrupts_total", - "Total number of received and serviced " \ - "clock tick interrupts", - "core"), - - WE_PERFLIB_COUNTER_SPEC("idle_break_events_total", - "Total number of time processor was woken " \ - "from idle", - "core"), - - WE_PERFLIB_GAUGE_SPEC("parkings_status", - "Parking Status represents whether a " \ - "processor is parked or not", - "core"), - - WE_PERFLIB_GAUGE_SPEC("core_frequency_mhz", - "Core frequency in megahertz", - "core"), - - WE_PERFLIB_GAUGE_SPEC("processor_performance", - "Processor Performance is the average " \ - "performance of the processor while it is " \ - "executing instructions, as a percentage of" \ - " the nominal performance of the processor." \ - " On some processors, Processor Performance" \ - " may exceed 100%", - "core"), - - WE_PERFLIB_TERMINATOR_SPEC() - }; - - -int we_cpu_init(struct flb_we *ctx) -{ - struct we_perflib_metric_source *metric_sources; - int result; - - ctx->cpu.operational = FLB_FALSE; - - ctx->cpu.metrics = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, 64, 128); - - if (ctx->cpu.metrics == NULL) { - flb_plg_error(ctx->ins, "could not create metrics hash table"); - - return -1; - } - - result = we_initialize_perflib_metric_specs(ctx->cmt, - ctx->cpu.metrics, - "windows", - "cpu", - &ctx->cpu.metric_specs, - full_metric_specs); - - if (result != 0) { - flb_plg_error(ctx->ins, "could not initialize metric specs"); - - return -2; - } - - if (fabsf(ctx->windows_version - 6.05) > FLT_EPSILON) { - metric_sources = full_metric_sources; - ctx->cpu.query = (char *) "Processor Information"; - } - else { - metric_sources = basic_metric_sources; - ctx->cpu.query = (char *) "Processor"; - } - - result = we_initialize_perflib_metric_sources(ctx->cpu.metrics, - &ctx->cpu.metric_sources, - metric_sources); - - if (result != 0) { - flb_plg_error(ctx->ins, "could not initialize metric sources"); - - we_deinitialize_perflib_metric_specs(ctx->cpu.metric_specs); - flb_free(ctx->cpu.metric_specs); - - return -3; - } - - ctx->cpu.operational = FLB_TRUE; - - return 0; -} - -int we_cpu_exit(struct flb_we *ctx) -{ - we_deinitialize_perflib_metric_sources(ctx->cpu.metric_sources); - we_deinitialize_perflib_metric_specs(ctx->cpu.metric_specs); - - flb_free(ctx->cpu.metric_sources); - flb_free(ctx->cpu.metric_specs); - - ctx->cpu.operational = FLB_FALSE; - - return 0; -} - -int we_cpu_instance_hook(char *instance_name, struct flb_we *ctx) -{ - return (strcasestr(instance_name, "Total") != NULL); -} - -int we_cpu_label_prepend_hook(char **label_list, - size_t label_list_size, - size_t *label_count, - struct we_perflib_metric_source *metric_source, - char *instance_name, - struct we_perflib_counter *counter) -{ - if (label_count == NULL) { - return -1; - } - - if (*label_count >= label_list_size) { - return -2; - } - - label_list[(*label_count)++] = instance_name; - - return 0; -} - -int we_cpu_update(struct flb_we *ctx) -{ - if (!ctx->cpu.operational) { - flb_plg_error(ctx->ins, "cpu collector not yet in operational state"); - - return -1; - } - - return we_perflib_update_counters(ctx, - ctx->cpu.query, - ctx->cpu.metric_sources, - we_cpu_instance_hook, - we_cpu_label_prepend_hook); -} diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_cpu.h b/fluent-bit/plugins/in_windows_exporter_metrics/we_cpu.h deleted file mode 100644 index f2b040977..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_cpu.h +++ /dev/null @@ -1,30 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2021 The Fluent Bit Authors - * Copyright (C) 2015-2018 Treasure Data Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_WE_CPU_H -#define FLB_WE_CPU_H - -#include "we.h" - -int we_cpu_init(struct flb_we *ctx); -int we_cpu_exit(struct flb_we *ctx); -int we_cpu_update(struct flb_we *ctx); - -#endif \ No newline at end of file diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_cs.c b/fluent-bit/plugins/in_windows_exporter_metrics/we_cs.c deleted file mode 100644 index 9ed4a1ca1..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_cs.c +++ /dev/null @@ -1,112 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "we.h" -#include "we_cs.h" -#include "we_util.h" -#include "we_metric.h" - -int we_cs_init(struct flb_we *ctx) -{ - ctx->cs.operational = FLB_FALSE; - - struct cmt_gauge *g; - - g = cmt_gauge_create(ctx->cmt, "windows", "cs", "logical_processors", - "Number of logical processors", - 0, NULL); - - if (!g) { - return -1; - } - ctx->cs.logical_processors = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "cs", "physical_memory_bytes", - "Amount of bytes of physical memory", - 0, NULL); - - if (!g) { - return -1; - } - ctx->cs.physical_memory_bytes = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "cs", "hostname", - "Value of Local Time", - 3, (char *[]) {"hostname", "domain", "fqdn"}); - if (!g) { - return -1; - } - ctx->cs.hostname = g; - - ctx->cs.operational = FLB_TRUE; - - return 0; -} - -int we_cs_exit(struct flb_we *ctx) -{ - return 0; -} - -int we_cs_update(struct flb_we *ctx) -{ - SYSTEM_INFO system_info; - MEMORYSTATUSEX statex; - char hostname[256] = "", domain[256] = "", fqdn[256] = ""; - DWORD size = 0; - uint64_t timestamp = 0; - - if (!ctx->cs.operational) { - flb_plg_error(ctx->ins, "cs collector not yet in operational state"); - - return -1; - } - - timestamp = cfl_time_now(); - - statex.dwLength = sizeof (statex); - GlobalMemoryStatusEx(&statex); - - GetSystemInfo(&system_info); - - size = _countof(hostname); - if (!GetComputerNameExA(ComputerNameDnsHostname, hostname, &size)) { - flb_plg_warn(ctx->ins, "Failed to retrieve hostname info"); - } - size = _countof(domain); - if (!GetComputerNameExA(ComputerNameDnsDomain, domain, &size)) { - flb_plg_warn(ctx->ins, "Failed to retrieve domain info"); - } - size = _countof(fqdn); - if (!GetComputerNameExA(ComputerNameDnsFullyQualified, fqdn, &size)) { - flb_plg_warn(ctx->ins, "Failed to retrieve fqdn info"); - } - - cmt_gauge_set(ctx->cs.logical_processors, timestamp, (double)system_info.dwNumberOfProcessors, 0, NULL); - cmt_gauge_set(ctx->cs.physical_memory_bytes, timestamp, (double)statex.ullTotalPhys, 0, NULL); - cmt_gauge_set(ctx->cs.hostname, timestamp, 1.0, 3, (char *[]) { hostname, domain, fqdn }); - - return 0; -} diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_cs.h b/fluent-bit/plugins/in_windows_exporter_metrics/we_cs.h deleted file mode 100644 index 111672891..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_cs.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_WE_CS_H -#define FLB_WE_CS_H - -#include "we.h" - -int we_cs_init(struct flb_we *ctx); -int we_cs_exit(struct flb_we *ctx); -int we_cs_update(struct flb_we *ctx); - -#endif diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_logical_disk.c b/fluent-bit/plugins/in_windows_exporter_metrics/we_logical_disk.c deleted file mode 100644 index df2e09c41..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_logical_disk.c +++ /dev/null @@ -1,272 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "we.h" -#include "we_logical_disk.h" -#include "we_util.h" -#include "we_metric.h" -#include "we_perflib.h" - - -struct we_perflib_metric_source logical_disk_metric_sources[] = { - WE_PERFLIB_METRIC_SOURCE("requests_queued", - "Current Disk Queue Length", - NULL), - - WE_PERFLIB_METRIC_SOURCE("read_bytes_total", - "Disk Read Bytes/sec", - NULL), - - WE_PERFLIB_METRIC_SOURCE("read_total", - "Disk Reads/sec", - NULL), - - WE_PERFLIB_METRIC_SOURCE("write_bytes_total", - "Disk Write Bytes/sec", - NULL), - - WE_PERFLIB_METRIC_SOURCE("write_total", - "Disk Writes/sec", - NULL), - - WE_PERFLIB_METRIC_SOURCE("read_seconds_total", - "% Disk Read Time", - NULL), - - WE_PERFLIB_METRIC_SOURCE("write_seconds_total", - "% Disk Write Time", - NULL), - - WE_PERFLIB_METRIC_SOURCE("free_megabytes", - "Free Megabytes", - NULL), - - /* FIXME: Prometheus windows exporter uses '% Free Space_Base' as - * query for size_(mega)bytes metrics, but it does not work. */ - /* WE_PERFLIB_METRIC_SOURCE("size_megabytes", */ - /* "% Free Space_Base", */ - /* NULL), */ - - WE_PERFLIB_METRIC_SOURCE("idle_seconds_total", - "% Idle Time", - NULL), - - WE_PERFLIB_METRIC_SOURCE("split_ios_total", - "Split IO/Sec", - NULL), - - WE_PERFLIB_METRIC_SOURCE("read_latency_seconds_total", - "Avg. Disk sec/Read", - NULL), - - WE_PERFLIB_METRIC_SOURCE("write_latency_seconds_total", - "Avg. Disk sec/Write", - NULL), - - WE_PERFLIB_METRIC_SOURCE("read_write_latency_seconds_total", - "Avg. Disk sec/Transfer", - NULL), - - WE_PERFLIB_TERMINATOR_SOURCE() - }; - -struct we_perflib_metric_spec logical_disk_metric_specs[] = { - WE_PERFLIB_GAUGE_SPEC("requests_queued", - "Number of queued requests on the disk", - "volume"), - - WE_PERFLIB_COUNTER_SPEC("read_bytes_total", - "Number of read bytes from the disk", - "volume"), - - WE_PERFLIB_COUNTER_SPEC("read_total", - "Number of read from the disk", - "volume"), - - WE_PERFLIB_COUNTER_SPEC("write_bytes_total", - "Number of write bytes to the disk", - "volume"), - - WE_PERFLIB_COUNTER_SPEC("write_total", - "Number of write from to disk", - "volume"), - - WE_PERFLIB_COUNTER_SPEC("read_seconds_total", - "Total amount of reading time from the disk", - "volume"), - - WE_PERFLIB_COUNTER_SPEC("write_seconds_total", - "Total amount of writeing time to the disk", - "volume"), - - WE_PERFLIB_GAUGE_SPEC("free_megabytes", - "Free megabytes on the disk", - "volume"), - - /* WE_PERFLIB_COUNTER_SPEC("size_megabytes", */ - /* "Total amount of free megabytes on the disk", */ - /* "volume"), */ - - WE_PERFLIB_COUNTER_SPEC("idle_seconds_total", - "Total amount of idling time on the disk", - "volume"), - - WE_PERFLIB_COUNTER_SPEC("split_ios_total", - "Total amount of split I/O operations on the disk", - "volume"), - - WE_PERFLIB_COUNTER_SPEC("read_latency_seconds_total", - "Average latency, in seconds, to read from the disk", - "volume"), - - WE_PERFLIB_COUNTER_SPEC("write_latency_seconds_total", - "Average latency, in seconds, to write into the disk", - "volume"), - - WE_PERFLIB_COUNTER_SPEC("read_write_latency_seconds_total", - "Average latency, in seconds, to transfer operations on the disk", - "volume"), - - WE_PERFLIB_TERMINATOR_SPEC() - }; - - -int we_logical_disk_init(struct flb_we *ctx) -{ - struct we_perflib_metric_source *metric_sources; - int result; - - ctx->logical_disk.operational = FLB_FALSE; - - ctx->logical_disk.metrics = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, 32, 128); - - if (ctx->logical_disk.metrics == NULL) { - flb_plg_error(ctx->ins, "could not create metrics hash table for logical_disk metrics"); - - return -1; - } - - result = we_initialize_perflib_metric_specs(ctx->cmt, - ctx->logical_disk.metrics, - "windows", - "logical_disk", - &ctx->logical_disk.metric_specs, - logical_disk_metric_specs); - - if (result != 0) { - flb_plg_error(ctx->ins, "could not initialize logical_disk metric specs"); - - return -2; - } - - ctx->logical_disk.query = (char *) "LogicalDisk"; - - result = we_initialize_perflib_metric_sources(ctx->logical_disk.metrics, - &ctx->logical_disk.metric_sources, - logical_disk_metric_sources); - - if (result != 0) { - flb_plg_error(ctx->ins, "could not initialize logical_disk metric sources"); - - we_deinitialize_perflib_metric_specs(ctx->logical_disk.metric_specs); - flb_free(ctx->logical_disk.metric_specs); - - return -3; - } - - ctx->logical_disk.operational = FLB_TRUE; - - return 0; -} - -int we_logical_disk_exit(struct flb_we *ctx) -{ - we_deinitialize_perflib_metric_sources(ctx->logical_disk.metric_sources); - we_deinitialize_perflib_metric_specs(ctx->logical_disk.metric_specs); - - flb_free(ctx->logical_disk.metric_sources); - flb_free(ctx->logical_disk.metric_specs); - - ctx->logical_disk.operational = FLB_FALSE; - - return 0; -} - -static int logical_disk_regex_match(struct flb_regex *regex, char *instance_name) -{ - if (regex == NULL) { - return 0; - } - return flb_regex_match(regex, instance_name, strlen(instance_name)); -} - - -int we_logical_disk_instance_hook(char *instance_name, struct flb_we *ctx) -{ - if (strcasestr(instance_name, "Total") != NULL) { - return 1; - } - if (logical_disk_regex_match(ctx->denying_disk_regex, instance_name) || - !logical_disk_regex_match(ctx->allowing_disk_regex, instance_name)) { - return 1; - } - - return 0; -} - -int we_logical_disk_label_prepend_hook(char **label_list, - size_t label_list_size, - size_t *label_count, - struct we_perflib_metric_source *metric_source, - char *instance_name, - struct we_perflib_counter *counter) -{ - if (label_count == NULL) { - return -1; - } - - if (*label_count >= label_list_size) { - return -2; - } - - label_list[(*label_count)++] = instance_name; - - return 0; -} - -int we_logical_disk_update(struct flb_we *ctx) -{ - if (!ctx->logical_disk.operational) { - flb_plg_error(ctx->ins, "logical_disk collector not yet in operational state"); - - return -1; - } - - return we_perflib_update_counters(ctx, - ctx->logical_disk.query, - ctx->logical_disk.metric_sources, - we_logical_disk_instance_hook, - we_logical_disk_label_prepend_hook); -} diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_logical_disk.h b/fluent-bit/plugins/in_windows_exporter_metrics/we_logical_disk.h deleted file mode 100644 index d63678bae..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_logical_disk.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_WE_LOGICAL_DISK_H -#define FLB_WE_LOGICAL_DISK_H - -#include "we.h" - -int we_logical_disk_init(struct flb_we *ctx); -int we_logical_disk_exit(struct flb_we *ctx); -int we_logical_disk_update(struct flb_we *ctx); - -#endif diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_metric.c b/fluent-bit/plugins/in_windows_exporter_metrics/we_metric.c deleted file mode 100644 index 3475cdad3..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_metric.c +++ /dev/null @@ -1,368 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2021 The Fluent Bit Authors - * Copyright (C) 2015-2018 Treasure Data Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "we.h" -#include "we_cpu.h" -#include "we_util.h" -#include "we_perflib.h" - -static int we_expand_perflib_label_set(char *input_raw_label_set, - char ***output_label_set, - size_t *output_label_set_size) -{ - flb_sds_t raw_label_set; - size_t label_index; - size_t label_count; - char *label_name; - char **label_set; - int result; - - raw_label_set = flb_sds_create(input_raw_label_set); - - if (raw_label_set == NULL) { - return -1; - } - - label_count = 0; - label_name = (char *) raw_label_set; - - while (label_name != NULL) { - result = mk_string_char_search(label_name, ',', -1); - - if (result != -1) { - label_name[result] = '\0'; - label_name = &label_name[result + 1]; - } - else { - label_name = NULL; - } - - label_count++; - } - - label_set = (char **) flb_calloc(label_count, sizeof(char *)); - - if (label_set == NULL) { - flb_sds_destroy(raw_label_set); - - return -2; - } - - label_name = (char *) raw_label_set; - - for (label_index = 0 ; label_index < label_count ; label_index++) { - label_set[label_index] = label_name; - label_name = &label_name[strlen(label_name) + 1]; - } - - *output_label_set = label_set; - *output_label_set_size = label_count; - - return 0; -} - -static int we_expand_perflib_metric_source_labels( - struct we_perflib_metric_source *source) -{ - source->label_set_size = 0; - source->label_set = NULL; - - if (source->raw_label_set == NULL) { - return 0; - } - - return we_expand_perflib_label_set(source->raw_label_set, - &source->label_set, - &source->label_set_size); -} - -static int we_expand_perflib_metric_spec_labels( - struct we_perflib_metric_spec *spec) -{ - spec->label_set_size = 0; - spec->label_set = NULL; - - if (spec->raw_label_set == NULL) { - return 0; - } - - return we_expand_perflib_label_set(spec->raw_label_set, - &spec->label_set, - &spec->label_set_size); -} - -static int we_match_perflib_metric_source_to_parent( - struct flb_hash_table *lookup_table, - struct we_perflib_metric_source *source) -{ - struct we_perflib_metric_spec *spec; - - spec = flb_hash_table_get_ptr(lookup_table, - source->parent_name, - strlen(source->parent_name)); - - if (spec == NULL) { - return -1; - } - - source->parent = spec; - - return 0; -} - -static int we_create_perflib_metric_instance( - struct cmt *context, - struct flb_hash_table *lookup_table, - char *namespace, - char *subsystem, - struct we_perflib_metric_spec *spec) -{ - void *metric_instance; - int result; - - if (spec->type == CMT_COUNTER) { - metric_instance = (void *) cmt_counter_create(context, - namespace, - subsystem, - spec->name, - spec->description, - spec->label_set_size, - spec->label_set); - if (metric_instance == NULL) { - return -1; - } - } - else if (spec->type == CMT_GAUGE) { - metric_instance = (void *) cmt_gauge_create(context, - namespace, - subsystem, - spec->name, - spec->description, - spec->label_set_size, - spec->label_set); - - if (metric_instance == NULL) { - return -2; - } - } - else { - return -3; - } - - result = flb_hash_table_add(lookup_table, - spec->name, - strlen(spec->name), - spec, - 0); - - if (result < 0) { - if (spec->type == CMT_COUNTER) { - cmt_counter_destroy(metric_instance); - } - else { - cmt_gauge_destroy(metric_instance); - } - - return -4; - } - - spec->metric_instance = metric_instance; - - return 0; -} - -void we_deinitialize_perflib_metric_sources(struct we_perflib_metric_source *sources) -{ - size_t source_index; - - for (source_index = 0 ; - sources[source_index].name != NULL; - source_index++) { - if (sources[source_index].label_set_size) { - flb_sds_destroy(sources[source_index].label_set[0]); - flb_free(sources[source_index].label_set); - } - } -} - -int we_initialize_perflib_metric_sources( - struct flb_hash_table *lookup_table, - struct we_perflib_metric_source **out_sources, - struct we_perflib_metric_source *in_sources) -{ - size_t source_array_size; - struct we_perflib_metric_source *source_array_copy; - struct we_perflib_metric_spec *source_entry; - size_t source_index; - size_t source_count; - int result; - - if (out_sources == NULL) { - return -1; - } - - if (in_sources == NULL) { - return -2; - } - - source_count = 0; - - while (in_sources[source_count].name != NULL) { - source_count++; - } - - if (source_count == 0) { - return -3; - } - - source_array_size = sizeof(struct we_perflib_metric_source); - source_array_size *= (source_count + 1); - - source_array_copy = (struct we_perflib_metric_spec *) flb_calloc(1, source_array_size); - - if (source_array_copy == NULL) { - return -4; - } - - memcpy(source_array_copy, in_sources, source_array_size); - - for (source_index = 0 ; source_index < source_count; source_index++) { - source_entry = &source_array_copy[source_index]; - - result = we_expand_perflib_metric_source_labels(source_entry); - - if (result != 0) { - we_deinitialize_perflib_metric_sources(source_array_copy); - flb_free(source_array_copy); - - return -5; - } - - result = we_match_perflib_metric_source_to_parent(lookup_table, - source_entry); - - if (result != 0) { - we_deinitialize_perflib_metric_sources(source_array_copy); - flb_free(source_array_copy); - - return -6; - } - } - - *out_sources = source_array_copy; - - return 0; -} - -void we_deinitialize_perflib_metric_specs(struct we_perflib_metric_spec *specs) -{ - size_t spec_index; - - for (spec_index = 0 ; - specs[spec_index].name != NULL; - spec_index++) { - if (specs[spec_index].label_set_size) { - flb_sds_destroy(specs[spec_index].label_set[0]); - flb_free(specs[spec_index].label_set); - } - } -} - -int we_initialize_perflib_metric_specs( - struct cmt *context, - struct flb_hash_table *lookup_table, - char *namespace, - char *subsystem, - struct we_perflib_metric_spec **out_specs, - struct we_perflib_metric_spec *in_specs) -{ - size_t spec_array_size; - struct we_perflib_metric_spec *spec_array_copy; - struct we_perflib_metric_spec *spec_entry; - size_t spec_index; - size_t spec_count; - int result; - - if (out_specs == NULL) { - return -1; - } - - if (in_specs == NULL) { - return -2; - } - - spec_count = 0; - - while (in_specs[spec_count].name != NULL) { - spec_count++; - } - - if (spec_count == 0) { - return -3; - } - - spec_array_size = sizeof(struct we_perflib_metric_spec); - spec_array_size *= spec_count + 1; - - spec_array_copy = (struct we_perflib_metric_spec *) flb_calloc(1, spec_array_size); - - if (spec_array_copy == NULL) { - return -4; - } - - memcpy(spec_array_copy, in_specs, spec_array_size); - - for (spec_index = 0 ; spec_index < spec_count; spec_index++) { - spec_entry = &spec_array_copy[spec_index]; - - result = we_expand_perflib_metric_spec_labels(spec_entry); - - if (result) { - we_deinitialize_perflib_metric_specs(spec_array_copy); - flb_free(spec_array_copy); - - return -5; - } - - result = we_create_perflib_metric_instance(context, - lookup_table, - namespace, - subsystem, - spec_entry); - - if (result) { - we_deinitialize_perflib_metric_specs(spec_array_copy); - flb_free(spec_array_copy); - - return -6; - } - } - - *out_specs = spec_array_copy; - - return 0; -} - diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_metric.h b/fluent-bit/plugins/in_windows_exporter_metrics/we_metric.h deleted file mode 100644 index 7c9611a69..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_metric.h +++ /dev/null @@ -1,98 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2021 The Fluent Bit Authors - * Copyright (C) 2015-2018 Treasure Data Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_METRIC_H -#define FLB_METRIC_H - -#include -#include -#include -#include - -struct we_perflib_metric_spec { - int type; - char *name; - char *description; - char *raw_label_set; - char **label_set; - size_t label_set_size; - void *metric_instance; -}; - -struct we_perflib_metric_source { - struct we_perflib_metric_spec *parent; - char *parent_name; - char *name; - char *raw_label_set; - char **label_set; - size_t label_set_size; -}; - -#define WE_PERFLIB_SPEC(type_, name_, description_, raw_label_set_) \ - { \ - .type = type_, \ - .name = name_, \ - .description = description_, \ - .raw_label_set = raw_label_set_, \ - .label_set = NULL, \ - .label_set_size = 0, \ - .metric_instance = NULL \ - } - -#define WE_PERFLIB_COUNTER_SPEC(name_, description_, raw_label_set_) \ - WE_PERFLIB_SPEC(CMT_COUNTER, name_, description_, raw_label_set_) - -#define WE_PERFLIB_GAUGE_SPEC(name_, description_, raw_label_set_) \ - WE_PERFLIB_SPEC(CMT_GAUGE, name_, description_, raw_label_set_) - -#define WE_PERFLIB_TERMINATOR_SPEC() \ - WE_PERFLIB_SPEC(0, NULL, NULL, NULL) - -#define WE_PERFLIB_METRIC_SOURCE(parent_name_, name_, raw_label_set_) \ - { \ - .parent = NULL, \ - .parent_name = parent_name_, \ - .name = name_, \ - .raw_label_set = raw_label_set_, \ - .label_set = NULL, \ - .label_set_size = 0 \ - } - -#define WE_PERFLIB_TERMINATOR_SOURCE() \ - WE_PERFLIB_METRIC_SOURCE(NULL, NULL, NULL) - - -void we_deinitialize_perflib_metric_sources(struct we_perflib_metric_source *sources); -int we_initialize_perflib_metric_sources( - struct flb_hash *lookup_table, - struct we_perflib_metric_source **out_sources, - struct we_perflib_metric_source *in_sources); - - -void we_deinitialize_perflib_metric_specs(struct we_perflib_metric_spec *specs); -int we_initialize_perflib_metric_specs( - struct cmt *context, - struct flb_hash *lookup_table, - char *namespace, - char *subsystem, - struct we_perflib_metric_spec **out_specs, - struct we_perflib_metric_spec *in_specs); - -#endif diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_net.c b/fluent-bit/plugins/in_windows_exporter_metrics/we_net.c deleted file mode 100644 index 673d665e8..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_net.c +++ /dev/null @@ -1,253 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "we.h" -#include "we_net.h" -#include "we_util.h" -#include "we_metric.h" -#include "we_perflib.h" - - -struct we_perflib_metric_source net_metric_sources[] = { - WE_PERFLIB_METRIC_SOURCE("bytes_received_total", - "Bytes Received/sec", - NULL), - - WE_PERFLIB_METRIC_SOURCE("bytes_sent_total", - "Bytes Sent/sec", - NULL), - - WE_PERFLIB_METRIC_SOURCE("bytes_total", - "Bytes Total/sec", - NULL), - - WE_PERFLIB_METRIC_SOURCE("packets_outbound_discarded_total", - "Packets Outbound Discarded", - NULL), - - WE_PERFLIB_METRIC_SOURCE("packets_outbound_errors_total", - "Packets Outbound Errors", - NULL), - - WE_PERFLIB_METRIC_SOURCE("packets_received_discarded_total", - "Packets Received Discarded", - NULL), - - WE_PERFLIB_METRIC_SOURCE("packets_received_errors_total", - "Packets Received Errors", - NULL), - - WE_PERFLIB_METRIC_SOURCE("packets_received_total", - "Packets Received/sec", - NULL), - - WE_PERFLIB_METRIC_SOURCE("packets_received_unknown_total", - "Packets Received Unknown", - NULL), - - WE_PERFLIB_METRIC_SOURCE("packets_total", - "Packets/sec", - NULL), - - WE_PERFLIB_METRIC_SOURCE("packets_sent_total", - "Packets Sent/sec", - NULL), - - WE_PERFLIB_METRIC_SOURCE("current_bandwidth_bits", - "Current Bandwidth", - NULL), - - WE_PERFLIB_TERMINATOR_SOURCE() - }; - -struct we_perflib_metric_spec net_metric_specs[] = { - WE_PERFLIB_COUNTER_SPEC("bytes_received_total", - "Total amount of received bytes", - "nic"), - - WE_PERFLIB_COUNTER_SPEC("bytes_sent_total", - "Total amount of sent bytes", - "nic"), - - WE_PERFLIB_COUNTER_SPEC("bytes_total", - "Total amount of bytes", - "nic"), - - WE_PERFLIB_COUNTER_SPEC("packets_outbound_discarded_total", - "Total amount of outbound discarded bytes", - "nic"), - - WE_PERFLIB_COUNTER_SPEC("packets_outbound_errors_total", - "Total number of outbound errors", - "nic"), - - WE_PERFLIB_COUNTER_SPEC("packets_received_discarded_total", - "Total amount of received discarded bytes", - "nic"), - - WE_PERFLIB_COUNTER_SPEC("packets_received_errors_total", - "Total number of received packets' errors", - "nic"), - - WE_PERFLIB_COUNTER_SPEC("packets_received_total", - "Total number of received packets", - "nic"), - - WE_PERFLIB_COUNTER_SPEC("packets_received_unknown_total", - "Total number of received unknown", - "nic"), - - WE_PERFLIB_COUNTER_SPEC("packets_total", - "Total amount of packets", - "nic"), - - WE_PERFLIB_COUNTER_SPEC("packets_sent_total", - "Total amount of sent packets", - "nic"), - - WE_PERFLIB_GAUGE_SPEC("current_bandwidth_bits", - "Current Bandwidth /bits", - "nic"), - - WE_PERFLIB_TERMINATOR_SPEC() - }; - - -int we_net_init(struct flb_we *ctx) -{ - struct we_perflib_metric_source *metric_sources; - int result; - - ctx->net.operational = FLB_FALSE; - - ctx->net.metrics = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, 32, 128); - - if (ctx->net.metrics == NULL) { - flb_plg_error(ctx->ins, "could not create metrics hash table"); - - return -1; - } - - result = we_initialize_perflib_metric_specs(ctx->cmt, - ctx->net.metrics, - "windows", - "net", - &ctx->net.metric_specs, - net_metric_specs); - - if (result != 0) { - flb_plg_error(ctx->ins, "could not initialize net metric specs"); - - return -2; - } - - ctx->net.query = (char *) "Network Interface"; - - result = we_initialize_perflib_metric_sources(ctx->net.metrics, - &ctx->net.metric_sources, - net_metric_sources); - - if (result != 0) { - flb_plg_error(ctx->ins, "could not initialize net metric sources"); - - we_deinitialize_perflib_metric_specs(ctx->net.metric_specs); - flb_free(ctx->net.metric_specs); - - return -3; - } - - ctx->net.operational = FLB_TRUE; - - return 0; -} - -int we_net_exit(struct flb_we *ctx) -{ - we_deinitialize_perflib_metric_sources(ctx->net.metric_sources); - we_deinitialize_perflib_metric_specs(ctx->net.metric_specs); - - flb_free(ctx->net.metric_sources); - flb_free(ctx->net.metric_specs); - - ctx->net.operational = FLB_FALSE; - - return 0; -} - -static int net_regex_match(struct flb_regex *regex, char *instance_name) -{ - if (regex == NULL) { - return 0; - } - return flb_regex_match(regex, instance_name, strlen(instance_name)); -} - -int we_net_instance_hook(char *instance_name, struct flb_we *ctx) -{ - if (strcasestr(instance_name, "Total") != NULL) { - return 1; - } - - if (!net_regex_match(ctx->allowing_nic_regex, instance_name)) { - return 1; - } - - return 0; -} - -int we_net_label_prepend_hook(char **label_list, - size_t label_list_size, - size_t *label_count, - struct we_perflib_metric_source *metric_source, - char *instance_name, - struct we_perflib_counter *counter) -{ - if (label_count == NULL) { - return -1; - } - - if (*label_count >= label_list_size) { - return -2; - } - - label_list[(*label_count)++] = instance_name; - - return 0; -} - -int we_net_update(struct flb_we *ctx) -{ - if (!ctx->net.operational) { - flb_plg_error(ctx->ins, "net collector not yet in operational state"); - - return -1; - } - - return we_perflib_update_counters(ctx, - ctx->net.query, - ctx->net.metric_sources, - we_net_instance_hook, - we_net_label_prepend_hook); -} diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_net.h b/fluent-bit/plugins/in_windows_exporter_metrics/we_net.h deleted file mode 100644 index 52340ccfb..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_net.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_WE_NET_H -#define FLB_WE_NET_H - -#include "we.h" - -int we_net_init(struct flb_we *ctx); -int we_net_exit(struct flb_we *ctx); -int we_net_update(struct flb_we *ctx); - -#endif diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_os.c b/fluent-bit/plugins/in_windows_exporter_metrics/we_os.c deleted file mode 100644 index 299514d8c..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_os.c +++ /dev/null @@ -1,268 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#ifndef UNICODE -#define UNICODE -#endif -#include -#include -#include - -#include "we.h" -#include "we_os.h" -#include "we_util.h" -#include "we_metric.h" - -int we_os_init(struct flb_we *ctx) -{ - ctx->os = flb_calloc(1, sizeof(struct we_os_counters)); - if (!ctx->os) { - flb_errno(); - return -1; - } - ctx->os->operational = FLB_FALSE; - - struct cmt_gauge *g; - - g = cmt_gauge_create(ctx->cmt, "windows", "os", "info", - "Version information of OperatingSystem", - 5, (char *[]) {"product", "version", "major_version", "minor_version", "build_number"}); - - if (!g) { - return -1; - } - ctx->os->info = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "os", "physical_memory_free_bytes", - "Amount of free bytes of physical memory", - 0, NULL); - - if (!g) { - return -1; - } - ctx->os->physical_memory_free_bytes = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "os", "time", - "Value of Local Time", - 0, NULL); - if (!g) { - return -1; - } - ctx->os->time = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "os", "timezone", - "Name of Local Timezone", - 1, (char *[]) {"timezone"}); - if (!g) { - return -1; - } - ctx->os->tz = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "os", "virtual_memory_bytes", - "Total amount of bytes of virtual memory", - 0, NULL); - - if (!g) { - return -1; - } - ctx->os->virtual_memory_bytes = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "os", "processes_limit", - "Number of processes limit", - 0, NULL); - - if (!g) { - return -1; - } - ctx->os->processes_limit = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "os", "process_memory_limit_bytes", - "Limit of processes memory", - 0, NULL); - - if (!g) { - return -1; - } - ctx->os->process_memory_limit_bytes = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "os", "processes", - "Number of processes", - 0, NULL); - - if (!g) { - return -1; - } - ctx->os->processes = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "os", "users", - "Number of users", - 0, NULL); - - if (!g) { - return -1; - } - ctx->os->users = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "os", "visible_memory_bytes", - "Total amount of bytes of visibile memory", - 0, NULL); - - if (!g) { - return -1; - } - ctx->os->visible_memory_bytes = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "os", "virtual_memory_free_bytes", - "Amount of free bytes of virtual memory", - 0, NULL); - - if (!g) { - return -1; - } - ctx->os->virtual_memory_free_bytes = g; - - ctx->os->operational = FLB_TRUE; - - return 0; -} - -int we_os_exit(struct flb_we *ctx) -{ - flb_free(ctx->os); - return 0; -} - -int we_os_update(struct flb_we *ctx) -{ - DWORD level = 102; - LPWKSTA_INFO_102 wksta = NULL; - NET_API_STATUS status; - MEMORYSTATUSEX statex; - PERFORMANCE_INFORMATION perf; - DWORD size = 0; - char version[65] = {0}, major[32] = {0}, minor[32] = {0}; - int users = 0; - LONG ret; - HKEY hkey; - char caption[80], build_number[32]; - DWORD caption_len = sizeof(caption), build_len = sizeof(build_number); - uint64_t timestamp = 0; - char label_caption[90]; - TIME_ZONE_INFORMATION tzi; - DWORD tztype = 0; - char *displaytz; - - if (!ctx->os->operational) { - flb_plg_error(ctx->ins, "os collector not yet in operational state"); - - return -1; - } - - timestamp = cfl_time_now(); - - ret = RegOpenKeyExA(HKEY_LOCAL_MACHINE, WE_OS_CURRENT_VERSION_PATH, 0, KEY_QUERY_VALUE, &hkey); - if (ret != ERROR_SUCCESS) { - return -1; - } - ret = RegQueryValueExA(hkey, "ProductName", NULL, NULL, (LPBYTE)caption, &caption_len); - if (ret != ERROR_SUCCESS) { - return -1; - } - ret = RegQueryValueExA(hkey, "CurrentBuildNumber", NULL, NULL, (LPBYTE)build_number, &build_len); - if (ret != ERROR_SUCCESS) { - return -1; - } - RegCloseKey(hkey); - - status = NetWkstaGetInfo(NULL, - level, - (LPBYTE *)&wksta); - - if (status == NERR_Success) { - snprintf(version, 65, "%d.%d", wksta->wki102_ver_major, - wksta->wki102_ver_minor); - snprintf(major, 32, "%d", wksta->wki102_ver_major); - snprintf(minor, 32, "%d", wksta->wki102_ver_minor); - snprintf(label_caption, 90, "Microsoft %s", caption); - - users = wksta->wki102_logged_on_users; - - cmt_gauge_set(ctx->os->info, timestamp, 1.0, 5, - (char *[]) { label_caption, version, major, minor, build_number}); - cmt_gauge_set(ctx->os->users, timestamp, (double)users, 0, NULL); - } - else { - if (wksta != NULL) { - NetApiBufferFree(wksta); - } - flb_plg_error(ctx->ins, "A system error has occurred: %d\n", status); - return -1; - } - - cmt_gauge_set(ctx->os->time, timestamp, (double)timestamp/1000000000L, 0, NULL); - - tztype = GetTimeZoneInformation(&tzi); - switch (tztype) { - case TIME_ZONE_ID_STANDARD: - displaytz = we_convert_wstr(tzi.StandardName, CP_UTF8); - cmt_gauge_set(ctx->os->tz, timestamp, 1.0, 1, (char *[]) {displaytz}); - flb_free(displaytz); - break; - case TIME_ZONE_ID_DAYLIGHT: - displaytz = we_convert_wstr(tzi.DaylightName, CP_UTF8); - cmt_gauge_set(ctx->os->tz, timestamp, 1.0, 1, (char *[]) {displaytz}); - flb_free(displaytz); - break; - case TIME_ZONE_ID_UNKNOWN: - /* The current timezone does not use daylight saving time. */ - displaytz = we_convert_wstr(tzi.StandardName, CP_UTF8); - cmt_gauge_set(ctx->os->tz, timestamp, 1.0, 1, (char *[]) {displaytz}); - flb_free(displaytz); - break; - default: - flb_plg_error(ctx->ins, "Error to retrieve timezone information with status: %d", GetLastError()); - } - - statex.dwLength = sizeof (statex); - GlobalMemoryStatusEx(&statex); - - size = sizeof(perf); - GetPerformanceInfo(&perf, size); - - cmt_gauge_set(ctx->os->physical_memory_free_bytes, timestamp, (double)statex.ullAvailPhys, 0, NULL); - cmt_gauge_set(ctx->os->virtual_memory_free_bytes, timestamp, (double)statex.ullAvailPageFile, 0, NULL); - /* The result is from $(Get-WMIObject Win32_OperatingSystem).MaxNumberOfProcesses. */ - cmt_gauge_set(ctx->os->processes_limit, timestamp, (double)4294967295, 0, NULL); - cmt_gauge_set(ctx->os->process_memory_limit_bytes, timestamp, (double)statex.ullTotalVirtual, 0, NULL); - cmt_gauge_set(ctx->os->processes, timestamp, (double)perf.ProcessCount, 0, NULL); - cmt_gauge_set(ctx->os->virtual_memory_bytes, timestamp, (double)statex.ullTotalPageFile, 0, NULL); - cmt_gauge_set(ctx->os->visible_memory_bytes, timestamp, (double)statex.ullTotalPhys, 0, NULL); - - if (wksta != NULL) { - NetApiBufferFree(wksta); - } - - return 0; -} diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_os.h b/fluent-bit/plugins/in_windows_exporter_metrics/we_os.h deleted file mode 100644 index 6a62e6064..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_os.h +++ /dev/null @@ -1,32 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_WE_OS_H -#define FLB_WE_OS_H - -#include "we.h" - -#define WE_OS_CURRENT_VERSION_PATH \ - "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion" - -int we_os_init(struct flb_we *ctx); -int we_os_exit(struct flb_we *ctx); -int we_os_update(struct flb_we *ctx); - -#endif diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_perflib.c b/fluent-bit/plugins/in_windows_exporter_metrics/we_perflib.c deleted file mode 100644 index 0c140661d..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_perflib.c +++ /dev/null @@ -1,1048 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2021 The Fluent Bit Authors - * Copyright (C) 2015-2018 Treasure Data Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include - -#include "we.h" -#include "we_util.h" -#include "we_metric.h" -#include "we_perflib.h" - -double we_perflib_get_adjusted_counter_value(struct we_perflib_counter *counter) -{ - double result; - - result = (double) counter->primary_value.as_qword; - - switch(counter->definition->type) { - case PERF_ELAPSED_TIME: - result -= counter->parent->parent->time; - result /= counter->parent->parent->frequency; - break; - - case PERF_100NSEC_TIMER: - case PERF_PRECISION_100NS_TIMER: - result /= counter->parent->parent->frequency; - break; - } - - return result; -} - -char *we_perflib_get_counter_type_as_text(uint32_t counter_Type) -{ - switch (counter_Type) { - case PERF_100NSEC_TIMER: - return "PERF_100NSEC_TIMER"; - case PERF_100NSEC_TIMER_INV: - return "PERF_100NSEC_TIMER_INV"; - case PERF_100NSEC_MULTI_TIMER: - return "PERF_100NSEC_MULTI_TIMER"; - case PERF_100NSEC_MULTI_TIMER_INV: - return "PERF_100NSEC_MULTI_TIMER_INV"; - case PERF_AVERAGE_BASE: - return "PERF_AVERAGE_BASE"; - case PERF_AVERAGE_BULK: - return "PERF_AVERAGE_BULK"; - case PERF_AVERAGE_TIMER: - return "PERF_AVERAGE_TIMER"; - case PERF_COUNTER_100NS_QUEUELEN_TYPE: - return "PERF_COUNTER_100NS_QUEUELEN_TYPE"; - case PERF_COUNTER_BULK_COUNT: - return "PERF_COUNTER_BULK_COUNT"; - case PERF_COUNTER_COUNTER: - return "PERF_COUNTER_COUNTER"; - case PERF_COUNTER_DELTA: - return "PERF_COUNTER_DELTA"; - case PERF_COUNTER_HISTOGRAM_TYPE: - return "PERF_COUNTER_HISTOGRAM_TYPE"; - case PERF_COUNTER_LARGE_DELTA: - return "PERF_COUNTER_LARGE_DELTA"; - case PERF_COUNTER_LARGE_QUEUELEN_TYPE: - return "PERF_COUNTER_LARGE_QUEUELEN_TYPE"; - case PERF_COUNTER_LARGE_RAWCOUNT: - return "PERF_COUNTER_LARGE_RAWCOUNT"; - case PERF_COUNTER_LARGE_RAWCOUNT_HEX: - return "PERF_COUNTER_LARGE_RAWCOUNT_HEX"; - case PERF_COUNTER_MULTI_BASE: - return "PERF_COUNTER_MULTI_BASE"; - case PERF_COUNTER_MULTI_TIMER: - return "PERF_COUNTER_MULTI_TIMER"; - case PERF_COUNTER_MULTI_TIMER_INV: - return "PERF_COUNTER_MULTI_TIMER_INV"; - case PERF_COUNTER_NODATA: - return "PERF_COUNTER_NODATA"; - case PERF_COUNTER_OBJ_TIME_QUEUELEN_TYPE: - return "PERF_COUNTER_OBJ_TIME_QUEUELEN_TYPE"; - case PERF_COUNTER_QUEUELEN_TYPE: - return "PERF_COUNTER_QUEUELEN_TYPE"; - case PERF_COUNTER_RAWCOUNT: - return "PERF_COUNTER_RAWCOUNT"; - case PERF_COUNTER_RAWCOUNT_HEX: - return "PERF_COUNTER_RAWCOUNT_HEX"; - case PERF_COUNTER_TEXT: - return "PERF_COUNTER_TEXT"; - case PERF_COUNTER_TIMER: - return "PERF_COUNTER_TIMER"; - case PERF_COUNTER_TIMER_INV: - return "PERF_COUNTER_TIMER_INV"; - case PERF_ELAPSED_TIME: - return "PERF_ELAPSED_TIME"; - case PERF_LARGE_RAW_BASE: - return "PERF_LARGE_RAW_BASE"; - case PERF_LARGE_RAW_FRACTION: - return "PERF_LARGE_RAW_FRACTION"; - case PERF_OBJ_TIME_TIMER: - return "PERF_OBJ_TIME_TIMER"; - case PERF_PRECISION_100NS_TIMER: - return "PERF_PRECISION_100NS_TIMER"; - case PERF_PRECISION_OBJECT_TIMER: - return "PERF_PRECISION_OBJECT_TIMER"; - case PERF_PRECISION_SYSTEM_TIMER: - return "PERF_PRECISION_SYSTEM_TIMER"; - case PERF_RAW_BASE: - return "PERF_RAW_BASE"; - case PERF_RAW_FRACTION: - return "PERF_RAW_FRACTION"; - case PERF_SAMPLE_BASE: - return "PERF_SAMPLE_BASE"; - case PERF_SAMPLE_COUNTER: - return "PERF_SAMPLE_COUNTER"; - case PERF_SAMPLE_FRACTION: - return "PERF_SAMPLE_FRACTION"; - }; - - return "UNRECOGNIZED_COUNTER_TYPE"; -} - -void we_perflib_destroy_counter(struct we_perflib_counter *counter) -{ - flb_free(counter); -} - -void we_perflib_destroy_instance(struct we_perflib_instance *instance) -{ - struct flb_hash_table_entry *counter_hash_entry; - struct mk_list *counter_iterator; - struct we_perflib_counter *counter; - struct mk_list *tmp; - - mk_list_foreach_safe(counter_iterator, - tmp, - &instance->counters->entries) { - counter_hash_entry = mk_list_entry(counter_iterator, - struct flb_hash_table_entry, - _head_parent); - - counter = (struct we_perflib_counter *) counter_hash_entry->val; - - we_perflib_destroy_counter(counter); - } - - if (instance->name != NULL) { - flb_free(instance->name); - } - - flb_hash_table_destroy(instance->counters); - - flb_free(instance); -} - -void we_perflib_destroy_counter_definition( - struct we_perflib_counter_definition *definition) -{ - flb_sds_destroy(definition->name_index_str); - - mk_list_del(&definition->_head); - - flb_free(definition); -} - -void we_perflib_destroy_object(struct we_perflib_object *object) -{ - struct mk_list *definition_iterator; - struct flb_hash_table_entry *instance_hash_entry; - struct mk_list *instance_iterator; - struct we_perflib_counter_definition *definition; - struct we_perflib_instance *instance; - struct mk_list *tmp; - - mk_list_foreach_safe(definition_iterator, tmp, &object->counter_definitions) { - definition = mk_list_entry(definition_iterator, - struct we_perflib_counter_definition, - _head); - - we_perflib_destroy_counter_definition(definition); - } - - mk_list_foreach_safe(instance_iterator, tmp, &object->instances->entries) { - instance_hash_entry = mk_list_entry(instance_iterator, - struct flb_hash_table_entry, - _head_parent); - - instance = (struct we_perflib_instance *) instance_hash_entry->val; - - we_perflib_destroy_instance(instance); - } - - flb_hash_table_destroy(object->instances); - - flb_free(object); -} - -static int get_string_list(char *source, flb_sds_t *out_result_buffer) -{ - DWORD result_buffer_size; - flb_sds_t result_buffer; - LSTATUS result; - - result_buffer = NULL; - result_buffer_size = 0; - - if (out_result_buffer == NULL) { - return -1; - } - - result = RegQueryValueExA(HKEY_PERFORMANCE_TEXT, - source, - NULL, - NULL, - NULL, - &result_buffer_size); - - if (result != ERROR_SUCCESS) { - return -2; - } - - result_buffer = flb_sds_create_size(result_buffer_size); - - if (result_buffer == NULL) { - return -3; - } - - result = RegQueryValueExA(HKEY_PERFORMANCE_TEXT, - source, - NULL, - NULL, - (LPBYTE) result_buffer, - &result_buffer_size); - - if (result != ERROR_SUCCESS) - { - flb_sds_destroy(result_buffer); - - return -4; - } - - *out_result_buffer = result_buffer; - - return 0; -} - -static int get_number_of_string_entries(uint32_t *result_count) -{ - DWORD argument_size; - DWORD entry_count; - HKEY key_handle; - LSTATUS result; - - entry_count = 0; - argument_size = sizeof(DWORD); - - result = RegOpenKeyExA(HKEY_LOCAL_MACHINE, - WE_PERFLIB_REGISTRY_PATH, - 0, - KEY_READ, - &key_handle); - - if (result != ERROR_SUCCESS) { - return -1; - } - - result = RegQueryValueExA(key_handle, - WE_PERFLIB_STRING_COUNT_KEY, - NULL, - 0, - (LPBYTE) &entry_count, - &argument_size); - - RegCloseKey(key_handle); - - if (result != ERROR_SUCCESS) { - return -2; - } - - *result_count = (uint32_t) entry_count; - - return 0; -} - -static int get_text_mapping_table(struct flb_hash_table **out_mapping_table) -{ - char *current_counter_string; - flb_sds_t counter_strings; - char *counter_index; - char *counter_name; - uint32_t string_count; - int result; - - if (out_mapping_table == NULL) { - return -1; - } - - result = get_number_of_string_entries(&string_count); - - if (result) { - return -2; - } - - result = get_string_list(WE_PERFLIB_COUNTER_KEY_NAME, &counter_strings); - - if (result) { - return -3; - } - - *out_mapping_table = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, - 512, string_count * 2); - if (*out_mapping_table == NULL) { - flb_sds_destroy(counter_strings); - - return -4; - } - - current_counter_string = (char *) counter_strings; - - while (1) { - counter_index = current_counter_string; - current_counter_string = ¤t_counter_string[strlen(current_counter_string) + 1]; - - if (!current_counter_string[0]) { - break; - } - - counter_name = current_counter_string; - current_counter_string = ¤t_counter_string[strlen(current_counter_string) + 1]; - - if (!current_counter_string[0]) { - break; - } - - result = flb_hash_table_add(*out_mapping_table, - counter_name, strlen(counter_name), - counter_index, strlen(counter_index)); - - if (result < 0) { - flb_sds_destroy(counter_strings); - flb_hash_table_destroy(*out_mapping_table); - - *out_mapping_table = NULL; - - return -5; - } - - result = flb_hash_table_add(*out_mapping_table, - counter_index, strlen(counter_index), - counter_name, strlen(counter_name)); - - if (result < 0) { - flb_sds_destroy(counter_strings); - flb_hash_table_destroy(*out_mapping_table); - - *out_mapping_table = NULL; - - return -5; - } - } - - flb_sds_destroy(counter_strings); - - return 0; -} - -int we_perflib_query_raw_data(struct flb_we *ctx, char *source, - char **out_buffer, size_t *out_buffer_size) -{ - char *reallocated_buffer; - DWORD buffer_size; - DWORD data_size; - char *buffer; - LSTATUS result; - - buffer_size = WE_PERFLIB_QUERY_BUFFER_INITIAL_SIZE; - - result = ERROR_MORE_DATA; - - buffer = (char *) flb_malloc(buffer_size); - - if (buffer == NULL) { - return -1; - } - - while (result == ERROR_MORE_DATA) { - data_size = buffer_size; - - result = RegQueryValueExA(HKEY_PERFORMANCE_DATA, - source, - NULL, - NULL, - buffer, - &data_size); - - RegCloseKey(HKEY_PERFORMANCE_DATA); - - buffer_size += WE_PERFLIB_QUERY_BUFFER_INCREMENT_SIZE; - - reallocated_buffer = (char *) flb_realloc(buffer, buffer_size); - - if (reallocated_buffer == NULL) { - flb_free(buffer); - - return -2; - } - - buffer = reallocated_buffer; - } - - *out_buffer = buffer; - *out_buffer_size = data_size; - - return 0; -} - -static char *we_perflib_lookup_counter_index(struct flb_hash_table *mapping_table, - char *name) -{ - return flb_hash_table_get_ptr(mapping_table, - name, - strlen(name)); -} - -static char *we_perflib_lookup_counter_name(struct flb_hash_table *mapping_table, - uint32_t index) -{ - char hash_table_index[11]; - - sprintf(hash_table_index, "%" PRIu32, index); - - return flb_hash_table_get_ptr(mapping_table, - hash_table_index, - strlen(hash_table_index)); -} - -static int we_perflib_process_object_type( - struct we_perflib_context *context, - char *input_data_block, - struct we_perflib_object **out_perflib_object) -{ - char *input_object_block; - struct we_perflib_object *perflib_object; - PERF_OBJECT_TYPE *perf_object; - PERF_DATA_BLOCK *perf_data; - int result; - - perf_data = (PERF_DATA_BLOCK *) input_data_block; - - result = wcsncmp(perf_data->Signature, L"PERF", 4); - - if (result) { - return -1; - } - - input_object_block = &input_data_block[perf_data->HeaderLength]; - - perf_object = (PERF_OBJECT_TYPE *) input_object_block; - - perflib_object = (struct we_perflib_object *) \ - flb_calloc(1, sizeof(struct we_perflib_object)); - - if (perflib_object == NULL) { - return -2; - } - - perflib_object->name = we_perflib_lookup_counter_name( - context->counter_indexes, - perf_object->ObjectNameTitleIndex); - - if (perflib_object->name == NULL) { - flb_free(perflib_object); - - return -3; - } - - perflib_object->time = perf_data->PerfTime.QuadPart; - perflib_object->frequency = perf_data->PerfFreq.QuadPart; - perflib_object->hundred_ns_time = perf_data->PerfTime100nSec.QuadPart; - - perflib_object->counter_count = perf_object->NumCounters; - perflib_object->instance_count = perf_object->NumInstances; - - - perflib_object->instances = flb_hash_table_create( - FLB_HASH_TABLE_EVICT_NONE, - 64, - perflib_object->instance_count + 1); - - if (perflib_object->instances == NULL) { - flb_free(perflib_object); - - return -4; - } - - mk_list_init(&perflib_object->counter_definitions); - - *out_perflib_object = perflib_object; - - return perf_data->HeaderLength + perf_object->HeaderLength; -} - -static int we_perflib_process_counter_definition( - struct we_perflib_context *context, - char *input_data_block, - struct we_perflib_counter_definition **out_counter_definition) -{ - PERF_COUNTER_DEFINITION *perf_counter_definition; - struct we_perflib_counter_definition *counter_definition; - char name_index_str[12]; - - perf_counter_definition = (PERF_COUNTER_DEFINITION *) input_data_block; - - counter_definition = (struct we_perflib_counter_definition *) \ - flb_calloc(1, sizeof(struct we_perflib_counter_definition)); - - if (counter_definition == NULL) { - return -1; - } - - counter_definition->name_index = perf_counter_definition->CounterNameTitleIndex; - - counter_definition->name = we_perflib_lookup_counter_name( - context->counter_indexes, - counter_definition->name_index); - - snprintf(name_index_str, - sizeof(name_index_str), - "%" PRIu32, - counter_definition->name_index); - - counter_definition->name_index_str = flb_sds_create(name_index_str); - - if (counter_definition->name_index_str == NULL) { - flb_free(counter_definition); - - return -2; - } - - if (counter_definition->name == NULL) { - counter_definition->name = ""; - } - - if (counter_definition->name_index_str == NULL) { - counter_definition->name_index_str = flb_sds_create(""); - } - - counter_definition->help_index = perf_counter_definition->CounterHelpTitleIndex; - - counter_definition->type = perf_counter_definition->CounterType; - counter_definition->size = perf_counter_definition->CounterSize; - counter_definition->offset = perf_counter_definition->CounterOffset; - counter_definition->detail_level = perf_counter_definition->DetailLevel; - - *out_counter_definition = counter_definition; - - return perf_counter_definition->ByteLength; -} - -static int we_perflib_process_counter_definitions( - struct we_perflib_context *context, - struct we_perflib_object *perflib_object, - char *input_data_block) -{ - size_t counter_definition_index; - struct we_perflib_counter_definition *counter_definition; - size_t offset; - int result; - - offset = 0; - - for (counter_definition_index = 0 ; - counter_definition_index < perflib_object->counter_count ; - counter_definition_index++) { - result = we_perflib_process_counter_definition(context, - &input_data_block[offset], - &counter_definition); - - if (result <= 0) { - return -1; - } - - offset += result; - - mk_list_add(&counter_definition->_head, &perflib_object->counter_definitions); - } - - return offset; -} - -static struct we_perflib_counter * we_perflib_create_counter( - struct we_perflib_counter_definition *counter_definition) -{ - struct we_perflib_counter *counter; - - counter = (struct we_perflib_counter *) \ - flb_calloc(1, sizeof(struct we_perflib_counter)); - - if (counter == NULL) { - return NULL; - } - - counter->definition = counter_definition; - - return counter; -} - -static int we_perflib_process_counter( - struct we_perflib_context *context, - struct we_perflib_counter_definition *counter_definition, - char *input_data_block, - struct we_perflib_counter **out_counter) -{ - struct we_perflib_counter *perflib_instance_counter; - - perflib_instance_counter = we_perflib_create_counter(counter_definition); - - if (perflib_instance_counter == NULL) { - return -1; - } - - memcpy(&perflib_instance_counter->primary_value, - &input_data_block[counter_definition->offset], - counter_definition->size); - - if (counter_definition->size > sizeof(union we_perflib_value)) { - we_perflib_destroy_counter(perflib_instance_counter); - - return -2; - } - - *out_counter = perflib_instance_counter; - - return 0; -} - -static int we_perflib_process_counters(struct we_perflib_context *context, - struct we_perflib_object *perflib_object, - struct we_perflib_instance *instance, - char *input_data_block) -{ - struct mk_list *counter_definition_iterator; - struct we_perflib_counter *perflib_instance_counter; - PERF_COUNTER_BLOCK *perf_counter_block; - struct we_perflib_counter_definition *counter_definition; - int result; - int offset; - - perf_counter_block = (PERF_COUNTER_BLOCK *) input_data_block; - - mk_list_foreach(counter_definition_iterator, - &perflib_object->counter_definitions) { - counter_definition = mk_list_entry(counter_definition_iterator, - struct we_perflib_counter_definition, - _head); - - if (!counter_definition->name_index) { - continue; - } - - result = we_perflib_process_counter(context, - counter_definition, - input_data_block, - &perflib_instance_counter); - - if (result < 0) { - return -1; - } - - perflib_instance_counter->parent = instance; - - result = -1; - - if (counter_definition->name[0]) { - result = flb_hash_table_add(instance->counters, - counter_definition->name, - strlen(counter_definition->name), - perflib_instance_counter, - 0); - } - else - { - result = flb_hash_table_add(instance->counters, - counter_definition->name_index_str, - strlen(counter_definition->name_index_str), - perflib_instance_counter, - 0); - } - - if (result < 0) { - we_perflib_destroy_counter(perflib_instance_counter); - - return -2; - } - } - - return perf_counter_block->ByteLength; -} - -static struct we_perflib_instance *we_perflib_create_instance(size_t counter_count) -{ - struct we_perflib_instance *instance; - - instance = (struct we_perflib_instance *) \ - flb_calloc(1, sizeof(struct we_perflib_instance)); - - if (instance == NULL) { - return NULL; - } - - instance->counters = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, - 64, - counter_count + 1); - - if (instance->counters == NULL) { - flb_free(instance); - - return NULL; - } - - return instance; -} - -static int we_perflib_process_instance(struct we_perflib_context *context, - struct we_perflib_object *perflib_object, - char *input_data_block, - struct we_perflib_instance **out_instance) -{ - PERF_INSTANCE_DEFINITION *perf_instance_definition; - struct we_perflib_instance *perflib_instance; - int offset; - int result; - - perflib_instance = we_perflib_create_instance(perflib_object->counter_count); - - if (perflib_instance == NULL) { - return -1; - } - - offset = 0; - - if (perflib_object->instance_count >= 1) { - perf_instance_definition = (PERF_INSTANCE_DEFINITION *) input_data_block; - - if (perf_instance_definition->NameLength > 0) { - perflib_instance->name = \ - we_convert_wstr(&input_data_block[perf_instance_definition->NameOffset], CP_UTF8); - if (perflib_instance->name == NULL) { - we_perflib_destroy_instance(perflib_instance); - - return -2; - } - } - else { - perflib_instance->name = flb_strdup("DEFAULT"); - } - - offset = perf_instance_definition->ByteLength; - } - - perflib_instance->parent = perflib_object; - - result = we_perflib_process_counters(context, - perflib_object, - perflib_instance, - &input_data_block[offset]); - - if (result < 0) { - we_perflib_destroy_instance(perflib_instance); - - return -3; - } - - offset += result; - - *out_instance = perflib_instance; - - return offset; -} - -static int we_perflib_process_instances(struct we_perflib_context *context, - struct we_perflib_object *perflib_object, - char *input_data_block) -{ - struct we_perflib_instance *perflib_instance; - size_t instance_index; - int result; - int offset; - - offset = 0; - - for (instance_index = 0 ; - instance_index < perflib_object->instance_count ; - instance_index++) { - - result = we_perflib_process_instance(context, - perflib_object, - &input_data_block[offset], - &perflib_instance); - - if (result <= 0) { - return -1; - } - - offset += result; - - result = flb_hash_table_add(perflib_object->instances, - perflib_instance->name, - strlen(perflib_instance->name), - perflib_instance, - 0); - - if (result < 0) { - we_perflib_destroy_instance(perflib_instance); - - return -2; - } - } - - return offset; -} - -int we_perflib_query(struct flb_we *ctx, - char *counter_name, - struct we_perflib_object **out_object) -{ - char *counter_name_index; - char *raw_data_buffer; - size_t raw_data_offset; - struct we_perflib_object *perflib_object; - size_t raw_data_size; - int result; - - - counter_name_index = we_perflib_lookup_counter_index( - ctx->perflib_context.counter_indexes, counter_name); - - if (counter_name_index == NULL) { - return -1; - } - - result = we_perflib_query_raw_data(ctx, - counter_name_index, - &raw_data_buffer, - &raw_data_size); - - if (result) { - return -2; - } - - raw_data_offset = 0; - - result = we_perflib_process_object_type(&ctx->perflib_context, - &raw_data_buffer[raw_data_offset], - &perflib_object); - - if (result < 0) { - flb_free(raw_data_buffer); - - return -3; - } - - raw_data_offset += result; - - result = we_perflib_process_counter_definitions(&ctx->perflib_context, - perflib_object, - &raw_data_buffer[raw_data_offset]); - - if (result < 0) { - we_perflib_destroy_object(perflib_object); - flb_free(raw_data_buffer); - - return -4; - } - - raw_data_offset += result; - - result = we_perflib_process_instances(&ctx->perflib_context, - perflib_object, - &raw_data_buffer[raw_data_offset]); - - if (result < 0) { - we_perflib_destroy_object(perflib_object); - flb_free(raw_data_buffer); - - return -5; - } - - flb_free(raw_data_buffer); - - *out_object = perflib_object; - - return 0; -} - -int we_perflib_update_counters(struct flb_we *ctx, - char *query, - struct we_perflib_metric_source *metric_sources, - we_perflib_instance_filter filter_hook, - we_perflib_label_prepend_hook label_prepend_hook) -{ - char *metric_label_list[WE_PERFLIB_METRIC_LABEL_LIST_SIZE]; - struct flb_hash_table_entry *instance_hash_entry; - size_t metric_label_count; - struct mk_list *instance_iterator; - struct we_perflib_metric_source *metric_source; - size_t metric_index; - void *metric_entry; - size_t label_index; - struct we_perflib_object *measurement; - uint64_t timestamp; - struct we_perflib_counter *counter; - int result; - - - timestamp = cfl_time_now(); - - result = we_perflib_query(ctx, query, &measurement); - - if (result) { - return -1; - } - - mk_list_foreach_r (instance_iterator, &measurement->instances->entries) { - instance_hash_entry = mk_list_entry(instance_iterator, - struct flb_hash_table_entry, - _head_parent); - - if (filter_hook(instance_hash_entry->key, ctx) == 0) { - for (metric_index = 0 ; - metric_sources[metric_index].name != NULL ; - metric_index++) { - - metric_source = &metric_sources[metric_index]; - - counter = we_perflib_get_counter(measurement, - instance_hash_entry->key, - metric_source->name); - - if (counter == NULL) { - return -2; - } - - metric_label_count = 0; - - result = label_prepend_hook(metric_label_list, - WE_PERFLIB_METRIC_LABEL_LIST_SIZE, - &metric_label_count, - metric_source, - instance_hash_entry->key, - counter); - - if (result != 0) { - return -3; - } - - for (label_index = 0 ; - label_index < metric_source->label_set_size; - label_index++) { - metric_label_list[metric_label_count++] = \ - metric_source->label_set[label_index]; - } - - metric_entry = metric_source->parent->metric_instance; - - if (metric_source->parent->type == CMT_COUNTER) { - cmt_counter_set(metric_entry, timestamp, - we_perflib_get_adjusted_counter_value(counter), - metric_label_count, metric_label_list); - } - else if (metric_source->parent->type == CMT_GAUGE) { - cmt_gauge_set(metric_entry, timestamp, - we_perflib_get_adjusted_counter_value(counter), - metric_label_count, metric_label_list); - } - } - } - } - - we_perflib_destroy_object(measurement); - - return 0; -} - -struct we_perflib_counter *we_perflib_get_counter(struct we_perflib_object *object, - char *instance_name, - char *counter_name) -{ - struct we_perflib_instance *instance; - struct we_perflib_counter *counter; - - if (instance_name == NULL) { - instance_name = "DEFAULT"; - } - - instance = flb_hash_table_get_ptr(object->instances, - instance_name, - strlen(instance_name)); - - if (instance == NULL) { - return NULL; - } - - counter = flb_hash_table_get_ptr(instance->counters, - counter_name, - strlen(counter_name)); - - return counter; -} - -int we_perflib_init(struct flb_we *ctx) -{ - int result; - - result = get_text_mapping_table(&ctx->perflib_context.counter_indexes); - - if (result) { - return -1; - } - - return 0; -} - -int we_perflib_exit(struct flb_we *ctx) -{ - if (ctx->perflib_context.counter_indexes != NULL) { - flb_hash_table_destroy(ctx->perflib_context.counter_indexes); - ctx->perflib_context.counter_indexes = NULL; - } - - return 0; -} - -/* -https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-server-2003/cc785636(v=ws.10) -*/ - diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_perflib.h b/fluent-bit/plugins/in_windows_exporter_metrics/we_perflib.h deleted file mode 100644 index 46059bb8d..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_perflib.h +++ /dev/null @@ -1,72 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2021 The Fluent Bit Authors - * Copyright (C) 2015-2018 Treasure Data Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_WE_PERFLIB_H -#define FLB_WE_PERFLIB_H - -#include "we.h" -#include "we_metric.h" - -#define WE_PERFLIB_REGISTRY_PATH \ - "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Perflib" - -#define WE_PERFLIB_STRING_COUNT_KEY "Last Help" -#define WE_PERFLIB_COUNTER_KEY_NAME "Counter 009" - -#define WE_PERFLIB_METRIC_LABEL_LIST_SIZE 64 - -#define WE_PERFLIB_QUERY_BUFFER_INITIAL_SIZE (32 * 1024) -#define WE_PERFLIB_QUERY_BUFFER_INCREMENT_SIZE (16 * 1024) - -#define WE_PERFLIB_WINDOWS_EPOCH ((double) 1 / 1e7) -#define WE_PERFLIB_TICKS_TO_SECONDS_SCALE_FACTOR ((double) 116444736000000000) - -typedef int (*we_perflib_instance_filter)(char *, struct flb_we *); -typedef int (*we_perflib_label_prepend_hook)(char **, - size_t, - size_t *, - struct we_perflib_metric_source *, - char *, - struct we_perflib_counter *); - -int we_perflib_init(struct flb_we *ctx); -int we_perflib_exit(struct flb_we *ctx); - -int we_perflib_query(struct flb_we *ctx, - char *counter_name, - struct we_perflib_object **out_object); - -struct we_perflib_counter *we_perflib_get_counter(struct we_perflib_object *object, - char *instance_name, - char *counter_name); - -void we_perflib_destroy_object(struct we_perflib_object *object); - -char *we_perflib_get_counter_type_as_text(uint32_t counter_Type); - -int we_perflib_update_counters(struct flb_we *ctx, - char *query, - struct we_perflib_metric_source *metric_sources, - we_perflib_instance_filter filter_hook, - we_perflib_label_prepend_hook label_prepend_hook); - -double we_perflib_get_adjusted_counter_value(struct we_perflib_counter *counter); - -#endif \ No newline at end of file diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_util.c b/fluent-bit/plugins/in_windows_exporter_metrics/we_util.c deleted file mode 100644 index 625872709..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_util.c +++ /dev/null @@ -1,167 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2021 The Fluent Bit Authors - * Copyright (C) 2015-2018 Treasure Data Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include "we.h" -#include "we_util.h" - -int we_get_windows_version(double *version_number) -{ - LSTATUS result; - DWORD data_size; - HKEY key_handle; - char version_text[8]; - - data_size = sizeof(version_text); - - result = RegOpenKeyExA(HKEY_LOCAL_MACHINE, - WE_VERSION_REGISTRY_PATH, - 0, - KEY_READ, - &key_handle); - - if (result != ERROR_SUCCESS) { - return FLB_FALSE; - } - - result = RegQueryValueExA(key_handle, - WE_VERSION_KEY_NAME, - NULL, - 0, - version_text, - &data_size); - - RegCloseKey(key_handle); - - if (result != ERROR_SUCCESS) - { - return FLB_FALSE; - } - - *version_number = strtod(version_text, NULL); - - return FLB_TRUE; -} - -void we_hexdump(uint8_t *buffer, size_t buffer_length, size_t line_length) { - char *printable_line; - size_t buffer_index; - size_t filler_index; - - if (40 < line_length) - { - line_length = 40; - } - - printable_line = malloc(line_length + 1); - - if (NULL == printable_line) - { - printf("Alloca returned NULL\n"); - - return; - } - - memset(printable_line, '\0', line_length + 1); - - for (buffer_index = 0 ; buffer_index < buffer_length ; buffer_index++) { - if (0 != buffer_index && - 0 == (buffer_index % line_length)) { - - printf("%s\n", printable_line); - - memset(printable_line, '\0', line_length + 1); - } - - if (0 != isprint(buffer[buffer_index])) { - printable_line[(buffer_index % line_length)] = buffer[buffer_index]; - } - else { - printable_line[(buffer_index % line_length)] = '.'; - } - - printf("%02X ", buffer[buffer_index]); - } - - if (0 != buffer_index && - 0 != (buffer_index % line_length)) { - - for (filler_index = 0 ; - filler_index < (line_length - (buffer_index % line_length)) ; - filler_index++) { - printf(" "); - } - - printf("%s\n", printable_line); - - memset(printable_line, '.', line_length); - } - - free(printable_line); -} - -char* we_convert_wstr(wchar_t *wstr, UINT codePage) -{ - int size = 0; - char *buf = NULL; - - size = WideCharToMultiByte(codePage, 0, wstr, -1, NULL, 0, NULL, NULL); - if (size == 0) { - return NULL; - } - - buf = flb_calloc(1, size); - if (buf == NULL) { - flb_errno(); - return NULL; - } - size = WideCharToMultiByte(codePage, 0, wstr, -1, buf, size, NULL, NULL); - if (size == 0) { - flb_free(buf); - return NULL; - } - - return buf; -} - -wchar_t* we_convert_str(char *str) -{ - int size = 0; - wchar_t *buf = NULL; - - size = MultiByteToWideChar(CP_UTF8, 0, str, -1, NULL, 0); - if (size == 0) { - return NULL; - } - - buf = flb_calloc(1, sizeof(PWSTR) * size); - if (buf == NULL) { - flb_errno(); - return NULL; - } - size = MultiByteToWideChar(CP_UTF8, 0, str, -1, buf, size); - if (size == 0) { - flb_free(buf); - return NULL; - } - - return buf; -} diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_util.h b/fluent-bit/plugins/in_windows_exporter_metrics/we_util.h deleted file mode 100644 index 1f556d2cb..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_util.h +++ /dev/null @@ -1,37 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2021 The Fluent Bit Authors - * Copyright (C) 2015-2018 Treasure Data Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_WE_UTIL_H -#define FLB_WE_UTIL_H - -#include "we.h" - -#define WE_VERSION_REGISTRY_PATH \ - "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion" - -#define WE_VERSION_KEY_NAME "CurrentVersion" - -int we_get_windows_version(double *version_number); -void we_hexdump(uint8_t *buffer, size_t buffer_length, size_t line_length); -/* Utilites for char/wchar_t conversion */ -wchar_t* we_convert_str(char *str); -char* we_convert_wstr(wchar_t *wstr, UINT codePage); - -#endif diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi.c b/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi.c deleted file mode 100644 index 03505c4bc..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi.c +++ /dev/null @@ -1,572 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2021 The Fluent Bit Authors - * Copyright (C) 2015-2018 Treasure Data Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "we.h" -#include "we_util.h" -#include "we_wmi.h" - -static int wmi_coinitialize(struct flb_we *ctx, char* wmi_namespace) -{ - IWbemLocator *locator = 0; - IWbemServices *service = 0; - HRESULT hr; - wchar_t *wnamespace; - - flb_plg_debug(ctx->ins, "initializing WMI instance...."); - - /* Initialize COM library */ - hr = CoInitializeEx(0, COINIT_MULTITHREADED); - if (FAILED(hr)) { - flb_plg_error(ctx->ins, "Failed to initialize COM library. Error code = %x", hr); - return -1; - } - - /* Initialize COM security */ - hr = CoInitializeSecurity(NULL, - -1, - NULL, - NULL, - RPC_C_AUTHN_LEVEL_DEFAULT, - RPC_C_IMP_LEVEL_IMPERSONATE, - NULL, - EOAC_NONE, - NULL); - - if (FAILED(hr)) { - return hr; - } - - /* Create WMI instance */ - hr = CoCreateInstance(&CLSID_WbemLocator, 0, - CLSCTX_INPROC_SERVER, &IID_IWbemLocator, (LPVOID *) &locator); - if (FAILED(hr)) { - flb_plg_error(ctx->ins, "Failed to create IWbemLocator object. Error code = %x", hr); - CoUninitialize(); - return hr; - } - ctx->locator = locator; - - if (wmi_namespace == NULL) { - wnamespace = we_convert_str("ROOT\\CIMV2"); - } - else { - wnamespace = we_convert_str(wmi_namespace); - } - /* Connect WMI server */ - hr = locator->lpVtbl->ConnectServer(locator, - wnamespace, - NULL, - NULL, - 0, - 0, - 0, - NULL, - &service); - flb_free(wnamespace); - - if (FAILED(hr)) { - flb_plg_error(ctx->ins, "Could not connect. Error code = %x", hr); - locator->lpVtbl->Release(locator); - CoUninitialize(); - return hr; - } - ctx->service = service; - - /* Set up ProxyBlanket */ - hr = CoSetProxyBlanket(service, - RPC_C_AUTHN_WINNT, - RPC_C_AUTHZ_NONE, - NULL, - RPC_C_AUTHN_LEVEL_CALL, - RPC_C_IMP_LEVEL_IMPERSONATE, - NULL, - EOAC_NONE - ); - if (FAILED(hr)) { - flb_plg_error(ctx->ins, "Could not set proxy blanket. Error code = %x", hr); - service->lpVtbl->Release(service); - locator->lpVtbl->Release(locator); - CoUninitialize(); - return hr; - } - - return 0; -} - - - -int wmi_utils_str_to_double(char *str, double *out_val) -{ - double val; - char *end; - - errno = 0; - val = strtod(str, &end); - if (errno != 0 || *end != '\0') { - return -1; - } - *out_val = val; - return 0; -} - -static int wmi_update_counters(struct wmi_query_spec *spec, uint64_t timestamp, double val, int metric_label_count, char **metric_label_set) -{ - val = spec->value_adjuster(val); - - if (spec->type == CMT_GAUGE) { - cmt_gauge_set((struct cmt_gauge *)spec->metric_instance, timestamp, - val, - metric_label_count, metric_label_set); - } - else if (spec->type == CMT_COUNTER) { - cmt_counter_set((struct cmt_counter *)spec->metric_instance, timestamp, - val, - metric_label_count, metric_label_set); - } - - return 0; -} - -static char *convert_prop_to_str(VARIANT *prop, int handle_null) -{ - char *strlabel = NULL; - char *newstr = NULL; - - if (handle_null == FLB_TRUE && prop->vt == VT_NULL) { - newstr = strdup(""); - if (newstr == NULL) { - return NULL; - } - } - else { - if (VariantChangeType(prop, prop, 0, VT_BSTR) != S_OK) { - return NULL; - } - strlabel = we_convert_wstr(prop->bstrVal, CP_UTF8); - if (strlabel == NULL) { - return NULL; - } - newstr = strdup(strlabel); - if (newstr == NULL) { - free(strlabel); - return NULL; - } - free(strlabel); - } - return newstr; -} - -static double wmi_get_value(struct flb_we *ctx, struct wmi_query_spec *spec, IWbemClassObject *class_obj) -{ - VARIANT prop; - char *strprop; - double val = 1.0; - HRESULT hr; - wchar_t *wproperty; - - VariantInit(&prop); - wproperty = we_convert_str(spec->wmi_property); - hr = class_obj->lpVtbl->Get(class_obj, wproperty, 0, &prop, 0, 0); - if (FAILED(hr)) { - flb_plg_warn(ctx->ins, "Retrive prop '%s' failed. Error code = %x", spec->wmi_property, hr); - } - strprop = convert_prop_to_str(&prop, FLB_FALSE); - if (strprop == NULL) { - return 0; - } - wmi_utils_str_to_double(strprop, &val); - flb_free(strprop); - VariantClear(&prop); - flb_free(wproperty); - - return val; -} - -static double wmi_get_property_value(struct flb_we *ctx, char *raw_property_key, IWbemClassObject *class_obj) -{ - VARIANT prop; - char *strprop; - double val = 1.0; - HRESULT hr; - wchar_t *wproperty; - - VariantInit(&prop); - wproperty = we_convert_str(raw_property_key); - hr = class_obj->lpVtbl->Get(class_obj, wproperty, 0, &prop, 0, 0); - if (FAILED(hr)) { - flb_plg_warn(ctx->ins, "Retrive prop '%s' failed. Error code = %x", raw_property_key, hr); - } - strprop = convert_prop_to_str(&prop, FLB_FALSE); - if (strprop == NULL) { - return 0; - } - wmi_utils_str_to_double(strprop, &val); - flb_free(strprop); - VariantClear(&prop); - flb_free(wproperty); - - return val; -} - -static char *wmi_get_property_str_value(struct flb_we *ctx, char *raw_property_key, - IWbemClassObject *class_obj) -{ - VARIANT prop; - char *strprop; - char *str_val = NULL; - HRESULT hr; - wchar_t *wproperty; - - - VariantInit(&prop); - wproperty = we_convert_str(raw_property_key); - hr = class_obj->lpVtbl->Get(class_obj, wproperty, 0, &prop, 0, 0); - if (FAILED(hr)) { - flb_plg_warn(ctx->ins, "Retrive prop '%s' failed. Error code = %x", raw_property_key, hr); - } - str_val = convert_prop_to_str(&prop, FLB_TRUE); - VariantClear(&prop); - flb_free(wproperty); - - return str_val; -} - -static inline int wmi_update_metrics(struct flb_we *ctx, struct wmi_query_spec *spec, - double val, IWbemClassObject *class_obj, uint64_t timestamp) -{ - - VARIANT prop; - int label_index = 0; - HRESULT hr; - char *metric_label_set[WE_WMI_METRIC_LABEL_LIST_SIZE]; - int metric_label_count = 0; - char buf[16] = {0}; - wchar_t *wlabel; - char *newstr = NULL; - - VariantInit(&prop); - metric_label_count = 0; - for (label_index = 0; label_index < spec->label_property_count; label_index++) { - wlabel = we_convert_str(spec->label_property_keys[label_index]); - hr = class_obj->lpVtbl->Get(class_obj, wlabel, 0, &prop, 0, 0); - if (FAILED(hr)) { - flb_plg_warn(ctx->ins, "Retrive prop failed. Error code = %x", hr); - } - newstr = convert_prop_to_str(&prop, FLB_TRUE); - if (newstr == NULL) { - continue; - } - metric_label_set[label_index] = newstr; - metric_label_count++; - VariantClear(&prop); - flb_free(wlabel); - } - - wmi_update_counters(spec, timestamp, val, metric_label_count, metric_label_set); - - VariantClear(&prop); - - return 0; -} - -static inline int wmi_execute_query(struct flb_we *ctx, struct wmi_query_spec *spec, IEnumWbemClassObject **out_enumerator) -{ - HRESULT hr; - wchar_t *wquery; - char *query = NULL; - IEnumWbemClassObject* enumerator = NULL; - size_t size; - - size = 14 + strlen(spec->wmi_counter); - if (spec->where_clause != NULL) { - size += 7 + strlen(spec->where_clause); - } - query = flb_calloc(size, sizeof(char *)); - if (!query) { - flb_errno(); - return -1; - } - if (spec->where_clause != NULL) { - snprintf(query, size, "SELECT * FROM %s WHERE %s", spec->wmi_counter, spec->where_clause); - } - else { - snprintf(query, size, "SELECT * FROM %s", spec->wmi_counter); - } - flb_trace("[wmi] query = %s", query); - wquery = we_convert_str(query); - flb_free(query); - - hr = ctx->service->lpVtbl->ExecQuery( - ctx->service, - L"WQL", - wquery, - WBEM_FLAG_FORWARD_ONLY | WBEM_FLAG_RETURN_IMMEDIATELY, - NULL, - &enumerator); - - flb_free(wquery); - - if (FAILED(hr)) { - flb_plg_error(ctx->ins, "Query for %s %s failed. Error code = %x", - spec->wmi_counter, spec->wmi_counter, hr); - ctx->service->lpVtbl->Release(ctx->service); - ctx->locator->lpVtbl->Release(ctx->locator); - CoUninitialize(); - return -1; - } - - *out_enumerator = enumerator; - - return 0; -} - -static int wmi_exec_query_fixed_val(struct flb_we *ctx, struct wmi_query_spec *spec) -{ - IEnumWbemClassObject* enumerator = NULL; - HRESULT hr; - - IWbemClassObject *class_obj = NULL; - ULONG ret = 0; - uint64_t timestamp = 0; - - timestamp = cfl_time_now(); - - if (FAILED(wmi_execute_query(ctx, spec, &enumerator))) { - return -1; - } - - while (enumerator) { - hr = enumerator->lpVtbl->Next(enumerator, WBEM_INFINITE, 1, - &class_obj, &ret); - - if(0 == ret) { - break; - } - - wmi_update_metrics(ctx, spec, 1.0, class_obj, timestamp); - - class_obj->lpVtbl->Release(class_obj); - } - - enumerator->lpVtbl->Release(enumerator); - - return 0; -} - -static int wmi_exec_query(struct flb_we *ctx, struct wmi_query_spec *spec) -{ - IEnumWbemClassObject* enumerator = NULL; - HRESULT hr; - - IWbemClassObject *class_obj = NULL; - ULONG ret = 0; - double val = 0; - uint64_t timestamp = 0; - - timestamp = cfl_time_now(); - - if (FAILED(wmi_execute_query(ctx, spec, &enumerator))) { - return -1; - } - - while (enumerator) { - hr = enumerator->lpVtbl->Next(enumerator, WBEM_INFINITE, 1, - &class_obj, &ret); - - if(0 == ret) { - break; - } - - val = wmi_get_value(ctx, spec, class_obj); - - wmi_update_metrics(ctx, spec, val, class_obj, timestamp); - - class_obj->lpVtbl->Release(class_obj); - } - - enumerator->lpVtbl->Release(enumerator); - - return 0; -} - -static int wmi_cleanup(struct flb_we *ctx) -{ - flb_plg_debug(ctx->ins, "deinitializing WMI instance...."); - - /* Clean up */ - if (ctx->service != NULL) { - ctx->service->lpVtbl->Release(ctx->service); - ctx->service = NULL; - } - if (ctx->locator != NULL) { - ctx->locator->lpVtbl->Release(ctx->locator); - ctx->locator = NULL; - } - CoUninitialize(); - - return 0; -} - -static int wmi_query(struct flb_we *ctx, struct wmi_query_spec *spec) -{ - if (FAILED(wmi_coinitialize(ctx, NULL))) { - return -1; - } - if (FAILED(wmi_exec_query(ctx, spec))) { - return -1; - } - - wmi_cleanup(ctx); - - return 0; -} - -static int wmi_query_namespace(struct flb_we *ctx, struct wmi_query_spec *spec, char *namespace) -{ - if (FAILED(wmi_coinitialize(ctx, namespace))) { - return -1; - } - if (FAILED(wmi_exec_query(ctx, spec))) { - return -1; - } - - wmi_cleanup(ctx); - - return 0; -} - -static int wmi_query_fixed_val(struct flb_we *ctx, struct wmi_query_spec *spec) -{ - if (FAILED(wmi_coinitialize(ctx, NULL))) { - return -1; - } - if (FAILED(wmi_exec_query_fixed_val(ctx, spec))) { - return -1; - } - - wmi_cleanup(ctx); - - return 0; -} - -int we_wmi_init(struct flb_we *ctx) -{ - ctx->locator = NULL; - ctx->service = NULL; - - return 0; -} - -int we_wmi_cleanup(struct flb_we *ctx) -{ - wmi_cleanup(ctx); - - return 0; -} - -int we_wmi_exit(struct flb_we *ctx) -{ - return 0; -} - -/* Abstract APIs */ -int we_wmi_query_fixed_val(struct flb_we *ctx, struct wmi_query_specs *spec) -{ - if (FAILED(wmi_query_fixed_val(ctx, spec))) { - return -1; - } - return 0; -} - -int we_wmi_query(struct flb_we *ctx, struct wmi_query_specs *spec) -{ - if (FAILED(wmi_query(ctx, spec))) { - return -1; - } - return 0; -} - -int we_wmi_query_namespace(struct flb_we *ctx, struct wmi_query_specs *spec, char *namespace) -{ - if (FAILED(wmi_query_namespace(ctx, spec, namespace))) { - return -1; - } - return 0; -} - -/* Concreate APIs */ -int we_wmi_coinitialize(struct flb_we *ctx) -{ - if (FAILED(wmi_coinitialize(ctx, NULL))) { - return -1; - } - - return 0; -} - -int we_wmi_execute_query(struct flb_we *ctx, struct wmi_query_spec *spec, IEnumWbemClassObject **out_enumerator) -{ - IEnumWbemClassObject* enumerator = NULL; - - if (FAILED(wmi_execute_query(ctx, spec, &enumerator))) { - return -1; - } - - *out_enumerator = enumerator; - - return 0; -} - -double we_wmi_get_value(struct flb_we *ctx, struct wmi_query_spec *spec, IWbemClassObject *class_obj) -{ - return wmi_get_value(ctx, spec, class_obj); -} - -double we_wmi_get_property_value(struct flb_we *ctx, char *raw_property_key, IWbemClassObject *class_obj) -{ - return wmi_get_property_value(ctx, raw_property_key, class_obj); -} - -char *we_wmi_get_property_str_value(struct flb_we *ctx, char *raw_property_key, - IWbemClassObject *class_obj) -{ - return wmi_get_property_str_value(ctx, raw_property_key, class_obj); -} - -int we_wmi_update_counters(struct flb_we *ctx, struct wmi_query_spec *spec, uint64_t timestamp, double val, int metric_label_count, char **metric_label_set) -{ - wmi_update_counters(spec, timestamp, val, metric_label_count, metric_label_set); - - return 0; -} - -/* -https://stackoverflow.com/questions/33033111/create-com-object-using-plain-c -https://stackoverflow.com/questions/1431103/how-to-obtain-data-from-wmi-using-a-c-application -https://stackoverflow.com/questions/626674/wmi-queries-in-c -https://github.com/MicrosoftDocs/win32/blob/docs/desktop-src/WmiSdk/example--getting-wmi-data-from-the-local-computer.md -https://docs.microsoft.com/en-us/windows/win32/wmisdk/creating-wmi-clients -*/ diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi.h b/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi.h deleted file mode 100644 index 2999f5764..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi.h +++ /dev/null @@ -1,59 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2021 The Fluent Bit Authors - * Copyright (C) 2015-2018 Treasure Data Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_WE_WMI_H -#define FLB_WE_WMI_H - -#include "we.h" - -#define WE_WMI_METRIC_LABEL_LIST_SIZE 64 - -typedef double (*we_wmi_value_adjuster) (double); -struct wmi_query_spec { - void *metric_instance; - int type; - we_wmi_value_adjuster value_adjuster; - char *wmi_counter; - char *wmi_property; - int label_property_count; - char **label_property_keys; - char *where_clause; -}; - -int we_wmi_init(struct flb_we *ctx); -int we_wmi_cleanup(struct flb_we *ctx); -int we_wmi_exit(struct flb_we *ctx); - -/* Abstract APIs */ -int we_wmi_query(struct flb_we *ctx, struct wmi_query_specs *spec); -int we_wmi_query_fixed_val(struct flb_we *ctx, struct wmi_query_specs *spec); -int we_wmi_query_namespace(struct flb_we *ctx, struct wmi_query_specs *spec, char *namepsace); - -/* Concrete APIs */ -int we_wmi_coinitialize(struct flb_we *ctx); -int we_wmi_execute_query(struct flb_we *ctx, struct wmi_query_spec *spec, IEnumWbemClassObject **out_enumerator); -double we_wmi_get_value(struct flb_we *ctx, struct wmi_query_spec *spec, IWbemClassObject *class_obj); -double we_wmi_get_property_value(struct flb_we *ctx, char *raw_property_key, IWbemClassObject *class_obj); -char *we_wmi_get_property_str_value(struct flb_we *ctx, char *raw_property_key, - IWbemClassObject *class_obj); -int we_wmi_update_counters(struct flb_we *ctx, struct wmi_query_spec *spec, - uint64_t timestamp, double val, int metric_label_count, char **metric_label_set); - -#endif diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_cpu_info.c b/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_cpu_info.c deleted file mode 100644 index 9e8e96e1a..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_cpu_info.c +++ /dev/null @@ -1,116 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "we.h" -#include "we_wmi.h" -#include "we_wmi_cpu_info.h" -#include "we_util.h" -#include "we_metric.h" - -static double nop_adjust(double value) -{ - return value; -} - -int we_wmi_cpu_info_init(struct flb_we *ctx) -{ - ctx->wmi_cpu_info = flb_calloc(1, sizeof(struct we_wmi_cpu_info_counters)); - if (!ctx->wmi_cpu_info) { - flb_errno(); - return -1; - } - ctx->wmi_cpu_info->operational = FLB_FALSE; - - struct cmt_gauge *g; - - g = cmt_gauge_create(ctx->cmt, "windows", "", "cpu_info", - "Labeled CPU information provided by WMI Win32_Processor", - 7, (char *[]) {"architecture", - "device_id", - "description", - "family", - "l2_cache_size", - "l3_cache_size", - "name"}); - if (!g) { - return -1; - } - - ctx->wmi_cpu_info->info = flb_calloc(1, sizeof(struct wmi_query_spec)); - if (!ctx->wmi_cpu_info->info) { - flb_errno(); - return -1; - } - ctx->wmi_cpu_info->info->label_property_keys = (char **) flb_calloc(7, sizeof(char *)); - if (!ctx->wmi_cpu_info->info->label_property_keys) { - flb_errno(); - return -1; - } - - ctx->wmi_cpu_info->info->metric_instance = (void *)g; - ctx->wmi_cpu_info->info->type = CMT_GAUGE; - ctx->wmi_cpu_info->info->value_adjuster = nop_adjust; - ctx->wmi_cpu_info->info->wmi_counter = "Win32_Processor"; - /* This metrics does not retrieve metrics values. Filled out as - * 1.0. */ - ctx->wmi_cpu_info->info->wmi_property = ""; - ctx->wmi_cpu_info->info->label_property_count = 7; - ctx->wmi_cpu_info->info->label_property_keys[0] = "architecture" ; - ctx->wmi_cpu_info->info->label_property_keys[1] = "deviceid" ; - ctx->wmi_cpu_info->info->label_property_keys[2] = "description" ; - ctx->wmi_cpu_info->info->label_property_keys[3] = "family" ; - ctx->wmi_cpu_info->info->label_property_keys[4] = "l2cachesize" ; - ctx->wmi_cpu_info->info->label_property_keys[5] = "l3cachesize" ; - ctx->wmi_cpu_info->info->label_property_keys[6] = "name" ; - ctx->wmi_cpu_info->info->where_clause = NULL; - - ctx->wmi_cpu_info->operational = FLB_TRUE; - - return 0; -} - -int we_wmi_cpu_info_exit(struct flb_we *ctx) -{ - flb_free(ctx->wmi_cpu_info->info->label_property_keys); - flb_free(ctx->wmi_cpu_info->info); - flb_free(ctx->wmi_cpu_info); - - return 0; -} - -int we_wmi_cpu_info_update(struct flb_we *ctx) -{ - if (!ctx->wmi_cpu_info->operational) { - flb_plg_error(ctx->ins, "cpu_info collector not yet in operational state"); - - return -1; - } - - if (FAILED(we_wmi_query_fixed_val(ctx, ctx->wmi_cpu_info->info))) { - return -1; - } - - return 0; -} diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_cpu_info.h b/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_cpu_info.h deleted file mode 100644 index 8120b4983..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_cpu_info.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_WE_WMI_CPU_INFO_H -#define FLB_WE_WMI_CPU_INFO_H - -#include "we.h" - -int we_wmi_cpu_info_init(struct flb_we *ctx); -int we_wmi_cpu_info_exit(struct flb_we *ctx); -int we_wmi_cpu_info_update(struct flb_we *ctx); - -#endif diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_logon.c b/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_logon.c deleted file mode 100644 index 761d4ac7a..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_logon.c +++ /dev/null @@ -1,198 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "we.h" -#include "we_wmi.h" -#include "we_wmi_cpu_info.h" -#include "we_util.h" -#include "we_metric.h" - -static double nop_adjust(double value) -{ - return value; -} - -int we_wmi_logon_init(struct flb_we *ctx) -{ - ctx->wmi_logon = flb_calloc(1, sizeof(struct we_wmi_logon_counters)); - if (!ctx->wmi_logon) { - flb_errno(); - return -1; - } - ctx->wmi_logon->operational = FLB_FALSE; - - struct cmt_gauge *g; - - g = cmt_gauge_create(ctx->cmt, "windows", "logon", "logon_type", - "Number of active logon sessions (LogonSession.LogonType) by WMI Win32_LogonSession", - 1, (char *[]) {"status"}); - if (!g) { - return -1; - } - - ctx->wmi_logon->info = flb_calloc(1, sizeof(struct wmi_query_spec)); - if (!ctx->wmi_logon->info) { - flb_errno(); - return -1; - } - ctx->wmi_logon->info->label_property_keys = (char **) flb_calloc(1, sizeof(char *)); - if (!ctx->wmi_logon->info->label_property_keys) { - flb_errno(); - return -1; - } - - ctx->wmi_logon->info->metric_instance = (void *)g; - ctx->wmi_logon->info->type = CMT_GAUGE; - ctx->wmi_logon->info->value_adjuster = nop_adjust; - ctx->wmi_logon->info->wmi_counter = "Win32_LogonSession"; - ctx->wmi_logon->info->wmi_property = "LogonType"; - ctx->wmi_logon->info->label_property_count = 1; - ctx->wmi_logon->info->label_property_keys[0] = "status" ; - ctx->wmi_logon->info->where_clause = NULL; - - ctx->wmi_logon->operational = FLB_TRUE; - - return 0; -} - -int we_wmi_logon_exit(struct flb_we *ctx) -{ - flb_free(ctx->wmi_logon->info->label_property_keys); - flb_free(ctx->wmi_logon->info); - flb_free(ctx->wmi_logon); - - return 0; -} - -int we_wmi_logon_update(struct flb_we *ctx) -{ - IEnumWbemClassObject* enumerator = NULL; - HRESULT hr; - - IWbemClassObject *class_obj = NULL; - ULONG ret = 0; - double val = 0; - int type = 0; - uint64_t timestamp = 0; - /* Init counters for logon */ - uint64_t system = 0, interactive = 0, network = 0, batch = 0, service = 0, - proxy = 0, unlock = 0, networkcleartext = 0, newcredentials = 0, remoteinteractive = 0, - cachedinteractive = 0, cachedremoteinteractive = 0, cachedunlock = 0; - struct wmi_query_spec *spec; - - if (!ctx->wmi_logon->operational) { - flb_plg_error(ctx->ins, "logon collector not yet in operational state"); - - return -1; - } - - if (FAILED(we_wmi_coinitialize(ctx))) { - return -1; - } - - timestamp = cfl_time_now(); - - if (FAILED(we_wmi_execute_query(ctx, ctx->wmi_logon->info, &enumerator))) { - return -1; - } - - while(enumerator) { - hr = enumerator->lpVtbl->Next(enumerator, WBEM_INFINITE, 1, - &class_obj, &ret); - - if(0 == ret) { - break; - } - - val = we_wmi_get_value(ctx, ctx->wmi_logon->info, class_obj); - type = (int)val; - - switch(type) { - case 0: - system++; - break; - case 2: - interactive++; - break; - case 3: - network++; - break; - case 4: - batch++; - break; - case 5: - service++; - break; - case 6: - proxy++; - break; - case 7: - unlock++; - break; - case 8: - networkcleartext++; - break; - case 9: - newcredentials++; - break; - case 10: - remoteinteractive++; - break; - case 11: - cachedinteractive++; - break; - case 12: - cachedremoteinteractive++; - break; - case 13: - cachedunlock++; - break; - } - - class_obj->lpVtbl->Release(class_obj); - } - - enumerator->lpVtbl->Release(enumerator); - - spec = ctx->wmi_logon->info; - - we_wmi_update_counters(ctx, spec, timestamp, (double)system, 1, (char *[]) {"system"} ); - we_wmi_update_counters(ctx, spec, timestamp, (double)interactive, 1, (char *[]) {"interactive"} ); - we_wmi_update_counters(ctx, spec, timestamp, (double)network, 1, (char *[]) {"network"} ); - we_wmi_update_counters(ctx, spec, timestamp, (double)batch, 1, (char *[]) {"batch"} ); - we_wmi_update_counters(ctx, spec, timestamp, (double)service, 1, (char *[]) {"service"} ); - we_wmi_update_counters(ctx, spec, timestamp, (double)proxy, 1, (char *[]) {"proxy"} ); - we_wmi_update_counters(ctx, spec, timestamp, (double)unlock, 1, (char *[]) {"unlock"} ); - we_wmi_update_counters(ctx, spec, timestamp, (double)networkcleartext, 1, (char *[]) {"network_clear_text"} ); - we_wmi_update_counters(ctx, spec, timestamp, (double)newcredentials, 1, (char *[]) {"new_credentials"} ); - we_wmi_update_counters(ctx, spec, timestamp, (double)remoteinteractive, 1, (char *[]) {"remote_interactive"} ); - we_wmi_update_counters(ctx, spec, timestamp, (double)cachedinteractive, 1, (char *[]) {"cached_interactive"} ); - we_wmi_update_counters(ctx, spec, timestamp, (double)cachedremoteinteractive, 1, (char *[]) {"cached_remote_interactive"} ); - we_wmi_update_counters(ctx, spec, timestamp, (double)cachedunlock, 1, (char *[]) {"cached_unlock"} ); - - we_wmi_cleanup(ctx); - - return 0; -} diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_logon.h b/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_logon.h deleted file mode 100644 index 9fc8c5318..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_logon.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_WE_WMI_LOGON_H -#define FLB_WE_WMI_LOGON_H - -#include "we.h" - -int we_wmi_logon_init(struct flb_we *ctx); -int we_wmi_logon_exit(struct flb_we *ctx); -int we_wmi_logon_update(struct flb_we *ctx); - -#endif diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_memory.c b/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_memory.c deleted file mode 100644 index ec1a13e96..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_memory.c +++ /dev/null @@ -1,557 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "we.h" -#include "we_wmi.h" -#include "we_wmi_memory.h" -#include "we_util.h" -#include "we_metric.h" - -static double nop_adjust(double value) -{ - return value; -} - -int we_wmi_memory_init(struct flb_we *ctx) -{ - struct cmt_gauge *g; - - ctx->wmi_memory = flb_calloc(1, sizeof(struct we_wmi_memory_counters)); - if (!ctx->wmi_memory) { - flb_errno(); - return -1; - } - ctx->wmi_memory->operational = FLB_FALSE; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "available_bytes", - "The amount of physical memory, in bytes, immediately available " \ - "for allocation to a process or for system use. (AvailableBytes)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->available_bytes = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "cache_bytes", - "The size, in bytes, of the portion of the system file cache " \ - "which is currently resident and active in physical memory "\ - "(CacheBytes)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->cache_bytes = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "cache_bytes_peak", - "the maximum number of bytes used by the system file cache " \ - "since the system was last restarted (CacheBytesPeak)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->cache_bytes_peak = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "cache_faults_total", - "The rate at which faults occur when a page sought in " \ - "the file system cache is not found and must be retrieved " \ - "from elsewhere in memory (a soft fault) or from disk (a hard fault)" \ - "(CacheFaultsPersec)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->cache_faults_total = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "commit_limit", - "The amount of virtual memory that can be committed " \ - "without having to extend the paging file(s) " \ - "(CommitLimit)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->commit_limit = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "committed_bytes", - "The amount of committed virtual memory, in bytes " \ - "(CommittedBytes)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->committed_bytes = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "demand_zero_faults_total", - "The rate at which a zeroed page is required to satisfy the fault " \ - "(DemandZeroFaultsPersec)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->demand_zero_faults_total = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "free_and_zero_page_list_bytes", - "the amount of physical memory, in bytes, that is assigned to " \ - "the free and zero page lists " \ - "(FreeAndZeroPageListBytes)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->free_and_zero_page_list_bytes = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "free_system_page_table_entries", - "The number of page table entries not currently in used by the system " \ - "(FreeSystemPageTableEntries)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->free_system_page_table_entries = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "modified_page_list_bytes", - "The amount of physical memory, in bytes, that is assigned to " \ - "the modified page list " \ - "(ModifiedPageListBytes)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->modified_page_list_bytes = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "page_faults_total", - "The average number of pages faulted per second. " \ - "It is measured in number of pages faulted per second " \ - "because only one page is faulted in each fault operation, " \ - "hence this is also equal to the number of page fault operations " \ - "(PageFaultsPersec)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->page_faults_total = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "swap_page_reads_total", - "The rate at which the disk was read to resolve hard page faults " \ - "(PageReadsPersec)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->swap_page_reads_total = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "swap_pages_read_total", - "The rate at which pages are read from disk to resolve hard page faults " \ - "(PagesInputPersec)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->swap_pages_read_total = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "swap_pages_written_total", - "the rate at which pages are written to disk to free up space "\ - "in physical memory (PagesOutputPersec)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->swap_pages_written_total = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "swap_page_operations_total", - "the rate at which pages are read from or written " \ - "to disk to resolve hard page faults (PagesPersec)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->swap_page_operations_total = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "swap_page_writes_total", - "the rate at which pages are written to disk to free up space " \ - "in physical memory (PageWritesPersec)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->swap_page_writes_total = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "pool_nonpaged_allocs_total", - "Number of calls to allocate space in the nonpaged pool (PoolNonpagedAllocs)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->pool_nonpaged_allocs_total = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "pool_nonpaged_bytes", - "the size, in bytes, of the nonpaged pool, an area of " \ - "the system virtual memory that is used for objects " \ - "that cannot be written to disk, but must remain " \ - "in physical memory as long as they are allocated " \ - "(PoolNonpagedBytes)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->pool_nonpaged_bytes = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "pool_nonpaged_allocs_total", - "Number of bytes of allocated space in paged pool (PoolPagedAllocs)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->pool_paged_allocs_total = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "pool_paged_bytes", - "the size, in bytes, of the paged pool, an area of the system " \ - "virtual memory that is used for objects that can be written " \ - "to disk when they are not being used (PoolPagedBytes)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->pool_paged_bytes = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "pool_paged_resident_bytes", - "the size, in bytes, of the portion of the paged pool " \ - "that is currently resident and active in physical memory " \ - "(PoolPagedResidentBytes)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->pool_paged_resident_bytes = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "standby_cache_core_bytes", - "The amount of physical memory, in bytes, that is assigned " \ - "to the core standby cache page lists (StandbyCacheCoreBytes)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->standby_cache_core_bytes = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "standby_cache_normal_priority_bytes", - " the amount of physical memory, in bytes, that is assigned " \ - "to the normal priority standby cache page lists " \ - "(StandbyCacheNormalPriorityBytes)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->standby_cache_normal_priority_bytes = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "standby_cache_reserve_bytes", - "Number of physical memory size(bytes) which is assigned to " \ - "the reserve standby cache page lists (StandbyCacheReserveBytes)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->standby_cache_reserve_bytes = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "system_cache_resident_bytes", - "Number of physical memory size(bytes) of the portion of " \ - "the system file cache which is currently resident and active " \ - "(SystemCacheResidentBytes)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->system_cache_resident_bytes = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "system_code_resident_bytes", - "Number of physical memory size(bytes) of the pageable operating system code "\ - "which is currently resident and active (SystemCodeResidentBytes)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->system_code_resident_bytes = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "system_code_total_bytes", - "Number of virtual memory size(bytes) of the pageable operating system code " \ - "which is mapped into virtual address (SystemCodeTotalBytes)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->system_code_total_bytes = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "system_driver_resident_bytes", - "Number of pagable physical memory size(bytes) by used device drivers "\ - "(SystemDriverResidentBytes)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->system_driver_resident_bytes = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "system_driver_total_bytes", - "Number of virtual memory size(bytes) by used device drivers " \ - "(SystemDriverTotalBytes)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->system_driver_total_bytes = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "transition_faults_total", - "Number of the rate at which page faults are resolved by recovering pages " \ - "that were being used by another process sharing the page, " \ - "or were on the modified page list or the standby list, " \ - "or were being written to disk at the time of the page fault " \ - "(TransitionFaultsPersec)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->transition_faults_total = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "transition_pages_repurposed_total", - "Number of the rate at which the number of transition cache " \ - "pages were reused for a different purpose " \ - "(TransitionPagesRePurposedPersec)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->transition_pages_repurposed_total = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "memory", "write_copies_total", - "Number of the rate at which page faults are caused by "\ - "attempts to write that have been satisfied by coping " \ - "of the page from elsewhere in physical memory " \ - "(WriteCopiesPersec)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_memory->write_copies_total = g; - - ctx->wmi_memory->info = flb_calloc(1, sizeof(struct wmi_query_spec)); - if (!ctx->wmi_memory->info) { - flb_errno(); - return -1; - } - ctx->wmi_memory->info->metric_instance = (void *)g; - ctx->wmi_memory->info->type = CMT_GAUGE; - ctx->wmi_memory->info->value_adjuster = nop_adjust; - ctx->wmi_memory->info->wmi_counter = "Win32_PerfRawData_PerfOS_Memory"; - ctx->wmi_memory->info->wmi_property = ""; - ctx->wmi_memory->info->label_property_count = 0; - ctx->wmi_memory->info->label_property_keys = NULL; - ctx->wmi_memory->info->where_clause = NULL; - - ctx->wmi_memory->operational = FLB_TRUE; - - return 0; -} - -int we_wmi_memory_exit(struct flb_we *ctx) -{ - ctx->wmi_memory->operational = FLB_FALSE; - - flb_free(ctx->wmi_memory->info); - flb_free(ctx->wmi_memory); - - return 0; -} - -int we_wmi_memory_update(struct flb_we *ctx) -{ - uint64_t timestamp = 0; - IEnumWbemClassObject* enumerator = NULL; - HRESULT hr; - - IWbemClassObject *class_obj = NULL; - ULONG ret = 0; - double val = 0; - - if (!ctx->wmi_memory->operational) { - flb_plg_error(ctx->ins, "memory collector not yet in operational state"); - - return -1; - } - - if (FAILED(we_wmi_coinitialize(ctx))) { - return -1; - } - - timestamp = cfl_time_now(); - - if (FAILED(we_wmi_execute_query(ctx, ctx->wmi_memory->info, &enumerator))) { - return -1; - } - - while(enumerator) { - hr = enumerator->lpVtbl->Next(enumerator, WBEM_INFINITE, 1, &class_obj, &ret); - - if(ret == 0) { - break; - } - - val = we_wmi_get_property_value(ctx, "AvailableBytes", class_obj); - cmt_gauge_set(ctx->wmi_memory->available_bytes, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "CacheBytes", class_obj); - cmt_gauge_set(ctx->wmi_memory->cache_bytes, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "CacheBytesPeak", class_obj); - cmt_gauge_set(ctx->wmi_memory->cache_bytes_peak, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "CacheFaultsPersec", class_obj); - cmt_gauge_set(ctx->wmi_memory->cache_faults_total, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "CommitLimit", class_obj); - cmt_gauge_set(ctx->wmi_memory->commit_limit, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "CommittedBytes", class_obj); - cmt_gauge_set(ctx->wmi_memory->committed_bytes, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "DemandZeroFaultsPersec", class_obj); - cmt_gauge_set(ctx->wmi_memory->demand_zero_faults_total, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "FreeAndZeroPageListBytes", class_obj); - cmt_gauge_set(ctx->wmi_memory->free_and_zero_page_list_bytes, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "FreeSystemPageTableEntries", class_obj); - cmt_gauge_set(ctx->wmi_memory->free_system_page_table_entries, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "ModifiedPageListBytes", class_obj); - cmt_gauge_set(ctx->wmi_memory->modified_page_list_bytes, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "PageFaultsPersec", class_obj); - cmt_gauge_set(ctx->wmi_memory->page_faults_total, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "PageReadsPersec", class_obj); - cmt_gauge_set(ctx->wmi_memory->swap_page_reads_total, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "PagesInputPersec", class_obj); - cmt_gauge_set(ctx->wmi_memory->swap_pages_read_total, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "PagesOutputPersec", class_obj); - cmt_gauge_set(ctx->wmi_memory->swap_pages_written_total, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "PagesPersec", class_obj); - cmt_gauge_set(ctx->wmi_memory->swap_page_operations_total, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "PageWritesPersec", class_obj); - cmt_gauge_set(ctx->wmi_memory->swap_page_writes_total, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "PoolNonpagedAllocs", class_obj); - cmt_gauge_set(ctx->wmi_memory->pool_nonpaged_allocs_total, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "PoolNonpagedBytes", class_obj); - cmt_gauge_set(ctx->wmi_memory->pool_nonpaged_bytes, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "PoolPagedAllocs", class_obj); - cmt_gauge_set(ctx->wmi_memory->pool_paged_allocs_total, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "PoolPagedBytes", class_obj); - cmt_gauge_set(ctx->wmi_memory->pool_paged_bytes, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "PoolPagedResidentBytes", class_obj); - cmt_gauge_set(ctx->wmi_memory->pool_paged_resident_bytes, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "StandbyCacheCoreBytes", class_obj); - cmt_gauge_set(ctx->wmi_memory->standby_cache_core_bytes, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "StandbyCacheNormalPriorityBytes", class_obj); - cmt_gauge_set(ctx->wmi_memory->standby_cache_normal_priority_bytes, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "StandbyCacheReserveBytes", class_obj); - cmt_gauge_set(ctx->wmi_memory->standby_cache_reserve_bytes, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "SystemCacheResidentBytes", class_obj); - cmt_gauge_set(ctx->wmi_memory->system_cache_resident_bytes, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "SystemCacheResidentBytes", class_obj); - cmt_gauge_set(ctx->wmi_memory->system_cache_resident_bytes, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "SystemCodeResidentBytes", class_obj); - cmt_gauge_set(ctx->wmi_memory->system_code_resident_bytes, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "SystemCodeTotalBytes", class_obj); - cmt_gauge_set(ctx->wmi_memory->system_code_total_bytes, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "SystemDriverResidentBytes", class_obj); - cmt_gauge_set(ctx->wmi_memory->system_driver_resident_bytes, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "SystemDriverTotalBytes", class_obj); - cmt_gauge_set(ctx->wmi_memory->system_driver_total_bytes, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "TransitionFaultsPersec", class_obj); - cmt_gauge_set(ctx->wmi_memory->transition_faults_total, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "TransitionPagesRePurposedPersec", class_obj); - cmt_gauge_set(ctx->wmi_memory->transition_pages_repurposed_total, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "WriteCopiesPersec", class_obj); - cmt_gauge_set(ctx->wmi_memory->write_copies_total, timestamp, val, 0, NULL); - - class_obj->lpVtbl->Release(class_obj); - } - - enumerator->lpVtbl->Release(enumerator); - - we_wmi_cleanup(ctx); - - return 0; -} diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_memory.h b/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_memory.h deleted file mode 100644 index fd6f08d54..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_memory.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_WE_WMI_MEMORY_H -#define FLB_WE_WMI_MEMORY_H - -#include "we.h" - -int we_wmi_memory_init(struct flb_we *ctx); -int we_wmi_memory_exit(struct flb_we *ctx); -int we_wmi_memory_update(struct flb_we *ctx); - -#endif diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_paging_file.c b/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_paging_file.c deleted file mode 100644 index ed5853811..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_paging_file.c +++ /dev/null @@ -1,156 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "we.h" -#include "we_wmi.h" -#include "we_wmi_paging_file.h" -#include "we_util.h" -#include "we_metric.h" - -static double nop_adjust(double value) -{ - return value; -} - -int we_wmi_paging_file_init(struct flb_we *ctx) -{ - struct cmt_gauge *g; - - ctx->wmi_paging_file = flb_calloc(1, sizeof(struct we_wmi_paging_file_counters)); - if (!ctx->wmi_paging_file) { - flb_errno(); - return -1; - } - ctx->wmi_paging_file->operational = FLB_FALSE; - - g = cmt_gauge_create(ctx->cmt, "windows", "paging_file", "allocated_base_size_megabytes", - "The value indicates the actual amount of disk space allocated "\ - "for use with this page file (AllocatedBaseSize)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_paging_file->allocated_base_size_megabytes = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "paging_file", "current_usage_megabytes", - "The value indicates how much of the total reserved page file " \ - "is currently in use (CurrentUsage)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_paging_file->current_usage_megabytes = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "paging_file", "peak_usage_megabytes", - "The value indicates the highest use page file (PeakUsage)", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_paging_file->peak_usage_megabytes = g; - - ctx->wmi_paging_file->info = flb_calloc(1, sizeof(struct wmi_query_spec)); - if (!ctx->wmi_paging_file->info) { - flb_errno(); - return -1; - } - ctx->wmi_paging_file->info->metric_instance = (void *)g; - ctx->wmi_paging_file->info->type = CMT_GAUGE; - ctx->wmi_paging_file->info->value_adjuster = nop_adjust; - ctx->wmi_paging_file->info->wmi_counter = "Win32_PageFileUsage"; - ctx->wmi_paging_file->info->wmi_property = ""; - ctx->wmi_paging_file->info->label_property_count = 0; - ctx->wmi_paging_file->info->label_property_keys = NULL; - ctx->wmi_paging_file->info->where_clause = NULL; - - ctx->wmi_paging_file->operational = FLB_TRUE; - - return 0; -} - -int we_wmi_paging_file_exit(struct flb_we *ctx) -{ - ctx->wmi_paging_file->operational = FLB_FALSE; - - flb_free(ctx->wmi_paging_file->info); - flb_free(ctx->wmi_paging_file); - - return 0; -} - -int we_wmi_paging_file_update(struct flb_we *ctx) -{ - uint64_t timestamp = 0; - IEnumWbemClassObject* enumerator = NULL; - HRESULT hr; - - IWbemClassObject *class_obj = NULL; - ULONG ret = 0; - double val = 0; - - if (!ctx->wmi_paging_file->operational) { - flb_plg_error(ctx->ins, "paging_file collector not yet in operational state"); - - return -1; - } - - if (FAILED(we_wmi_coinitialize(ctx))) { - return -1; - } - - timestamp = cfl_time_now(); - - if (FAILED(we_wmi_execute_query(ctx, ctx->wmi_paging_file->info, &enumerator))) { - return -1; - } - - while(enumerator) { - hr = enumerator->lpVtbl->Next(enumerator, WBEM_INFINITE, 1, &class_obj, &ret); - - if(ret == 0) { - break; - } - - val = we_wmi_get_property_value(ctx, "AllocatedBaseSize", class_obj); - cmt_gauge_set(ctx->wmi_paging_file->allocated_base_size_megabytes, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "CurrentUsage", class_obj); - cmt_gauge_set(ctx->wmi_paging_file->current_usage_megabytes, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "PeakUsage", class_obj); - cmt_gauge_set(ctx->wmi_paging_file->peak_usage_megabytes, timestamp, val, 0, NULL); - - class_obj->lpVtbl->Release(class_obj); - } - - enumerator->lpVtbl->Release(enumerator); - - we_wmi_cleanup(ctx); - - return 0; -} diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_paging_file.h b/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_paging_file.h deleted file mode 100644 index 5ed74d292..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_paging_file.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_WE_WMI_PAGING_FILE_H -#define FLB_WE_WMI_PAGING_FILE_H - -#include "we.h" - -int we_wmi_paging_file_init(struct flb_we *ctx); -int we_wmi_paging_file_exit(struct flb_we *ctx); -int we_wmi_paging_file_update(struct flb_we *ctx); - -#endif diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_process.c b/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_process.c deleted file mode 100644 index 97226c274..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_process.c +++ /dev/null @@ -1,417 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "we.h" -#include "we_wmi.h" -#include "we_wmi_process.h" -#include "we_util.h" -#include "we_metric.h" - -static double nop_adjust(double value) -{ - return value; -} - -int we_wmi_process_init(struct flb_we *ctx) -{ - struct cmt_gauge *g; - - ctx->wmi_process = flb_calloc(1, sizeof(struct we_wmi_process_counters)); - if (!ctx->wmi_process) { - flb_errno(); - return -1; - } - ctx->wmi_process->operational = FLB_FALSE; - - g = cmt_gauge_create(ctx->cmt, "windows", "process", "start_time", - "Time of process start", - 3, (char *[]) {"process", "process_id", "creating_process_id"}); - - if (!g) { - return -1; - } - ctx->wmi_process->start_time = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "process", "handles", - "Total number of handles the process has open. " \ - "This number is the sum of the handles currently " \ - "open by each thread in the process.", - 3, (char *[]) {"process", "process_id", "creating_process_id"}); - - if (!g) { - return -1; - } - ctx->wmi_process->handles = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "process", "cpu_time_total", - "Returns elapsed time that all of the threads of this process " \ - "used the processor to execute instructions by mode " \ - "(privileged, user). An instruction is the basic unit " \ - "of execution in a computer, a thread is the object " \ - "that executes instructions, and a process is " \ - "the object created when a program is run. " \ - "Code executed to handle some hardware interrupts " \ - "and trap conditions is included in this count.", - 4, (char *[]) {"process", "process_id", "creating_process_id", "mode"}); - - if (!g) { - return -1; - } - ctx->wmi_process->cpu_time_total = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "process", "io_bytes_total", - "Bytes issued to I/O operations in different modes "\ - "(read, write, other).", - 4, (char *[]) {"process", "process_id", "creating_process_id", "mode"}); - - if (!g) { - return -1; - } - ctx->wmi_process->io_bytes_total = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "process", "io_operations_total", - "I/O operations issued in different modes (read, write, other).", - 4, (char *[]) {"process", "process_id", "creating_process_id", "mode"}); - - if (!g) { - return -1; - } - ctx->wmi_process->io_operations_total = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "process", "page_faults_total", - "Page faults by the threads executing in this process.", - 3, (char *[]) {"process", "process_id", "creating_process_id"}); - - if (!g) { - return -1; - } - ctx->wmi_process->page_faults_total = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "process", "page_file_bytes", - "Current number of bytes this process has used " \ - "in the paging file(s).", - 3, (char *[]) {"process", "process_id", "creating_process_id"}); - - if (!g) { - return -1; - } - ctx->wmi_process->page_file_bytes = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "process", "pool_bytes", - "Pool Bytes is the last observed number of bytes " \ - "in the paged or nonpaged pool.", - 4, (char *[]) {"process", "process_id", "creating_process_id", "pool"}); - - if (!g) { - return -1; - } - ctx->wmi_process->pool_bytes = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "process", "priority_base", - "Current base priority of this process. " \ - "Threads within a process can raise and " \ - "lower their own base priority relative to " \ - "the process base priority of the process.", - 3, (char *[]) {"process", "process_id", "creating_process_id"}); - - if (!g) { - return -1; - } - ctx->wmi_process->priority_base = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "process", "private_bytes", - "Current number of bytes this process has allocated " \ - "that cannot be shared with other processes.", - 3, (char *[]) {"process", "process_id", "creating_process_id"}); - - if (!g) { - return -1; - } - ctx->wmi_process->private_bytes = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "process", "thread_count", - "Number of threads currently active in this process.", - 3, (char *[]) {"process", "process_id", "creating_process_id"}); - - if (!g) { - return -1; - } - ctx->wmi_process->thread_count = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "process", "virtual_bytes", - "Current size, in bytes, of the virtual address space " \ - "that the process is using.", - 3, (char *[]) {"process", "process_id", "creating_process_id"}); - - if (!g) { - return -1; - } - ctx->wmi_process->virtual_bytes = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "process", "working_set_private_bytes", - "Size of the working set, in bytes, that is " \ - "use for this process only and not shared nor " \ - "shareable by other processes.", - 3, (char *[]) {"process", "process_id", "creating_process_id"}); - - if (!g) { - return -1; - } - ctx->wmi_process->working_set_private_bytes = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "process", "working_set_peak_bytes", - "Maximum size, in bytes, of the Working Set of " \ - "this process at any point in time. " \ - "The Working Set is the set of memory pages touched recently " \ - "by the threads in the process.", - 3, (char *[]) {"process", "process_id", "creating_process_id"}); - - if (!g) { - return -1; - } - ctx->wmi_process->working_set_peak_bytes = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "process", "working_set_bytes", - "Maximum number of bytes in the working set of " \ - "this process at any point in time. " \ - "The working set is the set of memory pages touched recently " \ - "by the threads in the process.", - 3, (char *[]) {"process", "process_id", "creating_process_id"}); - - if (!g) { - return -1; - } - ctx->wmi_process->working_set_bytes = g; - - ctx->wmi_process->info = flb_calloc(1, sizeof(struct wmi_query_spec)); - if (!ctx->wmi_process->info) { - flb_errno(); - return -1; - } - ctx->wmi_process->info->metric_instance = (void *)g; - ctx->wmi_process->info->type = CMT_GAUGE; - ctx->wmi_process->info->value_adjuster = nop_adjust; - ctx->wmi_process->info->wmi_counter = "Win32_PerfRawData_PerfProc_Process"; - ctx->wmi_process->info->wmi_property = ""; - ctx->wmi_process->info->label_property_count = 0; - ctx->wmi_process->info->label_property_keys = NULL; - ctx->wmi_process->info->where_clause = NULL; - - ctx->wmi_process->operational = FLB_TRUE; - - return 0; -} - -int we_wmi_process_exit(struct flb_we *ctx) -{ - ctx->wmi_process->operational = FLB_FALSE; - - flb_free(ctx->wmi_process->info); - flb_free(ctx->wmi_process); - - return 0; -} - -static int wmi_process_regex_match(struct flb_regex *regex, char *name) -{ - if (regex == NULL) { - return 0; - } - - if (name == NULL) { - return 0; - } - - return flb_regex_match(regex, name, strlen(name)); -} - -int we_wmi_process_filter(char *name, struct flb_we *ctx) -{ - if (strcasestr(name, "_Total") != NULL) { - return 1; - } - - if (wmi_process_regex_match(ctx->denying_process_regex, name) || - !wmi_process_regex_match(ctx->allowing_process_regex, name)) { - return 1; - } - - return 0; -} - -int we_wmi_process_update(struct flb_we *ctx) -{ - uint64_t timestamp = 0; - IEnumWbemClassObject* enumerator = NULL; - HRESULT hr; - - IWbemClassObject *class_obj = NULL; - ULONG ret = 0; - double val = 0; - char *name = NULL; - char *process_name = NULL; - char *process_id = NULL; - char *creating_process_id = NULL; - double freq = 0; - double ticks_to_seconds = 1 / 1e7; - char *state; - - if (!ctx->wmi_process->operational) { - flb_plg_error(ctx->ins, "process collector not yet in operational state"); - - return -1; - } - - if (FAILED(we_wmi_coinitialize(ctx))) { - return -1; - } - - timestamp = cfl_time_now(); - - if (FAILED(we_wmi_execute_query(ctx, ctx->wmi_process->info, &enumerator))) { - return -1; - } - - while(enumerator) { - hr = enumerator->lpVtbl->Next(enumerator, WBEM_INFINITE, 1, &class_obj, &ret); - - if(ret == 0) { - break; - } - - name = we_wmi_get_property_str_value(ctx, "Name", class_obj); - if (!name) { - continue; - } - /* Remove # from the duplicated process names */ - process_name = strtok_s(name, "#", &state); - if (we_wmi_process_filter(process_name, ctx) == 1) { - flb_free(name); - - continue; - } - - process_id = we_wmi_get_property_str_value(ctx, "IDProcess", class_obj); - creating_process_id = we_wmi_get_property_str_value(ctx, "CreatingProcessID", class_obj); - freq = we_wmi_get_property_value(ctx, "Frequency_Object", class_obj); - - val = we_wmi_get_property_value(ctx, "ElapsedTime", class_obj); - cmt_gauge_set(ctx->wmi_process->start_time, timestamp, - (double)((val-116444736000000000)/freq), - 3, (char *[]) {process_name, process_id, creating_process_id}); - - val = we_wmi_get_property_value(ctx, "HandleCount", class_obj); - cmt_gauge_set(ctx->wmi_process->handles, timestamp, val, - 3, (char *[]) {process_name, process_id, creating_process_id}); - - val = we_wmi_get_property_value(ctx, "PercentUserTime", class_obj); - cmt_gauge_set(ctx->wmi_process->cpu_time_total, timestamp, val * ticks_to_seconds, - 4, (char *[]) {process_name, process_id, creating_process_id, "user"}); - - val = we_wmi_get_property_value(ctx, "PercentPrivilegedTime", class_obj); - cmt_gauge_set(ctx->wmi_process->cpu_time_total, timestamp, val * ticks_to_seconds, - 4, (char *[]) {process_name, process_id, creating_process_id, "privileged"}); - - val = we_wmi_get_property_value(ctx, "IOOtherBytesPersec", class_obj); - cmt_gauge_set(ctx->wmi_process->io_bytes_total, timestamp, val, - 4, (char *[]) {process_name, process_id, creating_process_id, "other"}); - - val = we_wmi_get_property_value(ctx, "IOOtherOperationsPersec", class_obj); - cmt_gauge_set(ctx->wmi_process->io_operations_total, timestamp, val, - 4, (char *[]) {process_name, process_id, creating_process_id, "other"}); - - val = we_wmi_get_property_value(ctx, "IOReadBytesPersec", class_obj); - cmt_gauge_set(ctx->wmi_process->io_bytes_total, timestamp, val, - 4, (char *[]) {process_name, process_id, creating_process_id, "read"}); - - val = we_wmi_get_property_value(ctx, "IOReadOperationsPersec", class_obj); - cmt_gauge_set(ctx->wmi_process->io_operations_total, timestamp, val, - 4, (char *[]) {process_name, process_id, creating_process_id, "read"}); - - val = we_wmi_get_property_value(ctx, "IOWriteBytesPersec", class_obj); - cmt_gauge_set(ctx->wmi_process->io_bytes_total, timestamp, val, - 4, (char *[]) {process_name, process_id, creating_process_id, "write"}); - - val = we_wmi_get_property_value(ctx, "IOWriteOperationsPersec", class_obj); - cmt_gauge_set(ctx->wmi_process->io_operations_total, timestamp, val, - 4, (char *[]) {process_name, process_id, creating_process_id, "write"}); - - val = we_wmi_get_property_value(ctx, "PageFaultsPersec", class_obj); - cmt_gauge_set(ctx->wmi_process->page_faults_total, timestamp, val, - 3, (char *[]) {process_name, process_id, creating_process_id}); - - val = we_wmi_get_property_value(ctx, "PageFileBytes", class_obj); - cmt_gauge_set(ctx->wmi_process->page_file_bytes, timestamp, val, - 3, (char *[]) {process_name, process_id, creating_process_id}); - - val = we_wmi_get_property_value(ctx, "PoolNonpagedBytes", class_obj); - cmt_gauge_set(ctx->wmi_process->pool_bytes, timestamp, val, - 4, (char *[]) {process_name, process_id, creating_process_id, "nonpaged"}); - - val = we_wmi_get_property_value(ctx, "PoolPagedBytes", class_obj); - cmt_gauge_set(ctx->wmi_process->pool_bytes, timestamp, val, - 4, (char *[]) {process_name, process_id, creating_process_id, "paged"}); - - val = we_wmi_get_property_value(ctx, "PriorityBase", class_obj); - cmt_gauge_set(ctx->wmi_process->priority_base, timestamp, val, - 3, (char *[]) {process_name, process_id, creating_process_id}); - - val = we_wmi_get_property_value(ctx, "ThreadCount", class_obj); - cmt_gauge_set(ctx->wmi_process->thread_count, timestamp, val, - 3, (char *[]) {process_name, process_id, creating_process_id}); - - val = we_wmi_get_property_value(ctx, "PrivateBytes", class_obj); - cmt_gauge_set(ctx->wmi_process->private_bytes, timestamp, val, - 3, (char *[]) {process_name, process_id, creating_process_id}); - - val = we_wmi_get_property_value(ctx, "VirtualBytes", class_obj); - cmt_gauge_set(ctx->wmi_process->virtual_bytes, timestamp, val, - 3, (char *[]) {process_name, process_id, creating_process_id}); - - val = we_wmi_get_property_value(ctx, "WorkingSetPrivate", class_obj); - cmt_gauge_set(ctx->wmi_process->working_set_private_bytes, timestamp, val, - 3, (char *[]) {process_name, process_id, creating_process_id}); - - val = we_wmi_get_property_value(ctx, "WorkingSetPeak", class_obj); - cmt_gauge_set(ctx->wmi_process->working_set_peak_bytes, timestamp, val, - 3, (char *[]) {process_name, process_id, creating_process_id}); - - val = we_wmi_get_property_value(ctx, "WorkingSet", class_obj); - cmt_gauge_set(ctx->wmi_process->working_set_bytes, timestamp, val, - 3, (char *[]) {process_name, process_id, creating_process_id}); - - class_obj->lpVtbl->Release(class_obj); - - flb_free(name); - flb_free(process_id); - flb_free(creating_process_id); - } - - enumerator->lpVtbl->Release(enumerator); - - we_wmi_cleanup(ctx); - - return 0; -} diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_process.h b/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_process.h deleted file mode 100644 index 0ad921ea6..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_process.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_WE_WMI_PROCESS_INFO_H -#define FLB_WE_WMI_PROCESS_INFO_H - -#include "we.h" - -int we_wmi_process_info_init(struct flb_we *ctx); -int we_wmi_process_info_exit(struct flb_we *ctx); -int we_wmi_process_info_update(struct flb_we *ctx); - -#endif diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_service.c b/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_service.c deleted file mode 100644 index e31a4943b..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_service.c +++ /dev/null @@ -1,493 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "we.h" -#include "we_wmi.h" -#include "we_wmi_service.h" -#include "we_util.h" -#include "we_metric.h" - -static double nop_adjust(double value) -{ - return value; -} - -static int construct_include_clause(struct flb_we *ctx, flb_sds_t *clause) -{ - int ret = -1; - size_t off = 0; - msgpack_unpacked result; - msgpack_object key; - msgpack_object val; - msgpack_object map; - int map_size; - int i; - int idx = 0; - int use_like = FLB_FALSE; - char *key_str = NULL; - size_t key_str_size = 0; - char *val_str = NULL; - size_t val_str_size = 0; - flb_sds_t val_buf; - - msgpack_unpacked_init(&result); - while (msgpack_unpack_next(&result, - ctx->service_include_buffer, - ctx->service_include_buffer_size, - &off) == MSGPACK_UNPACK_SUCCESS) { - if (result.data.type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "Invalid include buffer"); - ret = -2; - - goto cleanup; - } - - map = result.data; - map_size = map.via.map.size; - - for (i = 0; i < map_size; i++) { - use_like = FLB_FALSE; - if (idx == 0) { - flb_sds_cat_safe(clause, "(", 1); - } - else { - flb_sds_cat_safe(clause, " OR ", 4); - } - - key = map.via.map.ptr[i].key; - val = map.via.map.ptr[i].val; - if (key.type == MSGPACK_OBJECT_BIN) { - key_str = (char *) key.via.bin.ptr; - key_str_size = key.via.bin.size; - } - else if (key.type == MSGPACK_OBJECT_STR) { - key_str = (char *) key.via.str.ptr; - key_str_size = key.via.str.size; - } - if (val.type == MSGPACK_OBJECT_BIN) { - val_str = (char *) val.via.bin.ptr; - val_str_size = val.via.bin.size; - val_buf = flb_sds_create_len(val_str, val_str_size); - if (val_buf == NULL) { - flb_plg_error(ctx->ins, "val_buf creation is failed"); - ret = -3; - - goto cleanup; - } - } - else if (val.type == MSGPACK_OBJECT_STR) { - val_str = (char *) val.via.str.ptr; - val_str_size = val.via.str.size; - val_buf = flb_sds_create_len(val_str, val_str_size); - if (val_buf == NULL) { - flb_plg_error(ctx->ins, "val_buf creation is failed"); - ret = -3; - - goto cleanup; - } - } - - if (val_str != NULL && strstr(val_buf, "%") != NULL) { - use_like = FLB_TRUE; - flb_sds_destroy(val_buf); - } - flb_sds_cat_safe(clause, key_str, key_str_size); - if (use_like == FLB_TRUE) { - flb_sds_cat_safe(clause, " LIKE ", 6); - } - else { - flb_sds_cat_safe(clause, "=", 1); - } - flb_sds_cat_safe(clause, "'", 1); - flb_sds_cat_safe(clause, val_str, val_str_size); - flb_sds_cat_safe(clause, "'", 1); - idx++; - } - flb_sds_cat_safe(clause, ")", 1); - } - msgpack_unpacked_destroy(&result); - - return 0; - -cleanup: - msgpack_unpacked_destroy(&result); - - return ret; -} - -static int construct_exclude_clause(struct flb_we *ctx, flb_sds_t *clause) -{ - int ret = -1; - size_t off = 0; - msgpack_unpacked result; - msgpack_object key; - msgpack_object val; - msgpack_object map; - int map_size; - int i; - int idx = 0; - int use_like = FLB_FALSE; - char *key_str = NULL; - size_t key_str_size = 0; - char *val_str = NULL; - size_t val_str_size = 0; - flb_sds_t val_buf; - - msgpack_unpacked_init(&result); - while (msgpack_unpack_next(&result, - ctx->service_exclude_buffer, - ctx->service_exclude_buffer_size, - &off) == MSGPACK_UNPACK_SUCCESS) { - if (result.data.type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "Invalid exclude buffer"); - ret = -2; - - goto cleanup; - } - - map = result.data; - map_size = map.via.map.size; - - for (i = 0; i < map_size; i++) { - use_like = FLB_FALSE; - if (idx == 0) { - flb_sds_cat_safe(clause, "(", 1); - } - else { - flb_sds_cat_safe(clause, " AND ", 5); - } - - key = map.via.map.ptr[i].key; - val = map.via.map.ptr[i].val; - if (key.type == MSGPACK_OBJECT_BIN) { - key_str = (char *) key.via.bin.ptr; - key_str_size = key.via.bin.size; - } - else if (key.type == MSGPACK_OBJECT_STR) { - key_str = (char *) key.via.str.ptr; - key_str_size = key.via.str.size; - } - if (val.type == MSGPACK_OBJECT_BIN) { - val_str = (char *) val.via.bin.ptr; - val_str_size = val.via.bin.size; - val_buf = flb_sds_create_len(val_str, val_str_size); - if (val_buf == NULL) { - flb_plg_error(ctx->ins, "val_buf creation is failed"); - ret = -3; - - goto cleanup; - } - } - else if (val.type == MSGPACK_OBJECT_STR) { - val_str = (char *) val.via.str.ptr; - val_str_size = val.via.str.size; - val_buf = flb_sds_create_len(val_str, val_str_size); - if (val_buf == NULL) { - flb_plg_error(ctx->ins, "val_buf creation is failed"); - ret = -3; - - goto cleanup; - } - } - - if (val_str != NULL && strstr(val_buf, "%") != NULL) { - use_like = FLB_TRUE; - flb_sds_destroy(val_buf); - } - if (use_like == FLB_TRUE) { - flb_sds_cat_safe(clause, "NOT ", 4); - } - flb_sds_cat_safe(clause, key_str, key_str_size); - if (use_like == FLB_TRUE) { - flb_sds_cat_safe(clause, " LIKE ", 6); - } - else { - flb_sds_cat_safe(clause, "!=", 2); - } - flb_sds_cat_safe(clause, "'", 1); - flb_sds_cat_safe(clause, val_str, val_str_size); - flb_sds_cat_safe(clause, "'", 1); - idx++; - } - flb_sds_cat_safe(clause, ")", 1); - } - msgpack_unpacked_destroy(&result); - - return 0; - -cleanup: - msgpack_unpacked_destroy(&result); - - return ret; -} - -static int construct_where_clause(struct flb_we *ctx) -{ - int ret; - flb_sds_t clause; - - clause = flb_sds_create_size(256); - if (!clause) { - return -1; - } - - if (ctx->service_include_buffer != NULL && ctx->service_include_buffer_size > 0) { - ret = construct_include_clause(ctx, &clause); - if (ret != 0) { - goto cleanup; - } - } - - if (ctx->service_exclude_buffer != NULL && ctx->service_exclude_buffer_size > 0) { - if (flb_sds_len(clause) > 0) { - flb_sds_cat_safe(&clause, " AND ", 5); - } - ret = construct_exclude_clause(ctx, &clause); - if (ret != 0) { - goto cleanup; - } - } - - if (ctx->raw_where_clause != NULL){ - if (flb_sds_len(clause) > 0) { - flb_sds_cat_safe(&clause, " AND (", 6); - flb_sds_cat_safe(&clause, ctx->raw_where_clause, strlen(ctx->raw_where_clause)); - flb_sds_cat_safe(&clause, ")", 1); - } - else { - flb_sds_cat_safe(&clause, ctx->raw_where_clause, strlen(ctx->raw_where_clause)); - } - } - - if (flb_sds_len(clause) > 0) { - ctx->wmi_service->info->where_clause = clause; - } - - return 0; - -cleanup: - flb_sds_destroy(clause); - - return ret; -} - -int we_wmi_service_init(struct flb_we *ctx) -{ - int ret; - struct cmt_gauge *g; - - ctx->wmi_service = flb_calloc(1, sizeof(struct we_wmi_service_counters)); - if (!ctx->wmi_service) { - flb_errno(); - return -1; - } - ctx->wmi_service->operational = FLB_FALSE; - - g = cmt_gauge_create(ctx->cmt, "windows", "service", "info", - "A metric for Windows Service information", - 4, (char *[]) {"name", "display_name", "process_id", "run_as"}); - - if (!g) { - return -1; - } - ctx->wmi_service->information = g; - - - g = cmt_gauge_create(ctx->cmt, "windows", "service", "state", - "A state of the service", - 2, (char *[]){"name", "state"}); - if (!g) { - return -1; - } - ctx->wmi_service->state = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "service", "start_mode", - "A start mode of the service", - 2, (char *[]){"name", "start_mode"}); - if (!g) { - return -1; - } - ctx->wmi_service->start_mode = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "service", "status", - "A status of the service", - 2, (char *[]){"name", "status"}); - if (!g) { - return -1; - } - ctx->wmi_service->status = g; - - ctx->wmi_service->info = flb_calloc(1, sizeof(struct wmi_query_spec)); - if (!ctx->wmi_service->info) { - flb_errno(); - return -1; - } - ctx->wmi_service->info->metric_instance = (void *)g; - ctx->wmi_service->info->type = CMT_GAUGE; - ctx->wmi_service->info->value_adjuster = nop_adjust; - ctx->wmi_service->info->wmi_counter = "Win32_Service"; - ctx->wmi_service->info->wmi_property = ""; - ctx->wmi_service->info->label_property_count = 0; - ctx->wmi_service->info->label_property_keys = NULL; - ctx->wmi_service->info->where_clause = NULL; - ret = construct_where_clause(ctx); - if (ret != 0) { - return ret; - } - - ctx->wmi_service->operational = FLB_TRUE; - - return 0; -} - -int we_wmi_service_exit(struct flb_we *ctx) -{ - ctx->wmi_service->operational = FLB_FALSE; - - if (ctx->wmi_service->info->where_clause != NULL) { - flb_sds_destroy(ctx->wmi_service->info->where_clause); - } - flb_free(ctx->wmi_service->info); - flb_free(ctx->wmi_service); - - return 0; -} - -int we_wmi_service_update(struct flb_we *ctx) -{ - IEnumWbemClassObject* enumerator = NULL; - HRESULT hr; - - IWbemClassObject *class_obj = NULL; - ULONG ret = 0; - int i = 0; - uint64_t timestamp = 0; - char *service_name = NULL; - char *display_name = NULL; - char *pid = NULL; - char *run_as = NULL; - char *str_prop = NULL; - char *state = NULL; - char *start_mode = NULL; - char *status = NULL; - char **states = (char *[]){ - "stopped", "start pending", "stop pending", "running", - "continue pending", "pause pending", "paused", "unknown", NULL - }; - char **statuses = (char *[]){ - "ok", "error", "degraded", "unknown", - "pred fail", "starting", "stopping", "service", - "stressed", "nonrecover", "no contact", "lost comm", NULL - }; - char **start_modes = (char *[]) { - "boot", "system", "auto", "manual", "disabled", NULL - }; - - if (!ctx->wmi_service->operational) { - flb_plg_error(ctx->ins, "windows_service collector not yet in operational state"); - - return -1; - } - - if (FAILED(we_wmi_coinitialize(ctx))) { - return -1; - } - - timestamp = cfl_time_now(); - - if (FAILED(we_wmi_execute_query(ctx, ctx->wmi_service->info, &enumerator))) { - return -1; - } - - while (enumerator) { - hr = enumerator->lpVtbl->Next(enumerator, WBEM_INFINITE, 1, - &class_obj, &ret); - - if (0 == ret) { - break; - } - - service_name = we_wmi_get_property_str_value(ctx, "Name", class_obj); - display_name = we_wmi_get_property_str_value(ctx, "DisplayName", class_obj); - pid = we_wmi_get_property_str_value(ctx, "ProcessID", class_obj); - run_as = we_wmi_get_property_str_value(ctx, "StartName", class_obj); - state = we_wmi_get_property_str_value(ctx, "State", class_obj); - start_mode = we_wmi_get_property_str_value(ctx, "StartMode", class_obj); - status = we_wmi_get_property_str_value(ctx, "Status", class_obj); - - /* Information */ - cmt_gauge_set(ctx->wmi_service->information, timestamp, 1.0, - 4, (char *[]){ service_name, display_name, pid, run_as}); - - /* State */ - for (i = 0; states[i] != NULL; i++) { - if (strcasecmp(state, states[i]) == 0) { - cmt_gauge_set(ctx->wmi_service->state, timestamp, 1.0, - 2, (char *[]){ service_name, states[i]}); - } - else { - cmt_gauge_set(ctx->wmi_service->state, timestamp, 0.0, - 2, (char *[]){ service_name, states[i]}); - } - } - /* Start Mode */ - for (i = 0; start_modes[i] != NULL; i++) { - if (strcasecmp(start_mode, start_modes[i]) == 0) { - cmt_gauge_set(ctx->wmi_service->start_mode, timestamp, 1.0, - 2, (char *[]){ service_name, start_modes[i]}); - } - else { - cmt_gauge_set(ctx->wmi_service->start_mode, timestamp, 0.0, - 2, (char *[]){ service_name, start_modes[i]}); - } - } - - /* Status */ - for (i = 0; statuses[i] != NULL; i++) { - if (strcasecmp(status, statuses[i]) == 0) { - cmt_gauge_set(ctx->wmi_service->status, timestamp, 1.0, - 2, (char *[]){ service_name, statuses[i]}); - } else { - cmt_gauge_set(ctx->wmi_service->status, timestamp, 0.0, - 2, (char *[]){ service_name, statuses[i]}); - } - } - - class_obj->lpVtbl->Release(class_obj); - - flb_free(service_name); - flb_free(display_name); - flb_free(pid); - flb_free(run_as); - flb_free(state); - flb_free(start_mode); - flb_free(status); - } - - enumerator->lpVtbl->Release(enumerator); - we_wmi_cleanup(ctx); - - return 0; -} diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_service.h b/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_service.h deleted file mode 100644 index d9b3efea9..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_service.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_WE_WMI_SERVICE_H -#define FLB_WE_WMI_SERVICE_H - -#include "we.h" - -int we_wmi_service_init(struct flb_we *ctx); -int we_wmi_service_exit(struct flb_we *ctx); -int we_wmi_service_update(struct flb_we *ctx); - -#endif diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_system.c b/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_system.c deleted file mode 100644 index 0eb7fffaf..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_system.c +++ /dev/null @@ -1,190 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "we.h" -#include "we_wmi.h" -#include "we_wmi_system.h" -#include "we_util.h" -#include "we_metric.h" - -static double nop_adjust(double value) -{ - return value; -} - -int we_wmi_system_init(struct flb_we *ctx) -{ - ctx->wmi_system = flb_calloc(1, sizeof(struct we_wmi_system_counters)); - if (!ctx->wmi_system) { - flb_errno(); - return -1; - } - ctx->wmi_system->operational = FLB_FALSE; - - struct cmt_gauge *g; - - g = cmt_gauge_create(ctx->cmt, "windows", "system", "context_switches_total", - "Total number of context switches", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_system->context_switches = g; - - g = cmt_counter_create(ctx->cmt, "windows", "system", "exception_dispatches_total", - "Total number of exception_dispatches", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_system->exception_dispatches = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "system", "processor_queue", - "Length of processor queues", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_system->processor_queue = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "system", "system_calls_total", - "Total number of system calls", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_system->system_calls = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "system", "system_up_time", - "System boot time", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_system->system_up_time = g; - - g = cmt_gauge_create(ctx->cmt, "windows", "system", "threads", - "Current number of threads", - 0, NULL); - - if (!g) { - return -1; - } - ctx->wmi_system->threads = g; - - ctx->wmi_system->info = flb_calloc(1, sizeof(struct wmi_query_spec)); - if (!ctx->wmi_system->info) { - flb_errno(); - return -1; - } - ctx->wmi_system->info->metric_instance = (void *)g; - ctx->wmi_system->info->type = CMT_GAUGE; - ctx->wmi_system->info->value_adjuster = nop_adjust; - ctx->wmi_system->info->wmi_counter = "Win32_PerfFormattedData_PerfOS_System"; - ctx->wmi_system->info->wmi_property = ""; - ctx->wmi_system->info->label_property_count = 0; - ctx->wmi_system->info->label_property_keys = NULL; - ctx->wmi_system->info->where_clause = NULL; - - ctx->wmi_system->operational = FLB_TRUE; - - return 0; -} - -int we_wmi_system_exit(struct flb_we *ctx) -{ - ctx->wmi_system->operational = FLB_FALSE; - - flb_free(ctx->wmi_system->info); - flb_free(ctx->wmi_system); - - return 0; -} - -int we_wmi_system_update(struct flb_we *ctx) -{ - uint64_t timestamp = 0; - IEnumWbemClassObject* enumerator = NULL; - HRESULT hr; - - IWbemClassObject *class_obj = NULL; - ULONG ret = 0; - double val = 0; - - if (!ctx->wmi_system->operational) { - flb_plg_error(ctx->ins, "system collector not yet in operational state"); - - return -1; - } - - if (FAILED(we_wmi_coinitialize(ctx))) { - return -1; - } - - timestamp = cfl_time_now(); - - if (FAILED(we_wmi_execute_query(ctx, ctx->wmi_system->info, &enumerator))) { - return -1; - } - - while(enumerator) { - hr = enumerator->lpVtbl->Next(enumerator, WBEM_INFINITE, 1, &class_obj, &ret); - - if(0 == ret) { - break; - } - - val = we_wmi_get_property_value(ctx, "ContextSwitchesPersec", class_obj); - cmt_gauge_set(ctx->wmi_system->context_switches, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "ExceptionDispatchesPersec", class_obj); - cmt_gauge_set(ctx->wmi_system->exception_dispatches, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "ProcessorQueueLength", class_obj); - cmt_gauge_set(ctx->wmi_system->processor_queue, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "SystemCallsPersec", class_obj); - cmt_gauge_set(ctx->wmi_system->system_calls, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "SystemUpTime", class_obj); - cmt_gauge_set(ctx->wmi_system->system_up_time, timestamp, val, 0, NULL); - - val = we_wmi_get_property_value(ctx, "Threads", class_obj); - cmt_gauge_set(ctx->wmi_system->threads, timestamp, val, 0, NULL); - - class_obj->lpVtbl->Release(class_obj); - } - - enumerator->lpVtbl->Release(enumerator); - - we_wmi_cleanup(ctx); - - return 0; -} diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_system.h b/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_system.h deleted file mode 100644 index abc8ea6f5..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_system.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_WE_WMI_SYSTEM_H -#define FLB_WE_WMI_SYSTEM_H - -#include "we.h" - -int we_wmi_system_init(struct flb_we *ctx); -int we_wmi_system_exit(struct flb_we *ctx); -int we_wmi_system_update(struct flb_we *ctx); - -#endif diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_thermalzone.c b/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_thermalzone.c deleted file mode 100644 index 1766c5907..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_thermalzone.c +++ /dev/null @@ -1,171 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "we.h" -#include "we_wmi.h" -#include "we_wmi_thermalzone.h" -#include "we_util.h" -#include "we_metric.h" - -static double adjust_celsius(double value) -{ - return (value/10.0) - 273.15; -} - -static double nop_adjust(double value) -{ - return value; -} - -int we_wmi_thermalzone_init(struct flb_we *ctx) -{ - ctx->wmi_thermals = flb_calloc(1, sizeof(struct we_wmi_thermal_counters)); - if (!ctx->wmi_thermals) { - flb_errno(); - return -1; - } - ctx->wmi_thermals->operational = FLB_FALSE; - - struct cmt_gauge *g; - - g = cmt_gauge_create(ctx->cmt, "windows", "thermalzone", "temperature_celsius", - "Temperature of the sensor device.", - 1, (char *[]) {"name"}); - if (!g) { - return -1; - } - - ctx->wmi_thermals->temperature_celsius = flb_calloc(1, sizeof(struct wmi_query_spec)); - if (!ctx->wmi_thermals->temperature_celsius) { - return -1; - } - ctx->wmi_thermals->temperature_celsius->label_property_keys = (char **) flb_calloc(1, sizeof(char *)); - if (!ctx->wmi_thermals->temperature_celsius->label_property_keys) { - return -1; - } - - ctx->wmi_thermals->temperature_celsius->metric_instance = (void *)g; - ctx->wmi_thermals->temperature_celsius->type = CMT_GAUGE; - ctx->wmi_thermals->temperature_celsius->value_adjuster = adjust_celsius; - ctx->wmi_thermals->temperature_celsius->wmi_counter = "Win32_PerfRawData_Counters_ThermalZoneInformation"; - ctx->wmi_thermals->temperature_celsius->wmi_property = "HighPrecisionTemperature"; - ctx->wmi_thermals->temperature_celsius->label_property_count = 1; - ctx->wmi_thermals->temperature_celsius->label_property_keys[0] = "name" ; - ctx->wmi_thermals->temperature_celsius->where_clause = NULL; - - g = cmt_gauge_create(ctx->cmt, "windows", "thermalzone", "percent_passive_limit", - "The limit of passive limit (percent).", - 1, (char *[]) {"name"}); - if (!g) { - return -1; - } - - ctx->wmi_thermals->percent_passive_limit = flb_calloc(1, sizeof(struct wmi_query_spec)); - if (!ctx->wmi_thermals->percent_passive_limit) { - flb_errno(); - return -1; - } - - ctx->wmi_thermals->percent_passive_limit->label_property_keys = (char **) flb_calloc(1, sizeof(char *)); - if (!ctx->wmi_thermals->percent_passive_limit->label_property_keys) { - flb_errno(); - return -1; - } - - ctx->wmi_thermals->percent_passive_limit->metric_instance = (void *)g; - ctx->wmi_thermals->percent_passive_limit->type = CMT_GAUGE; - ctx->wmi_thermals->percent_passive_limit->value_adjuster = nop_adjust; - ctx->wmi_thermals->percent_passive_limit->wmi_counter = "Win32_PerfRawData_Counters_ThermalZoneInformation"; - ctx->wmi_thermals->percent_passive_limit->wmi_property = "PercentPassiveLimit"; - ctx->wmi_thermals->percent_passive_limit->label_property_count = 1; - ctx->wmi_thermals->percent_passive_limit->label_property_keys[0] = "name"; - ctx->wmi_thermals->percent_passive_limit->where_clause = NULL; - - g = cmt_gauge_create(ctx->cmt, "windows", "thermalzone", "throttle_reasons", - "The reason of throttle.", - 1, (char *[]) {"name"}); - if (!g) { - return -1; - } - ctx->wmi_thermals->throttle_reasons = flb_calloc(1, sizeof(struct wmi_query_spec)); - if (!ctx->wmi_thermals->throttle_reasons) { - flb_errno(); - return -1; - } - ctx->wmi_thermals->throttle_reasons->label_property_keys = (char **) flb_calloc(1, sizeof(char *)); - if (!ctx->wmi_thermals->throttle_reasons->label_property_keys) { - flb_errno(); - return -1; - } - - ctx->wmi_thermals->throttle_reasons->metric_instance = (void *)g; - ctx->wmi_thermals->throttle_reasons->type = CMT_GAUGE; - ctx->wmi_thermals->throttle_reasons->value_adjuster = nop_adjust; - ctx->wmi_thermals->throttle_reasons->wmi_counter = "Win32_PerfRawData_Counters_ThermalZoneInformation"; - ctx->wmi_thermals->throttle_reasons->wmi_property = "ThrottleReasons"; - ctx->wmi_thermals->throttle_reasons->label_property_count = 1; - ctx->wmi_thermals->throttle_reasons->label_property_keys[0] = "name"; - ctx->wmi_thermals->throttle_reasons->where_clause = NULL; - - ctx->wmi_thermals->operational = FLB_TRUE; - - return 0; -} - -int we_wmi_thermalzone_exit(struct flb_we *ctx) -{ - flb_free(ctx->wmi_thermals->temperature_celsius->label_property_keys); - flb_free(ctx->wmi_thermals->temperature_celsius); - flb_free(ctx->wmi_thermals->percent_passive_limit->label_property_keys); - flb_free(ctx->wmi_thermals->percent_passive_limit); - flb_free(ctx->wmi_thermals->throttle_reasons->label_property_keys); - flb_free(ctx->wmi_thermals->throttle_reasons); - flb_free(ctx->wmi_thermals); - - return 0; -} - -int we_wmi_thermalzone_update(struct flb_we *ctx) -{ - if (!ctx->wmi_thermals->operational) { - flb_plg_error(ctx->ins, "thermalzone collector not yet in operational state"); - - return -1; - } - - if (FAILED(we_wmi_query(ctx, ctx->wmi_thermals->temperature_celsius))) { - return -1; - } - - if (FAILED(we_wmi_query(ctx, ctx->wmi_thermals->percent_passive_limit))) { - return -1; - } - - if (FAILED(we_wmi_query(ctx, ctx->wmi_thermals->throttle_reasons))) { - return -1; - } - - return 0; -} diff --git a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_thermalzone.h b/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_thermalzone.h deleted file mode 100644 index a94d6dc36..000000000 --- a/fluent-bit/plugins/in_windows_exporter_metrics/we_wmi_thermalzone.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_WE_WMI_THERMAL_ZONE_H -#define FLB_WE_WMI_THERMAL_ZONE_H - -#include "we.h" - -int we_wmi_thermalzone_init(struct flb_we *ctx); -int we_wmi_thermalzone_exit(struct flb_we *ctx); -int we_wmi_thermalzone_update(struct flb_we *ctx); - -#endif diff --git a/fluent-bit/plugins/in_winevtlog/CMakeLists.txt b/fluent-bit/plugins/in_winevtlog/CMakeLists.txt deleted file mode 100644 index 8f4c0e233..000000000 --- a/fluent-bit/plugins/in_winevtlog/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(src - in_winevtlog.c - pack.c - winevtlog.c) - -FLB_PLUGIN(in_winevtlog "${src}" "wevtapi") diff --git a/fluent-bit/plugins/in_winevtlog/in_winevtlog.c b/fluent-bit/plugins/in_winevtlog/in_winevtlog.c deleted file mode 100644 index 60c5bcecb..000000000 --- a/fluent-bit/plugins/in_winevtlog/in_winevtlog.c +++ /dev/null @@ -1,279 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2021 The Fluent Bit Authors - * Copyright (C) 2015-2018 Treasure Data Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include "winevtlog.h" - -#define DEFAULT_INTERVAL_SEC 1 -#define DEFAULT_INTERVAL_NSEC 0 -#define DEFAULT_THRESHOLD_SIZE 0x7ffff /* Default reading buffer size (512kb) */ - -static int in_winevtlog_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context); - -static int in_winevtlog_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - int ret; - const char *tmp; - int read_existing_events = FLB_FALSE; - struct mk_list *head; - struct winevtlog_channel *ch; - struct winevtlog_config *ctx; - - /* Initialize context */ - ctx = flb_calloc(1, sizeof(struct winevtlog_config)); - if (ctx == NULL) { - flb_errno(); - return -1; - } - ctx->ins = in; - - ctx->log_encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); - - if (ctx->log_encoder == NULL) { - flb_plg_error(in, "could not initialize event encoder"); - flb_free(ctx); - - return NULL; - } - - /* Load the config map */ - ret = flb_input_config_map_set(in, (void *) ctx); - if (ret == -1) { - flb_log_event_encoder_destroy(ctx->log_encoder); - flb_free(ctx); - return -1; - } - - /* Set up total reading size threshold */ - ctx->total_size_threshold = DEFAULT_THRESHOLD_SIZE; - - /* Open channels */ - tmp = flb_input_get_property("channels", in); - if (!tmp) { - flb_plg_debug(ctx->ins, "no channel provided. listening to 'Application'"); - tmp = "Application"; - } - - ctx->active_channel = winevtlog_open_all(tmp, ctx); - if (!ctx->active_channel) { - flb_plg_error(ctx->ins, "failed to open channels"); - flb_log_event_encoder_destroy(ctx->log_encoder); - flb_free(ctx); - return -1; - } - - /* Initialize SQLite DB (optional) */ - tmp = flb_input_get_property("db", in); - if (tmp) { - ctx->db = flb_sqldb_open(tmp, in->name, config); - if (!ctx->db) { - flb_plg_error(ctx->ins, "could not open/create database"); - winevtlog_close_all(ctx->active_channel); - flb_log_event_encoder_destroy(ctx->log_encoder); - flb_free(ctx); - return -1; - } - - ret = flb_sqldb_query(ctx->db, SQL_CREATE_CHANNELS, NULL, NULL); - if (ret != FLB_OK) { - flb_plg_error(ctx->ins, "could not create 'channels' table"); - flb_sqldb_close(ctx->db); - winevtlog_close_all(ctx->active_channel); - flb_log_event_encoder_destroy(ctx->log_encoder); - flb_free(ctx); - return -1; - } - - mk_list_foreach(head, ctx->active_channel) { - ch = mk_list_entry(head, struct winevtlog_channel, _head); - winevtlog_sqlite_load(ch, ctx->db); - flb_plg_debug(ctx->ins, "load channel<%s time=%u>", - ch->name, ch->time_created); - } - } - - /* Set the context */ - flb_input_set_context(in, ctx); - - /* Set the collector */ - ret = flb_input_set_collector_time(in, - in_winevtlog_collect, - ctx->interval_sec, - ctx->interval_nsec, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not set up a collector"); - } - ctx->coll_fd = ret; - - return 0; -} - -static int in_winevtlog_read_channel(struct flb_input_instance *ins, - struct winevtlog_config *ctx, - struct winevtlog_channel *ch) -{ - unsigned int read; - - if (winevtlog_read(ch, ctx, &read)) { - flb_plg_error(ctx->ins, "failed to read '%s'", ch->name); - return -1; - } - if (read == 0) { - return 0; - } - flb_plg_debug(ctx->ins, "read %u bytes from '%s'", read, ch->name); - - if (ctx->db) { - ch->time_updated = time(NULL); - flb_plg_debug(ctx->ins, "save channel<%s time=%u>", - ch->name, ch->time_updated); - winevtlog_sqlite_save(ch, ctx->db); - } - - if (ctx->log_encoder->output_length > 0) { - flb_input_log_append(ctx->ins, NULL, 0, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); - } - - flb_log_event_encoder_reset(ctx->log_encoder); - - return 0; -} - -static int in_winevtlog_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct winevtlog_config *ctx = in_context; - struct mk_list *head; - struct winevtlog_channel *ch; - - mk_list_foreach(head, ctx->active_channel) { - ch = mk_list_entry(head, struct winevtlog_channel, _head); - in_winevtlog_read_channel(ins, ctx, ch); - } - return 0; -} - -static void in_winevtlog_pause(void *data, struct flb_config *config) -{ - struct winevtlog_config *ctx = data; - flb_input_collector_pause(ctx->coll_fd, ctx->ins); -} - -static void in_winevtlog_resume(void *data, struct flb_config *config) -{ - struct winevtlog_config *ctx = data; - flb_input_collector_resume(ctx->coll_fd, ctx->ins); -} - -static int in_winevtlog_exit(void *data, struct flb_config *config) -{ - struct winevtlog_config *ctx = data; - - if (!ctx) { - return 0; - } - - winevtlog_close_all(ctx->active_channel); - - if (ctx->db) { - flb_sqldb_close(ctx->db); - } - flb_free(ctx); - - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "channels", NULL, - 0, FLB_FALSE, 0, - "Specify a comma-separated list of channels to read from" - }, - { - FLB_CONFIG_MAP_STR, "db", NULL, - 0, FLB_FALSE, 0, - "Specify DB file to save read offsets" - }, - { - FLB_CONFIG_MAP_TIME, "interval_sec", "1s", - 0, FLB_TRUE, offsetof(struct winevtlog_config, interval_sec), - "Set the polling interval for each channel" - }, - { - FLB_CONFIG_MAP_INT, "interval_nsec", "0", - 0, FLB_TRUE, offsetof(struct winevtlog_config, interval_nsec), - "Set the polling interval for each channel (sub seconds)" - }, - { - FLB_CONFIG_MAP_BOOL, "string_inserts", "true", - 0, FLB_TRUE, offsetof(struct winevtlog_config, string_inserts), - "Whether to include StringInserts in output records" - }, - { - FLB_CONFIG_MAP_BOOL, "read_existing_events", "false", - 0, FLB_TRUE, offsetof(struct winevtlog_config, read_existing_events), - "Whether to consume at oldest records in channels" - }, - { - FLB_CONFIG_MAP_BOOL, "render_event_as_xml", "false", - 0, FLB_TRUE, offsetof(struct winevtlog_config, render_event_as_xml), - "Whether to consume at oldest records in channels" - }, - { - FLB_CONFIG_MAP_BOOL, "use_ansi", "false", - 0, FLB_TRUE, offsetof(struct winevtlog_config, use_ansi), - "Use ANSI encoding on eventlog messages" - }, - { - FLB_CONFIG_MAP_BOOL, "ignore_missing_channels", "false", - 0, FLB_TRUE, offsetof(struct winevtlog_config, ignore_missing_channels), - "Whether to ignore channels missing in eventlog" - }, - { - FLB_CONFIG_MAP_STR, "event_query", "*", - 0, FLB_TRUE, offsetof(struct winevtlog_config, event_query), - "Specify XML query for filtering events" - }, - /* EOF */ - {0} -}; - -struct flb_input_plugin in_winevtlog_plugin = { - .name = "winevtlog", - .description = "Windows EventLog using winevt.h API", - .cb_init = in_winevtlog_init, - .cb_pre_run = NULL, - .cb_collect = in_winevtlog_collect, - .cb_flush_buf = NULL, - .cb_pause = in_winevtlog_pause, - .cb_resume = in_winevtlog_resume, - .cb_exit = in_winevtlog_exit, - .config_map = config_map -}; diff --git a/fluent-bit/plugins/in_winevtlog/pack.c b/fluent-bit/plugins/in_winevtlog/pack.c deleted file mode 100644 index 28075436b..000000000 --- a/fluent-bit/plugins/in_winevtlog/pack.c +++ /dev/null @@ -1,625 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2021 The Fluent Bit Authors - * Copyright (C) 2015-2018 Treasure Data Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include "winevtlog.h" - -#define FORMAT_ISO8601 "%Y-%m-%d %H:%M:%S %z" - -#define BINDATA(evt) ((unsigned char *) (evt) + (evt)->DataOffset) - -static int pack_nullstr(struct winevtlog_config *ctx) -{ - flb_log_event_encoder_append_body_cstring(ctx->log_encoder, ""); -} - -static int pack_wstr(struct winevtlog_config *ctx, const wchar_t *wstr) -{ - int size; - char *buf; - UINT code_page = CP_UTF8; - LPCSTR defaultChar = L" "; - - if (ctx->use_ansi) { - code_page = CP_ACP; - } - - /* Compute the buffer size first */ - size = WideCharToMultiByte(code_page, 0, wstr, -1, NULL, 0, NULL, NULL); - if (size == 0) { - return -1; - } - - buf = flb_malloc(size); - if (buf == NULL) { - flb_errno(); - return -1; - } - - /* Convert UTF-16 into UTF-8 */ - size = WideCharToMultiByte(code_page, 0, wstr, -1, buf, size, defaultChar, NULL); - if (size == 0) { - flb_free(buf); - return -1; - } - - /* Pack buf except the trailing '\0' */ - flb_log_event_encoder_append_body_string(ctx->log_encoder, buf, size - 1); - - flb_free(buf); - return 0; -} - -static int pack_binary(struct winevtlog_config *ctx, PBYTE bin, size_t length) -{ - const char *HEX_TABLE = "0123456789ABCDEF"; - char *buffer; - int size = length * 2; - size_t i, j; - unsigned int idx = 0; - - if (length == 0) { - pack_nullstr(ctx->log_encoder); - return 0; - } - - buffer = flb_malloc(size); - if (buffer == NULL) { - flb_errno(); - return -1; - } - - for (i = 0; i < length; i++) { - for (j = 0; j < 2; j++) { - idx = (unsigned int)(bin[i] >> (j * 4) & 0x0F); - buffer[2*i+(1-j)] = HEX_TABLE[idx]; - } - } - - flb_log_event_encoder_append_body_string(ctx->log_encoder, buffer, size); - - flb_free(buffer); - - return 0; -} - -static int pack_guid(struct winevtlog_config *ctx, const GUID *guid) -{ - LPOLESTR p = NULL; - - if (FAILED(StringFromCLSID(guid, &p))) { - return -1; - } - if (pack_wstr(ctx, p)) { - CoTaskMemFree(p); - return -1; - } - - CoTaskMemFree(p); - - return 0; -} - -static int pack_hex32(struct winevtlog_config *ctx, int32_t hex) -{ - CHAR buffer[32]; - size_t size = _countof(buffer); - - _snprintf_s(buffer, - size, - _TRUNCATE, - "0x%lx", - hex); - size = strlen(buffer); - if (size > 0) { - flb_log_event_encoder_append_body_cstring(ctx->log_encoder, buffer); - - return 0; - } - - return -1; -} - -static int pack_hex64(struct winevtlog_config *ctx, int64_t hex) -{ - CHAR buffer[32]; - size_t size = _countof(buffer); - - _snprintf_s(buffer, - size, - _TRUNCATE, - "0x%llx", - hex); - - size = strlen(buffer); - if (size > 0) { - flb_log_event_encoder_append_body_cstring(ctx->log_encoder, buffer); - - return 0; - } - - return -1; -} - - -static int pack_keywords(struct winevtlog_config *ctx, uint64_t keywords) -{ - CHAR buffer[32]; - size_t size = _countof(buffer); - - _snprintf_s(buffer, - size, - _TRUNCATE, - "0x%llx", - keywords); - - size = strlen(buffer); - - flb_log_event_encoder_append_body_cstring(ctx->log_encoder, buffer); - - return 0; -} - -static int pack_systemtime(struct winevtlog_config *ctx, SYSTEMTIME *st) -{ - CHAR buf[64]; - size_t len = 0; - _locale_t locale; - TIME_ZONE_INFORMATION tzi; - SYSTEMTIME st_local; - - GetTimeZoneInformation(&tzi); - - locale = _get_current_locale(); - if (locale == NULL) { - return -1; - } - if (st != NULL) { - SystemTimeToTzSpecificLocalTime(&tzi, st, &st_local); - - struct tm tm = {st_local.wSecond, - st_local.wMinute, - st_local.wHour, - st_local.wDay, - st_local.wMonth-1, - st_local.wYear-1900, - st_local.wDayOfWeek, 0, 0}; - len = _strftime_l(buf, 64, FORMAT_ISO8601, &tm, locale); - if (len == 0) { - flb_errno(); - _free_locale(locale); - return -1; - } - _free_locale(locale); - - flb_log_event_encoder_append_body_string(ctx->log_encoder, buf, len); - } - else { - return -1; - } - - return 0; -} - -static int pack_filetime(struct winevtlog_config *ctx, ULONGLONG filetime) -{ - LARGE_INTEGER timestamp; - CHAR buf[64]; - size_t len = 0; - FILETIME ft, ft_local; - SYSTEMTIME st; - _locale_t locale; - - locale = _get_current_locale(); - if (locale == NULL) { - return -1; - } - timestamp.QuadPart = filetime; - ft.dwHighDateTime = timestamp.HighPart; - ft.dwLowDateTime = timestamp.LowPart; - FileTimeToLocalFileTime(&ft, &ft_local); - if (FileTimeToSystemTime(&ft_local, &st)) { - struct tm tm = {st.wSecond, st.wMinute, st.wHour, st.wDay, st.wMonth-1, st.wYear-1900, st.wDayOfWeek, 0, 0}; - len = _strftime_l(buf, 64, FORMAT_ISO8601, &tm, locale); - if (len == 0) { - flb_errno(); - _free_locale(locale); - return -1; - } - _free_locale(locale); - - flb_log_event_encoder_append_body_string(ctx->log_encoder, buf, len); - } - else { - return -1; - } - - return 0; -} - -static int pack_sid(struct winevtlog_config *ctx, PSID sid) -{ - size_t size; - LPWSTR wide_sid = NULL; - int ret = -1; - - if (ConvertSidToStringSidW(sid, &wide_sid)) { - ret = pack_wstr(ctx, wide_sid); - - LocalFree(wide_sid); - return ret; - } - - return ret; -} - -static void pack_string_inserts(struct winevtlog_config *ctx, PEVT_VARIANT values, DWORD count) -{ - int i; - int ret; - - ret = flb_log_event_encoder_body_begin_array(ctx->log_encoder); - - for (i = 0; i < count; i++) { - if (values[i].Type & EVT_VARIANT_TYPE_ARRAY) { - continue; - } - - switch (values[i].Type & EVT_VARIANT_TYPE_MASK) { - case EvtVarTypeNull: - pack_nullstr(ctx); - break; - case EvtVarTypeString: - if (pack_wstr(ctx, values[i].StringVal)) { - pack_nullstr(ctx); - } - break; - case EvtVarTypeAnsiString: - if (pack_wstr(ctx, values[i].AnsiStringVal)) { - pack_nullstr(ctx); - } - break; - case EvtVarTypeSByte: - flb_log_event_encoder_append_body_int8(ctx->log_encoder, values[i].SByteVal); - break; - case EvtVarTypeByte: - flb_log_event_encoder_append_body_uint8(ctx->log_encoder, values[i].ByteVal); - break; - case EvtVarTypeInt16: - flb_log_event_encoder_append_body_int16(ctx->log_encoder, values[i].Int16Val); - break; - case EvtVarTypeUInt16: - flb_log_event_encoder_append_body_uint16(ctx->log_encoder, values[i].UInt16Val); - break; - case EvtVarTypeInt32: - flb_log_event_encoder_append_body_int32(ctx->log_encoder, values[i].Int32Val); - break; - case EvtVarTypeUInt32: - flb_log_event_encoder_append_body_uint32(ctx->log_encoder, values[i].UInt32Val); - break; - case EvtVarTypeInt64: - flb_log_event_encoder_append_body_int64(ctx->log_encoder, values[i].Int64Val); - break; - case EvtVarTypeUInt64: - flb_log_event_encoder_append_body_uint64(ctx->log_encoder, values[i].UInt64Val); - break; - case EvtVarTypeSingle: - flb_log_event_encoder_append_body_double(ctx->log_encoder, values[i].SingleVal); - break; - case EvtVarTypeDouble: - flb_log_event_encoder_append_body_double(ctx->log_encoder, values[i].DoubleVal); - break; - case EvtVarTypeBoolean: - flb_log_event_encoder_append_body_boolean(ctx->log_encoder, (int) values[i].BooleanVal); - break; - case EvtVarTypeGuid: - if (pack_guid(ctx, values[i].GuidVal)) { - pack_nullstr(ctx); - } - break; - case EvtVarTypeSizeT: - flb_log_event_encoder_append_body_uint64(ctx->log_encoder, values[i].SizeTVal); - break; - case EvtVarTypeFileTime: - if (pack_filetime(ctx, values[i].FileTimeVal)) { - pack_nullstr(ctx); - } - break; - case EvtVarTypeSysTime: - if (pack_systemtime(ctx, values[i].SysTimeVal)) { - pack_nullstr(ctx); - } - break; - case EvtVarTypeSid: - if (pack_sid(ctx, values[i].SidVal)) { - pack_nullstr(ctx); - } - break; - case EvtVarTypeHexInt32: - if (pack_hex32(ctx, values[i].Int32Val)) { - pack_nullstr(ctx); - } - break; - case EvtVarTypeHexInt64: - if (pack_hex64(ctx, values[i].Int64Val)) { - pack_nullstr(ctx); - } - break; - case EvtVarTypeEvtXml: - if (pack_wstr(ctx, values[i].XmlVal, ctx)) { - pack_nullstr(ctx); - } - break; - case EvtVarTypeBinary: - if (pack_binary(ctx, values[i].BinaryVal, values[i].Count)) { - pack_nullstr(ctx); - } - break; - default: - flb_log_event_encoder_append_body_cstring(ctx->log_encoder, "?"); - } - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_body_commit_array(ctx->log_encoder); - } - -} - -void winevtlog_pack_xml_event(WCHAR *system_xml, WCHAR *message, - PEVT_VARIANT string_inserts, UINT count_inserts, struct winevtlog_channel *ch, - struct winevtlog_config *ctx) -{ - int ret; - - ret = flb_log_event_encoder_begin_record(ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp(ctx->log_encoder); - } - - - ret = flb_log_event_encoder_append_body_cstring(ctx->log_encoder, "System"); - - if (pack_wstr(ctx, system_xml)) { - pack_nullstr(ctx); - } - - ret = flb_log_event_encoder_append_body_cstring(ctx->log_encoder, "Message"); - - if (pack_wstr(ctx, message)) { - pack_nullstr(ctx); - } - - if (ctx->string_inserts) { - ret = flb_log_event_encoder_append_body_cstring(ctx->log_encoder, "StringInserts"); - - pack_string_inserts(ctx, string_inserts, count_inserts); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(ctx->log_encoder); - } -} - -void winevtlog_pack_event(PEVT_VARIANT system, WCHAR *message, - PEVT_VARIANT string_inserts, UINT count_inserts, struct winevtlog_channel *ch, - struct winevtlog_config *ctx) -{ - int ret; - size_t len; - int count = 19; - - if (ctx->string_inserts) { - count++; - } - - ret = flb_log_event_encoder_begin_record(ctx->log_encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_current_timestamp(ctx->log_encoder); - } - - /* ProviderName */ - ret = flb_log_event_encoder_append_body_cstring(ctx->log_encoder, "ProviderName"); - - if (pack_wstr(ctx, system[EvtSystemProviderName].StringVal)) { - pack_nullstr(ctx); - } - - /* ProviderGuid */ - ret = flb_log_event_encoder_append_body_cstring(ctx->log_encoder, "ProviderGuid"); - - if (EvtVarTypeNull != system[EvtSystemProviderGuid].Type) { - if (pack_guid(ctx, system[EvtSystemProviderGuid].GuidVal)) { - pack_nullstr(ctx); - } - } - else { - pack_nullstr(ctx); - } - - /* Qualifiers */ - ret = flb_log_event_encoder_append_body_cstring(ctx->log_encoder, "Qualifiers"); - - if (EvtVarTypeNull != system[EvtSystemQualifiers].Type) { - flb_log_event_encoder_append_body_uint16(ctx->log_encoder, system[EvtSystemQualifiers].UInt16Val); - } - else { - pack_nullstr(ctx); - } - - /* EventID */ - ret = flb_log_event_encoder_append_body_cstring(ctx->log_encoder, "EventID"); - - if (EvtVarTypeNull != system[EvtSystemEventID].Type) { - flb_log_event_encoder_append_body_uint16(ctx->log_encoder, system[EvtSystemEventID].UInt16Val); - } - else { - pack_nullstr(ctx); - } - - /* Version */ - ret = flb_log_event_encoder_append_body_cstring(ctx->log_encoder, "Version"); - - if (EvtVarTypeNull != system[EvtSystemVersion].Type) { - flb_log_event_encoder_append_body_uint8(ctx->log_encoder, system[EvtSystemVersion].ByteVal); - } - else { - flb_log_event_encoder_append_body_uint8(ctx->log_encoder, 0); - } - - /* Level */ - ret = flb_log_event_encoder_append_body_cstring(ctx->log_encoder, "Level"); - - if (EvtVarTypeNull != system[EvtSystemLevel].Type) { - flb_log_event_encoder_append_body_uint8(ctx->log_encoder, system[EvtSystemLevel].ByteVal); - } - else { - flb_log_event_encoder_append_body_uint8(ctx->log_encoder, 0); - } - - /* Task */ - ret = flb_log_event_encoder_append_body_cstring(ctx->log_encoder, "Task"); - - if (EvtVarTypeNull != system[EvtSystemTask].Type) { - flb_log_event_encoder_append_body_uint16(ctx->log_encoder, system[EvtSystemTask].UInt16Val); - } - else { - flb_log_event_encoder_append_body_uint16(ctx->log_encoder, 0); - } - - /* Opcode */ - ret = flb_log_event_encoder_append_body_cstring(ctx->log_encoder, "Opcode"); - - if (EvtVarTypeNull != system[EvtSystemOpcode].Type) { - flb_log_event_encoder_append_body_uint8(ctx->log_encoder, system[EvtSystemOpcode].ByteVal); - } - else { - flb_log_event_encoder_append_body_uint8(ctx->log_encoder, 0); - } - - /* Keywords */ - ret = flb_log_event_encoder_append_body_cstring(ctx->log_encoder, "Keywords"); - - if (EvtVarTypeNull != system[EvtSystemKeywords].Type) { - pack_keywords(ctx, system[EvtSystemKeywords].UInt64Val); - } - else { - flb_log_event_encoder_append_body_uint64(ctx->log_encoder, 0); - } - - /* TimeCreated */ - ret = flb_log_event_encoder_append_body_cstring(ctx->log_encoder, "TimeCreated"); - - if (pack_filetime(ctx, system[EvtSystemTimeCreated].FileTimeVal)) { - pack_nullstr(ctx); - } - - /* EventRecordID */ - ret = flb_log_event_encoder_append_body_cstring(ctx->log_encoder, "EventRecordID"); - - if (EvtVarTypeNull != system[EvtSystemEventRecordId].Type) { - flb_log_event_encoder_append_body_uint64(ctx->log_encoder, system[EvtSystemEventRecordId].UInt64Val); - } - else { - flb_log_event_encoder_append_body_uint64(ctx->log_encoder, 0); - } - - /* ActivityID */ - ret = flb_log_event_encoder_append_body_cstring(ctx->log_encoder, "ActivityID"); - - if (pack_guid(ctx, system[EvtSystemActivityID].GuidVal)) { - pack_nullstr(ctx); - } - - /* Related ActivityID */ - ret = flb_log_event_encoder_append_body_cstring(ctx->log_encoder, "RelatedActivityID"); - - if (pack_guid(ctx, system[EvtSystemRelatedActivityID].GuidVal)) { - pack_nullstr(ctx); - } - - /* ProcessID */ - ret = flb_log_event_encoder_append_body_cstring(ctx->log_encoder, "ProcessID"); - - if (EvtVarTypeNull != system[EvtSystemProcessID].Type) { - flb_log_event_encoder_append_body_uint32(ctx->log_encoder, system[EvtSystemProcessID].UInt32Val); - } - else { - flb_log_event_encoder_append_body_uint32(ctx->log_encoder, 0); - } - - /* ThreadID */ - ret = flb_log_event_encoder_append_body_cstring(ctx->log_encoder, "ThreadID"); - - if (EvtVarTypeNull != system[EvtSystemThreadID].Type) { - flb_log_event_encoder_append_body_uint32(ctx->log_encoder, system[EvtSystemThreadID].UInt32Val); - } - else { - flb_log_event_encoder_append_body_uint32(ctx->log_encoder, 0); - } - - /* Channel */ - ret = flb_log_event_encoder_append_body_cstring(ctx->log_encoder, "Channel"); - - if (pack_wstr(ctx, system[EvtSystemChannel].StringVal)) { - pack_nullstr(ctx); - } - - /* Computer */ - ret = flb_log_event_encoder_append_body_cstring(ctx->log_encoder, "Computer"); - - if (pack_wstr(ctx, system[EvtSystemComputer].StringVal)) { - pack_nullstr(ctx); - } - - /* UserID */ - ret = flb_log_event_encoder_append_body_cstring(ctx->log_encoder, "UserID"); - - if (pack_sid(ctx, system[EvtSystemUserID].SidVal)) { - pack_nullstr(ctx); - } - - /* Message */ - ret = flb_log_event_encoder_append_body_cstring(ctx->log_encoder, "Message"); - - if (pack_wstr(ctx, message)) { - pack_nullstr(ctx); - } - - /* String Inserts */ - if (ctx->string_inserts) { - ret = flb_log_event_encoder_append_body_cstring(ctx->log_encoder, "StringInserts"); - - pack_string_inserts(ctx, string_inserts, count_inserts); - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(ctx->log_encoder); - } -} diff --git a/fluent-bit/plugins/in_winevtlog/winevtlog.c b/fluent-bit/plugins/in_winevtlog/winevtlog.c deleted file mode 100644 index 09d3f9624..000000000 --- a/fluent-bit/plugins/in_winevtlog/winevtlog.c +++ /dev/null @@ -1,840 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2021 The Fluent Bit Authors - * Copyright (C) 2015-2018 Treasure Data Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include "winevtlog.h" - -#define EVENT_PROVIDER_NAME_LENGTH 256 - -static char* convert_wstr(wchar_t *wstr, UINT codePage); -static wchar_t* convert_str(char *str); - -struct winevtlog_channel *winevtlog_subscribe(const char *channel, int read_existing_events, - EVT_HANDLE stored_bookmark, const char *query) -{ - struct winevtlog_channel *ch; - EVT_HANDLE bookmark = NULL; - HANDLE signal_event = NULL; - DWORD len; - DWORD flags = 0L; - PWSTR wide_channel = NULL; - PWSTR wide_query = NULL; - void *buf; - - ch = flb_calloc(1, sizeof(struct winevtlog_channel)); - if (ch == NULL) { - flb_errno(); - return NULL; - } - - ch->name = flb_strdup(channel); - if (!ch->name) { - flb_errno(); - flb_free(ch); - return NULL; - } - ch->query = NULL; - - signal_event = CreateEvent(NULL, FALSE, FALSE, NULL); - - // channel : To wide char - len = MultiByteToWideChar(CP_UTF8, 0, channel, -1, NULL, 0); - wide_channel = flb_malloc(sizeof(PWSTR) * len); - MultiByteToWideChar(CP_UTF8, 0, channel, -1, wide_channel, len); - if (query != NULL) { - // query : To wide char - len = MultiByteToWideChar(CP_UTF8, 0, query, -1, NULL, 0); - wide_query = flb_malloc(sizeof(PWSTR) * len); - MultiByteToWideChar(CP_UTF8, 0, query, -1, wide_query, len); - ch->query = flb_strdup(query); - } - - if (stored_bookmark) { - flags |= EvtSubscribeStartAfterBookmark; - } else if (read_existing_events) { - flags |= EvtSubscribeStartAtOldestRecord; - } else { - flags |= EvtSubscribeToFutureEvents; - } - - /* The wide_query parameter can handle NULL as `*` for retrieving all events. - * ref. https://learn.microsoft.com/en-us/windows/win32/api/winevt/nf-winevt-evtsubscribe - */ - ch->subscription = EvtSubscribe(NULL, signal_event, wide_channel, wide_query, - stored_bookmark, NULL, NULL, flags); - if (!ch->subscription) { - flb_error("[in_winevtlog] cannot subscribe '%s' (%i)", channel, GetLastError()); - flb_free(ch->name); - if (ch->query != NULL) { - flb_free(ch->query); - } - flb_free(ch); - return NULL; - } - ch->signal_event = signal_event; - - if (stored_bookmark) { - ch->bookmark = stored_bookmark; - } - else { - bookmark = EvtCreateBookmark(NULL); - if (bookmark) { - ch->bookmark = bookmark; - } - else { - if (ch->subscription) { - EvtClose(ch->subscription); - } - if (signal_event) { - CloseHandle(signal_event); - } - flb_error("[in_winevtlog] cannot subscribe '%s' (%i)", channel, GetLastError()); - flb_free(wide_channel); - flb_free(ch->name); - if (ch->query != NULL) { - flb_free(ch->query); - } - flb_free(ch); - return NULL; - } - } - - flb_free(wide_channel); - if (wide_query != NULL) { - flb_free(wide_query); - } - - return ch; -} - -BOOL cancel_subscription(struct winevtlog_channel *ch) -{ - return EvtCancel(ch->subscription); -} - -static void close_handles(struct winevtlog_channel *ch) -{ - int i; - - if (ch->subscription) { - EvtClose(ch->subscription); - ch->subscription = NULL; - } - if (ch->signal_event) { - CloseHandle(ch->signal_event); - ch->signal_event = NULL; - } - if (ch->bookmark) { - EvtClose(ch->bookmark); - ch->bookmark = NULL; - } - for (i = 0; i < ch->count; i++) { - if (ch->events[i]) { - EvtClose(ch->events[i]); - ch->events[i] = NULL; - } - } - ch->count = 0; -} - - -void winevtlog_close(struct winevtlog_channel *ch) -{ - flb_free(ch->name); - if (ch->query != NULL) { - flb_free(ch->query); - } - close_handles(ch); - - flb_free(ch); -} - -// Render the event as an XML string and print it. -PWSTR render_event(EVT_HANDLE hEvent, DWORD flags, unsigned int *event_size) -{ - DWORD status = ERROR_SUCCESS; - DWORD buffer_size = 0; - DWORD buffer_used = 0; - DWORD count = 0; - LPWSTR event_xml = NULL; - - if (flags != EvtRenderEventXml && flags != EvtRenderBookmark) { - flb_error("Invalid flags is specified: %d", flags); - return NULL; - } - - if (!EvtRender(NULL, hEvent, flags, buffer_size, event_xml, &buffer_used, &count)) { - status = GetLastError(); - if (status == ERROR_INSUFFICIENT_BUFFER) { - buffer_size = buffer_used; - /* return buffer size */ - *event_size = buffer_size; - event_xml = (LPWSTR)flb_malloc(buffer_size); - if (event_xml) { - EvtRender(NULL, hEvent, flags, buffer_size, event_xml, &buffer_used, &count); - } - else { - flb_error("malloc failed"); - goto cleanup; - } - } - - status = GetLastError(); - if (status != ERROR_SUCCESS) { - flb_error("EvtRender failed with %d", GetLastError()); - goto cleanup; - } - } - - return event_xml; - -cleanup: - - if (event_xml) { - flb_free(event_xml); - } - - return NULL; -} - -DWORD render_system_event(EVT_HANDLE event, PEVT_VARIANT *system, unsigned int *system_size) -{ - DWORD status = ERROR_SUCCESS; - EVT_HANDLE context = NULL; - DWORD buffer_size = 0; - DWORD buffer_used = 0; - DWORD count = 0; - PEVT_VARIANT rendered_system = NULL; - - context = EvtCreateRenderContext(0, NULL, EvtRenderContextSystem); - if (NULL == context) { - status = GetLastError(); - flb_error("failed to create RenderContext with %d", status); - - goto cleanup; - } - if (!EvtRender(context, - event, - EvtRenderEventValues, - buffer_size, - rendered_system, - &buffer_used, - &count)) { - status = GetLastError(); - - if (status == ERROR_INSUFFICIENT_BUFFER) { - buffer_size = buffer_used; - rendered_system = (PEVT_VARIANT)flb_malloc(buffer_size); - if (rendered_system) { - EvtRender(context, - event, - EvtRenderEventValues, - buffer_size, - rendered_system, - &buffer_used, - &count); - status = GetLastError(); - *system_size = buffer_used; - } else { - if (rendered_system) - flb_free(rendered_system); - - flb_error("failed to malloc memory with %d", status); - - goto cleanup; - } - } - - if (ERROR_SUCCESS != status) { - EvtClose(context); - flb_free(rendered_system); - - return status; - } - } - - *system = rendered_system; - -cleanup: - - if (context) { - EvtClose(context); - } - - return status; -} - - -PWSTR get_message(EVT_HANDLE metadata, EVT_HANDLE handle, unsigned int *message_size) -{ - WCHAR* buffer = NULL; - DWORD status = ERROR_SUCCESS; - DWORD buffer_size = 0; - DWORD buffer_used = 0; - LPVOID format_message_buffer; - WCHAR* message = NULL; - char *error_message = NULL; - - // Get the size of the buffer - if (!EvtFormatMessage(metadata, handle, 0, 0, NULL, - EvtFormatMessageEvent, buffer_size, buffer, &buffer_used)) { - status = GetLastError(); - if (ERROR_INSUFFICIENT_BUFFER == status) { - buffer_size = buffer_used; - buffer = flb_malloc(sizeof(WCHAR) * buffer_size); - if (!buffer) { - flb_error("failed to malloc message buffer"); - - goto cleanup; - } - if (!EvtFormatMessage(metadata, - handle, - 0xffffffff, - 0, - NULL, - EvtFormatMessageEvent, - buffer_size, - buffer, - &buffer_used)) { - status = GetLastError(); - *message_size = buffer_used; - - if (status != ERROR_EVT_UNRESOLVED_VALUE_INSERT) { - switch (status) { - case ERROR_EVT_MESSAGE_NOT_FOUND: - case ERROR_EVT_MESSAGE_ID_NOT_FOUND: - case ERROR_EVT_MESSAGE_LOCALE_NOT_FOUND: - case ERROR_RESOURCE_DATA_NOT_FOUND: - case ERROR_RESOURCE_TYPE_NOT_FOUND: - case ERROR_RESOURCE_NAME_NOT_FOUND: - case ERROR_RESOURCE_LANG_NOT_FOUND: - case ERROR_MUI_FILE_NOT_FOUND: - case ERROR_EVT_UNRESOLVED_PARAMETER_INSERT: - { - if (FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, - status, - MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), - (WCHAR*)(&format_message_buffer), - 0, - NULL) == 0) - FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, - status, - MAKELANGID(LANG_ENGLISH, SUBLANG_ENGLISH_US), - (WCHAR*)(&format_message_buffer), - 0, - NULL); - error_message = convert_wstr((WCHAR*)format_message_buffer, CP_ACP); - flb_error("Failed to get message with %d, err = %s", status, error_message); - flb_free(error_message); - - message = _wcsdup((WCHAR*)format_message_buffer); - LocalFree(format_message_buffer); - - goto cleanup; - } - } - - if (status != ERROR_INSUFFICIENT_BUFFER) { - flb_error("failed with %d", status); - goto cleanup; - } - } - } - } - } - - message = _wcsdup(buffer); - -cleanup: - if (buffer) { - flb_free(buffer); - } - - return message; -} - -PWSTR get_description(EVT_HANDLE handle, LANGID langID, unsigned int *message_size) -{ - WCHAR *buffer[EVENT_PROVIDER_NAME_LENGTH]; - PEVT_VARIANT values = NULL; - DWORD buffer_used = 0; - DWORD status = ERROR_SUCCESS; - DWORD count = 0; - WCHAR *message = NULL; - EVT_HANDLE metadata = NULL; - - PCWSTR properties[] = { L"Event/System/Provider/@Name" }; - EVT_HANDLE context = - EvtCreateRenderContext(1, properties, EvtRenderContextValues); - if (context == NULL) { - flb_error("Failed to create renderContext"); - goto cleanup; - } - - if (EvtRender(context, - handle, - EvtRenderEventValues, - EVENT_PROVIDER_NAME_LENGTH, - buffer, - &buffer_used, - &count) != FALSE){ - status = ERROR_SUCCESS; - } - else { - status = GetLastError(); - } - - if (status != ERROR_SUCCESS) { - flb_error("failed to query RenderContextValues"); - goto cleanup; - } - values = (PEVT_VARIANT)buffer; - - metadata = EvtOpenPublisherMetadata( - NULL, // TODO: Remote handle - values[0].StringVal, - NULL, - MAKELCID(langID, SORT_DEFAULT), - 0); - if (metadata == NULL) { - goto cleanup; - } - - message = get_message(metadata, handle, message_size); - -cleanup: - if (context) { - EvtClose(context); - } - - if (metadata) { - EvtClose(metadata); - } - - return message; -} - -int get_string_inserts(EVT_HANDLE handle, PEVT_VARIANT *string_inserts_values, - UINT *prop_count, unsigned int *string_inserts_size) -{ - PEVT_VARIANT values; - DWORD buffer_size = 0; - DWORD buffer_size_used = 0; - DWORD count = 0; - BOOL succeeded = FLB_TRUE; - - EVT_HANDLE context = EvtCreateRenderContext(0, NULL, EvtRenderContextUser); - if (context == NULL) { - flb_error("Failed to create renderContext"); - succeeded = FLB_FALSE; - goto cleanup; - } - - // Get the size of the buffer - EvtRender(context, handle, EvtRenderEventValues, 0, NULL, &buffer_size, &count); - values = (PEVT_VARIANT)flb_malloc(buffer_size); - - succeeded = EvtRender(context, - handle, - EvtRenderContextValues, - buffer_size, - values, - &buffer_size_used, - &count); - - if (!succeeded) { - flb_error("Failed to get string inserts with %d\n", GetLastError()); - goto cleanup; - } - - *prop_count = count; - *string_inserts_values = values; - *string_inserts_size = buffer_size; - -cleanup: - - if (context != NULL) { - EvtClose(context); - } - - return succeeded; -} - -static int winevtlog_next(struct winevtlog_channel *ch, int hit_threshold) -{ - EVT_HANDLE events[SUBSCRIBE_ARRAY_SIZE]; - DWORD count = 0; - DWORD status = ERROR_SUCCESS; - BOOL has_next = FALSE; - int i; - - /* If subscription handle is NULL, it should return false. */ - if (!ch->subscription) { - flb_error("Invalid subscription is passed"); - return FLB_FALSE; - } - - if (hit_threshold) { - return FLB_FALSE; - } - - has_next = EvtNext(ch->subscription, SUBSCRIBE_ARRAY_SIZE, - events, INFINITE, 0, &count); - - if (!has_next) { - status = GetLastError(); - if (ERROR_CANCELLED == status) { - return FLB_FALSE; - } - if (ERROR_NO_MORE_ITEMS != status) { - return FLB_FALSE; - } - } - - if (status == ERROR_SUCCESS) { - ch->count = count; - for (i = 0; i < count; i++) { - ch->events[i] = events[i]; - EvtUpdateBookmark(ch->bookmark, ch->events[i]); - } - - return FLB_TRUE; - } - - return FLB_FALSE; -} - -/* - * Read from an open Windows Event Log channel. - */ -int winevtlog_read(struct winevtlog_channel *ch, struct winevtlog_config *ctx, - unsigned int *read) -{ - DWORD status = ERROR_SUCCESS; - PWSTR system_xml = NULL; - unsigned int system_size = 0; - unsigned int message_size = 0; - unsigned int string_inserts_size = 0; - int hit_threshold = FLB_FALSE; - unsigned int read_size = 0; - PWSTR message = NULL; - PEVT_VARIANT rendered_system = NULL; - PEVT_VARIANT string_inserts = NULL; - UINT count_inserts = 0; - DWORD i = 0; - - while (winevtlog_next(ch, hit_threshold)) { - for (i = 0; i < ch->count; i++) { - if (ctx->render_event_as_xml) { - system_xml = render_event(ch->events[i], EvtRenderEventXml, &system_size); - message = get_description(ch->events[i], LANG_NEUTRAL, &message_size); - get_string_inserts(ch->events[i], &string_inserts, &count_inserts, &string_inserts_size); - if (system_xml) { - /* Caluculate total allocated size: system + message + string_inserts */ - read_size += (system_size + message_size + string_inserts_size); - winevtlog_pack_xml_event(system_xml, message, string_inserts, - count_inserts, ch, ctx); - - flb_free(string_inserts); - flb_free(system_xml); - if (message) - flb_free(message); - } - } - else { - render_system_event(ch->events[i], &rendered_system, &system_size); - message = get_description(ch->events[i], LANG_NEUTRAL, &message_size); - get_string_inserts(ch->events[i], &string_inserts, &count_inserts, &string_inserts_size); - if (rendered_system) { - /* Caluculate total allocated size: system + message + string_inserts */ - read_size += (system_size + message_size + string_inserts_size); - winevtlog_pack_event(rendered_system, message, string_inserts, - count_inserts, ch, ctx); - - flb_free(string_inserts); - flb_free(rendered_system); - if (message) - flb_free(message); - } - } - } - - /* Closes any events in case an error occurred above. */ - for (i = 0; i < ch->count; i++) { - if (NULL != ch->events[i]) { - EvtClose(ch->events[i]); - ch->events[i] = NULL; - } - } - - if (read_size > ctx->total_size_threshold) { - hit_threshold = FLB_TRUE; - /* hit reading threshold on read, then break. */ - break; - } - } - - *read = read_size; - - return 0; -} - -/* - * Open multiple channels at once. The return value is a linked - * list of winevtlog_channel objects. - * - * "channels" are comma-separated names like "Setup,Security". - */ -struct mk_list *winevtlog_open_all(const char *channels, struct winevtlog_config *ctx) -{ - char *tmp; - char *channel; - char *state; - struct winevtlog_channel *ch; - struct mk_list *list; - - tmp = flb_strdup(channels); - if (!tmp) { - flb_errno(); - return NULL; - } - - list = flb_malloc(sizeof(struct mk_list)); - if (!list) { - flb_errno(); - flb_free(tmp); - return NULL; - } - mk_list_init(list); - - channel = strtok_s(tmp , ",", &state); - while (channel) { - ch = winevtlog_subscribe(channel, ctx->read_existing_events, NULL, ctx->event_query); - if (ch) { - mk_list_add(&ch->_head, list); - } - else { - if (ctx->ignore_missing_channels) { - flb_debug("[in_winevtlog] channel '%s' does not exist", channel); - } - else { - flb_free(tmp); - winevtlog_close_all(list); - return NULL; - } - } - channel = strtok_s(NULL, ",", &state); - } - - if (mk_list_size(list) == 0) { - flb_free(tmp); - winevtlog_close_all(list); - return NULL; - } - - flb_free(tmp); - return list; -} - -void winevtlog_close_all(struct mk_list *list) -{ - struct winevtlog_channel *ch; - struct mk_list *head; - struct mk_list *tmp; - - mk_list_foreach_safe(head, tmp, list) { - ch = mk_list_entry(head, struct winevtlog_channel, _head); - mk_list_del(&ch->_head); - winevtlog_close(ch); - } - flb_free(list); -} - -/* - * Callback function for flb_sqldb_query(). - */ -static int winevtlog_sqlite_callback(void *data, int argc, char **argv, char **cols) -{ - struct winevtlog_sqlite_record *p = data; - - p->name = argv[0]; - p->bookmark_xml = strdup(argv[1]); - p->time_updated = (unsigned int) strtoul(argv[2], NULL, 10); - p->created = (unsigned int) strtoul(argv[3], NULL, 10); - return 0; -} - -static wchar_t* convert_str(char *str) -{ - int size = 0; - wchar_t *buf = NULL; - - size = MultiByteToWideChar(CP_UTF8, 0, str, -1, NULL, 0); - if (size == 0) { - return NULL; - } - - buf = flb_malloc(sizeof(PWSTR) * size); - if (buf == NULL) { - flb_errno(); - return NULL; - } - size = MultiByteToWideChar(CP_UTF8, 0, str, -1, buf, size); - if (size == 0) { - flb_free(buf); - return NULL; - } - - return buf; -} - -static char* convert_wstr(wchar_t *wstr, UINT codePage) -{ - int size = 0; - char *buf = NULL; - - size = WideCharToMultiByte(codePage, 0, wstr, -1, NULL, 0, NULL, NULL); - if (size == 0) { - return NULL; - } - - buf = flb_malloc(size); - if (buf == NULL) { - flb_errno(); - return NULL; - } - size = WideCharToMultiByte(codePage, 0, wstr, -1, buf, size, NULL, NULL); - if (size == 0) { - flb_free(buf); - return NULL; - } - - return buf; -} - -/* - * Load the bookmark from SQLite DB. - */ -int winevtlog_sqlite_load(struct winevtlog_channel *ch, struct flb_sqldb *db) -{ - int ret; - char query[1024]; - struct winevtlog_sqlite_record record = {0}; - EVT_HANDLE bookmark = NULL; - PWSTR bookmark_xml = NULL; - struct winevtlog_channel *re_ch = NULL; - - snprintf(query, sizeof(query) - 1, SQL_GET_CHANNEL, ch->name); - - ret = flb_sqldb_query(db, query, winevtlog_sqlite_callback, &record); - if (ret == FLB_ERROR) { - return -1; - } - - if (record.created) { - ch->time_created = record.created; - } - if (record.time_updated) { - ch->time_updated = record.time_updated; - } - - if (record.name) { - bookmark_xml = convert_str(record.bookmark_xml); - if (bookmark_xml) { - bookmark = EvtCreateBookmark(bookmark_xml); - if (bookmark) { - /* re-create subscription handles */ - re_ch = winevtlog_subscribe(ch->name, FLB_FALSE, bookmark, ch->query); - if (re_ch != NULL) { - close_handles(ch); - - ch->bookmark = re_ch->bookmark; - ch->subscription = re_ch->subscription; - ch->signal_event = re_ch->signal_event; - } - else { - flb_error("Failed to subscribe with bookmark XML: %s\n", record.bookmark_xml); - ch->bookmark = EvtCreateBookmark(NULL); - } - } - else { - flb_error("Failed to load bookmark XML with %d\n", GetLastError()); - ch->bookmark = EvtCreateBookmark(NULL); - } - } - if (bookmark_xml) { - flb_free(bookmark_xml); - } - } - return 0; -} - -/* - * Save the bookmark into SQLite DB. - */ -int winevtlog_sqlite_save(struct winevtlog_channel *ch, struct flb_sqldb *db) -{ - int ret; - char query[1024]; - PWSTR wide_bookmark_xml = NULL; - char *bookmark_xml; - int used_size = 0; - - wide_bookmark_xml = render_event(ch->bookmark, EvtRenderBookmark, &used_size); - if (wide_bookmark_xml == NULL) { - flb_error("failed to render bookmark with %d", GetLastError()); - flb_free(wide_bookmark_xml); - - return -1; - } - bookmark_xml = convert_wstr(wide_bookmark_xml, CP_UTF8); - if (bookmark_xml == NULL) { - flb_error("failed to convert Wider string with %d", GetLastError()); - flb_free(wide_bookmark_xml); - flb_free(bookmark_xml); - - return -1; - } - - snprintf(query, sizeof(query) - 1, SQL_UPDATE_CHANNEL, - ch->name, bookmark_xml, ch->time_updated, time(NULL)); - - ret = flb_sqldb_query(db, query, NULL, NULL); - if (ret == FLB_ERROR) { - flb_error("failed to save db with %d", GetLastError()); - flb_free(wide_bookmark_xml); - flb_free(bookmark_xml); - - return -1; - } - - flb_free(wide_bookmark_xml); - flb_free(bookmark_xml); - - return 0; -} diff --git a/fluent-bit/plugins/in_winevtlog/winevtlog.h b/fluent-bit/plugins/in_winevtlog/winevtlog.h deleted file mode 100644 index 10ef3e457..000000000 --- a/fluent-bit/plugins/in_winevtlog/winevtlog.h +++ /dev/null @@ -1,134 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2019-2021 The Fluent Bit Authors - * Copyright (C) 2015-2018 Treasure Data Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_WINEVTLOG_H -#define FLB_WINEVTLOG_H - -#include -#include - -struct winevtlog_config { - unsigned int interval_sec; - unsigned int interval_nsec; - unsigned int total_size_threshold; - int string_inserts; - int read_existing_events; - int render_event_as_xml; - int use_ansi; - int ignore_missing_channels; - flb_sds_t event_query; - - struct mk_list *active_channel; - struct flb_sqldb *db; - flb_pipefd_t coll_fd; - struct flb_input_instance *ins; - struct flb_log_event_encoder *log_encoder; -}; - -/* Some channels has very heavy contents for 10 events at same time. - * For now, we specify simultaneous subscribe size to 5. - */ -#define SUBSCRIBE_ARRAY_SIZE 5 - -struct winevtlog_channel { - EVT_HANDLE subscription; - EVT_HANDLE bookmark; - HANDLE signal_event; - EVT_HANDLE events[SUBSCRIBE_ARRAY_SIZE]; - int count; - - char *name; - char *query; - unsigned int time_updated; - unsigned int time_created; - struct mk_list _head; -}; - -struct winevtlog_sqlite_record { - char *name; - char *bookmark_xml; - unsigned int time_updated; - unsigned int created; -}; - -/* - * Open a Windows Event Log channel. - */ -struct winevtlog_channel *winevtlog_open(const char *channel); -void winevtlog_close(struct winevtlog_channel *ch); - -/* - * Read records from a channel. - */ -int winevtlog_read(struct winevtlog_channel *ch, - struct winevtlog_config *ctx, unsigned int *read); - -/* - * A bulk API to handle multiple channels at once using mk_list. - * - * "channels" are comma-separated names like "Setup,Security". - */ -struct mk_list *winevtlog_open_all(const char *channels, struct winevtlog_config *ctx); -void winevtlog_close_all(struct mk_list *list); - -void winevtlog_pack_xml_event(WCHAR *system_xml, WCHAR *message, - PEVT_VARIANT string_inserts, UINT count_inserts, struct winevtlog_channel *ch, - struct winevtlog_config *ctx); -void winevtlog_pack_event(PEVT_VARIANT system, WCHAR *message, - PEVT_VARIANT string_inserts, UINT count_inserts, struct winevtlog_channel *ch, - struct winevtlog_config *ctx); - -/* - * Save the read offset to disk. - */ -int winevtlog_sqlite_load(struct winevtlog_channel *ch, struct flb_sqldb *db); -int winevtlog_sqlite_save(struct winevtlog_channel *ch, struct flb_sqldb *db); - -/* - * SQL templates - */ -#define SQL_CREATE_CHANNELS \ - "CREATE TABLE IF NOT EXISTS in_winevtlog_channels (" \ - " name TEXT PRIMARY KEY," \ - " bookmark_xml TEXT," \ - " time_updated INTEGER," \ - " created INTEGER" \ - ");" - -#define SQL_GET_CHANNEL \ - "SELECT name, bookmark_xml, time_updated, created" \ - " FROM in_winevtlog_channels WHERE name = '%s';" - -/* - * This uses UPCERT i.e. execute INSERT first and fall back to - * UPDATE if the entry already exists. It saves the trouble of - * doing an existence check manually. - * - * https://www.sqlite.org/lang_UPSERT.html - */ -#define SQL_UPDATE_CHANNEL \ - "INSERT INTO in_winevtlog_channels" \ - " (name, bookmark_xml, time_updated, created)" \ - " VALUES ('%s', \"%s\", %u, %llu)" \ - " ON CONFLICT(name) DO UPDATE" \ - " SET bookmark_xml = excluded.bookmark_xml," \ - " time_updated = excluded.time_updated" \ - -#endif diff --git a/fluent-bit/plugins/in_winlog/CMakeLists.txt b/fluent-bit/plugins/in_winlog/CMakeLists.txt deleted file mode 100644 index 7b8b31563..000000000 --- a/fluent-bit/plugins/in_winlog/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(src - in_winlog.c - pack.c - winlog.c) - -FLB_PLUGIN(in_winlog "${src}" "advapi32") diff --git a/fluent-bit/plugins/in_winlog/in_winlog.c b/fluent-bit/plugins/in_winlog/in_winlog.c deleted file mode 100644 index 1d5398635..000000000 --- a/fluent-bit/plugins/in_winlog/in_winlog.c +++ /dev/null @@ -1,267 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include "winlog.h" - -#define DEFAULT_INTERVAL_SEC 1 -#define DEFAULT_INTERVAL_NSEC 0 -#define DEFAULT_BUFFER_SIZE 0x7ffff /* Max size allowed by Win32 (512kb) */ - -static int in_winlog_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context); - -static int in_winlog_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - int ret; - const char *tmp; - struct mk_list *head; - struct winlog_channel *ch; - struct winlog_config *ctx; - - /* Initialize context */ - ctx = flb_calloc(1, sizeof(struct winlog_config)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->ins = in; - - /* Load the config map */ - ret = flb_input_config_map_set(in, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return -1; - } - - /* Read Buffer */ - ctx->bufsize = DEFAULT_BUFFER_SIZE; - ctx->buf = flb_malloc(ctx->bufsize); - if (!ctx->buf) { - flb_errno(); - flb_free(ctx); - } - - /* Open channels */ - tmp = flb_input_get_property("channels", in); - if (!tmp) { - flb_plg_debug(ctx->ins, "no channel provided. listening to 'Application'"); - tmp = "Application"; - } - - ctx->active_channel = winlog_open_all(tmp); - if (!ctx->active_channel) { - flb_plg_error(ctx->ins, "failed to open channels"); - flb_free(ctx->buf); - flb_free(ctx); - return -1; - } - - /* Initialize SQLite DB (optional) */ - tmp = flb_input_get_property("db", in); - if (tmp) { - ctx->db = flb_sqldb_open(tmp, in->name, config); - if (!ctx->db) { - flb_plg_error(ctx->ins, "could not open/create database"); - winlog_close_all(ctx->active_channel); - flb_free(ctx->buf); - flb_free(ctx); - return -1; - } - - ret = flb_sqldb_query(ctx->db, SQL_CREATE_CHANNELS, NULL, NULL); - if (ret != FLB_OK) { - flb_plg_error(ctx->ins, "could not create 'channels' table"); - flb_sqldb_close(ctx->db); - winlog_close_all(ctx->active_channel); - flb_free(ctx->buf); - flb_free(ctx); - return -1; - } - - mk_list_foreach(head, ctx->active_channel) { - ch = mk_list_entry(head, struct winlog_channel, _head); - winlog_sqlite_load(ch, ctx->db); - flb_plg_debug(ctx->ins, "load channel<%s record=%u time=%u>", - ch->name, ch->record_number, ch->time_written); - } - } - - /* Set the context */ - flb_input_set_context(in, ctx); - - /* Set the collector */ - ret = flb_input_set_collector_time(in, - in_winlog_collect, - ctx->interval_sec, - ctx->interval_nsec, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not set up a collector"); - } - ctx->coll_fd = ret; - - return 0; -} - -static int in_winlog_read_channel(struct flb_input_instance *ins, - struct winlog_config *ctx, - struct winlog_channel *ch) -{ - unsigned int read; - char *ptr; - PEVENTLOGRECORD evt; - msgpack_packer mp_pck; - msgpack_sbuffer mp_sbuf; - - if (winlog_read(ch, ctx->buf, ctx->bufsize, &read)) { - flb_plg_error(ctx->ins, "failed to read '%s'", ch->name); - return -1; - } - if (read == 0) { - return 0; - } - flb_plg_debug(ctx->ins, "read %u bytes from '%s'", read, ch->name); - - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - ptr = ctx->buf; - while (ptr < ctx->buf + read) { - evt = (PEVENTLOGRECORD) ptr; - - winlog_pack_event(&mp_pck, evt, ch, ctx); - - ch->record_number = evt->RecordNumber; - ch->time_written = evt->TimeWritten; - - ptr += evt->Length; - } - - if (ctx->db) { - flb_plg_debug(ctx->ins, "save channel<%s record=%u time=%u>", - ch->name, ch->record_number, ch->time_written); - winlog_sqlite_save(ch, ctx->db); - } - - flb_input_log_append(ins, NULL, 0, mp_sbuf.data, mp_sbuf.size); - - msgpack_sbuffer_destroy(&mp_sbuf); - return 0; -} - -static int in_winlog_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct winlog_config *ctx = in_context; - struct mk_list *head; - struct winlog_channel *ch; - - mk_list_foreach(head, ctx->active_channel) { - ch = mk_list_entry(head, struct winlog_channel, _head); - in_winlog_read_channel(ins, ctx, ch); - } - return 0; -} - -static void in_winlog_pause(void *data, struct flb_config *config) -{ - struct winlog_config *ctx = data; - flb_input_collector_pause(ctx->coll_fd, ctx->ins); -} - -static void in_winlog_resume(void *data, struct flb_config *config) -{ - struct winlog_config *ctx = data; - flb_input_collector_resume(ctx->coll_fd, ctx->ins); -} - -static int in_winlog_exit(void *data, struct flb_config *config) -{ - struct winlog_config *ctx = data; - - if (!ctx) { - return 0; - } - - winlog_close_all(ctx->active_channel); - - if (ctx->db) { - flb_sqldb_close(ctx->db); - } - flb_free(ctx->buf); - flb_free(ctx); - - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "channels", NULL, - 0, FLB_FALSE, 0, - "Specify a comma-separated list of channels to read from" - }, - { - FLB_CONFIG_MAP_STR, "db", NULL, - 0, FLB_FALSE, 0, - "Specify DB file to save read offsets" - }, - { - FLB_CONFIG_MAP_TIME, "interval_sec", "1s", - 0, FLB_TRUE, offsetof(struct winlog_config, interval_sec), - "Set the polling interval for each channel" - }, - { - FLB_CONFIG_MAP_INT, "interval_nsec", "0", - 0, FLB_TRUE, offsetof(struct winlog_config, interval_nsec), - "Set the polling interval for each channel (sub seconds)" - }, - { - FLB_CONFIG_MAP_BOOL, "string_inserts", "true", - 0, FLB_TRUE, offsetof(struct winlog_config, string_inserts), - "Whether to include StringInserts in output records" - }, - { - FLB_CONFIG_MAP_BOOL, "use_ansi", "false", - 0, FLB_TRUE, offsetof(struct winlog_config, use_ansi), - "Use ANSI encoding on eventlog messages" - }, - - /* EOF */ - {0} -}; - -struct flb_input_plugin in_winlog_plugin = { - .name = "winlog", - .description = "Windows Event Log", - .cb_init = in_winlog_init, - .cb_pre_run = NULL, - .cb_collect = in_winlog_collect, - .cb_flush_buf = NULL, - .cb_pause = in_winlog_pause, - .cb_resume = in_winlog_resume, - .cb_exit = in_winlog_exit, - .config_map = config_map -}; diff --git a/fluent-bit/plugins/in_winlog/pack.c b/fluent-bit/plugins/in_winlog/pack.c deleted file mode 100644 index 4547d5514..000000000 --- a/fluent-bit/plugins/in_winlog/pack.c +++ /dev/null @@ -1,451 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include "winlog.h" - -#define REGKEY_MAXLEN 256 -#define FMT_ISO8601 "%Y-%m-%d %H:%M:%S %z" -#define FMT_EVTLOG L"SYSTEM\\CurrentControlSet\\Services\\EventLog\\%S\\%s" -#define FMT_EVTALT L"SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\WINEVT\\Publishers\\%s" - -/* 127 is the max number of function params */ -#define PARAM_MAXNUM 127 - -#define SRCNAME(evt) ((wchar_t *) ((char *) (evt) + sizeof(EVENTLOGRECORD))) -#define BINDATA(evt) ((unsigned char *) (evt) + (evt)->DataOffset) - -static void pack_nullstr(msgpack_packer *mp_pck) -{ - msgpack_pack_str(mp_pck, 0); - msgpack_pack_str_body(mp_pck, "", 0); -} - -static int pack_wstr(msgpack_packer *mp_pck, wchar_t *wstr, int use_ansi) -{ - int size; - char *buf; - UINT codePage = CP_UTF8; - if (use_ansi) { - codePage = CP_ACP; - } - - /* Compute the buffer size first */ - size = WideCharToMultiByte(codePage, 0, wstr, -1, NULL, 0, NULL, NULL); - if (size == 0) { - return -1; - } - - buf = flb_malloc(size); - if (buf == NULL) { - flb_errno(); - return -1; - } - - /* Convert UTF-16 into UTF-8/System code Page encoding */ - size = WideCharToMultiByte(codePage, 0, wstr, -1, buf, size, NULL, NULL); - if (size == 0) { - flb_free(buf); - return -1; - } - - /* Pack buf except the trailing '\0' */ - msgpack_pack_str(mp_pck, size - 1); - msgpack_pack_str_body(mp_pck, buf, size - 1); - flb_free(buf); - return 0; -} - -static int pack_time(msgpack_packer *mp_pck, int time) -{ - size_t len; - struct tm tm; - char buf[64]; - _locale_t locale; - - if (_localtime32_s(&tm, &time)) { - flb_errno(); - return -1; - } - - locale = _get_current_locale(); - if (locale == NULL) { - return -1; - } - - len = _strftime_l(buf, 64, FMT_ISO8601, &tm, locale); - if (len == 0) { - flb_errno(); - _free_locale(locale); - return -1; - } - _free_locale(locale); - msgpack_pack_str(mp_pck, len); - msgpack_pack_str_body(mp_pck, buf, len); - - return 0; -} - -static int pack_event_type(msgpack_packer *mp_pck, int type) -{ - switch (type) { - case EVENTLOG_SUCCESS: - msgpack_pack_str(mp_pck, 7); - msgpack_pack_str_body(mp_pck, "Success", 7); - break; - case EVENTLOG_INFORMATION_TYPE: - msgpack_pack_str(mp_pck, 11); - msgpack_pack_str_body(mp_pck, "Information", 11); - break; - case EVENTLOG_WARNING_TYPE: - msgpack_pack_str(mp_pck, 7); - msgpack_pack_str_body(mp_pck, "Warning", 7); - break; - case EVENTLOG_ERROR_TYPE: - msgpack_pack_str(mp_pck, 5); - msgpack_pack_str_body(mp_pck, "Error", 5); - break; - case EVENTLOG_AUDIT_SUCCESS: - msgpack_pack_str(mp_pck, 12); - msgpack_pack_str_body(mp_pck, "SuccessAudit", 12); - break; - case EVENTLOG_AUDIT_FAILURE: - msgpack_pack_str(mp_pck, 12); - msgpack_pack_str_body(mp_pck, "FailureAudit", 12); - break; - default: - return -1; - } - return 0; -} - -static int pack_binary(msgpack_packer *mp_pck, unsigned char *bin, int len) -{ - const char *hex = "0123456789abcdef"; - char *buf; - int size = len * 2; - int i; - - if (len == 0) { - pack_nullstr(mp_pck); - return 0; - } - - buf = flb_malloc(size); - if (buf == NULL) { - flb_errno(); - return -1; - } - - for (i = 0; i < len; i++) { - buf[2*i] = hex[bin[i] / 16]; - buf[2*i+1] = hex[bin[i] % 16]; - } - msgpack_pack_str(mp_pck, size); - msgpack_pack_str_body(mp_pck, buf, size); - flb_free(buf); - return 0; -} - -static int pack_sid(msgpack_packer *mp_pck, PEVENTLOGRECORD evt, - struct winlog_config *ctx) -{ - size_t size; - char *buf; - char *sid = (char *) evt + evt->UserSidOffset; - - if (evt->UserSidLength == 0) { - pack_nullstr(mp_pck); - return 0; - } - - if (!ConvertSidToStringSidA(sid, &buf)) { - flb_plg_error(ctx->ins, "fail to convert SID: %i", GetLastError()); - return -1; - } - - size = strlen(buf); - msgpack_pack_str(mp_pck, size); - msgpack_pack_str_body(mp_pck, buf, size); - - LocalFree(buf); - return 0; -} - -static wchar_t *read_registry(HKEY hkey, wchar_t *key, wchar_t *val) -{ - int ret; - int size; - wchar_t *buf; - unsigned int flags = RRF_RT_REG_EXPAND_SZ | RRF_RT_REG_SZ; - - /* Get the buffer size first */ - ret = RegGetValueW(hkey, key, val, flags, NULL, NULL, &size); - if (ret != ERROR_SUCCESS) { - return NULL; - } - - buf = flb_malloc(size); - if (buf == NULL) { - flb_errno(); - return NULL; - } - - /* Read data into buffer */ - ret = RegGetValueW(hkey, key, val, flags, NULL, buf, &size); - if (ret != ERROR_SUCCESS) { - flb_free(buf); - return NULL; - } - return buf; -} - -static wchar_t *query_guid(wchar_t *guid) -{ - int ret; - wchar_t key[REGKEY_MAXLEN]; - - ret = swprintf_s(key, REGKEY_MAXLEN, FMT_EVTALT, guid); - if (ret == -1) { - flb_errno(); - return NULL; - } - - return read_registry(HKEY_LOCAL_MACHINE, key, L"MessageFileName"); -} - -static int pack_message(msgpack_packer *mp_pck, PEVENTLOGRECORD evt, - struct winlog_channel *ch, struct winlog_config *ctx) -{ - int ret; - int i; - HMODULE hfile; - wchar_t key[REGKEY_MAXLEN]; - wchar_t *msg; - wchar_t *paths; - wchar_t *path; - wchar_t *guid; - wchar_t *state; - wchar_t *tmp; - DWORD_PTR *args = NULL; - - ret = swprintf_s(key, REGKEY_MAXLEN, FMT_EVTLOG, ch->name, SRCNAME(evt)); - if (ret == -1) { - flb_errno(); - return -1; - } - - guid = read_registry(HKEY_LOCAL_MACHINE, key, L"ProviderGuid"); - if (guid) { - paths = query_guid(guid); - flb_free(guid); - } - else { - paths = read_registry(HKEY_LOCAL_MACHINE, key, L"EventMessageFile"); - } - - if (paths == NULL) { - return -1; - } - - if (evt->NumStrings) { - args = flb_calloc(PARAM_MAXNUM, sizeof(DWORD_PTR)); - if (args == NULL) { - flb_errno(); - flb_free(paths); - return -1; - } - - tmp = (wchar_t *) ((char *) evt + evt->StringOffset); - for (i = 0; i < evt->NumStrings; i++) { - args[i] = (DWORD_PTR) tmp; - tmp += wcslen(tmp) + 1; - } - } - - path = paths; - wcstok_s(path, L";", &state); - while (path) { - hfile = LoadLibraryExW(path, NULL, LOAD_LIBRARY_AS_DATAFILE); - if (hfile == NULL) { - path = wcstok_s(NULL , L";", &state); - continue; - } - - ret = FormatMessageW(FORMAT_MESSAGE_FROM_HMODULE | - FORMAT_MESSAGE_ALLOCATE_BUFFER | - FORMAT_MESSAGE_ARGUMENT_ARRAY, - hfile, /* lpSource */ - evt->EventID, /* dwMessageId */ - 0, /* dwLanguageId */ - (LPWSTR) &msg,/* lpBuffer */ - 0, /* nSize */ - (va_list *) args); - if (ret > 0) { - ret = pack_wstr(mp_pck, msg, ctx->use_ansi); - LocalFree(msg); - FreeLibrary(hfile); - flb_free(paths); - flb_free(args); - return ret; - } - FreeLibrary(hfile); - path = wcstok_s(NULL , L";", &state); - } - - flb_free(paths); - flb_free(args); - return -1; -} - -static void pack_strings(msgpack_packer *mp_pck, PEVENTLOGRECORD evt, int use_ansi) -{ - int i; - int len; - wchar_t *wstr = (wchar_t *) ((char *) evt + evt->StringOffset); - - msgpack_pack_array(mp_pck, evt->NumStrings); - - for (i = 0; i < evt->NumStrings; i++) { - if (pack_wstr(mp_pck, wstr, use_ansi)) { - pack_nullstr(mp_pck); - } - wstr += wcslen(wstr) + 1; - } -} - -void winlog_pack_event(msgpack_packer *mp_pck, PEVENTLOGRECORD evt, - struct winlog_channel *ch, struct winlog_config *ctx) -{ - wchar_t *source_name = SRCNAME(evt); - wchar_t *computer_name = source_name + wcslen(source_name) + 1; - size_t len; - int count = 13; - - if (ctx->string_inserts) { - count++; - } - - msgpack_pack_array(mp_pck, 2); - flb_pack_time_now(mp_pck); - - msgpack_pack_map(mp_pck, count); - - /* RecordNumber */ - msgpack_pack_str(mp_pck, 12); - msgpack_pack_str_body(mp_pck, "RecordNumber", 12); - msgpack_pack_uint32(mp_pck, evt->RecordNumber); - - /* TimeGenerated */ - msgpack_pack_str(mp_pck, 13); - msgpack_pack_str_body(mp_pck, "TimeGenerated", 13); - if (pack_time(mp_pck, evt->TimeGenerated)) { - flb_plg_error(ctx->ins, "invalid TimeGenerated %i", evt->TimeGenerated); - pack_nullstr(mp_pck); - } - - /* TimeWritten */ - msgpack_pack_str(mp_pck, 11); - msgpack_pack_str_body(mp_pck, "TimeWritten", 11); - if (pack_time(mp_pck, evt->TimeWritten)) { - flb_plg_error(ctx->ins, "invalid TimeWritten %i", evt->TimeWritten); - pack_nullstr(mp_pck); - } - - /* EventId */ - msgpack_pack_str(mp_pck, 7); - msgpack_pack_str_body(mp_pck, "EventID", 7); - msgpack_pack_uint16(mp_pck, evt->EventID & 0xffff); - - /* Qualifiers */ - msgpack_pack_str(mp_pck, 10); - msgpack_pack_str_body(mp_pck, "Qualifiers", 10); - msgpack_pack_uint16(mp_pck, evt->EventID >> 16); - - /* EventType */ - msgpack_pack_str(mp_pck, 9); - msgpack_pack_str_body(mp_pck, "EventType", 9); - if (pack_event_type(mp_pck, evt->EventType)) { - flb_plg_error(ctx->ins, "invalid EventType %i", evt->EventType); - pack_nullstr(mp_pck); - } - - /* EventCategory */ - msgpack_pack_str(mp_pck, 13); - msgpack_pack_str_body(mp_pck, "EventCategory", 13); - msgpack_pack_uint16(mp_pck, evt->EventCategory); - - /* Channel */ - len = strlen(ch->name); - msgpack_pack_str(mp_pck, 7); - msgpack_pack_str_body(mp_pck, "Channel", 7); - msgpack_pack_str(mp_pck, len); - msgpack_pack_str_body(mp_pck, ch->name, len); - - /* Source Name */ - msgpack_pack_str(mp_pck, 10); - msgpack_pack_str_body(mp_pck, "SourceName", 10); - if (pack_wstr(mp_pck, source_name, ctx->use_ansi)) { - flb_plg_error(ctx->ins, "invalid SourceName '%ls'", source_name); - pack_nullstr(mp_pck); - } - - /* Computer Name */ - msgpack_pack_str(mp_pck, 12); - msgpack_pack_str_body(mp_pck, "ComputerName", 12); - if (pack_wstr(mp_pck, computer_name, ctx->use_ansi)) { - flb_plg_error(ctx->ins, "invalid ComputerName '%ls'", computer_name); - pack_nullstr(mp_pck); - } - - /* Event-specific Data */ - msgpack_pack_str(mp_pck, 4); - msgpack_pack_str_body(mp_pck, "Data", 4); - if (pack_binary(mp_pck, BINDATA(evt), evt->DataLength)) { - pack_nullstr(mp_pck); - } - - /* Sid */ - msgpack_pack_str(mp_pck, 3); - msgpack_pack_str_body(mp_pck, "Sid", 3); - if (pack_sid(mp_pck, evt, ctx)) { - pack_nullstr(mp_pck); - } - - /* Message */ - msgpack_pack_str(mp_pck, 7); - msgpack_pack_str_body(mp_pck, "Message", 7); - if (pack_message(mp_pck, evt, ch, ctx)) { - pack_nullstr(mp_pck); - } - - /* StringInserts (optional) */ - if (ctx->string_inserts) { - msgpack_pack_str(mp_pck, 13); - msgpack_pack_str_body(mp_pck, "StringInserts", 13); - pack_strings(mp_pck, evt, ctx->use_ansi); - } -} diff --git a/fluent-bit/plugins/in_winlog/winlog.c b/fluent-bit/plugins/in_winlog/winlog.c deleted file mode 100644 index b6064b1f1..000000000 --- a/fluent-bit/plugins/in_winlog/winlog.c +++ /dev/null @@ -1,300 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include "winlog.h" - -struct winlog_channel *winlog_open(const char *channel) -{ - struct winlog_channel *ch; - - ch = flb_calloc(1, sizeof(struct winlog_channel)); - if (!ch) { - flb_errno(); - return NULL; - } - - ch->name = flb_strdup(channel); - if (!ch->name) { - flb_errno(); - flb_free(ch); - return NULL; - } - - ch->h = OpenEventLogA(NULL, channel); - if (!ch->h) { - flb_error("[in_winlog] cannot open '%s' (%i)", channel, GetLastError()); - flb_free(ch->name); - flb_free(ch); - return NULL; - } - - return ch; -} - -void winlog_close(struct winlog_channel *ch) -{ - flb_free(ch->name); - CloseEventLog(ch->h); - flb_free(ch); -} - -/* - * This routine is called when Windows Event Log was cleared - * while reading (e.g. running Clear-EventLog on PowerShell). - * - * In such a case, the only neat thing to do is to reopen the - * channel and start reading from the beginning. - */ -int winlog_on_cleared(struct winlog_channel *ch) -{ - HANDLE h; - - h = OpenEventLogA(NULL, ch->name); - if (!h) { - flb_error("[in_winlog] cannot open '%s' (%i)", ch->name, GetLastError()); - return -1; - } - - if (ch->h) { - CloseEventLog(ch->h); - } - - ch->h = h; - ch->seek = 0; - return 0; -} - - -/* - * ReadEventLog() has a known bug that SEEK_READ fails when the log file - * is too big. - * - * winlog_seek() is a workaround for the issue, which emulates seek - * by reading the stream until it reaches the target record. - * - * https://support.microsoft.com/en-hk/help/177199/ - */ -static int winlog_seek(struct winlog_channel *ch, char *buf, - unsigned int size, unsigned int *read) -{ - char *p; - char *end; - PEVENTLOGRECORD evt; - - ch->seek = 0; - while (1) { - if (winlog_read(ch, buf, size, read)) { - return -1; - } - if (*read == 0) { - flb_trace("[in_winlog] seek '%s' to EOF", ch->name); - return 0; - } - - p = buf; - end = buf + *read; - while (p < end) { - evt = (PEVENTLOGRECORD) p; - - /* If the record is newer than the last record we've read, - * stop immediately. - */ - if (evt->TimeWritten > ch->time_written) { - *read = (end - p); - memmove(buf, p, *read); - flb_trace("[in_winlog] seek '%s' to RecordNumber=%u (time)", - ch->name, evt->RecordNumber); - return 0; - } - if (evt->TimeWritten == ch->time_written) { - - /* If the record was written at the same time, compare - * the record number. - * - * Note! Since Windows would reset RecordNumber occasionally, - * this comparison is not completely reliable. - */ - if (evt->RecordNumber > ch->record_number) { - *read = (end - p); - memmove(buf, p, *read); - flb_trace("[in_winlog] seek '%s' to RecordNumber=%u", - ch->name, evt->RecordNumber); - return 0; - } - } - p += evt->Length; - } - } -} - -/* - * Read from an open Windows Event Log channel. - */ -int winlog_read(struct winlog_channel *ch, char *buf, unsigned int size, - unsigned int *read) -{ - unsigned int flags; - unsigned int req; - unsigned int err; - - if (ch->seek) { - flags = EVENTLOG_SEEK_READ; - } else { - flags = EVENTLOG_SEQUENTIAL_READ | EVENTLOG_FORWARDS_READ; - } - - /* - * Note: ReadEventLogW() ignores `ch->record_number` (dwRecordOffset) - * if EVENTLOG_SEEK_READ is not set. - */ - if (!ReadEventLogW(ch->h, flags, ch->record_number, buf, size, read, &req)) { - switch (err = GetLastError()) { - case ERROR_HANDLE_EOF: - break; - case ERROR_INVALID_PARAMETER: - return winlog_seek(ch, buf, size, read); - case ERROR_EVENTLOG_FILE_CHANGED: - flb_info("[in_winlog] channel '%s' is cleared. reopen it.", ch->name); - return winlog_on_cleared(ch); - default: - flb_error("[in_winlog] cannot read '%s' (%i)", ch->name, err); - return -1; - } - } - ch->seek = 0; - return 0; -} - -/* - * Open multiple channels at once. The return value is a linked - * list of window_channel objects. - * - * "channels" are comma-separated names like "Setup,Security". - */ -struct mk_list *winlog_open_all(const char *channels) -{ - char *tmp; - char *channel; - char *state; - struct winlog_channel *ch; - struct mk_list *list; - - tmp = flb_strdup(channels); - if (!tmp) { - flb_errno(); - return NULL; - } - - list = flb_malloc(sizeof(struct mk_list)); - if (!list) { - flb_errno(); - flb_free(tmp); - return NULL; - } - mk_list_init(list); - - channel = strtok_s(tmp , ",", &state); - while (channel) { - ch = winlog_open(channel); - if (!ch) { - flb_free(tmp); - winlog_close_all(list); - return NULL; - } - mk_list_add(&ch->_head, list); - channel = strtok_s(NULL, ",", &state); - } - flb_free(tmp); - return list; -} - -void winlog_close_all(struct mk_list *list) -{ - struct winlog_channel *ch; - struct mk_list *head; - struct mk_list *tmp; - - mk_list_foreach_safe(head, tmp, list) { - ch = mk_list_entry(head, struct winlog_channel, _head); - mk_list_del(&ch->_head); - winlog_close(ch); - } - flb_free(list); -} - -/* - * Callback function for flb_sqldb_query(). - */ -static int winlog_sqlite_callback(void *data, int argc, char **argv, char **cols) -{ - struct winlog_sqlite_record *p = data; - - p->name = argv[0]; - p->record_number = (unsigned int) strtoul(argv[1], NULL, 10); - p->time_written = (unsigned int) strtoul(argv[2], NULL, 10); - p->created = (unsigned int) strtoul(argv[3], NULL, 10); - return 0; -} - -/* - * Load the read offset from SQLite DB. - */ -int winlog_sqlite_load(struct winlog_channel *ch, struct flb_sqldb *db) -{ - int ret; - char query[1024]; - struct winlog_sqlite_record record = {0}; - - snprintf(query, sizeof(query) - 1, SQL_GET_CHANNEL, ch->name); - - ret = flb_sqldb_query(db, query, winlog_sqlite_callback, &record); - if (ret == FLB_ERROR) { - return -1; - } - - if (record.name) { - ch->record_number = record.record_number; - ch->time_written = record.time_written; - ch->seek = 1; - } - return 0; -} - -/* - * Save the read offset into SQLite DB. - */ -int winlog_sqlite_save(struct winlog_channel *ch, struct flb_sqldb *db) -{ - int ret; - char query[1024]; - - snprintf(query, sizeof(query) - 1, SQL_UPDATE_CHANNEL, - ch->name, ch->record_number, ch->time_written, time(NULL)); - - ret = flb_sqldb_query(db, query, NULL, NULL); - if (ret == FLB_ERROR) { - return -1; - } - return 0; -} diff --git a/fluent-bit/plugins/in_winlog/winlog.h b/fluent-bit/plugins/in_winlog/winlog.h deleted file mode 100644 index 007a996ed..000000000 --- a/fluent-bit/plugins/in_winlog/winlog.h +++ /dev/null @@ -1,110 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_WINLOG_H -#define FLB_WINLOG_H - -struct winlog_config { - unsigned int interval_sec; - unsigned int interval_nsec; - unsigned int bufsize; - int string_inserts; - int use_ansi; - char *buf; - struct mk_list *active_channel; - struct flb_sqldb *db; - flb_pipefd_t coll_fd; - struct flb_input_instance *ins; -}; - -struct winlog_channel { - HANDLE h; - char *name; - unsigned int record_number; - unsigned int time_written; - unsigned int seek; - struct mk_list _head; -}; - -struct winlog_sqlite_record { - char *name; - unsigned int record_number; - unsigned int time_written; - unsigned int created; -}; - -/* - * Open a Windows Event Log channel. - */ -struct winlog_channel *winlog_open(const char *channel); -void winlog_close(struct winlog_channel *ch); - -/* - * Read records from a channel. - */ -int winlog_read(struct winlog_channel *ch, char *buf, unsigned int size, unsigned int *read); - -/* - * A bulk API to handle multiple channels at once using mk_list. - * - * "channels" are comma-separated names like "Setup,Security". - */ -struct mk_list *winlog_open_all(const char *channels); -void winlog_close_all(struct mk_list *list); - -void winlog_pack_event(msgpack_packer *mp_pck, PEVENTLOGRECORD evt, - struct winlog_channel *ch, struct winlog_config *ctx); - -/* - * Save the read offset to disk. - */ -int winlog_sqlite_load(struct winlog_channel *ch, struct flb_sqldb *db); -int winlog_sqlite_save(struct winlog_channel *ch, struct flb_sqldb *db); - -/* - * SQL templates - */ -#define SQL_CREATE_CHANNELS \ - "CREATE TABLE IF NOT EXISTS in_winlog_channels (" \ - " name TEXT PRIMARY KEY," \ - " record_number INTEGER," \ - " time_written INTEGER," \ - " created INTEGER" \ - ");" - -#define SQL_GET_CHANNEL \ - "SELECT name, record_number, time_written, created" \ - " FROM in_winlog_channels WHERE name = '%s';" - -/* - * This uses UPCERT i.e. execute INSERT first and fall back to - * UPDATE if the entry already exists. It saves the trouble of - * doing an existence check manually. - * - * https://www.sqlite.org/lang_UPSERT.html - */ -#define SQL_UPDATE_CHANNEL \ - "INSERT INTO in_winlog_channels" \ - " (name, record_number, time_written, created)" \ - " VALUES ('%s', %u, %u, %llu)" \ - " ON CONFLICT(name) DO UPDATE" \ - " SET record_number = excluded.record_number," \ - " time_written = excluded.time_written" - -#endif diff --git a/fluent-bit/plugins/in_winstat/CMakeLists.txt b/fluent-bit/plugins/in_winstat/CMakeLists.txt deleted file mode 100644 index ed36a5615..000000000 --- a/fluent-bit/plugins/in_winstat/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - winstat.c) - -FLB_PLUGIN(in_winstat "${src}" "") diff --git a/fluent-bit/plugins/in_winstat/winstat.c b/fluent-bit/plugins/in_winstat/winstat.c deleted file mode 100644 index b6fd2e173..000000000 --- a/fluent-bit/plugins/in_winstat/winstat.c +++ /dev/null @@ -1,340 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include - -#include - -struct stat_cache { - int64_t processes; - int64_t threads; - int64_t handles; - int64_t commit_total; - int64_t commit_limit; - int64_t kernel_total; - int64_t kernel_paged; - int64_t kernel_nonpaged; - int64_t physical_available; - int64_t physical_total; - int64_t physical_used; - uint64_t idletime; - uint64_t kerneltime; - uint64_t usertime; - uint64_t cpu_idle; - uint64_t cpu_user; - uint64_t cpu_kernel; - float cpu_utilization; - char uptime_human[32]; - uint64_t uptime_msec; -}; - -struct flb_winstat { - int coll_fd; - int interval_sec; - int interval_nsec; - struct flb_input_instance *ins; - struct stat_cache cache; -}; - -#define filetime64(ft) \ - ((((uint64_t) (ft)->dwHighDateTime) << 32) + (ft)->dwLowDateTime) - -#define KB(n, page) ((n) * (page) / 1024) - -static int query_processor(struct stat_cache *cache) -{ - uint64_t prev_idletime = cache->idletime; - uint64_t prev_usertime = cache->usertime; - uint64_t prev_kerneltime = cache->kerneltime; - FILETIME idletime; - FILETIME kerneltime; - FILETIME usertime; - uint64_t total; - - if (!GetSystemTimes(&idletime, &kerneltime, &usertime)) { - return -1; - } - cache->idletime = filetime64(&idletime); - cache->kerneltime = filetime64(&kerneltime) - cache->idletime; - cache->usertime = filetime64(&usertime); - - cache->cpu_idle = cache->idletime - prev_idletime; - cache->cpu_user = cache->usertime - prev_usertime; - cache->cpu_kernel = cache->kerneltime - prev_kerneltime; - - total = cache->cpu_user + cache->cpu_kernel + cache->cpu_idle; - cache->cpu_utilization = 100 - 100.0 * cache->cpu_idle / total; - - return 0; -} - -static int query_performance_info(struct stat_cache *cache) -{ - PERFORMANCE_INFORMATION perf; - - if (!GetPerformanceInfo(&perf, sizeof(perf))) { - return -1; - } - - cache->processes = perf.ProcessCount; - cache->threads = perf.ThreadCount; - cache->handles = perf.HandleCount; - - cache->physical_total = KB(perf.PhysicalTotal, perf.PageSize); - cache->physical_available = KB(perf.PhysicalAvailable, perf.PageSize); - cache->physical_used = cache->physical_total - cache->physical_available; - - cache->commit_total = KB(perf.CommitTotal, perf.PageSize); - cache->commit_limit = KB(perf.CommitLimit, perf.PageSize); - - cache->kernel_total = KB(perf.KernelTotal, perf.PageSize); - cache->kernel_paged = KB(perf.KernelPaged, perf.PageSize); - cache->kernel_nonpaged = KB(perf.KernelNonpaged, perf.PageSize); - return 0; -} - -static int query_uptime(struct stat_cache *cache) -{ - int ret; - - cache->uptime_msec = GetTickCount64(); - - /* Emulate Windows Task Manager (DD:HH:MM:SS) */ - ret = sprintf_s(cache->uptime_human, 32, "%d:%02d:%02d:%02d", - (int) (cache->uptime_msec / 1000 / 60 / 60 / 24), - (int) ((cache->uptime_msec / 1000 / 60 / 60) % 24), - (int) ((cache->uptime_msec / 1000 / 60) % 60), - (int) ((cache->uptime_msec / 1000) % 60)); - if (ret == -1) { - return -1; - } - return 0; -} - -/* - * Input Plugin API - */ -static int in_winstat_collect(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - struct flb_winstat *ctx = data; - struct stat_cache *cache = &ctx->cache; - int uptime_len; - - msgpack_packer mp_pck; - msgpack_sbuffer mp_sbuf; - - /* Query Windows metrics */ - if (query_performance_info(cache)) { - flb_plg_error(ctx->ins, "cannot query Performance info"); - return -1; - } - - if (query_processor(cache)) { - flb_plg_error(ctx->ins, "cannot query Processor info"); - return -1; - } - - if (query_uptime(cache)) { - flb_plg_error(ctx->ins, "cannot query uptime"); - return -1; - } - - /* Pack the data */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - msgpack_pack_array(&mp_pck, 2); - flb_pack_time_now(&mp_pck); - msgpack_pack_map(&mp_pck, 17); - - /* Processes/Threads/Handles */ - msgpack_pack_str(&mp_pck, 9); - msgpack_pack_str_body(&mp_pck, "processes", 9); - msgpack_pack_int64(&mp_pck, cache->processes); - - msgpack_pack_str(&mp_pck, 7); - msgpack_pack_str_body(&mp_pck, "threads", 7); - msgpack_pack_int64(&mp_pck, cache->threads); - - msgpack_pack_str(&mp_pck, 7); - msgpack_pack_str_body(&mp_pck, "handles", 7); - msgpack_pack_int64(&mp_pck, cache->handles); - - /* System performance info */ - msgpack_pack_str(&mp_pck, 14); - msgpack_pack_str_body(&mp_pck, "physical_total", 14); - msgpack_pack_int64(&mp_pck, cache->physical_total); - - msgpack_pack_str(&mp_pck, 13); - msgpack_pack_str_body(&mp_pck, "physical_used", 13); - msgpack_pack_int64(&mp_pck, cache->physical_used); - - msgpack_pack_str(&mp_pck, 18); - msgpack_pack_str_body(&mp_pck, "physical_available", 18); - msgpack_pack_int64(&mp_pck, cache->physical_available); - - msgpack_pack_str(&mp_pck, 12); - msgpack_pack_str_body(&mp_pck, "commit_total", 12); - msgpack_pack_int64(&mp_pck, cache->commit_total); - - msgpack_pack_str(&mp_pck, 12); - msgpack_pack_str_body(&mp_pck, "commit_limit", 12); - msgpack_pack_int64(&mp_pck, cache->commit_limit); - - msgpack_pack_str(&mp_pck, 12); - msgpack_pack_str_body(&mp_pck, "kernel_total", 12); - msgpack_pack_int64(&mp_pck, cache->kernel_total); - - msgpack_pack_str(&mp_pck, 12); - msgpack_pack_str_body(&mp_pck, "kernel_paged", 12); - msgpack_pack_int64(&mp_pck, cache->kernel_paged); - - msgpack_pack_str(&mp_pck, 15); - msgpack_pack_str_body(&mp_pck, "kernel_nonpaged", 15); - msgpack_pack_int64(&mp_pck, cache->kernel_nonpaged); - - /* Processors */ - msgpack_pack_str(&mp_pck, 8); - msgpack_pack_str_body(&mp_pck, "cpu_user", 8); - msgpack_pack_uint64(&mp_pck, cache->cpu_user); - - msgpack_pack_str(&mp_pck, 8); - msgpack_pack_str_body(&mp_pck, "cpu_idle", 8); - msgpack_pack_uint64(&mp_pck, cache->cpu_idle); - - msgpack_pack_str(&mp_pck, 10); - msgpack_pack_str_body(&mp_pck, "cpu_kernel", 10); - msgpack_pack_uint64(&mp_pck, cache->cpu_kernel); - - msgpack_pack_str(&mp_pck, 15); - msgpack_pack_str_body(&mp_pck, "cpu_utilization", 15); - msgpack_pack_float(&mp_pck, cache->cpu_utilization); - - /* Uptime */ - msgpack_pack_str(&mp_pck, 11); - msgpack_pack_str_body(&mp_pck, "uptime_msec", 11); - msgpack_pack_uint64(&mp_pck, cache->uptime_msec); - - uptime_len = strlen(cache->uptime_human); - msgpack_pack_str(&mp_pck, 12); - msgpack_pack_str_body(&mp_pck, "uptime_human", 12); - msgpack_pack_str(&mp_pck, uptime_len); - msgpack_pack_str_body(&mp_pck, cache->uptime_human, uptime_len); - - flb_input_log_append(in, NULL, 0, mp_sbuf.data, mp_sbuf.size); - msgpack_sbuffer_destroy(&mp_sbuf); - - return 0; -} - -static int in_winstat_init(struct flb_input_instance *in, - struct flb_config *config, void *data) -{ - int ret; - struct flb_winstat *ctx; - - /* Initialize context */ - ctx = flb_calloc(1, sizeof(struct flb_winstat)); - if (!ctx) { - return -1; - } - ctx->ins = in; - - /* Load the config map */ - ret = flb_input_config_map_set(in, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return -1; - } - - /* Preload CPU usage */ - if (query_processor(&ctx->cache)) { - flb_plg_warn(ctx->ins, "cannot preload CPU times."); - } - - /* Set the context */ - flb_input_set_context(in, ctx); - - /* Set the collector */ - ret = flb_input_set_collector_time(in, - in_winstat_collect, - ctx->interval_sec, - ctx->interval_nsec, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not set up a collector"); - flb_free(ctx); - return -1; - } - ctx->coll_fd = ret; - - return 0; -} - -static int in_winstat_exit(void *data, struct flb_config *config) -{ - struct flb_winstat *ctx = data; - flb_free(ctx); - return 0; -} - -static void in_winstat_pause(void *data, struct flb_config *config) -{ - struct flb_winstat *ctx = data; - flb_input_collector_pause(ctx->coll_fd, ctx->ins); -} - -static void in_winstat_resume(void *data, struct flb_config *config) -{ - struct flb_winstat *ctx = data; - flb_input_collector_resume(ctx->coll_fd, ctx->ins); -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_TIME, "interval_sec", "1s", - 0, FLB_TRUE, offsetof(struct flb_winstat, interval_sec), - "Set the emitter interval" - }, - { - FLB_CONFIG_MAP_INT, "interval_nsec", "0", - 0, FLB_TRUE, offsetof(struct flb_winstat, interval_nsec), - "Set the emitter interval (sub seconds)" - }, - {0} -}; - -struct flb_input_plugin in_winstat_plugin = { - .name = "winstat", - .description = "Windows System Statistics", - .cb_init = in_winstat_init, - .cb_pre_run = NULL, - .cb_collect = in_winstat_collect, - .cb_flush_buf = NULL, - .cb_pause = in_winstat_pause, - .cb_resume = in_winstat_resume, - .cb_exit = in_winstat_exit, - .config_map = config_map -}; diff --git a/fluent-bit/plugins/out_azure/CMakeLists.txt b/fluent-bit/plugins/out_azure/CMakeLists.txt deleted file mode 100644 index d8a5ba5e7..000000000 --- a/fluent-bit/plugins/out_azure/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(src - azure_conf.c - azure.c - ) - -FLB_PLUGIN(out_azure "${src}" "") diff --git a/fluent-bit/plugins/out_azure/azure.c b/fluent-bit/plugins/out_azure/azure.c deleted file mode 100644 index d4322fb65..000000000 --- a/fluent-bit/plugins/out_azure/azure.c +++ /dev/null @@ -1,452 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "azure.h" -#include "azure_conf.h" - -static int cb_azure_init(struct flb_output_instance *ins, - struct flb_config *config, void *data) -{ - struct flb_azure *ctx; - - ctx = flb_azure_conf_create(ins, config); - if (!ctx) { - flb_plg_error(ins, "configuration failed"); - return -1; - } - - flb_output_set_context(ins, ctx); - return 0; -} - -static int azure_format(const void *in_buf, size_t in_bytes, - flb_sds_t tag, flb_sds_t *tag_val_out, - char **out_buf, size_t *out_size, - struct flb_azure *ctx) -{ - int i; - int array_size = 0; - int map_size; - double t; - msgpack_object map; - msgpack_object k; - msgpack_object v; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - msgpack_sbuffer tmp_sbuf; - msgpack_packer tmp_pck; - flb_sds_t record; - char time_formatted[32]; - size_t s; - struct tm tms; - int len; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - flb_sds_t tmp = NULL; - int ret; - - /* Count number of items */ - array_size = flb_mp_count(in_buf, in_bytes); - - ret = flb_log_event_decoder_init(&log_decoder, (char *) in_buf, in_bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return -1; - } - - /* Create temporary msgpack buffer */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - msgpack_pack_array(&mp_pck, array_size); - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - /* Create temporary msgpack buffer */ - msgpack_sbuffer_init(&tmp_sbuf); - msgpack_packer_init(&tmp_pck, &tmp_sbuf, msgpack_sbuffer_write); - - map = *log_event.body; - map_size = map.via.map.size; - - if (ctx->log_type_key) { - tmp = flb_ra_translate(ctx->ra_prefix_key, - tag, flb_sds_len(tag), - map, NULL); - if (!tmp) { - flb_plg_error(ctx->ins, "Tagged record translation failed!"); - } - else if (flb_sds_is_empty(tmp)) { - flb_plg_warn(ctx->ins, "Record accessor key not matched"); - flb_sds_destroy(tmp); - } - else { - /* tag_val_out must be destroyed by the caller */ - *tag_val_out = tmp; - } - } - - msgpack_pack_map(&mp_pck, map_size + 1); - - /* Append the time key */ - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->time_key)); - msgpack_pack_str_body(&mp_pck, - ctx->time_key, - flb_sds_len(ctx->time_key)); - - if (ctx->time_generated == FLB_TRUE) { - /* Append the time value as ISO 8601 */ - gmtime_r(&log_event.timestamp.tm.tv_sec, &tms); - - s = strftime(time_formatted, sizeof(time_formatted) - 1, - FLB_PACK_JSON_DATE_ISO8601_FMT, &tms); - - len = snprintf(time_formatted + s, - sizeof(time_formatted) - 1 - s, - ".%03" PRIu64 "Z", - (uint64_t) log_event.timestamp.tm.tv_nsec / 1000000); - - s += len; - msgpack_pack_str(&mp_pck, s); - msgpack_pack_str_body(&mp_pck, time_formatted, s); - } else { - /* Append the time value as millis.nanos */ - t = flb_time_to_double(&log_event.timestamp); - - msgpack_pack_double(&mp_pck, t); - } - - /* Append original map k/v */ - for (i = 0; i < map_size; i++) { - k = map.via.map.ptr[i].key; - v = map.via.map.ptr[i].val; - - msgpack_pack_object(&tmp_pck, k); - msgpack_pack_object(&tmp_pck, v); - } - msgpack_sbuffer_write(&mp_sbuf, tmp_sbuf.data, tmp_sbuf.size); - msgpack_sbuffer_destroy(&tmp_sbuf); - } - - record = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); - if (!record) { - flb_errno(); - - flb_log_event_decoder_destroy(&log_decoder); - msgpack_sbuffer_destroy(&mp_sbuf); - - return -1; - } - - flb_log_event_decoder_destroy(&log_decoder); - - msgpack_sbuffer_destroy(&mp_sbuf); - - *out_buf = record; - *out_size = flb_sds_len(record); - - return 0; -} - -static int build_headers(struct flb_http_client *c, - flb_sds_t log_type, - size_t content_length, - struct flb_azure *ctx) -{ - int len; - char *auth; - char tmp[256]; - time_t t; - size_t size; - size_t olen; - flb_sds_t rfc1123date; - flb_sds_t str_hash; - struct tm tm = {0}; - unsigned char hmac_hash[32] = {0}; - int result; - - /* Format Date */ - rfc1123date = flb_sds_create_size(32); - if (!rfc1123date) { - flb_errno(); - return -1; - } - - t = time(NULL); - if (!gmtime_r(&t, &tm)) { - flb_errno(); - flb_sds_destroy(rfc1123date); - return -1; - } - size = strftime(rfc1123date, - flb_sds_alloc(rfc1123date) - 1, - "%a, %d %b %Y %H:%M:%S GMT", &tm); - if (size <= 0) { - flb_errno(); - flb_sds_destroy(rfc1123date); - return -1; - } - flb_sds_len_set(rfc1123date, size); - - /* Compose source string for the hash */ - str_hash = flb_sds_create_size(256); - if (!str_hash) { - flb_errno(); - flb_sds_destroy(rfc1123date); - return -1; - } - - len = snprintf(tmp, sizeof(tmp) - 1, "%zu\n", content_length); - flb_sds_cat(str_hash, "POST\n", 5); - flb_sds_cat(str_hash, tmp, len); - flb_sds_cat(str_hash, "application/json\n", 17); - flb_sds_cat(str_hash, "x-ms-date:", 10); - flb_sds_cat(str_hash, rfc1123date, flb_sds_len(rfc1123date)); - flb_sds_cat(str_hash, "\n", 1); - flb_sds_cat(str_hash, FLB_AZURE_RESOURCE, sizeof(FLB_AZURE_RESOURCE) - 1); - - /* Authorization signature */ - result = flb_hmac_simple(FLB_HASH_SHA256, - (unsigned char *) ctx->dec_shared_key, - flb_sds_len(ctx->dec_shared_key), - (unsigned char *) str_hash, - flb_sds_len(str_hash), - hmac_hash, - sizeof(hmac_hash)); - - if (result != FLB_CRYPTO_SUCCESS) { - flb_sds_destroy(rfc1123date); - flb_sds_destroy(str_hash); - return -1; - } - - /* Encoded hash */ - result = flb_base64_encode((unsigned char *) &tmp, sizeof(tmp) - 1, &olen, - hmac_hash, sizeof(hmac_hash)); - tmp[olen] = '\0'; - - /* Append headers */ - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - flb_http_add_header(c, "Log-Type", 8, - log_type, flb_sds_len(log_type)); - flb_http_add_header(c, "Content-Type", 12, "application/json", 16); - flb_http_add_header(c, "x-ms-date", 9, rfc1123date, - flb_sds_len(rfc1123date)); - if (ctx->time_generated == FLB_TRUE) { - /* Use time value as time-generated within azure */ - flb_http_add_header(c, "time-generated-field", 20, ctx->time_key, flb_sds_len(ctx->time_key)); - } - - size = 32 + flb_sds_len(ctx->customer_id) + olen; - auth = flb_malloc(size); - if (!auth) { - flb_errno(); - flb_sds_destroy(rfc1123date); - flb_sds_destroy(str_hash); - return -1; - } - - - len = snprintf(auth, size, "SharedKey %s:%s", - ctx->customer_id, tmp); - flb_http_add_header(c, "Authorization", 13, auth, len); - - /* release resources */ - flb_sds_destroy(rfc1123date); - flb_sds_destroy(str_hash); - flb_free(auth); - - return 0; -} - -static void cb_azure_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - int ret; - size_t b_sent; - char *buf_data; - size_t buf_size; - struct flb_azure *ctx = out_context; - struct flb_connection *u_conn; - struct flb_http_client *c; - flb_sds_t payload; - flb_sds_t final_log_type = NULL; - (void) i_ins; - (void) config; - - /* Get upstream connection */ - u_conn = flb_upstream_conn_get(ctx->u); - if (!u_conn) { - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Convert binary logs into a JSON payload */ - ret = azure_format(event_chunk->data, event_chunk->size, - event_chunk->tag, &final_log_type, &buf_data, &buf_size, ctx); - /* If cannot get matching record using log_type_prefix, use log_type directly */ - if (!final_log_type) { - final_log_type = ctx->log_type; - } - - if (ret == -1) { - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_ERROR); - } - payload = (flb_sds_t) buf_data; - - /* Compose HTTP Client request */ - c = flb_http_client(u_conn, FLB_HTTP_POST, ctx->uri, - buf_data, buf_size, NULL, 0, NULL, 0); - flb_http_buffer_size(c, FLB_HTTP_DATA_SIZE_MAX); - - /* Append headers and Azure signature */ - ret = build_headers(c, final_log_type, flb_sds_len(payload), ctx); - if (ret == -1) { - flb_plg_error(ctx->ins, "error composing signature"); - flb_sds_destroy(payload); - flb_http_client_destroy(c); - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_ERROR); - } - - ret = flb_http_do(c, &b_sent); - if (ret != 0) { - flb_plg_warn(ctx->ins, "http_do=%i", ret); - goto retry; - } - else { - if (c->resp.status >= 200 && c->resp.status <= 299) { - flb_plg_info(ctx->ins, "customer_id=%s, HTTP status=%i", - ctx->customer_id, c->resp.status); - } - else { - if (c->resp.payload_size > 0) { - flb_plg_warn(ctx->ins, "http_status=%i:\n%s", - c->resp.status, c->resp.payload); - } - else { - flb_plg_warn(ctx->ins, "http_status=%i", c->resp.status); - } - goto retry; - } - } - - /* Cleanup */ - if (final_log_type != ctx->log_type) { - flb_sds_destroy(final_log_type); - } - flb_http_client_destroy(c); - flb_sds_destroy(payload); - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_OK); - - /* Issue a retry */ - retry: - flb_http_client_destroy(c); - flb_sds_destroy(payload); - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_RETRY); -} - -static int cb_azure_exit(void *data, struct flb_config *config) -{ - struct flb_azure *ctx = data; - - flb_azure_conf_destroy(ctx); - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "customer_id", NULL, - 0, FLB_TRUE, offsetof(struct flb_azure, customer_id), - "Customer ID or WorkspaceID string." - }, - - { - FLB_CONFIG_MAP_STR, "shared_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_azure, shared_key), - "The primary or the secondary Connected Sources client authentication key." - }, - - { - FLB_CONFIG_MAP_STR, "log_type", FLB_AZURE_LOG_TYPE, - 0, FLB_TRUE, offsetof(struct flb_azure, log_type), - "The name of the event type." - }, - - { - FLB_CONFIG_MAP_STR, "log_type_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_azure, log_type_key), - "If included, the value for this key will be looked upon in the record " - "and if present, will over-write the `log_type`. If the key/value " - "is not found in the record then the `log_type` option will be used. " - }, - - { - FLB_CONFIG_MAP_STR, "time_key", FLB_AZURE_TIME_KEY, - 0, FLB_TRUE, offsetof(struct flb_azure, time_key), - "Optional parameter to specify the key name where the timestamp will be stored." - }, - - { - FLB_CONFIG_MAP_BOOL, "time_generated", "false", - 0, FLB_TRUE, offsetof(struct flb_azure, time_generated), - "If enabled, the HTTP request header 'time-generated-field' will be included " - "so Azure can override the timestamp with the key specified by 'time_key' " - "option." - }, - - /* EOF */ - {0} -}; - -struct flb_output_plugin out_azure_plugin = { - .name = "azure", - .description = "Send events to Azure HTTP Event Collector", - .cb_init = cb_azure_init, - .cb_flush = cb_azure_flush, - .cb_exit = cb_azure_exit, - - /* Configuration */ - .config_map = config_map, - - /* Plugin flags */ - .flags = FLB_OUTPUT_NET | FLB_IO_TLS, -}; diff --git a/fluent-bit/plugins/out_azure/azure.h b/fluent-bit/plugins/out_azure/azure.h deleted file mode 100644 index 192d41ac8..000000000 --- a/fluent-bit/plugins/out_azure/azure.h +++ /dev/null @@ -1,62 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_AZURE -#define FLB_OUT_AZURE - -#define FLB_AZURE_API_VERSION "?api-version=2016-04-01" -#define FLB_AZURE_HOST ".ods.opinsights.azure.com" -#define FLB_AZURE_PORT 443 -#define FLB_AZURE_RESOURCE "/api/logs" -#define FLB_AZURE_LOG_TYPE "fluentbit" -#define FLB_AZURE_TIME_KEY "@timestamp" - -#include -#include -#include -#include - -struct flb_azure { - /* account setup */ - flb_sds_t customer_id; - flb_sds_t log_type; - flb_sds_t log_type_key; - flb_sds_t shared_key; - flb_sds_t dec_shared_key; - - /* networking */ - int port; - flb_sds_t host; - flb_sds_t uri; - - /* records */ - flb_sds_t time_key; - struct flb_record_accessor *ra_prefix_key; - - /* time_generated: on/off */ - int time_generated; - - /* Upstream connection to the backend server */ - struct flb_upstream *u; - - /* Plugin instance reference */ - struct flb_output_instance *ins; -}; - -#endif diff --git a/fluent-bit/plugins/out_azure/azure_conf.c b/fluent-bit/plugins/out_azure/azure_conf.c deleted file mode 100644 index 9f8f8a05f..000000000 --- a/fluent-bit/plugins/out_azure/azure_conf.c +++ /dev/null @@ -1,219 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "azure.h" -#include "azure_conf.h" - -struct flb_azure *flb_azure_conf_create(struct flb_output_instance *ins, - struct flb_config *config) -{ - int ret; - size_t size; - size_t olen; - const char *tmp; - struct flb_upstream *upstream; - struct flb_azure *ctx; - struct flb_record_accessor *ra_prefix_key = NULL; - - /* Allocate config context */ - ctx = flb_calloc(1, sizeof(struct flb_azure)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - - /* Set context */ - flb_output_set_context(ins, ctx); - - /* Load config map */ - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - return NULL; - } - - if (!ctx->customer_id) { - flb_plg_error(ctx->ins, "property 'customer_id' is not defined"); - flb_azure_conf_destroy(ctx); - return NULL; - } - - /* config: 'shared_key' */ - if (!ctx->shared_key) { - flb_plg_error(ctx->ins, "property 'shared_key' is not defined"); - flb_azure_conf_destroy(ctx); - return NULL; - } - - /* decode shared key */ - size = flb_sds_len(ctx->shared_key) * 1.2; - ctx->dec_shared_key = flb_sds_create_size(size); - if (!ctx->dec_shared_key) { - flb_errno(); - flb_azure_conf_destroy(ctx); - return NULL; - } - - ret = flb_base64_decode((unsigned char *) ctx->dec_shared_key, size, - &olen, - (unsigned char *) ctx->shared_key, - flb_sds_len(ctx->shared_key)); - if (ret != 0) { - flb_plg_error(ctx->ins, "error decoding shared_key"); - flb_azure_conf_destroy(ctx); - return NULL; - } - flb_sds_len_set(ctx->dec_shared_key, olen); - - /* config: 'log_type_key' */ - if (ctx->log_type_key) { - ra_prefix_key = flb_ra_create(ctx->log_type_key, FLB_TRUE); - - if (!ra_prefix_key) { - flb_plg_error(ctx->ins, "invalid log_type_key pattern '%s'", ctx->log_type_key); - flb_azure_conf_destroy(ctx); - return NULL; - } - else { - ctx->ra_prefix_key = ra_prefix_key; - } - } - - /* Validate hostname given by command line or 'Host' property */ - if (!ins->host.name && !ctx->customer_id) { - flb_plg_error(ctx->ins, "property 'customer_id' is not defined"); - flb_free(ctx); - return NULL; - } - - /* Lookup customer id from given host name */ - if (!ctx->customer_id) { - tmp = strchr(ins->host.name, '.'); - if (!tmp) { - flb_plg_error(ctx->ins, "invalid hostname"); - flb_free(ctx); - return NULL; - } - else { - ctx->customer_id = flb_sds_create_len(ins->host.name, - tmp - ins->host.name); - if (!ctx->customer_id) { - flb_errno(); - flb_free(ctx); - return NULL; - } - } - } - - /* Compose real host */ - ctx->host = flb_sds_create_size(256); - if (!ctx->host) { - flb_errno(); - flb_free(ctx); - return NULL; - } - - if (!ins->host.name) { - flb_sds_cat(ctx->host, ctx->customer_id, - flb_sds_len(ctx->customer_id)); - flb_sds_cat(ctx->host, FLB_AZURE_HOST, sizeof(FLB_AZURE_HOST) - 1); - } - else { - if (!strstr(ins->host.name, ctx->customer_id)) { - flb_sds_cat(ctx->host, ctx->customer_id, - flb_sds_len(ctx->customer_id)); - if (ins->host.name[0] != '.') { - flb_sds_cat(ctx->host, ".", 1); - } - } - flb_sds_cat(ctx->host, ins->host.name, strlen(ins->host.name)); - } - - - /* TCP Port */ - if (ins->host.port == 0) { - ctx->port = FLB_AZURE_PORT; - } - else { - ctx->port = ins->host.port; - } - - /* Prepare an upstream handler */ - upstream = flb_upstream_create(config, - ctx->host, - ctx->port, - FLB_IO_TLS, - ins->tls); - if (!upstream) { - flb_plg_error(ctx->ins, "cannot create Upstream context"); - flb_azure_conf_destroy(ctx); - return NULL; - } - ctx->u = upstream; - flb_output_upstream_set(ctx->u, ins); - - /* Compose uri */ - ctx->uri = flb_sds_create_size(1024); - if (!ctx->uri) { - flb_errno(); - flb_azure_conf_destroy(ctx); - return NULL; - } - flb_sds_cat(ctx->uri, FLB_AZURE_RESOURCE, sizeof(FLB_AZURE_RESOURCE) - 1); - flb_sds_cat(ctx->uri, FLB_AZURE_API_VERSION, - sizeof(FLB_AZURE_API_VERSION) - 1); - - flb_plg_info(ctx->ins, "customer_id='%s' host='%s:%i'", - ctx->customer_id, ctx->host, ctx->port); - - return ctx; -} - -int flb_azure_conf_destroy(struct flb_azure *ctx) -{ - if (!ctx) { - return -1; - } - - if (ctx->dec_shared_key) { - flb_sds_destroy(ctx->dec_shared_key); - } - - if (ctx->host) { - flb_sds_destroy(ctx->host); - } - if (ctx->uri) { - flb_sds_destroy(ctx->uri); - } - if (ctx->ra_prefix_key) { - flb_ra_destroy(ctx->ra_prefix_key); - } - if (ctx->u) { - flb_upstream_destroy(ctx->u); - } - flb_free(ctx); - - return 0; -} diff --git a/fluent-bit/plugins/out_azure/azure_conf.h b/fluent-bit/plugins/out_azure/azure_conf.h deleted file mode 100644 index 626b217ce..000000000 --- a/fluent-bit/plugins/out_azure/azure_conf.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_AZURE_CONF_H -#define FLB_OUT_AZURE_CONF_H - -#include "azure.h" - -struct flb_azure *flb_azure_conf_create(struct flb_output_instance *ins, - struct flb_config *config); -int flb_azure_conf_destroy(struct flb_azure *ctx); - -#endif diff --git a/fluent-bit/plugins/out_azure_blob/CMakeLists.txt b/fluent-bit/plugins/out_azure_blob/CMakeLists.txt deleted file mode 100644 index 3624480e6..000000000 --- a/fluent-bit/plugins/out_azure_blob/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -set(src - azure_blob.c - azure_blob_uri.c - azure_blob_conf.c - azure_blob_http.c - azure_blob_appendblob.c - azure_blob_blockblob.c - ) - -FLB_PLUGIN(out_azure_blob "${src}" "") diff --git a/fluent-bit/plugins/out_azure_blob/azure_blob.c b/fluent-bit/plugins/out_azure_blob/azure_blob.c deleted file mode 100644 index 3f539826d..000000000 --- a/fluent-bit/plugins/out_azure_blob/azure_blob.c +++ /dev/null @@ -1,594 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "azure_blob.h" -#include "azure_blob_uri.h" -#include "azure_blob_conf.h" -#include "azure_blob_appendblob.h" -#include "azure_blob_blockblob.h" -#include "azure_blob_http.h" - -#define CREATE_BLOB 1337 - -static int azure_blob_format(struct flb_config *config, - struct flb_input_instance *ins, - void *plugin_context, - void *flush_ctx, - int event_type, - const char *tag, int tag_len, - const void *data, size_t bytes, - void **out_data, size_t *out_size) -{ - flb_sds_t out_buf; - struct flb_azure_blob *ctx = plugin_context; - - out_buf = flb_pack_msgpack_to_json_format(data, bytes, - FLB_PACK_JSON_FORMAT_LINES, - FLB_PACK_JSON_DATE_ISO8601, - ctx->date_key); - if (!out_buf) { - return -1; - } - - *out_data = out_buf; - *out_size = flb_sds_len(out_buf); - return 0; -} - -static int send_blob(struct flb_config *config, - struct flb_input_instance *i_ins, - struct flb_azure_blob *ctx, char *name, - char *tag, int tag_len, void *data, size_t bytes) -{ - int ret; - int compressed = FLB_FALSE; - int content_encoding = FLB_FALSE; - int content_type = FLB_FALSE; - uint64_t ms = 0; - size_t b_sent; - void *out_buf; - size_t out_size; - flb_sds_t uri = NULL; - flb_sds_t blockid = NULL; - void *payload_buf; - size_t payload_size; - struct flb_http_client *c; - struct flb_connection *u_conn; - - if (ctx->btype == AZURE_BLOB_APPENDBLOB) { - uri = azb_append_blob_uri(ctx, tag); - } - else if (ctx->btype == AZURE_BLOB_BLOCKBLOB) { - blockid = azb_block_blob_id(&ms); - if (!blockid) { - flb_plg_error(ctx->ins, "could not generate block id"); - return FLB_RETRY; - } - uri = azb_block_blob_uri(ctx, tag, blockid, ms); - } - - if (!uri) { - flb_free(blockid); - return FLB_RETRY; - } - - /* Get upstream connection */ - u_conn = flb_upstream_conn_get(ctx->u); - if (!u_conn) { - flb_plg_error(ctx->ins, - "cannot create upstream connection for append_blob"); - flb_sds_destroy(uri); - flb_free(blockid); - return FLB_RETRY; - } - - /* Format the data */ - ret = azure_blob_format(config, i_ins, - ctx, NULL, - FLB_EVENT_TYPE_LOGS, - tag, tag_len, - data, bytes, - &out_buf, &out_size); - if (ret != 0) { - flb_upstream_conn_release(u_conn); - flb_sds_destroy(uri); - flb_free(blockid); - return FLB_RETRY; - } - - /* Map buffer */ - payload_buf = out_buf; - payload_size = out_size; - - if (ctx->compress_gzip == FLB_TRUE || ctx->compress_blob == FLB_TRUE) { - ret = flb_gzip_compress((void *) out_buf, out_size, - &payload_buf, &payload_size); - if (ret == -1) { - flb_plg_error(ctx->ins, - "cannot gzip payload, disabling compression"); - } - else { - compressed = FLB_TRUE; - /* JSON buffer is not longer needed */ - flb_sds_destroy(out_buf); - } - } - - if (ctx->compress_blob == FLB_TRUE) { - content_encoding = AZURE_BLOB_CE_NONE; - content_type = AZURE_BLOB_CT_GZIP; - } - else if (compressed == FLB_TRUE) { - content_encoding = AZURE_BLOB_CE_GZIP; - content_type = AZURE_BLOB_CT_JSON; - } - - /* Create HTTP client context */ - c = flb_http_client(u_conn, FLB_HTTP_PUT, - uri, - payload_buf, payload_size, NULL, 0, NULL, 0); - if (!c) { - flb_plg_error(ctx->ins, "cannot create HTTP client context"); - flb_sds_destroy(out_buf); - flb_upstream_conn_release(u_conn); - flb_free(blockid); - return FLB_RETRY; - } - - /* Prepare headers and authentication */ - azb_http_client_setup(ctx, c, (ssize_t) payload_size, FLB_FALSE, - content_type, content_encoding); - - /* Send HTTP request */ - ret = flb_http_do(c, &b_sent); - flb_sds_destroy(uri); - - /* Release */ - if (compressed == FLB_FALSE) { - flb_sds_destroy(out_buf); - } - else { - flb_free(payload_buf); - } - - flb_upstream_conn_release(u_conn); - - /* Validate HTTP status */ - if (ret == -1) { - flb_plg_error(ctx->ins, "error sending append_blob"); - flb_free(blockid); - return FLB_RETRY; - } - - if (c->resp.status == 201) { - flb_plg_info(ctx->ins, "content appended to blob successfully"); - flb_http_client_destroy(c); - - if (ctx->btype == AZURE_BLOB_BLOCKBLOB) { - ret = azb_block_blob_commit(ctx, blockid, tag, ms); - flb_free(blockid); - return ret; - } - flb_free(blockid); - return FLB_OK; - } - else if (c->resp.status == 404) { - flb_plg_info(ctx->ins, "blob not found: %s", c->uri); - flb_http_client_destroy(c); - return CREATE_BLOB; - } - else if (c->resp.payload_size > 0) { - flb_plg_error(ctx->ins, "cannot append content to blob\n%s", - c->resp.payload); - if (strstr(c->resp.payload, "must be 0 for Create Append")) { - flb_http_client_destroy(c); - return CREATE_BLOB; - } - } - else { - flb_plg_error(ctx->ins, "cannot append content to blob"); - } - flb_http_client_destroy(c); - - return FLB_RETRY; -} - -static int create_blob(struct flb_azure_blob *ctx, char *name) -{ - int ret; - size_t b_sent; - flb_sds_t uri = NULL; - struct flb_http_client *c; - struct flb_connection *u_conn; - - uri = azb_uri_create_blob(ctx, name); - if (!uri) { - return FLB_RETRY; - } - - /* Get upstream connection */ - u_conn = flb_upstream_conn_get(ctx->u); - if (!u_conn) { - flb_plg_error(ctx->ins, - "cannot create upstream connection for create_append_blob"); - flb_sds_destroy(uri); - return FLB_RETRY; - } - - /* Create HTTP client context */ - c = flb_http_client(u_conn, FLB_HTTP_PUT, - uri, - NULL, 0, NULL, 0, NULL, 0); - if (!c) { - flb_plg_error(ctx->ins, "cannot create HTTP client context"); - flb_upstream_conn_release(u_conn); - flb_sds_destroy(uri); - return FLB_RETRY; - } - - /* Prepare headers and authentication */ - azb_http_client_setup(ctx, c, -1, FLB_TRUE, - AZURE_BLOB_CT_NONE, AZURE_BLOB_CE_NONE); - - /* Send HTTP request */ - ret = flb_http_do(c, &b_sent); - flb_sds_destroy(uri); - - if (ret == -1) { - flb_plg_error(ctx->ins, "error sending append_blob"); - flb_http_client_destroy(c); - flb_upstream_conn_release(u_conn); - return FLB_RETRY; - } - - if (c->resp.status == 201) { - flb_plg_info(ctx->ins, "blob created successfully: %s", c->uri); - } - else { - if (c->resp.payload_size > 0) { - flb_plg_error(ctx->ins, "http_status=%i cannot create append blob\n%s", - c->resp.status, c->resp.payload); - } - else { - flb_plg_error(ctx->ins, "http_status=%i cannot create append blob", - c->resp.status); - } - flb_http_client_destroy(c); - flb_upstream_conn_release(u_conn); - return FLB_RETRY; - } - - flb_http_client_destroy(c); - flb_upstream_conn_release(u_conn); - return FLB_OK; -} - -static int create_container(struct flb_azure_blob *ctx, char *name) -{ - int ret; - size_t b_sent; - flb_sds_t uri; - struct flb_http_client *c; - struct flb_connection *u_conn; - - /* Get upstream connection */ - u_conn = flb_upstream_conn_get(ctx->u); - if (!u_conn) { - flb_plg_error(ctx->ins, - "cannot create upstream connection for container creation"); - return FLB_FALSE; - } - - /* URI */ - uri = azb_uri_ensure_or_create_container(ctx); - if (!uri) { - flb_upstream_conn_release(u_conn); - return FLB_FALSE; - } - - /* Create HTTP client context */ - c = flb_http_client(u_conn, FLB_HTTP_PUT, - uri, - NULL, 0, NULL, 0, NULL, 0); - if (!c) { - flb_plg_error(ctx->ins, "cannot create HTTP client context"); - flb_upstream_conn_release(u_conn); - return FLB_FALSE; - } - - /* Prepare headers and authentication */ - azb_http_client_setup(ctx, c, -1, FLB_FALSE, - AZURE_BLOB_CT_NONE, AZURE_BLOB_CE_NONE); - - /* Send HTTP request */ - ret = flb_http_do(c, &b_sent); - - /* Release URI */ - flb_sds_destroy(uri); - - /* Validate http response */ - if (ret == -1) { - flb_plg_error(ctx->ins, "error requesting container creation"); - flb_http_client_destroy(c); - flb_upstream_conn_release(u_conn); - return FLB_FALSE; - } - - if (c->resp.status == 201) { - flb_plg_info(ctx->ins, "container '%s' created sucessfully", name); - } - else { - if (c->resp.payload_size > 0) { - flb_plg_error(ctx->ins, "cannot create container '%s'\n%s", - name, c->resp.payload); - } - else { - flb_plg_error(ctx->ins, "cannot create container '%s'\n%s", - name, c->resp.payload); - } - flb_http_client_destroy(c); - flb_upstream_conn_release(u_conn); - return FLB_FALSE; - } - - flb_http_client_destroy(c); - flb_upstream_conn_release(u_conn); - return FLB_TRUE; -} - -/* - * Check that the container exists, if it doesn't and the configuration property - * auto_create_container is enabled, it will send a request to create it. If it - * could not be created or auto_create_container is disabled, it returns FLB_FALSE. - */ -static int ensure_container(struct flb_azure_blob *ctx) -{ - int ret; - int status; - size_t b_sent; - flb_sds_t uri; - struct flb_http_client *c; - struct flb_connection *u_conn; - - uri = azb_uri_ensure_or_create_container(ctx); - if (!uri) { - return FLB_FALSE; - } - - /* Get upstream connection */ - u_conn = flb_upstream_conn_get(ctx->u); - if (!u_conn) { - flb_plg_error(ctx->ins, - "cannot create upstream connection for container check"); - flb_sds_destroy(uri); - return FLB_FALSE; - } - - /* Create HTTP client context */ - c = flb_http_client(u_conn, FLB_HTTP_GET, - uri, - NULL, 0, NULL, 0, NULL, 0); - if (!c) { - flb_plg_error(ctx->ins, "cannot create HTTP client context"); - flb_upstream_conn_release(u_conn); - return FLB_FALSE; - } - flb_http_strip_port_from_host(c); - - /* Prepare headers and authentication */ - azb_http_client_setup(ctx, c, -1, FLB_FALSE, - AZURE_BLOB_CT_NONE, AZURE_BLOB_CE_NONE); - - /* Send HTTP request */ - ret = flb_http_do(c, &b_sent); - flb_sds_destroy(uri); - - if (ret == -1) { - flb_plg_error(ctx->ins, "error requesting container properties"); - flb_upstream_conn_release(u_conn); - return FLB_FALSE; - } - - status = c->resp.status; - flb_http_client_destroy(c); - - /* Release connection */ - flb_upstream_conn_release(u_conn); - - /* Request was successful, validate HTTP status code */ - if (status == 404) { - /* The container was not found, try to create it */ - flb_plg_info(ctx->ins, "container '%s' not found, trying to create it", - ctx->container_name); - ret = create_container(ctx, ctx->container_name); - return ret; - } - else if (status == 200) { - return FLB_TRUE; - } - - return FLB_FALSE; -} - -static int cb_azure_blob_init(struct flb_output_instance *ins, - struct flb_config *config, void *data) -{ - struct flb_azure_blob *ctx = NULL; - (void) ins; - (void) config; - (void) data; - - ctx = flb_azure_blob_conf_create(ins, config); - if (!ctx) { - return -1; - } - - flb_output_set_http_debug_callbacks(ins); - return 0; -} - -static void cb_azure_blob_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - int ret; - struct flb_azure_blob *ctx = out_context; - (void) i_ins; - (void) config; - - /* Validate the container exists, otherwise just create it */ - ret = ensure_container(ctx); - if (ret == FLB_FALSE) { - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - ret = send_blob(config, i_ins, ctx, - (char *) event_chunk->tag, /* use tag as 'name' */ - (char *) event_chunk->tag, flb_sds_len(event_chunk->tag), - (char *) event_chunk->data, event_chunk->size); - - if (ret == CREATE_BLOB) { - ret = create_blob(ctx, event_chunk->tag); - if (ret == FLB_OK) { - ret = send_blob(config, i_ins, ctx, - (char *) event_chunk->tag, /* use tag as 'name' */ - (char *) event_chunk->tag, - flb_sds_len(event_chunk->tag), - (char *) event_chunk->data, event_chunk->size); - } - } - - /* FLB_RETRY, FLB_OK, FLB_ERROR */ - FLB_OUTPUT_RETURN(ret); -} - -static int cb_azure_blob_exit(void *data, struct flb_config *config) -{ - struct flb_azure_blob *ctx = data; - - if (!ctx) { - return 0; - } - - flb_azure_blob_conf_destroy(ctx); - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "account_name", NULL, - 0, FLB_TRUE, offsetof(struct flb_azure_blob, account_name), - "Azure account name (mandatory)" - }, - - { - FLB_CONFIG_MAP_STR, "container_name", NULL, - 0, FLB_TRUE, offsetof(struct flb_azure_blob, container_name), - "Container name (mandatory)" - }, - - { - FLB_CONFIG_MAP_BOOL, "auto_create_container", "true", - 0, FLB_TRUE, offsetof(struct flb_azure_blob, auto_create_container), - "Auto create container if it don't exists" - }, - - { - FLB_CONFIG_MAP_STR, "blob_type", "appendblob", - 0, FLB_TRUE, offsetof(struct flb_azure_blob, blob_type), - "Set the block type: appendblob or blockblob" - }, - - { - FLB_CONFIG_MAP_STR, "compress", NULL, - 0, FLB_FALSE, 0, - "Set payload compression in network transfer. Option available is 'gzip'" - }, - - { - FLB_CONFIG_MAP_BOOL, "compress_blob", "false", - 0, FLB_TRUE, offsetof(struct flb_azure_blob, compress_blob), - "Enable block blob GZIP compression in the final blob file. This option is " - "not compatible with 'appendblob' block type" - }, - - { - FLB_CONFIG_MAP_BOOL, "emulator_mode", "false", - 0, FLB_TRUE, offsetof(struct flb_azure_blob, emulator_mode), - "Use emulator mode, enable it if you want to use Azurite" - }, - - { - FLB_CONFIG_MAP_STR, "shared_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_azure_blob, shared_key), - "Azure shared key" - }, - - { - FLB_CONFIG_MAP_STR, "endpoint", NULL, - 0, FLB_TRUE, offsetof(struct flb_azure_blob, endpoint), - "Custom full URL endpoint to use an emulator" - }, - - { - FLB_CONFIG_MAP_STR, "path", NULL, - 0, FLB_TRUE, offsetof(struct flb_azure_blob, path), - "Set a path for your blob" - }, - - { - FLB_CONFIG_MAP_STR, "date_key", "@timestamp", - 0, FLB_TRUE, offsetof(struct flb_azure_blob, date_key), - "Name of the key that will have the record timestamp" - }, - - /* EOF */ - {0} -}; - -/* Plugin registration */ -struct flb_output_plugin out_azure_blob_plugin = { - .name = "azure_blob", - .description = "Azure Blob Storage", - .cb_init = cb_azure_blob_init, - .cb_flush = cb_azure_blob_flush, - .cb_exit = cb_azure_blob_exit, - - /* Test */ - .test_formatter.callback = azure_blob_format, - - .config_map = config_map, - - /* Plugin flags */ - .flags = FLB_OUTPUT_NET | FLB_IO_OPT_TLS, -}; diff --git a/fluent-bit/plugins/out_azure_blob/azure_blob.h b/fluent-bit/plugins/out_azure_blob/azure_blob.h deleted file mode 100644 index 5cf8a2927..000000000 --- a/fluent-bit/plugins/out_azure_blob/azure_blob.h +++ /dev/null @@ -1,74 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_AZURE_BLOB_H -#define FLB_OUT_AZURE_BLOB_H - -#include -#include -#include - -/* Content-Type */ -#define AZURE_BLOB_CT "Content-Type" -#define AZURE_BLOB_CT_NONE 0 -#define AZURE_BLOB_CT_JSON 1 /* application/json */ -#define AZURE_BLOB_CT_GZIP 2 /* application/gzip */ - -/* Content-Encoding */ -#define AZURE_BLOB_CE "Content-Encoding" -#define AZURE_BLOB_CE_NONE 0 -#define AZURE_BLOB_CE_GZIP 1 /* gzip */ - -/* service endpoint */ -#define AZURE_ENDPOINT_PREFIX ".blob.core.windows.net" - -#define AZURE_BLOB_APPENDBLOB 0 -#define AZURE_BLOB_BLOCKBLOB 1 - -struct flb_azure_blob { - int auto_create_container; - int emulator_mode; - int compress_gzip; - int compress_blob; - flb_sds_t account_name; - flb_sds_t container_name; - flb_sds_t blob_type; - flb_sds_t shared_key; - flb_sds_t endpoint; - flb_sds_t path; - flb_sds_t date_key; - - /* - * Internal use - */ - int btype; /* blob type */ - flb_sds_t real_endpoint; - flb_sds_t base_uri; - flb_sds_t shared_key_prefix; - - /* Shared key */ - unsigned char *decoded_sk; /* decoded shared key */ - size_t decoded_sk_size; /* size of decoded shared key */ - - /* Upstream connection */ - struct flb_upstream *u; - struct flb_output_instance *ins; -}; - -#endif diff --git a/fluent-bit/plugins/out_azure_blob/azure_blob_appendblob.c b/fluent-bit/plugins/out_azure_blob/azure_blob_appendblob.c deleted file mode 100644 index 2d9a82171..000000000 --- a/fluent-bit/plugins/out_azure_blob/azure_blob_appendblob.c +++ /dev/null @@ -1,44 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include "azure_blob.h" -#include "azure_blob_conf.h" -#include "azure_blob_uri.h" - -flb_sds_t azb_append_blob_uri(struct flb_azure_blob *ctx, char *tag) -{ - flb_sds_t uri; - - uri = azb_uri_container(ctx); - if (!uri) { - return NULL; - } - - if (ctx->path) { - flb_sds_printf(&uri, "/%s/%s?comp=appendblock", ctx->path, tag); - } - else { - flb_sds_printf(&uri, "/%s?comp=appendblock", tag); - } - - return uri; -} diff --git a/fluent-bit/plugins/out_azure_blob/azure_blob_appendblob.h b/fluent-bit/plugins/out_azure_blob/azure_blob_appendblob.h deleted file mode 100644 index 9ab103b0f..000000000 --- a/fluent-bit/plugins/out_azure_blob/azure_blob_appendblob.h +++ /dev/null @@ -1,28 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef AZURE_BLOB_APPENDBLOB_H -#define AZURE_BLOB_APPENDBLOB_H - -#include -#include "azure_blob.h" - -flb_sds_t azb_append_blob_uri(struct flb_azure_blob *ctx, char *tag); - -#endif diff --git a/fluent-bit/plugins/out_azure_blob/azure_blob_blockblob.c b/fluent-bit/plugins/out_azure_blob/azure_blob_blockblob.c deleted file mode 100644 index a9b0e4a28..000000000 --- a/fluent-bit/plugins/out_azure_blob/azure_blob_blockblob.c +++ /dev/null @@ -1,238 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include - -#include "azure_blob.h" -#include "azure_blob_conf.h" -#include "azure_blob_uri.h" -#include "azure_blob_http.h" - -flb_sds_t azb_block_blob_uri(struct flb_azure_blob *ctx, char *tag, - char *blockid, uint64_t ms) -{ - int len; - flb_sds_t uri; - char *ext; - char *encoded_blockid; - - len = strlen(blockid); - encoded_blockid = azb_uri_encode(blockid, len); - if (!encoded_blockid) { - return NULL; - } - - uri = azb_uri_container(ctx); - if (!uri) { - flb_sds_destroy(encoded_blockid); - return NULL; - } - - if (ctx->compress_blob == FLB_TRUE) { - ext = ".gz"; - } - else { - ext = ""; - } - - if (ctx->path) { - flb_sds_printf(&uri, "/%s/%s.%" PRIu64 "%s?blockid=%s&comp=block", - ctx->path, tag, ms, ext, encoded_blockid); - } - else { - flb_sds_printf(&uri, "/%s.%" PRIu64 "%s?blockid=%s&comp=block", - tag, ms, ext, encoded_blockid); - } - - flb_sds_destroy(encoded_blockid); - return uri; -} - -flb_sds_t azb_block_blob_uri_commit(struct flb_azure_blob *ctx, - char *tag, uint64_t ms) -{ - char *ext; - flb_sds_t uri; - - uri = azb_uri_container(ctx); - if (!uri) { - return NULL; - } - - if (ctx->compress_blob == FLB_TRUE) { - ext = ".gz"; - } - else { - ext = ""; - } - - if (ctx->path) { - flb_sds_printf(&uri, "/%s/%s.%" PRIu64 "%s?comp=blocklist", ctx->path, tag, - ms, ext); - } - else { - flb_sds_printf(&uri, "/%s.%" PRIu64 "%s?comp=blocklist", tag, ms, ext); - } - - return uri; -} - -/* Generate a block id */ -char *azb_block_blob_id(uint64_t *ms) -{ - int len; - int ret; - double now; - char tmp[32]; - size_t size; - size_t o_len; - char *b64; - struct flb_time tm; - - /* Get current time */ - flb_time_get(&tm); - - /* - * Set outgoing time in milliseconds: this is used as a suffix for the - * block name - */ - *ms = ((tm.tm.tv_sec * 1000) + (tm.tm.tv_nsec / 1000000)); - - /* Convert time to double to format the block id */ - now = flb_time_to_double(&tm); - len = snprintf(tmp, sizeof(tmp), "flb-%.4f.id", now); - - /* Allocate space for the outgoing base64 buffer */ - size = (4 * ceil(((double) len / 3) + 1)); - b64 = flb_malloc(size); - if (!b64) { - return NULL; - } - - /* base64 encode block id */ - ret = flb_base64_encode((unsigned char *) b64, size, &o_len, - (unsigned char *) tmp, len); - if (ret != 0) { - flb_free(b64); - return NULL; - } - return b64; -} - -int azb_block_blob_commit(struct flb_azure_blob *ctx, char *blockid, char *tag, - uint64_t ms) -{ - int ret; - size_t b_sent; - flb_sds_t uri = NULL; - flb_sds_t payload; - struct flb_http_client *c; - struct flb_connection *u_conn; - - /* Get upstream connection */ - u_conn = flb_upstream_conn_get(ctx->u); - if (!u_conn) { - flb_plg_error(ctx->ins, - "cannot create upstream connection for blockblob commit"); - return FLB_RETRY; - } - - /* Compose commit URI */ - uri = azb_block_blob_uri_commit(ctx, tag, ms); - if (!uri) { - flb_upstream_conn_release(u_conn); - return FLB_ERROR; - } - - payload = flb_sds_create_size(256); - if (!payload) { - flb_sds_destroy(uri); - flb_upstream_conn_release(u_conn); - return FLB_ERROR; - } - - flb_sds_printf(&payload, - "" - "" - " %s" - "", - blockid); - - /* Create HTTP client context */ - c = flb_http_client(u_conn, FLB_HTTP_PUT, - uri, - payload, flb_sds_len(payload), NULL, 0, NULL, 0); - if (!c) { - flb_plg_error(ctx->ins, "cannot create HTTP client context"); - flb_sds_destroy(payload); - flb_sds_destroy(uri); - flb_upstream_conn_release(u_conn); - return FLB_RETRY; - } - - /* Prepare headers and authentication */ - azb_http_client_setup(ctx, c, flb_sds_len(payload), - FLB_FALSE, - AZURE_BLOB_CT_NONE, AZURE_BLOB_CE_NONE); - - /* Send HTTP request */ - ret = flb_http_do(c, &b_sent); - flb_sds_destroy(uri); - flb_sds_destroy(payload); - - /* Validate HTTP status */ - if (ret == -1) { - flb_plg_error(ctx->ins, "error sending append_blob"); - return FLB_RETRY; - } - - if (c->resp.status == 201) { - flb_plg_info(ctx->ins, "blob id %s committed successfully", blockid); - flb_http_client_destroy(c); - flb_upstream_conn_release(u_conn); - return FLB_OK; - } - else if (c->resp.status == 404) { - flb_plg_info(ctx->ins, "blob not found: %s", c->uri); - flb_http_client_destroy(c); - flb_upstream_conn_release(u_conn); - return FLB_RETRY; - } - else if (c->resp.payload_size > 0) { - flb_plg_error(ctx->ins, "cannot commit blob id %s\n%s", - blockid, c->resp.payload); - if (strstr(c->resp.payload, "must be 0 for Create Append")) { - flb_http_client_destroy(c); - flb_upstream_conn_release(u_conn); - return FLB_RETRY; - } - } - else { - flb_plg_error(ctx->ins, "cannot append content to blob"); - } - flb_http_client_destroy(c); - flb_upstream_conn_release(u_conn); - - return FLB_OK; -} diff --git a/fluent-bit/plugins/out_azure_blob/azure_blob_blockblob.h b/fluent-bit/plugins/out_azure_blob/azure_blob_blockblob.h deleted file mode 100644 index ee210d138..000000000 --- a/fluent-bit/plugins/out_azure_blob/azure_blob_blockblob.h +++ /dev/null @@ -1,32 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef AZURE_BLOB_BLOCKBLOB_H -#define AZURE_BLOB_BLOCKBLOB_H - -#include -#include "azure_blob.h" - -flb_sds_t azb_block_blob_uri(struct flb_azure_blob *ctx, char *tag, char *blockid, - uint64_t ms); -char *azb_block_blob_id(uint64_t *ms); -int azb_block_blob_commit(struct flb_azure_blob *ctx, char *blockid, char *tag, - uint64_t ms); - -#endif diff --git a/fluent-bit/plugins/out_azure_blob/azure_blob_conf.c b/fluent-bit/plugins/out_azure_blob/azure_blob_conf.c deleted file mode 100644 index 4437a6d2d..000000000 --- a/fluent-bit/plugins/out_azure_blob/azure_blob_conf.c +++ /dev/null @@ -1,245 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include "azure_blob.h" -#include "azure_blob_conf.h" - -#include -#include -#include - -static int set_shared_key(struct flb_azure_blob *ctx) -{ - int s; - int ret; - size_t o_len = 0; - - s = flb_sds_len(ctx->shared_key); - - /* buffer for final hex key */ - ctx->decoded_sk = flb_malloc(s * 2); - if (!ctx->decoded_sk) { - return -1; - } - - /* decode base64 */ - ret = flb_base64_decode(ctx->decoded_sk, s * 2, - &o_len, - (unsigned char *)ctx->shared_key, - flb_sds_len(ctx->shared_key)); - if (ret != 0) { - flb_plg_error(ctx->ins, "cannot decode shared_key"); - return -1; - } - - ctx->decoded_sk_size = o_len; - return 0; -} - -struct flb_azure_blob *flb_azure_blob_conf_create(struct flb_output_instance *ins, - struct flb_config *config) -{ - int ret; - int port; - int io_flags = 0; - flb_sds_t tmp; - struct flb_azure_blob *ctx; - - ctx = flb_calloc(1, sizeof(struct flb_azure_blob)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - - /* Set context */ - flb_output_set_context(ins, ctx); - - /* Load config map */ - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - return NULL; - } - - if (!ctx->container_name) { - flb_plg_error(ctx->ins, "'container_name' has not been set"); - return NULL; - } - - /* If the shared key is set decode it */ - if (ctx->shared_key) { - ret = set_shared_key(ctx); - if (ret == -1) { - return NULL; - } - } - - /* Set Blob type */ - tmp = (char *) flb_output_get_property("blob_type", ins); - if (!tmp) { - ctx->btype = AZURE_BLOB_APPENDBLOB; - } - else { - if (strcasecmp(tmp, "appendblob") == 0) { - ctx->btype = AZURE_BLOB_APPENDBLOB; - } - else if (strcasecmp(tmp, "blockblob") == 0) { - ctx->btype = AZURE_BLOB_BLOCKBLOB; - } - else { - flb_plg_error(ctx->ins, "invalid blob_type value '%s'", tmp); - return NULL; - } - } - - /* Compress (gzip) */ - tmp = (char *) flb_output_get_property("compress", ins); - ctx->compress_gzip = FLB_FALSE; - if (tmp) { - if (strcasecmp(tmp, "gzip") == 0) { - ctx->compress_gzip = FLB_TRUE; - } - } - - /* Compress Blob: only availabel for blockblob type */ - if (ctx->compress_blob == FLB_TRUE && ctx->btype != AZURE_BLOB_BLOCKBLOB) { - flb_plg_error(ctx->ins, - "the option 'compress_blob' is not compatible with 'appendblob' " - "blob_type"); - return NULL; - } - - /* - * Setting up the real endpoint: - * - * If the user provided a custom endpoint, just parse it. Here we need to - * discover if a TLS connection is required, just use the protocol prefix. - */ - if (ctx->endpoint) { - if (strncmp(ctx->endpoint, "https", 5) == 0) { - io_flags |= FLB_IO_TLS; - } - else { - io_flags |= FLB_IO_TCP; - } - - ctx->u = flb_upstream_create_url(config, ctx->endpoint, - io_flags, ins->tls); - if (!ctx->u) { - flb_plg_error(ctx->ins, "invalid endpoint '%s'", ctx->endpoint); - return NULL; - } - ctx->real_endpoint = flb_sds_create(ctx->endpoint); - } - else { - ctx->real_endpoint = flb_sds_create_size(256); - if (!ctx->real_endpoint) { - flb_plg_error(ctx->ins, "cannot create endpoint"); - return NULL; - } - flb_sds_printf(&ctx->real_endpoint, "%s%s", - ctx->account_name, - AZURE_ENDPOINT_PREFIX); - - /* use TLS ? */ - if (ins->use_tls == FLB_TRUE) { - port = 443; - io_flags = FLB_IO_TLS; - } - else { - port = 80; - io_flags = FLB_IO_TCP; - } - - ctx->u = flb_upstream_create(config, ctx->real_endpoint, port, io_flags, - ins->tls); - if (!ctx->u) { - flb_plg_error(ctx->ins, "cannot create upstream for endpoint '%s'", - ctx->real_endpoint); - return NULL; - } - } - flb_output_upstream_set(ctx->u, ins); - - /* Compose base uri */ - ctx->base_uri = flb_sds_create_size(256); - if (!ctx->base_uri) { - flb_plg_error(ctx->ins, "cannot create base_uri for endpoint '%s'", - ctx->real_endpoint); - return NULL; - } - - if (ctx->emulator_mode == FLB_TRUE) { - flb_sds_printf(&ctx->base_uri, "/%s/", ctx->account_name); - } - else { - flb_sds_printf(&ctx->base_uri, "/"); - } - - /* Prepare shared key buffer */ - ctx->shared_key_prefix = flb_sds_create_size(256); - if (!ctx->shared_key_prefix) { - flb_plg_error(ctx->ins, "cannot create shared key prefix"); - return NULL; - } - flb_sds_printf(&ctx->shared_key_prefix, "SharedKey %s:", ctx->account_name); - - /* Sanitize path: remove any ending slash */ - if (ctx->path) { - if (ctx->path[flb_sds_len(ctx->path) - 1] == '/') { - ctx->path[flb_sds_len(ctx->path) - 1] = '\0'; - } - } - - flb_plg_info(ctx->ins, - "account_name=%s, container_name=%s, blob_type=%s, emulator_mode=%s, endpoint=%s", - ctx->account_name, ctx->container_name, - ctx->btype == AZURE_BLOB_APPENDBLOB ? "appendblob": "blockblob", - ctx->emulator_mode ? "yes": "no", - ctx->real_endpoint ? ctx->real_endpoint: "no"); - return ctx; -} - -void flb_azure_blob_conf_destroy(struct flb_azure_blob *ctx) -{ - if (ctx->decoded_sk) { - flb_free(ctx->decoded_sk); - } - - if (ctx->base_uri) { - flb_sds_destroy(ctx->base_uri); - } - - if (ctx->real_endpoint) { - flb_sds_destroy(ctx->real_endpoint); - } - - if (ctx->shared_key_prefix) { - flb_sds_destroy(ctx->shared_key_prefix); - } - - if (ctx->u) { - flb_upstream_destroy(ctx->u); - } - - flb_free(ctx); -} diff --git a/fluent-bit/plugins/out_azure_blob/azure_blob_conf.h b/fluent-bit/plugins/out_azure_blob/azure_blob_conf.h deleted file mode 100644 index 32a85c678..000000000 --- a/fluent-bit/plugins/out_azure_blob/azure_blob_conf.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_AZURE_BLOB_CONF_H -#define FLB_OUT_AZURE_BLOB_CONF_H - -#include - -struct flb_azure_blob *flb_azure_blob_conf_create(struct flb_output_instance *ins, - struct flb_config *config); -void flb_azure_blob_conf_destroy(struct flb_azure_blob *ctx); - -#endif diff --git a/fluent-bit/plugins/out_azure_blob/azure_blob_http.c b/fluent-bit/plugins/out_azure_blob/azure_blob_http.c deleted file mode 100644 index 5ac81a9a1..000000000 --- a/fluent-bit/plugins/out_azure_blob/azure_blob_http.c +++ /dev/null @@ -1,361 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "azure_blob.h" -#include "azure_blob_uri.h" - -static int hmac_sha256_sign(unsigned char out[32], - unsigned char *key, size_t key_len, - unsigned char *msg, size_t msg_len) -{ - return flb_hmac_simple(FLB_HASH_SHA256, - key, key_len, - msg, msg_len, - out, 32); -} - -static flb_sds_t canonical_headers(struct flb_http_client *c) -{ - flb_sds_t ch; - flb_sds_t tmp; - struct flb_kv *kv; - struct mk_list *head; - - ch = flb_sds_create_size(mk_list_size(&c->headers) * 64); - if (!ch) { - return NULL; - } - - mk_list_foreach(head, &c->headers) { - kv = mk_list_entry(head, struct flb_kv, _head); - if (strncmp(kv->key, "x-ms-", 5) != 0) { - continue; - } - - /* key */ - tmp = flb_sds_cat(ch, kv->key, flb_sds_len(kv->key)); - if (!tmp) { - flb_sds_destroy(ch); - return NULL; - } - ch = tmp; - - /* sep */ - tmp = flb_sds_cat(ch, ":", 1); - if (!tmp) { - flb_sds_destroy(ch); - return NULL; - } - ch = tmp; - - /* value */ - tmp = flb_sds_cat(ch, kv->val, flb_sds_len(kv->val)); - if (!tmp) { - flb_sds_destroy(ch); - return NULL; - } - ch = tmp; - - tmp = flb_sds_cat(ch, "\n", 1); - if (!tmp) { - flb_sds_destroy(ch); - return NULL; - } - ch = tmp; - } - - return ch; -} - -static flb_sds_t canonical_resource(struct flb_azure_blob *ctx, - struct flb_http_client *c) -{ - int pos; - int len; - int kv_start; - char *p; - size_t size; - flb_sds_t cr; - flb_sds_t dec_uri; - flb_sds_t tmp; - - len = strlen(c->uri); - size = flb_sds_len(ctx->account_name) + len + 64; - - cr = flb_sds_create_size(size); - if (!cr) { - return NULL; - } - - dec_uri = azb_uri_decode(c->uri, len); - tmp = flb_sds_printf(&cr, "/%s%s", ctx->account_name, dec_uri); - if (!tmp) { - flb_sds_destroy(dec_uri); - flb_sds_destroy(cr); - return NULL; - } - flb_sds_destroy(dec_uri); - - pos = 1 + flb_sds_len(ctx->account_name); - - p = strchr(cr + pos, '?'); - if (p) { - kv_start = FLB_TRUE; - while (*p) { - if (*p == '?') { - *p = '\n'; - } - else if (*p == '=' && kv_start == FLB_TRUE) { - *p = ':'; - kv_start = FLB_FALSE; - } - else if (*p == '&') { - *p = '\n'; - kv_start = FLB_TRUE; - } - p++; - } - } - - return cr; -} - -flb_sds_t azb_http_canonical_request(struct flb_azure_blob *ctx, - struct flb_http_client *c, - ssize_t content_length, - int content_type, - int content_encoding) -{ - int ret; - size_t size; - size_t o_len = 0; - flb_sds_t can_req; - flb_sds_t can_res; - flb_sds_t can_headers; - flb_sds_t tmp = NULL; - char *b64 = NULL; - char *encoding; - char *ctype = ""; - unsigned char signature[32]; - - size = strlen(c->uri) + (mk_list_size(&c->headers) * 64) + 256; - can_req = flb_sds_create_size(size); - if (!can_req) { - flb_plg_error(ctx->ins, "cannot allocate buffer for canonical request"); - return NULL; - } - - switch (c->method) { - case FLB_HTTP_GET: - tmp = flb_sds_cat(can_req, "GET\n", 4); - break; - case FLB_HTTP_POST: - tmp = flb_sds_cat(can_req, "POST\n", 5); - break; - case FLB_HTTP_PUT: - tmp = flb_sds_cat(can_req, "PUT\n", 4); - break; - }; - - if (!tmp) { - flb_plg_error(ctx->ins, "invalid processing HTTP method"); - flb_sds_destroy(can_req); - return NULL; - } - - if (content_encoding == AZURE_BLOB_CE_GZIP) { - encoding = "gzip"; - } - else { - encoding = ""; - } - - flb_sds_printf(&can_req, - "%s\n" /* Content-Encoding */ - "\n", /* Content-Language */ - encoding - ); - - if (content_length >= 0) { - flb_sds_printf(&can_req, - "%zi\n" /* Content-Length */, - content_length); - } - else { - flb_sds_printf(&can_req, - "\n" /* Content-Length */ - ); - } - - if (content_type == AZURE_BLOB_CT_NONE) { - ctype = ""; - } - else if (content_type == AZURE_BLOB_CT_JSON) { - ctype = "application/json"; - } - else if (content_type == AZURE_BLOB_CT_GZIP) { - ctype = "application/gzip"; - } - - flb_sds_printf(&can_req, - "\n" /* Content-MD5 */ - "%s\n" /* Content-Type */ - "\n" /* Date */ - "\n" /* If-Modified-Since */ - "\n" /* If-Match */ - "\n" /* If-None-Match */ - "\n" /* If-Unmodified-Since */ - "\n" /* Range */, - ctype); - - /* Append canonicalized headers */ - can_headers = canonical_headers(c); - if (!can_headers) { - flb_sds_destroy(can_req); - return NULL; - } - tmp = flb_sds_cat(can_req, can_headers, flb_sds_len(can_headers)); - if (!tmp) { - flb_sds_destroy(can_req); - flb_sds_destroy(can_headers); - return NULL; - } - can_req = tmp; - flb_sds_destroy(can_headers); - - /* Append canonical resource */ - can_res = canonical_resource(ctx, c); - if (!can_res) { - flb_sds_destroy(can_req); - return NULL; - } - tmp = flb_sds_cat(can_req, can_res, flb_sds_len(can_res)); - if (!tmp) { - flb_sds_destroy(can_res); - flb_sds_destroy(can_req); - return NULL; - } - can_req = tmp; - flb_sds_destroy(can_res); - - flb_plg_trace(ctx->ins, "string to sign\n%s", can_req); - - /* Signature */ - hmac_sha256_sign(signature, ctx->decoded_sk, ctx->decoded_sk_size, - (unsigned char *) can_req, flb_sds_len(can_req)); - flb_sds_destroy(can_req); - - /* base64 decoded size */ - size = ((4 * ((sizeof(signature) + 1)) / 3) + 1); - b64 = flb_sds_create_size(size); - if (!b64) { - return NULL; - } - - ret = flb_base64_encode((unsigned char *) b64, size, &o_len, - signature, sizeof(signature)); - if (ret != 0) { - flb_sds_destroy(b64); - return NULL; - } - flb_sds_len_set(b64, o_len); - - return b64; -} - -int azb_http_client_setup(struct flb_azure_blob *ctx, struct flb_http_client *c, - ssize_t content_length, int blob_type, - int content_type, int content_encoding) -{ - int len; - time_t now; - struct tm tm; - char tmp[64]; - flb_sds_t can_req; - flb_sds_t auth; - - /* Header: User Agent */ - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - - /* Header: Content-Type */ - if (content_type == AZURE_BLOB_CT_JSON) { - flb_http_add_header(c, - AZURE_BLOB_CT, sizeof(AZURE_BLOB_CT) - 1, - "application/json", 16); - } - else if (content_type == AZURE_BLOB_CT_GZIP) { - flb_http_add_header(c, - AZURE_BLOB_CT, sizeof(AZURE_BLOB_CT) - 1, - "application/gzip", 16); - } - - if (content_encoding == AZURE_BLOB_CE_GZIP) { - flb_http_add_header(c, - AZURE_BLOB_CE, sizeof(AZURE_BLOB_CE) - 1, - "gzip", 4); - } - - /* Azure header: x-ms-blob-type */ - if (blob_type == FLB_TRUE) { - if (ctx->btype == AZURE_BLOB_APPENDBLOB) { - flb_http_add_header(c, "x-ms-blob-type", 14, "AppendBlob", 10); - } - else if (ctx->btype == AZURE_BLOB_BLOCKBLOB) { - flb_http_add_header(c, "x-ms-blob-type", 14, "BlockBlob", 9); - } - } - - /* Azure header: x-ms-date */ - now = time(NULL); - gmtime_r(&now, &tm); - len = strftime(tmp, sizeof(tmp) - 1, "%a, %d %b %Y %H:%M:%S GMT", &tm); - - flb_http_add_header(c, "x-ms-date", 9, tmp, len); - - /* Azure header: x-ms-version */ - flb_http_add_header(c, "x-ms-version", 12, "2019-12-12", 10); - - can_req = azb_http_canonical_request(ctx, c, content_length, content_type, - content_encoding); - - auth = flb_sds_create_size(64 + flb_sds_len(can_req)); - - flb_sds_cat(auth, ctx->shared_key_prefix, flb_sds_len(ctx->shared_key_prefix)); - flb_sds_cat(auth, can_req, flb_sds_len(can_req)); - - /* Azure header: authorization */ - flb_http_add_header(c, "Authorization", 13, auth, flb_sds_len(auth)); - - /* Release buffers */ - flb_sds_destroy(can_req); - flb_sds_destroy(auth); - - /* Set callback context to the HTTP client context */ - flb_http_set_callback_context(c, ctx->ins->callback); - - return 0; -} diff --git a/fluent-bit/plugins/out_azure_blob/azure_blob_http.h b/fluent-bit/plugins/out_azure_blob/azure_blob_http.h deleted file mode 100644 index 04f7cfd98..000000000 --- a/fluent-bit/plugins/out_azure_blob/azure_blob_http.h +++ /dev/null @@ -1,36 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef AZURE_BLOB_HTTP_H -#define AZURE_BLOB_HTTP_H - -#include -#include -#include "azure_blob.h" - -int azb_http_client_setup(struct flb_azure_blob *ctx, struct flb_http_client *c, - ssize_t content_length, int blob_type, - int content_type, int content_encoding); - -flb_sds_t azb_http_canonical_request(struct flb_azure_blob *ctx, - struct flb_http_client *c, - ssize_t content_length, - int content_type); - -#endif diff --git a/fluent-bit/plugins/out_azure_blob/azure_blob_uri.c b/fluent-bit/plugins/out_azure_blob/azure_blob_uri.c deleted file mode 100644 index c7a05e286..000000000 --- a/fluent-bit/plugins/out_azure_blob/azure_blob_uri.c +++ /dev/null @@ -1,150 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include "azure_blob.h" - -static inline int to_encode(char c) -{ - if ((c >= 48 && c <= 57) || /* 0-9 */ - (c >= 65 && c <= 90) || /* A-Z */ - (c >= 97 && c <= 122) || /* a-z */ - (c == '?' || c == '&' || c == '-' || c == '_' || c == '.' || - c == '~' || c == '/')) { - return FLB_FALSE; - } - - return FLB_TRUE; -} - -flb_sds_t azb_uri_encode(const char *uri, size_t len) -{ - int i; - flb_sds_t buf = NULL; - flb_sds_t tmp = NULL; - - buf = flb_sds_create_size(len * 2); - if (!buf) { - flb_error("[uri] cannot allocate buffer for URI encoding"); - return NULL; - } - - for (i = 0; i < len; i++) { - if (to_encode(uri[i]) == FLB_TRUE) { - tmp = flb_sds_printf(&buf, "%%%02X", (unsigned char) *(uri + i)); - if (!tmp) { - flb_sds_destroy(buf); - return NULL; - } - continue; - } - - /* Direct assignment, just copy the character */ - if (buf) { - tmp = flb_sds_cat(buf, uri + i, 1); - if (!tmp) { - flb_sds_destroy(buf); - return NULL; - } - buf = tmp; - } - } - - return buf; -} - -flb_sds_t azb_uri_decode(const char *uri, size_t len) -{ - int i; - int hex_result; - int c = 0; - char hex[3]; - flb_sds_t out; - - out = flb_sds_create_size(len); - if (!out) { - return NULL; - } - - for (i = 0; i < len; i++) { - if (uri[i] == '%') { - hex[0] = uri[i + 1]; - hex[1] = uri[i + 2]; - hex[2] = '\0'; - - hex_result = flb_utils_hex2int(hex, 2); - out[c++] = hex_result; - i += 2; - } - else { - out[c++] = uri[i]; - } - } - out[c++] = '\0'; - - return out; -} - -flb_sds_t azb_uri_container(struct flb_azure_blob *ctx) -{ - flb_sds_t uri; - - uri = flb_sds_create_size(256); - if (!uri) { - return NULL; - } - - flb_sds_printf(&uri, "%s%s", ctx->base_uri, ctx->container_name); - return uri; -} - -flb_sds_t azb_uri_ensure_or_create_container(struct flb_azure_blob *ctx) -{ - flb_sds_t uri; - - uri = azb_uri_container(ctx); - if (!uri) { - return NULL; - } - - flb_sds_printf(&uri, "?restype=container"); - return uri; -} - -flb_sds_t azb_uri_create_blob(struct flb_azure_blob *ctx, char *tag) -{ - flb_sds_t uri; - - uri = azb_uri_container(ctx); - if (!uri) { - return NULL; - } - - if (ctx->path) { - flb_sds_printf(&uri, "/%s/%s", ctx->path, tag); - } - else { - flb_sds_printf(&uri, "/%s", tag); - } - - return uri; -} diff --git a/fluent-bit/plugins/out_azure_blob/azure_blob_uri.h b/fluent-bit/plugins/out_azure_blob/azure_blob_uri.h deleted file mode 100644 index ffeed7636..000000000 --- a/fluent-bit/plugins/out_azure_blob/azure_blob_uri.h +++ /dev/null @@ -1,34 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_AZURE_BLOB_URI -#define FLB_AZURE_BLOB_URI - -#include -#include - -#include "azure_blob.h" - -flb_sds_t azb_uri_container(struct flb_azure_blob *ctx); -flb_sds_t azb_uri_ensure_or_create_container(struct flb_azure_blob *ctx); -flb_sds_t azb_uri_create_blob(struct flb_azure_blob *ctx, char *tag); -flb_sds_t azb_uri_encode(const char *uri, size_t len); -flb_sds_t azb_uri_decode(const char *uri, size_t len); - -#endif diff --git a/fluent-bit/plugins/out_azure_kusto/CMakeLists.txt b/fluent-bit/plugins/out_azure_kusto/CMakeLists.txt deleted file mode 100644 index 6803bee09..000000000 --- a/fluent-bit/plugins/out_azure_kusto/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -set(src - azure_kusto.c - azure_kusto_conf.c - azure_kusto_ingest.c - ) - -FLB_PLUGIN(out_azure_kusto "${src}" "") diff --git a/fluent-bit/plugins/out_azure_kusto/azure_kusto.c b/fluent-bit/plugins/out_azure_kusto/azure_kusto.c deleted file mode 100644 index 4b8ad9b82..000000000 --- a/fluent-bit/plugins/out_azure_kusto/azure_kusto.c +++ /dev/null @@ -1,477 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "azure_kusto.h" -#include "azure_kusto_conf.h" -#include "azure_kusto_ingest.h" - -/* Create a new oauth2 context and get a oauth2 token */ -static int azure_kusto_get_oauth2_token(struct flb_azure_kusto *ctx) -{ - int ret; - char *token; - - /* Clear any previous oauth2 payload content */ - flb_oauth2_payload_clear(ctx->o); - - ret = flb_oauth2_payload_append(ctx->o, "grant_type", 10, "client_credentials", 18); - if (ret == -1) { - flb_plg_error(ctx->ins, "error appending oauth2 params"); - return -1; - } - - ret = flb_oauth2_payload_append(ctx->o, "scope", 5, FLB_AZURE_KUSTO_SCOPE, 39); - if (ret == -1) { - flb_plg_error(ctx->ins, "error appending oauth2 params"); - return -1; - } - - ret = flb_oauth2_payload_append(ctx->o, "client_id", 9, ctx->client_id, -1); - if (ret == -1) { - flb_plg_error(ctx->ins, "error appending oauth2 params"); - return -1; - } - - ret = flb_oauth2_payload_append(ctx->o, "client_secret", 13, ctx->client_secret, -1); - if (ret == -1) { - flb_plg_error(ctx->ins, "error appending oauth2 params"); - return -1; - } - - /* Retrieve access token */ - token = flb_oauth2_token_get(ctx->o); - if (!token) { - flb_plg_error(ctx->ins, "error retrieving oauth2 access token"); - return -1; - } - - return 0; -} - -flb_sds_t get_azure_kusto_token(struct flb_azure_kusto *ctx) -{ - int ret = 0; - flb_sds_t output = NULL; - - if (pthread_mutex_lock(&ctx->token_mutex)) { - flb_plg_error(ctx->ins, "error locking mutex"); - return NULL; - } - - if (flb_oauth2_token_expired(ctx->o) == FLB_TRUE) { - ret = azure_kusto_get_oauth2_token(ctx); - } - - /* Copy string to prevent race conditions (get_oauth2 can free the string) */ - if (ret == 0) { - output = flb_sds_create_size(flb_sds_len(ctx->o->token_type) + - flb_sds_len(ctx->o->access_token) + 2); - if (!output) { - flb_plg_error(ctx->ins, "error creating token buffer"); - return NULL; - } - flb_sds_snprintf(&output, flb_sds_alloc(output), "%s %s", ctx->o->token_type, - ctx->o->access_token); - } - - if (pthread_mutex_unlock(&ctx->token_mutex)) { - flb_plg_error(ctx->ins, "error unlocking mutex"); - if (output) { - flb_sds_destroy(output); - } - return NULL; - } - - return output; -} - -/** - * Executes a control command against kusto's endpoint - * - * @param ctx Plugin's context - * @param csl Kusto's control command - * @return flb_sds_t Returns the response or NULL on error. - */ -flb_sds_t execute_ingest_csl_command(struct flb_azure_kusto *ctx, const char *csl) -{ - flb_sds_t token; - flb_sds_t body; - size_t b_sent; - int ret; - struct flb_connection *u_conn; - struct flb_http_client *c; - flb_sds_t resp = NULL; - - /* Get upstream connection */ - u_conn = flb_upstream_conn_get(ctx->u); - - if (u_conn) { - token = get_azure_kusto_token(ctx); - - if (token) { - /* Compose request body */ - body = flb_sds_create_size(sizeof(FLB_AZURE_KUSTO_MGMT_BODY_TEMPLATE) - 1 + - strlen(csl)); - - if (body) { - flb_sds_snprintf(&body, flb_sds_alloc(body), - FLB_AZURE_KUSTO_MGMT_BODY_TEMPLATE, csl); - - /* Compose HTTP Client request */ - c = flb_http_client(u_conn, FLB_HTTP_POST, FLB_AZURE_KUSTO_MGMT_URI_PATH, - body, flb_sds_len(body), NULL, 0, NULL, 0); - - if (c) { - /* Add headers */ - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - flb_http_add_header(c, "Content-Type", 12, "application/json", 16); - flb_http_add_header(c, "Accept", 6, "application/json", 16); - flb_http_add_header(c, "Authorization", 13, token, - flb_sds_len(token)); - flb_http_buffer_size(c, FLB_HTTP_DATA_SIZE_MAX * 10); - - /* Send HTTP request */ - ret = flb_http_do(c, &b_sent); - flb_plg_debug( - ctx->ins, - "Kusto ingestion command request http_do=%i, HTTP Status: %i", - ret, c->resp.status); - - if (ret == 0) { - if (c->resp.status == 200) { - /* Copy payload response to the response param */ - resp = - flb_sds_create_len(c->resp.payload, c->resp.payload_size); - } - else if (c->resp.payload_size > 0) { - flb_plg_debug(ctx->ins, "Request failed and returned: \n%s", - c->resp.payload); - } - else { - flb_plg_debug(ctx->ins, "Request failed"); - } - } - else { - flb_plg_error(ctx->ins, "cannot send HTTP request"); - } - - flb_http_client_destroy(c); - } - else { - flb_plg_error(ctx->ins, "cannot create HTTP client context"); - } - - flb_sds_destroy(body); - } - else { - flb_plg_error(ctx->ins, "cannot construct request body"); - } - - flb_sds_destroy(token); - } - else { - flb_plg_error(ctx->ins, "cannot retrieve oauth2 token"); - } - - flb_upstream_conn_release(u_conn); - } - else { - flb_plg_error(ctx->ins, "cannot create upstream connection"); - } - - return resp; -} - -static int cb_azure_kusto_init(struct flb_output_instance *ins, struct flb_config *config, - void *data) -{ - int io_flags = FLB_IO_TLS; - struct flb_azure_kusto *ctx; - - /* Create config context */ - ctx = flb_azure_kusto_conf_create(ins, config); - if (!ctx) { - flb_plg_error(ins, "configuration failed"); - return -1; - } - - flb_output_set_context(ins, ctx); - - /* Network mode IPv6 */ - if (ins->host.ipv6 == FLB_TRUE) { - io_flags |= FLB_IO_IPV6; - } - - /* Create mutex for acquiring oauth tokens and getting ingestion resources (they - * are shared across flush coroutines) - */ - pthread_mutex_init(&ctx->token_mutex, NULL); - pthread_mutex_init(&ctx->resources_mutex, NULL); - - /* - * Create upstream context for Kusto Ingestion endpoint - */ - ctx->u = flb_upstream_create_url(config, ctx->ingestion_endpoint, io_flags, ins->tls); - if (!ctx->u) { - flb_plg_error(ctx->ins, "upstream creation failed"); - return -1; - } - - /* Create oauth2 context */ - ctx->o = - flb_oauth2_create(ctx->config, ctx->oauth_url, FLB_AZURE_KUSTO_TOKEN_REFRESH); - if (!ctx->o) { - flb_plg_error(ctx->ins, "cannot create oauth2 context"); - return -1; - } - flb_output_upstream_set(ctx->u, ins); - - return 0; -} - -static int azure_kusto_format(struct flb_azure_kusto *ctx, const char *tag, int tag_len, - const void *data, size_t bytes, void **out_data, - size_t *out_size) -{ - int records = 0; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - /* for sub msgpack objs */ - int map_size; - struct tm tms; - char time_formatted[32]; - size_t s; - int len; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - int ret; - /* output buffer */ - flb_sds_t out_buf; - - /* Create array for all records */ - records = flb_mp_count(data, bytes); - if (records <= 0) { - flb_plg_error(ctx->ins, "error counting msgpack entries"); - return -1; - } - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return -1; - } - - /* Create temporary msgpack buffer */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - msgpack_pack_array(&mp_pck, records); - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - map_size = 1; - if (ctx->include_time_key == FLB_TRUE) { - map_size++; - } - - if (ctx->include_tag_key == FLB_TRUE) { - map_size++; - } - - msgpack_pack_map(&mp_pck, map_size); - - /* include_time_key */ - if (ctx->include_time_key == FLB_TRUE) { - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->time_key)); - msgpack_pack_str_body(&mp_pck, ctx->time_key, flb_sds_len(ctx->time_key)); - - /* Append the time value as ISO 8601 */ - gmtime_r(&log_event.timestamp.tm.tv_sec, &tms); - s = strftime(time_formatted, sizeof(time_formatted) - 1, - FLB_PACK_JSON_DATE_ISO8601_FMT, &tms); - - len = snprintf(time_formatted + s, sizeof(time_formatted) - 1 - s, - ".%03" PRIu64 "Z", - (uint64_t)log_event.timestamp.tm.tv_nsec / 1000000); - s += len; - msgpack_pack_str(&mp_pck, s); - msgpack_pack_str_body(&mp_pck, time_formatted, s); - } - - /* include_tag_key */ - if (ctx->include_tag_key == FLB_TRUE) { - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->tag_key)); - msgpack_pack_str_body(&mp_pck, ctx->tag_key, flb_sds_len(ctx->tag_key)); - msgpack_pack_str(&mp_pck, tag_len); - msgpack_pack_str_body(&mp_pck, tag, tag_len); - } - - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->log_key)); - msgpack_pack_str_body(&mp_pck, ctx->log_key, flb_sds_len(ctx->log_key)); - msgpack_pack_object(&mp_pck, *log_event.body); - } - - /* Convert from msgpack to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); - - /* Cleanup */ - flb_log_event_decoder_destroy(&log_decoder); - msgpack_sbuffer_destroy(&mp_sbuf); - - if (!out_buf) { - flb_plg_error(ctx->ins, "error formatting JSON payload"); - return -1; - } - - *out_data = out_buf; - *out_size = flb_sds_len(out_buf); - - return 0; -} - -static void cb_azure_kusto_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, void *out_context, - struct flb_config *config) -{ - int ret; - flb_sds_t json; - size_t json_size; - size_t tag_len; - struct flb_azure_kusto *ctx = out_context; - - (void)i_ins; - (void)config; - - flb_plg_trace(ctx->ins, "flushing bytes %zu", event_chunk->size); - - tag_len = flb_sds_len(event_chunk->tag); - - /* Load or refresh ingestion resources */ - ret = azure_kusto_load_ingestion_resources(ctx, config); - if (ret != 0) { - flb_plg_error(ctx->ins, "cannot load ingestion resources"); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Reformat msgpack to JSON payload */ - ret = azure_kusto_format(ctx, event_chunk->tag, tag_len, event_chunk->data, - event_chunk->size, (void **)&json, &json_size); - if (ret != 0) { - flb_plg_error(ctx->ins, "cannot reformat data into json"); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - ret = azure_kusto_queued_ingestion(ctx, event_chunk->tag, tag_len, json, json_size); - if (ret != 0) { - flb_plg_error(ctx->ins, "cannot perform queued ingestion"); - flb_sds_destroy(json); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Cleanup */ - flb_sds_destroy(json); - - /* Done */ - FLB_OUTPUT_RETURN(FLB_OK); -} - -static int cb_azure_kusto_exit(void *data, struct flb_config *config) -{ - struct flb_azure_kusto *ctx = data; - - if (!ctx) { - return -1; - } - - if (ctx->u) { - flb_upstream_destroy(ctx->u); - ctx->u = NULL; - } - - flb_azure_kusto_conf_destroy(ctx); - - return 0; -} - -static struct flb_config_map config_map[] = { - {FLB_CONFIG_MAP_STR, "tenant_id", (char *)NULL, 0, FLB_TRUE, - offsetof(struct flb_azure_kusto, tenant_id), - "Set the tenant ID of the AAD application used for authentication"}, - {FLB_CONFIG_MAP_STR, "client_id", (char *)NULL, 0, FLB_TRUE, - offsetof(struct flb_azure_kusto, client_id), - "Set the client ID (Application ID) of the AAD application used for authentication"}, - {FLB_CONFIG_MAP_STR, "client_secret", (char *)NULL, 0, FLB_TRUE, - offsetof(struct flb_azure_kusto, client_secret), - "Set the client secret (Application Password) of the AAD application used for " - "authentication"}, - {FLB_CONFIG_MAP_STR, "ingestion_endpoint", (char *)NULL, 0, FLB_TRUE, - offsetof(struct flb_azure_kusto, ingestion_endpoint), - "Set the Kusto cluster's ingestion endpoint URL (e.g. " - "https://ingest-mycluster.eastus.kusto.windows.net)"}, - {FLB_CONFIG_MAP_STR, "database_name", (char *)NULL, 0, FLB_TRUE, - offsetof(struct flb_azure_kusto, database_name), "Set the database name"}, - {FLB_CONFIG_MAP_STR, "table_name", (char *)NULL, 0, FLB_TRUE, - offsetof(struct flb_azure_kusto, table_name), "Set the table name"}, - {FLB_CONFIG_MAP_STR, "ingestion_mapping_reference", (char *)NULL, 0, FLB_TRUE, - offsetof(struct flb_azure_kusto, ingestion_mapping_reference), - "Set the ingestion mapping reference"}, - {FLB_CONFIG_MAP_STR, "log_key", FLB_AZURE_KUSTO_DEFAULT_LOG_KEY, 0, FLB_TRUE, - offsetof(struct flb_azure_kusto, log_key), "The key name of event payload"}, - {FLB_CONFIG_MAP_BOOL, "include_tag_key", "true", 0, FLB_TRUE, - offsetof(struct flb_azure_kusto, include_tag_key), - "If enabled, tag is appended to output. " - "The key name is used 'tag_key' property."}, - {FLB_CONFIG_MAP_STR, "tag_key", FLB_AZURE_KUSTO_DEFAULT_TAG_KEY, 0, FLB_TRUE, - offsetof(struct flb_azure_kusto, tag_key), - "The key name of tag. If 'include_tag_key' is false, " - "This property is ignored"}, - {FLB_CONFIG_MAP_BOOL, "include_time_key", "true", 0, FLB_TRUE, - offsetof(struct flb_azure_kusto, include_time_key), - "If enabled, time is appended to output. " - "The key name is used 'time_key' property."}, - {FLB_CONFIG_MAP_STR, "time_key", FLB_AZURE_KUSTO_DEFAULT_TIME_KEY, 0, FLB_TRUE, - offsetof(struct flb_azure_kusto, time_key), - "The key name of the time. If 'include_time_key' is false, " - "This property is ignored"}, - /* EOF */ - {0}}; - -struct flb_output_plugin out_azure_kusto_plugin = { - .name = "azure_kusto", - .description = "Send events to Kusto (Azure Data Explorer)", - .cb_init = cb_azure_kusto_init, - .cb_flush = cb_azure_kusto_flush, - .cb_exit = cb_azure_kusto_exit, - .config_map = config_map, - /* Plugin flags */ - .flags = FLB_OUTPUT_NET | FLB_IO_TLS, -}; diff --git a/fluent-bit/plugins/out_azure_kusto/azure_kusto.h b/fluent-bit/plugins/out_azure_kusto/azure_kusto.h deleted file mode 100644 index ac4eedfd0..000000000 --- a/fluent-bit/plugins/out_azure_kusto/azure_kusto.h +++ /dev/null @@ -1,110 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_AZURE_KUSTO -#define FLB_OUT_AZURE_KUSTO - -#include -#include -#include -#include -#include - -/* refresh token every 50 minutes */ -#define FLB_AZURE_KUSTO_TOKEN_REFRESH 3000 - -/* Kusto streaming inserts oauth scope */ -#define FLB_AZURE_KUSTO_SCOPE "https://help.kusto.windows.net/.default" - -/* MSAL authorization URL */ -#define FLB_MSAL_AUTH_URL_TEMPLATE \ - "https://login.microsoftonline.com/%s/oauth2/v2.0/token" - -#define FLB_AZURE_KUSTO_MGMT_URI_PATH "/v1/rest/mgmt" -#define FLB_AZURE_KUSTO_MGMT_BODY_TEMPLATE "{\"csl\":\"%s\", \"db\": \"NetDefaultDB\"}" - -#define FLB_AZURE_KUSTO_DEFAULT_TIME_KEY "timestamp" -#define FLB_AZURE_KUSTO_DEFAULT_TAG_KEY "tag" -#define FLB_AZURE_KUSTO_DEFAULT_LOG_KEY "log" - -#define AZURE_KUSTO_RESOURCE_STORAGE 0 -#define AZURE_KUSTO_RESOURCE_QUEUE 1 - -#define AZURE_KUSTO_RESOURCE_UPSTREAM_URI "uri" -#define AZURE_KUSTO_RESOURCE_UPSTREAM_SAS "sas" - -#define FLB_AZURE_KUSTO_RESOURCES_LOAD_INTERVAL_SEC 3600 - -struct flb_azure_kusto_resources { - struct flb_upstream_ha *blob_ha; - struct flb_upstream_ha *queue_ha; - flb_sds_t identity_token; - - /* used to reload resouces after some time */ - time_t load_time; -}; - -struct flb_azure_kusto { - /* azure_kusto configuration */ - flb_sds_t tenant_id; - flb_sds_t client_id; - flb_sds_t client_secret; - flb_sds_t ingestion_endpoint; - flb_sds_t database_name; - flb_sds_t table_name; - flb_sds_t ingestion_mapping_reference; - - /* records configuration */ - flb_sds_t log_key; - int include_tag_key; - flb_sds_t tag_key; - int include_time_key; - flb_sds_t time_key; - - /* --- internal data --- */ - - flb_sds_t ingestion_mgmt_endpoint; - - /* oauth2 context */ - flb_sds_t oauth_url; - struct flb_oauth2 *o; - - /* mutex for acquiring oauth tokens */ - pthread_mutex_t token_mutex; - - /* ingestion resources */ - struct flb_azure_kusto_resources *resources; - - /* mutex for loading reosurces */ - pthread_mutex_t resources_mutex; - - /* Upstream connection to the backend server */ - struct flb_upstream *u; - - /* Fluent Bit context */ - struct flb_config *config; - - /* Plugin output instance reference */ - struct flb_output_instance *ins; -}; - -flb_sds_t get_azure_kusto_token(struct flb_azure_kusto *ctx); -flb_sds_t execute_ingest_csl_command(struct flb_azure_kusto *ctx, const char *csl); - -#endif diff --git a/fluent-bit/plugins/out_azure_kusto/azure_kusto_conf.c b/fluent-bit/plugins/out_azure_kusto/azure_kusto_conf.c deleted file mode 100644 index 5303fef67..000000000 --- a/fluent-bit/plugins/out_azure_kusto/azure_kusto_conf.c +++ /dev/null @@ -1,665 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "azure_kusto.h" -#include "azure_kusto_conf.h" - -static struct flb_upstream_node *flb_upstream_node_create_url(struct flb_azure_kusto *ctx, - struct flb_config *config, - const char *url) -{ - int ret; - char *prot = NULL; - char *host = NULL; - char *port = NULL; - char *uri = NULL; - flb_sds_t sds_host = NULL; - flb_sds_t sds_port = NULL; - char *tmp; - struct flb_hash_table *kv = NULL; - struct flb_upstream_node *node = NULL; - int uri_length; - int sas_length; - - /* Parse and split URL */ - ret = flb_utils_url_split(url, &prot, &host, &port, &uri); - if (ret == -1) { - flb_plg_error(ctx->ins, "invalid URL: %s", url); - return NULL; - } - - /* find sas token in query */ - tmp = strchr(uri, '?'); - - if (tmp) { - uri_length = tmp - uri; - sas_length = strnlen(tmp + 1, 256); - - /* kv that will hold base uri, and sas token */ - kv = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, 2, 2); - - if (kv) { - ret = flb_hash_table_add(kv, AZURE_KUSTO_RESOURCE_UPSTREAM_URI, 3, uri, uri_length); - - if (ret != -1) { - ret = flb_hash_table_add(kv, AZURE_KUSTO_RESOURCE_UPSTREAM_SAS, 3, tmp + 1, - sas_length); - - if (ret != -1) { - /* if any/all of these creations would fail the node creation will fail and cleanup */ - sds_host = flb_sds_create(host); - sds_port = flb_sds_create(port); - - node = flb_upstream_node_create( - NULL, sds_host, sds_port, FLB_TRUE, ctx->ins->tls->verify, - ctx->ins->tls->debug, ctx->ins->tls->vhost, NULL, NULL, NULL, - NULL, NULL, kv, config); - - if (!node) { - flb_plg_error(ctx->ins, "error creating resource upstream node"); - } - } - else { - flb_plg_error(ctx->ins, "error storing resource sas token"); - } - } - else { - flb_plg_error(ctx->ins, "error storing resource uri"); - } - - /* avoid destorying if function is successful */ - if (!node) { - flb_hash_table_destroy(kv); - } - } - else { - flb_plg_error(ctx->ins, "error creating upstream node hash table"); - } - } - else { - flb_plg_error(ctx->ins, "uri has no sas token query: %s", uri); - } - - flb_free(prot); - flb_free(host); - flb_free(port); - flb_free(uri); - - return node; -} - -static int flb_azure_kusto_resources_clear(struct flb_azure_kusto_resources *resources) -{ - if (!resources) { - return -1; - } - - if (resources->blob_ha) { - flb_upstream_ha_destroy(resources->blob_ha); - resources->blob_ha = NULL; - } - - if (resources->queue_ha) { - flb_upstream_ha_destroy(resources->queue_ha); - resources->queue_ha = NULL; - } - - if (resources->identity_token) { - flb_sds_destroy(resources->identity_token); - resources->identity_token = NULL; - } - - resources->load_time = 0; - - return 0; -} - -/** - * Parses ".get ingestion resources" response into HA upstreams of the queue & blob - * resources in the response. - * - * @param ctx Pointer to the plugin's context - * @param config Pointer to the config - * @param response sds string containing the response body - * @param blob_ha Pointer to an HA upstream for the blob resources, that would be - * allocated here. - * @param queue_ha Pointer to an HA upstream for the queue resources, that would be - * allocated here. - * @return int 0 on success, -1 on failure - */ -static int parse_storage_resources(struct flb_azure_kusto *ctx, struct flb_config *config, - flb_sds_t response, struct flb_upstream_ha *blob_ha, - struct flb_upstream_ha *queue_ha) -{ - jsmn_parser parser; - jsmntok_t *t; - jsmntok_t *tokens; - int tok_size = 100; - int ret = -1; - int i; - int blob_count = 0; - int queue_count = 0; - char *token_str; - int token_str_len; - int resource_type; - struct flb_upstream_node *node; - struct flb_upstream_ha *ha; - flb_sds_t resource_uri; - - /* Response is a json in the form of - * { - * "Tables": [ - * { - * "TableName": "Table_0", - * "Columns": [...], - * "Rows": [ - * [ - * ("TempStorage" | "SecuredReadyForAggregationQueue" | - * "SuccessfulIngestionsQueue" | "FailedIngestionsQueue" | "IngestionsStatusTable"), - * - * ], - * ... - * ] - * } - * ] - * } - */ - - resource_uri = flb_sds_create(NULL); - if (!resource_uri) { - flb_plg_error(ctx->ins, "error allocating resource uri buffer"); - return -1; - } - - jsmn_init(&parser); - tokens = flb_calloc(1, sizeof(jsmntok_t) * tok_size); - - if (tokens) { - ret = jsmn_parse(&parser, response, flb_sds_len(response), tokens, tok_size); - - if (ret > 0) { - /* skip all tokens until we reach "Rows" */ - for (i = 0; i < ret - 1; i++) { - t = &tokens[i]; - - if (t->type != JSMN_STRING) { - continue; - } - - token_str = response + t->start; - token_str_len = (t->end - t->start); - - /** - * if we found the Rows key, skipping this token and the next one (key and - * wrapping array value) - */ - if (token_str_len == 4 && strncmp(token_str, "Rows", 4) == 0) { - i += 2; - break; - } - } - - /* iterating rows, each row will have 3 tokens: the array holding the column - * values, the first value containing the resource type, and the second value - * containing the resource uri */ - for (; i < ret; i++) { - t = &tokens[i]; - - /** - * each token should be an array with 2 strings: - * First will be the resource type (TempStorage, - * SecuredReadyForAggregationQueue, etc...) Second will be the SAS URI - */ - if (t->type != JSMN_ARRAY) { - break; - } - - /* move to the next token, first item in the array - resource type */ - i++; - t = &tokens[i]; - if (t->type != JSMN_STRING) { - break; - } - - token_str = response + t->start; - token_str_len = (t->end - t->start); - - flb_plg_debug(ctx->ins, "found resource of type: %.*s ", - t->end - t->start, response + t->start); - - if (token_str_len == 11 && strncmp(token_str, "TempStorage", 11) == 0) { - resource_type = AZURE_KUSTO_RESOURCE_STORAGE; - } - else if (token_str_len == 31 && - strncmp(token_str, "SecuredReadyForAggregationQueue", 31) == 0) { - resource_type = AZURE_KUSTO_RESOURCE_QUEUE; - } - /* we don't care about other resources so we just skip the next token and - move on to the next pair */ - else { - i++; - continue; - } - - /* move to the next token, second item in the array - resource URI */ - i++; - t = &tokens[i]; - - if (t->type != JSMN_STRING) { - break; - } - - token_str = response + t->start; - token_str_len = (t->end - t->start); - - resource_uri = flb_sds_copy(resource_uri, token_str, token_str_len); - if (resource_type == AZURE_KUSTO_RESOURCE_QUEUE) { - ha = queue_ha; - queue_count++; - } - else { - ha = blob_ha; - blob_count++; - } - - if (!ha) { - flb_plg_error(ctx->ins, "error creating HA upstream"); - ret = -1; - break; - } - - node = flb_upstream_node_create_url(ctx, config, resource_uri); - - if (!node) { - flb_plg_error(ctx->ins, "error creating HA upstream node"); - ret = -1; - break; - } - - flb_upstream_ha_node_add(ha, node); - } - - if (ret != -1) { - if (queue_count > 0 && blob_count > 0) { - flb_plg_debug(ctx->ins, - "parsed %d blob resources and %d queue resources", - blob_count, queue_count); - ret = 0; - } - else { - flb_plg_error(ctx->ins, "error parsing resources: missing resources"); - ret = -1; - } - } - } - else { - flb_plg_error(ctx->ins, "error parsing JSON response: %s", response); - ret = -1; - } - } - else { - flb_plg_error(ctx->ins, "error allocating tokens"); - ret = -1; - } - - flb_sds_destroy(resource_uri); - flb_free(tokens); - - return ret; -} - -/** - * Parses ".get kusto identity token" response and returns the token as an sds string - * - * @param ctx Pointer to the plugin's context - * @param response sds string containing the response body - * @return flb_sds_t The parsed token - */ -static flb_sds_t parse_ingestion_identity_token(struct flb_azure_kusto *ctx, - flb_sds_t response) -{ - flb_sds_t identity_token = NULL; - int tok_size = 19; - jsmn_parser parser; - jsmntok_t *t; - jsmntok_t *tokens; - int ret; - char *token_str; - int token_str_len; - - /** - * Response is a json in the form of - * { - * "Tables": [ - * { - * "TableName": "Table_0", - * "Columns": [{ - * "ColumnName": "AuthorizationContext", - * "DataType": "String", - * "ColumnType": "string" - * }], - * "Rows": [ - * [ - * , - * ] - * ] - * } - * ] - * } - * i.e. only one row and one column is expected (exactly 13 tokens) and the value - * should be the last - */ - - jsmn_init(&parser); - tokens = flb_calloc(1, sizeof(jsmntok_t) * tok_size); - if (!tokens) { - flb_plg_error(ctx->ins, "error allocating tokens"); - return NULL; - } - - ret = jsmn_parse(&parser, response, flb_sds_len(response), tokens, tok_size); - if (ret > 0) { - t = &tokens[tok_size - 1]; - - if (t->type == JSMN_STRING) { - t = &tokens[tok_size - 1]; - token_str = response + t->start; - token_str_len = (t->end - t->start); - - identity_token = flb_sds_create_len(token_str, token_str_len); - - if (identity_token) { - flb_plg_debug(ctx->ins, "parsed kusto identity token: '%s'", - identity_token); - } - else { - flb_plg_error(ctx->ins, "error parsing kusto identity token"); - } - } - else { - flb_plg_error(ctx->ins, "unexpected JSON response: %s", response); - } - } - else { - flb_plg_error(ctx->ins, "error parsing JSON response: %s", response); - } - - flb_free(tokens); - - return identity_token; -} - -int azure_kusto_load_ingestion_resources(struct flb_azure_kusto *ctx, - struct flb_config *config) -{ - int ret = -1; - flb_sds_t response = NULL; - flb_sds_t identity_token = NULL; - struct flb_upstream_ha *blob_ha = NULL; - struct flb_upstream_ha *queue_ha = NULL; - time_t now; - - if (pthread_mutex_lock(&ctx->resources_mutex)) { - flb_plg_error(ctx->ins, "error locking mutex"); - return -1; - } - - now = time(NULL); - - /* check if we have all resources and they are not stale */ - if (ctx->resources->blob_ha && ctx->resources->queue_ha && - ctx->resources->identity_token && - now - ctx->resources->load_time < FLB_AZURE_KUSTO_RESOURCES_LOAD_INTERVAL_SEC) { - flb_plg_debug(ctx->ins, "resources are already loaded and are not stale"); - ret = 0; - } - else { - flb_plg_info(ctx->ins, "loading kusto ingestion resourcs"); - response = execute_ingest_csl_command(ctx, ".get ingestion resources"); - - if (response) { - queue_ha = flb_upstream_ha_create("azure_kusto_queue_ha"); - - if (queue_ha) { - blob_ha = flb_upstream_ha_create("azure_kusto_blob_ha"); - - if (blob_ha) { - ret = - parse_storage_resources(ctx, config, response, blob_ha, queue_ha); - - if (ret == 0) { - flb_sds_destroy(response); - response = NULL; - - response = - execute_ingest_csl_command(ctx, ".get kusto identity token"); - - if (response) { - identity_token = - parse_ingestion_identity_token(ctx, response); - - if (identity_token) { - ret = flb_azure_kusto_resources_clear(ctx->resources); - - if (ret != -1) { - ctx->resources->blob_ha = blob_ha; - ctx->resources->queue_ha = queue_ha; - ctx->resources->identity_token = identity_token; - ctx->resources->load_time = now; - - ret = 0; - } - else { - flb_plg_error( - ctx->ins, - "error destroying previous ingestion resources"); - } - } - else { - flb_plg_error(ctx->ins, - "error parsing ingestion identity token"); - ret = -1; - } - } - else { - flb_plg_error(ctx->ins, "error getting kusto identity token"); - ret = -1; - } - } - else { - flb_plg_error(ctx->ins, - "error parsing ingestion storage resources"); - ret = -1; - } - - if (ret == -1) { - flb_upstream_ha_destroy(blob_ha); - } - } - else { - flb_plg_error(ctx->ins, "error creating storage resources upstreams"); - ret = -1; - } - - if (ret == -1) { - flb_upstream_ha_destroy(queue_ha); - } - } - else { - flb_plg_error(ctx->ins, "error creating storage resources upstreams"); - } - - if (response) { - flb_sds_destroy(response); - } - } - if (!response) { - flb_plg_error(ctx->ins, "error getting ingestion storage resources"); - } - } - - if (pthread_mutex_unlock(&ctx->resources_mutex)) { - flb_plg_error(ctx->ins, "error unlocking mutex"); - return -1; - } - - return ret; -} - -static int flb_azure_kusto_resources_destroy(struct flb_azure_kusto_resources *resources) -{ - int ret; - - if (!resources) { - return -1; - } - - ret = flb_azure_kusto_resources_clear(resources); - if (ret != 0) { - return -1; - } - - flb_free(resources); - - return 0; -} - -struct flb_azure_kusto *flb_azure_kusto_conf_create(struct flb_output_instance *ins, - struct flb_config *config) -{ - int ret; - struct flb_azure_kusto *ctx; - - /* Allocate config context */ - ctx = flb_calloc(1, sizeof(struct flb_azure_kusto)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - ctx->config = config; - - ret = flb_output_config_map_set(ins, (void *)ctx); - if (ret == -1) { - flb_plg_error(ins, "unable to load configuration"); - flb_free(ctx); - return NULL; - } - - /* config: 'tenant_id' */ - if (ctx->tenant_id == NULL) { - flb_plg_error(ctx->ins, "property 'tenant_id' is not defined."); - flb_azure_kusto_conf_destroy(ctx); - return NULL; - } - - /* config: 'client_id' */ - if (ctx->client_id == NULL) { - flb_plg_error(ctx->ins, "property 'client_id' is not defined"); - flb_azure_kusto_conf_destroy(ctx); - return NULL; - } - - /* config: 'client_secret' */ - if (ctx->client_secret == NULL) { - flb_plg_error(ctx->ins, "property 'client_secret' is not defined"); - flb_azure_kusto_conf_destroy(ctx); - return NULL; - } - - /* config: 'ingestion_endpoint' */ - if (ctx->ingestion_endpoint == NULL) { - flb_plg_error(ctx->ins, "property 'ingestion_endpoint' is not defined"); - flb_azure_kusto_conf_destroy(ctx); - return NULL; - } - - /* config: 'database_name' */ - if (ctx->database_name == NULL) { - flb_plg_error(ctx->ins, "property 'database_name' is not defined"); - flb_azure_kusto_conf_destroy(ctx); - return NULL; - } - - /* config: 'table_name' */ - if (ctx->table_name == NULL) { - flb_plg_error(ctx->ins, "property 'table_name' is not defined"); - flb_azure_kusto_conf_destroy(ctx); - return NULL; - } - - /* Create the auth URL */ - ctx->oauth_url = flb_sds_create_size(sizeof(FLB_MSAL_AUTH_URL_TEMPLATE) - 1 + - flb_sds_len(ctx->tenant_id)); - if (!ctx->oauth_url) { - flb_errno(); - flb_azure_kusto_conf_destroy(ctx); - return NULL; - } - flb_sds_snprintf(&ctx->oauth_url, flb_sds_alloc(ctx->oauth_url), - FLB_MSAL_AUTH_URL_TEMPLATE, ctx->tenant_id); - - ctx->resources = flb_calloc(1, sizeof(struct flb_azure_kusto_resources)); - if (!ctx->resources) { - flb_errno(); - flb_azure_kusto_conf_destroy(ctx); - return NULL; - } - - flb_plg_info(ctx->ins, "endpoint='%s', database='%s', table='%s'", - ctx->ingestion_endpoint, ctx->database_name, ctx->table_name); - - return ctx; -} - -int flb_azure_kusto_conf_destroy(struct flb_azure_kusto *ctx) -{ - if (!ctx) { - return -1; - } - - if (ctx->oauth_url) { - flb_sds_destroy(ctx->oauth_url); - ctx->oauth_url = NULL; - } - - if (ctx->o) { - flb_oauth2_destroy(ctx->o); - ctx->o = NULL; - } - - if (ctx->resources) { - flb_azure_kusto_resources_destroy(ctx->resources); - ctx->resources = NULL; - } - - flb_free(ctx); - return 0; -} diff --git a/fluent-bit/plugins/out_azure_kusto/azure_kusto_conf.h b/fluent-bit/plugins/out_azure_kusto/azure_kusto_conf.h deleted file mode 100644 index b4b2e3a39..000000000 --- a/fluent-bit/plugins/out_azure_kusto/azure_kusto_conf.h +++ /dev/null @@ -1,31 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_AZURE_KUSTO_CONF_H -#define FLB_OUT_AZURE_KUSTO_CONF_H - -#include "azure_kusto.h" - -int azure_kusto_load_ingestion_resources(struct flb_azure_kusto *ctx, - struct flb_config *config); -struct flb_azure_kusto *flb_azure_kusto_conf_create(struct flb_output_instance *ins, - struct flb_config *config); -int flb_azure_kusto_conf_destroy(struct flb_azure_kusto *ctx); - -#endif diff --git a/fluent-bit/plugins/out_azure_kusto/azure_kusto_ingest.c b/fluent-bit/plugins/out_azure_kusto/azure_kusto_ingest.c deleted file mode 100644 index d38d92e7f..000000000 --- a/fluent-bit/plugins/out_azure_kusto/azure_kusto_ingest.c +++ /dev/null @@ -1,496 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "azure_kusto_ingest.h" - -/* not really uuid but a random string in the form 00000000-0000-0000-0000-000000000000 */ -static char *generate_uuid() -{ - char *chars = "0123456789abcdef"; - char *uuid; - int i; - uint64_t rand; - - uuid = flb_malloc(37); - if (!uuid) { - flb_errno(); - return NULL; - } - - for (i = 0; i < 36; i++) { - if (i == 8 || i == 13 || i == 18 || i == 23) { - uuid[i] = '-'; - continue; - } - - if (flb_random_bytes((unsigned char *)&rand, sizeof(uint64_t))) { - rand = time(NULL); - } - uuid[i] = chars[rand % 16]; - } - uuid[36] = '\0'; - - return uuid; -} - -static char *base64_encode(flb_sds_t s, size_t len, size_t *out_len) -{ - char *b64; - int ret; - size_t buffer_len = 4 * ceil(((double)len / 3) + 1); - - b64 = flb_malloc(buffer_len); - if (!b64) { - flb_errno(); - return NULL; - } - - ret = flb_base64_encode((unsigned char *)b64, buffer_len, out_len, (unsigned char *)s, - len); - if (ret != 0) { - flb_error("cannot encode string %s into base64", s); - flb_free(b64); - return NULL; - } - - return b64; -} - -static flb_sds_t azure_kusto_create_blob_uri(struct flb_azure_kusto *ctx, - struct flb_upstream_node *u_node, - flb_sds_t blob_id) -{ - int ret; - flb_sds_t uri = NULL; - char *blob_uri; - size_t blob_uri_size; - char *blob_sas; - size_t blob_sas_size; - - ret = flb_hash_table_get(u_node->ht, AZURE_KUSTO_RESOURCE_UPSTREAM_URI, 3, - (void **)&blob_uri, &blob_uri_size); - if (ret == -1) { - flb_plg_error(ctx->ins, "error getting blob uri"); - return NULL; - } - - ret = flb_hash_table_get(u_node->ht, AZURE_KUSTO_RESOURCE_UPSTREAM_SAS, 3, - (void **)&blob_sas, &blob_sas_size); - if (ret == -1) { - flb_plg_error(ctx->ins, "error getting blob sas token"); - return NULL; - } - - /* uri will be https:////.multijson? */ - uri = flb_sds_create_size(flb_sds_len(u_node->host) + blob_uri_size + blob_sas_size + - flb_sds_len(blob_id) + 21); - - if (uri) { - flb_sds_snprintf(&uri, flb_sds_alloc(uri), "https://%s%s/%s.multijson?%s", - u_node->host, blob_uri, blob_id, blob_sas); - flb_plg_debug(ctx->ins, "created blob uri %s", uri); - } - else { - flb_plg_error(ctx->ins, "cannot create blob uri buffer"); - } - - return uri; -} - -static flb_sds_t azure_kusto_create_blob(struct flb_azure_kusto *ctx, flb_sds_t blob_id, - flb_sds_t payload, size_t payload_size) -{ - int ret = -1; - flb_sds_t uri = NULL; - struct flb_upstream_node *u_node; - struct flb_connection *u_conn; - struct flb_http_client *c; - size_t resp_size; - time_t now; - struct tm tm; - char tmp[64]; - int len; - - now = time(NULL); - gmtime_r(&now, &tm); - len = strftime(tmp, sizeof(tmp) - 1, "%a, %d %b %Y %H:%M:%S GMT", &tm); - - u_node = flb_upstream_ha_node_get(ctx->resources->blob_ha); - if (!u_node) { - flb_plg_error(ctx->ins, "error getting blob upstream"); - return NULL; - } - - u_conn = flb_upstream_conn_get(u_node->u); - - if (u_conn) { - uri = azure_kusto_create_blob_uri(ctx, u_node, blob_id); - - if (uri) { - flb_plg_debug(ctx->ins, "uploading payload to blob uri: %s", uri); - c = flb_http_client(u_conn, FLB_HTTP_PUT, uri, payload, payload_size, NULL, 0, - NULL, 0); - - if (c) { - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - flb_http_add_header(c, "Content-Type", 12, "application/json", 16); - flb_http_add_header(c, "x-ms-blob-type", 14, "BlockBlob", 9); - flb_http_add_header(c, "x-ms-date", 9, tmp, len); - flb_http_add_header(c, "x-ms-version", 12, "2019-12-12", 10); - - ret = flb_http_do(c, &resp_size); - flb_plg_debug(ctx->ins, - "kusto blob upload request http_do=%i, HTTP Status: %i", - ret, c->resp.status); - - if (ret == 0) { - /* Validate return status and HTTP status if set */ - if (c->resp.status != 201) { - ret = -1; - - if (c->resp.payload_size > 0) { - flb_plg_debug(ctx->ins, "Request failed and returned: \n%s", - c->resp.payload); - } - else { - flb_plg_debug(ctx->ins, "Request failed"); - } - } - } - else { - flb_plg_error(ctx->ins, "cannot send HTTP request"); - } - - flb_http_client_destroy(c); - } - else { - flb_plg_error(ctx->ins, - "cannot create HTTP client context for blob container"); - } - - if (ret != 0) { - flb_sds_destroy(uri); - uri = NULL; - } - } - else { - flb_plg_error(ctx->ins, "error creating blob container uri buffer"); - } - - flb_upstream_conn_release(u_conn); - } - else { - flb_plg_error(ctx->ins, "error getting blob container upstream connection"); - } - - return uri; -} - -static flb_sds_t create_ingestion_message(struct flb_azure_kusto *ctx, flb_sds_t blob_uri, - size_t payload_size) -{ - flb_sds_t message = NULL; - int ret = 0; - char *uuid; - char *message_b64; - size_t b64_len; - size_t message_len; - - uuid = generate_uuid(); - if (uuid) { - message = flb_sds_create(NULL); - - if (message) { - message_len = - flb_sds_snprintf(&message, 0, - "{\"Id\": \"%s\", \"BlobPath\": \"%s\", " - "\"RawDataSize\": %lu, \"DatabaseName\": " - "\"%s\", \"TableName\": \"%s\"," - "\"AdditionalProperties\": { \"format\": \"multijson\", " - "\"authorizationContext\": " - "\"%s\", \"jsonMappingReference\": \"%s\" }}%c", - uuid, blob_uri, payload_size, ctx->database_name, - ctx->table_name, ctx->resources->identity_token, - ctx->ingestion_mapping_reference == NULL - ? "" - : ctx->ingestion_mapping_reference, 0); - - if (message_len != -1) { - flb_plg_debug(ctx->ins, "created ingestion message:\n%s", message); - message_b64 = base64_encode(message, message_len, &b64_len); - - if (message_b64) { - ret = flb_sds_snprintf( - &message, flb_sds_alloc(message), - "%s%c", - message_b64, 0); - - if (ret == -1) { - flb_plg_error(ctx->ins, "error creating ingestion queue message"); - } - - flb_free(message_b64); - } - else { - flb_plg_error(ctx->ins, "error encoding ingestion message to base64"); - } - } - else { - flb_plg_error(ctx->ins, "error creating ingestion message"); - ret = -1; - } - - if (ret == -1) { - flb_sds_destroy(message); - message = NULL; - } - } - else { - flb_plg_error(ctx->ins, "error creating ingestion message buffer"); - } - - flb_free(uuid); - } - else { - flb_plg_error(ctx->ins, "error generating unique ingestion UUID"); - } - - return message; -} - -static flb_sds_t azure_kusto_create_queue_uri(struct flb_azure_kusto *ctx, - struct flb_upstream_node *u_node) -{ - int ret; - flb_sds_t uri = NULL; - char *queue_uri; - size_t queue_uri_size; - char *queue_sas; - size_t queue_sas_size; - - ret = flb_hash_table_get(u_node->ht, AZURE_KUSTO_RESOURCE_UPSTREAM_URI, 3, - (void **)&queue_uri, &queue_uri_size); - if (ret == -1) { - flb_plg_error(ctx->ins, "error getting queue uri"); - return NULL; - } - - ret = flb_hash_table_get(u_node->ht, AZURE_KUSTO_RESOURCE_UPSTREAM_SAS, 3, - (void **)&queue_sas, &queue_sas_size); - if (ret == -1) { - flb_plg_error(ctx->ins, "error getting queue sas token"); - return NULL; - } - - /* uri will be /messages? */ - uri = flb_sds_create_size(queue_uri_size + queue_sas_size + 11); - - if (uri) { - flb_sds_snprintf(&uri, flb_sds_alloc(uri), "%s/messages?%s", queue_uri, - queue_sas); - flb_plg_debug(ctx->ins, "created queue uri %s", uri); - } - else { - flb_plg_error(ctx->ins, "cannot create queue uri buffer"); - } - - return uri; -} - -static int azure_kusto_enqueue_ingestion(struct flb_azure_kusto *ctx, flb_sds_t blob_uri, - size_t payload_size) -{ - int ret = -1; - struct flb_upstream_node *u_node; - struct flb_connection *u_conn; - struct flb_http_client *c; - flb_sds_t uri; - flb_sds_t payload; - size_t resp_size; - time_t now; - struct tm tm; - char tmp[64]; - int len; - - now = time(NULL); - gmtime_r(&now, &tm); - len = strftime(tmp, sizeof(tmp) - 1, "%a, %d %b %Y %H:%M:%S GMT", &tm); - - u_node = flb_upstream_ha_node_get(ctx->resources->queue_ha); - if (!u_node) { - flb_plg_error(ctx->ins, "error getting queue upstream"); - return -1; - } - - u_conn = flb_upstream_conn_get(u_node->u); - - if (u_conn) { - uri = azure_kusto_create_queue_uri(ctx, u_node); - - if (uri) { - payload = create_ingestion_message(ctx, blob_uri, payload_size); - - if (payload) { - c = flb_http_client(u_conn, FLB_HTTP_POST, uri, payload, - flb_sds_len(payload), NULL, 0, NULL, 0); - - if (c) { - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - flb_http_add_header(c, "Content-Type", 12, "application/atom+xml", - 20); - flb_http_add_header(c, "x-ms-date", 9, tmp, len); - flb_http_add_header(c, "x-ms-version", 12, "2019-12-12", 10); - - ret = flb_http_do(c, &resp_size); - flb_plg_debug(ctx->ins, - "kusto queue request http_do=%i, HTTP Status: %i", ret, - c->resp.status); - - if (ret == 0) { - /* Validate return status and HTTP status if set */ - if (c->resp.status != 201) { - ret = -1; - - if (c->resp.payload_size > 0) { - flb_plg_debug(ctx->ins, - "Request failed and returned: \n%s", - c->resp.payload); - } - else { - flb_plg_debug(ctx->ins, "Request failed"); - } - } - } - else { - flb_plg_error(ctx->ins, "cannot send HTTP request"); - } - - flb_http_client_destroy(c); - } - else { - flb_plg_error(ctx->ins, - "cannot create HTTP client context for queue"); - } - - flb_sds_destroy(payload); - } - else { - flb_plg_error(ctx->ins, "error creating payload buffer"); - } - - flb_sds_destroy(uri); - } - else { - flb_plg_error(ctx->ins, "error creating queue uri buffer"); - } - - flb_upstream_conn_release(u_conn); - } - else { - flb_plg_error(ctx->ins, "error getting queue upstream connection"); - } - - return ret; -} - -static flb_sds_t azure_kusto_create_blob_id(struct flb_azure_kusto *ctx, flb_sds_t tag, - size_t tag_len) -{ - flb_sds_t blob_id = NULL; - struct flb_time tm; - uint64_t ms; - char *b64tag; - size_t b64_len; - - flb_time_get(&tm); - ms = ((tm.tm.tv_sec * 1000) + (tm.tm.tv_nsec / 1000000)); - - b64tag = base64_encode(tag, tag_len, &b64_len); - - if (b64tag) { - /* remove trailing '=' */ - while (b64_len && b64tag[b64_len - 1] == '=') { - b64tag[b64_len - 1] = '\0'; - b64_len--; - } - - blob_id = flb_sds_create_size(flb_sds_len(ctx->database_name) + - flb_sds_len(ctx->table_name) + b64_len + 24); - if (blob_id) { - flb_sds_snprintf(&blob_id, flb_sds_alloc(blob_id), "flb__%s__%s__%s__%lu", - ctx->database_name, ctx->table_name, b64tag, ms); - } - else { - flb_plg_error(ctx->ins, "cannot create blob id buffer"); - } - - flb_free(b64tag); - } - else { - flb_plg_error(ctx->ins, "error encoding tag '%s' to base64", tag); - } - - return blob_id; -} - -int azure_kusto_queued_ingestion(struct flb_azure_kusto *ctx, flb_sds_t tag, - size_t tag_len, flb_sds_t payload, size_t payload_size) -{ - int ret = -1; - flb_sds_t blob_id; - flb_sds_t blob_uri; - - /* flb________ */ - blob_id = azure_kusto_create_blob_id(ctx, tag, tag_len); - - if (blob_id) { - blob_uri = azure_kusto_create_blob(ctx, blob_id, payload, payload_size); - - if (blob_uri) { - ret = azure_kusto_enqueue_ingestion(ctx, blob_uri, payload_size); - - if (ret != 0) { - flb_plg_error(ctx->ins, "failed to enqueue ingestion blob to queue"); - ret = -1; - } - - flb_sds_destroy(blob_uri); - } - else { - flb_plg_error(ctx->ins, "failed to create payload blob uri"); - } - - flb_sds_destroy(blob_id); - } - else { - flb_plg_error(ctx->ins, "cannot create blob id"); - } - - return ret; -} diff --git a/fluent-bit/plugins/out_azure_kusto/azure_kusto_ingest.h b/fluent-bit/plugins/out_azure_kusto/azure_kusto_ingest.h deleted file mode 100644 index 60613919a..000000000 --- a/fluent-bit/plugins/out_azure_kusto/azure_kusto_ingest.h +++ /dev/null @@ -1,28 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_AZURE_KUSTO_INGEST_H -#define FLB_OUT_AZURE_KUSTO_INGEST_H - -#include "azure_kusto.h" - -int azure_kusto_queued_ingestion(struct flb_azure_kusto *ctx, flb_sds_t tag, - size_t tag_len, flb_sds_t payload, size_t payload_size); - -#endif \ No newline at end of file diff --git a/fluent-bit/plugins/out_azure_logs_ingestion/CMakeLists.txt b/fluent-bit/plugins/out_azure_logs_ingestion/CMakeLists.txt deleted file mode 100644 index b51308c70..000000000 --- a/fluent-bit/plugins/out_azure_logs_ingestion/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(src - azure_logs_ingestion.c - azure_logs_ingestion_conf.c - ) - -FLB_PLUGIN(out_azure_logs_ingestion "${src}" "") diff --git a/fluent-bit/plugins/out_azure_logs_ingestion/azure_logs_ingestion.c b/fluent-bit/plugins/out_azure_logs_ingestion/azure_logs_ingestion.c deleted file mode 100644 index 9b839ef7e..000000000 --- a/fluent-bit/plugins/out_azure_logs_ingestion/azure_logs_ingestion.c +++ /dev/null @@ -1,445 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "azure_logs_ingestion.h" -#include "azure_logs_ingestion_conf.h" - -static int cb_azure_logs_ingestion_init(struct flb_output_instance *ins, - struct flb_config *config, void *data) -{ - struct flb_az_li *ctx; - (void) config; - (void) ins; - (void) data; - - /* Allocate and initialize a context from configuration */ - ctx = flb_az_li_ctx_create(ins, config); - if (!ctx) { - flb_plg_error(ins, "configuration failed"); - return -1; - } - - return 0; -} - -/* A duplicate function copied from the azure log analytics plugin. - allocates sds string */ -static int az_li_format(const void *in_buf, size_t in_bytes, - char **out_buf, size_t *out_size, - struct flb_az_li *ctx) -{ - int i; - int array_size = 0; - int map_size; - size_t off = 0; - double t; - struct flb_time tm; - msgpack_unpacked result; - msgpack_object root; - msgpack_object *obj; - msgpack_object map; - msgpack_object k; - msgpack_object v; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - msgpack_sbuffer tmp_sbuf; - msgpack_packer tmp_pck; - flb_sds_t record; - char time_formatted[32]; - size_t s; - struct tm tms; - int len; - - /* Count number of items */ - array_size = flb_mp_count(in_buf, in_bytes); - msgpack_unpacked_init(&result); - - /* Create temporary msgpack buffer */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - msgpack_pack_array(&mp_pck, array_size); - - off = 0; - while (msgpack_unpack_next(&result, in_buf, in_bytes, &off) == MSGPACK_UNPACK_SUCCESS) { - root = result.data; - - /* Get timestamp */ - flb_time_pop_from_msgpack(&tm, &result, &obj); - - /* Create temporary msgpack buffer */ - msgpack_sbuffer_init(&tmp_sbuf); - msgpack_packer_init(&tmp_pck, &tmp_sbuf, msgpack_sbuffer_write); - - map = root.via.array.ptr[1]; - map_size = map.via.map.size; - - msgpack_pack_map(&mp_pck, map_size + 1); - - /* Append the time key */ - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->time_key)); - msgpack_pack_str_body(&mp_pck, - ctx->time_key, - flb_sds_len(ctx->time_key)); - - if (ctx->time_generated == FLB_TRUE) { - /* Append the time value as ISO 8601 */ - gmtime_r(&tm.tm.tv_sec, &tms); - s = strftime(time_formatted, sizeof(time_formatted) - 1, - FLB_PACK_JSON_DATE_ISO8601_FMT, &tms); - - len = snprintf(time_formatted + s, - sizeof(time_formatted) - 1 - s, - ".%03" PRIu64 "Z", - (uint64_t) tm.tm.tv_nsec / 1000000); - s += len; - msgpack_pack_str(&mp_pck, s); - msgpack_pack_str_body(&mp_pck, time_formatted, s); - } - else { - /* Append the time value as millis.nanos */ - t = flb_time_to_double(&tm); - msgpack_pack_double(&mp_pck, t); - } - - /* Append original map k/v */ - for (i = 0; i < map_size; i++) { - k = map.via.map.ptr[i].key; - v = map.via.map.ptr[i].val; - - msgpack_pack_object(&tmp_pck, k); - msgpack_pack_object(&tmp_pck, v); - } - msgpack_sbuffer_write(&mp_sbuf, tmp_sbuf.data, tmp_sbuf.size); - msgpack_sbuffer_destroy(&tmp_sbuf); - } - - record = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); - if (!record) { - flb_errno(); - msgpack_sbuffer_destroy(&mp_sbuf); - msgpack_unpacked_destroy(&result); - return -1; - } - - msgpack_sbuffer_destroy(&mp_sbuf); - msgpack_unpacked_destroy(&result); - - *out_buf = record; - *out_size = flb_sds_len(record); - - return 0; -} - -/* Gets OAuth token; (allocates sds string everytime, must deallocate) */ -flb_sds_t get_az_li_token(struct flb_az_li *ctx) -{ - int ret = 0; - char* token; - size_t token_len; - flb_sds_t token_return = NULL; - - if (pthread_mutex_lock(&ctx->token_mutex)) { - flb_plg_error(ctx->ins, "error locking mutex"); - return NULL; - } - /* Retrieve access token only if expired */ - if (flb_oauth2_token_expired(ctx->u_auth) == FLB_TRUE) { - flb_plg_debug(ctx->ins, "token expired. getting new token"); - /* Clear any previous oauth2 payload content */ - flb_oauth2_payload_clear(ctx->u_auth); - - ret = flb_oauth2_payload_append(ctx->u_auth, "grant_type", 10, - "client_credentials", 18); - if (ret == -1) { - flb_plg_error(ctx->ins, "error appending oauth2 params"); - goto token_cleanup; - } - - ret = flb_oauth2_payload_append(ctx->u_auth, "scope", 5, FLB_AZ_LI_AUTH_SCOPE, - sizeof(FLB_AZ_LI_AUTH_SCOPE) - 1); - if (ret == -1) { - flb_plg_error(ctx->ins, "error appending oauth2 params"); - goto token_cleanup; - } - - ret = flb_oauth2_payload_append(ctx->u_auth, "client_id", 9, - ctx->client_id, -1); - if (ret == -1) { - flb_plg_error(ctx->ins, "error appending oauth2 params"); - goto token_cleanup; - } - - ret = flb_oauth2_payload_append(ctx->u_auth, "client_secret", 13, - ctx->client_secret, -1); - if (ret == -1) { - flb_plg_error(ctx->ins, "error appending oauth2 params"); - goto token_cleanup; - } - - token = flb_oauth2_token_get(ctx->u_auth); - - /* Copy string to prevent race conditions */ - if (!token) { - flb_plg_error(ctx->ins, "error retrieving oauth2 access token"); - goto token_cleanup; - } - flb_plg_debug(ctx->ins, "got azure token"); - } - - /* Reached this code-block means, got new token or token not expired */ - /* Either way we copy the token to a new string */ - token_len = flb_sds_len(ctx->u_auth->token_type) + 2 + - flb_sds_len(ctx->u_auth->access_token); - flb_plg_debug(ctx->ins, "create token header string"); - /* Now create */ - token_return = flb_sds_create_size(token_len); - if (!token_return) { - flb_plg_error(ctx->ins, "error creating token buffer"); - goto token_cleanup; - } - flb_sds_snprintf(&token_return, flb_sds_alloc(token_return), "%s %s", - ctx->u_auth->token_type, ctx->u_auth->access_token); - -token_cleanup: - if (pthread_mutex_unlock(&ctx->token_mutex)) { - flb_plg_error(ctx->ins, "error unlocking mutex"); - return NULL; - } - - return token_return; -} - -static void cb_azure_logs_ingestion_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - int ret; - int flush_status; - size_t b_sent; - size_t json_payload_size; - void* final_payload; - size_t final_payload_size; - flb_sds_t token; - struct flb_connection *u_conn; - struct flb_http_client *c = NULL; - int is_compressed = FLB_FALSE; - flb_sds_t json_payload = NULL; - struct flb_az_li *ctx = out_context; - (void) i_ins; - (void) config; - - /* Get upstream connection */ - u_conn = flb_upstream_conn_get(ctx->u_dce); - if (!u_conn) { - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Convert binary logs into a JSON payload */ - ret = az_li_format(event_chunk->data, event_chunk->size, - &json_payload, &json_payload_size, ctx); - if (ret == -1) { - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_ERROR); - } - - /* Get OAuth2 token */ - token = get_az_li_token(ctx); - if (!token) { - flush_status = FLB_RETRY; - goto cleanup; - } - - /* Map buffer */ - final_payload = json_payload; - final_payload_size = json_payload_size; - if (ctx->compress_enabled == FLB_TRUE) { - ret = flb_gzip_compress((void *) json_payload, json_payload_size, - &final_payload, &final_payload_size); - if (ret == -1) { - flb_plg_error(ctx->ins, - "cannot gzip payload, disabling compression"); - } - else { - is_compressed = FLB_TRUE; - flb_plg_debug(ctx->ins, "enabled payload gzip compression"); - /* JSON buffer will be cleared at cleanup: */ - } - } - - /* Compose HTTP Client request */ - c = flb_http_client(u_conn, FLB_HTTP_POST, ctx->dce_u_url, - final_payload, final_payload_size, NULL, 0, NULL, 0); - - if (!c) { - flb_plg_warn(ctx->ins, "retrying payload bytes=%lu", final_payload_size); - flush_status = FLB_RETRY; - goto cleanup; - } - - /* Append headers */ - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - flb_http_add_header(c, "Content-Type", 12, "application/json", 16); - if (is_compressed) { - flb_http_add_header(c, "Content-Encoding", 16, "gzip", 4); - } - flb_http_add_header(c, "Authorization", 13, token, flb_sds_len(token)); - flb_http_buffer_size(c, FLB_HTTP_DATA_SIZE_MAX); - - /* Execute rest call */ - ret = flb_http_do(c, &b_sent); - if (ret != 0) { - flb_plg_warn(ctx->ins, "http_do=%i", ret); - flush_status = FLB_RETRY; - goto cleanup; - } - else { - if (c->resp.status >= 200 && c->resp.status <= 299) { - flb_plg_info(ctx->ins, "http_status=%i, dcr_id=%s, table=%s", - c->resp.status, ctx->dcr_id, ctx->table_name); - flush_status = FLB_OK; - goto cleanup; - } - else { - if (c->resp.payload_size > 0) { - flb_plg_warn(ctx->ins, "http_status=%i:\n%s", - c->resp.status, c->resp.payload); - } - else { - flb_plg_warn(ctx->ins, "http_status=%i", c->resp.status); - } - flb_plg_debug(ctx->ins, "retrying payload bytes=%lu", final_payload_size); - flush_status = FLB_RETRY; - goto cleanup; - } - } - -cleanup: - /* cleanup */ - if (json_payload) { - flb_sds_destroy(json_payload); - } - - /* release compressed payload */ - if (is_compressed == FLB_TRUE) { - flb_free(final_payload); - } - - if (c) { - flb_http_client_destroy(c); - } - if (u_conn) { - flb_upstream_conn_release(u_conn); - } - - /* destory token at last after HTTP call has finished */ - if (token) { - flb_sds_destroy(token); - } - FLB_OUTPUT_RETURN(flush_status); -} - -static int cb_azure_logs_ingestion_exit(void *data, struct flb_config *config) -{ - struct flb_az_li *ctx = data; - flb_plg_debug(ctx->ins, "exiting logs ingestion plugin"); - flb_az_li_ctx_destroy(ctx); - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "tenant_id", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_az_li, tenant_id), - "Set the tenant ID of the AAD application" - }, - { - FLB_CONFIG_MAP_STR, "client_id", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_az_li, client_id), - "Set the client/app ID of the AAD application" - }, - { - FLB_CONFIG_MAP_STR, "client_secret", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_az_li, client_secret), - "Set the client secret of the AAD application" - }, - { - FLB_CONFIG_MAP_STR, "dce_url", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_az_li, dce_url), - "Data Collection Endpoint(DCE) URI (e.g. " - "https://la-endpoint-q12a.eastus-1.ingest.monitor.azure.com)" - }, - { - FLB_CONFIG_MAP_STR, "dcr_id", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_az_li, dcr_id), - "Data Collection Rule (DCR) immutable ID" - }, - { - FLB_CONFIG_MAP_STR, "table_name", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_az_li, table_name), - "The name of the custom log table, including '_CL' suffix" - }, - /* optional params */ - { - FLB_CONFIG_MAP_STR, "time_key", FLB_AZ_LI_TIME_KEY, - 0, FLB_TRUE, offsetof(struct flb_az_li, time_key), - "[Optional] Specify the key name where the timestamp will be stored." - }, - { - FLB_CONFIG_MAP_BOOL, "time_generated", "false", - 0, FLB_TRUE, offsetof(struct flb_az_li, time_generated), - "If enabled, will generate a timestamp and append it to JSON. " - "The key name is set by the 'time_key' parameter" - }, - { - FLB_CONFIG_MAP_BOOL, "compress", "false", - 0, FLB_TRUE, offsetof(struct flb_az_li, compress_enabled), - "Enable HTTP payload compression (gzip)." - }, - /* EOF */ - {0} -}; - -struct flb_output_plugin out_azure_logs_ingestion_plugin = { - .name = "azure_logs_ingestion", - .description = "Send logs to Log Analytics with Log Ingestion API", - .cb_init = cb_azure_logs_ingestion_init, - .cb_flush = cb_azure_logs_ingestion_flush, - .cb_exit = cb_azure_logs_ingestion_exit, - - /* Configuration */ - .config_map = config_map, - - /* Plugin flags */ - .flags = FLB_OUTPUT_NET | FLB_IO_TLS, -}; diff --git a/fluent-bit/plugins/out_azure_logs_ingestion/azure_logs_ingestion.h b/fluent-bit/plugins/out_azure_logs_ingestion/azure_logs_ingestion.h deleted file mode 100644 index 15b2420b8..000000000 --- a/fluent-bit/plugins/out_azure_logs_ingestion/azure_logs_ingestion.h +++ /dev/null @@ -1,74 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_AZURE_LOGS_INGESTION -#define FLB_OUT_AZURE_LOGS_INGESTION - -#define FLB_AZ_LI_API_VERSION "api-version=2021-11-01-preview" -#define FLB_AZ_LI_TIME_KEY "@timestamp" -#define FLB_AZ_LI_AUTH_SCOPE "https://monitor.azure.com/.default" -/* auth url needs tenant_id */ -#define FLB_AZ_LI_AUTH_URL_TMPLT "https://login.microsoftonline.com/"\ - "%s/oauth2/v2.0/token" -/* DCE Full URL needs: dce_url, dcr_id, Log Analytics custom table name */ -#define FLB_AZ_LI_DCE_URL_TMPLT "%s/dataCollectionRules/%s/streams/"\ - "Custom-%s?"FLB_AZ_LI_API_VERSION -/* TLS Modes for upstream connection = FLB_IO_TLS or FLB_IO_OPT_TLS*/ -#define FLB_AZ_LI_TLS_MODE FLB_IO_TLS -/* refresh token every 60 minutes */ -#define FLB_AZ_LI_TOKEN_TIMEOUT 3600 - -#include -#include -#include - -/* Context structure for Azure Logs Ingestion API */ -struct flb_az_li { - /* log ingestion account setup */ - flb_sds_t tenant_id; - flb_sds_t client_id; - flb_sds_t client_secret; - flb_sds_t dce_url; - flb_sds_t dcr_id; - flb_sds_t table_name; - - /* time_generated: on/off */ - int time_generated; - /* time key name */ - flb_sds_t time_key; - - /* compress payload */ - int compress_enabled; - - /* mangement auth */ - flb_sds_t auth_url; - struct flb_oauth2 *u_auth; - /* mutex for acquiring tokens */ - pthread_mutex_t token_mutex; - - /* upstream connection to the data collection endpoint */ - struct flb_upstream *u_dce; - flb_sds_t dce_u_url; - - /* plugin output and config instance reference */ - struct flb_output_instance *ins; - struct flb_config *config; -}; - -#endif diff --git a/fluent-bit/plugins/out_azure_logs_ingestion/azure_logs_ingestion_conf.c b/fluent-bit/plugins/out_azure_logs_ingestion/azure_logs_ingestion_conf.c deleted file mode 100644 index 344a7f5ff..000000000 --- a/fluent-bit/plugins/out_azure_logs_ingestion/azure_logs_ingestion_conf.c +++ /dev/null @@ -1,172 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "azure_logs_ingestion.h" -#include "azure_logs_ingestion_conf.h" - -struct flb_az_li* flb_az_li_ctx_create(struct flb_output_instance *ins, - struct flb_config *config) -{ - int ret; - struct flb_az_li *ctx; - (void) ins; - (void) config; - - /* Allocate a new context object for this output instance */ - ctx = flb_calloc(1, sizeof(struct flb_az_li)); - if (!ctx) { - flb_errno(); - return NULL; - } - - /* Set the conext in output_instance so that we can retrieve it later */ - ctx->ins = ins; - ctx->config = config; - /* Set context */ - flb_output_set_context(ins, ctx); - - /* Load config map */ - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_plg_error(ins, "unable to load configuration"); - return NULL; - } - - /* config: 'client_id' */ - if (!ctx->client_id) { - flb_plg_error(ins, "property 'client_id' is not defined"); - flb_az_li_ctx_destroy(ctx); - return NULL; - } - /* config: 'tenant_id' */ - if (!ctx->tenant_id) { - flb_plg_error(ins, "property 'tenant_id' is not defined"); - flb_az_li_ctx_destroy(ctx); - return NULL; - } - /* config: 'client_secret' */ - if (!ctx->client_secret) { - flb_plg_error(ins, "property 'client_secret' is not defined"); - flb_az_li_ctx_destroy(ctx); - return NULL; - } - /* config: 'dce_url' */ - if (!ctx->dce_url) { - flb_plg_error(ins, "property 'dce_url' is not defined"); - flb_az_li_ctx_destroy(ctx); - return NULL; - } - /* config: 'dcr_id' */ - if (!ctx->dcr_id) { - flb_plg_error(ins, "property 'dcr_id' is not defined"); - flb_az_li_ctx_destroy(ctx); - return NULL; - } - /* config: 'table_name' */ - if (!ctx->table_name) { - flb_plg_error(ins, "property 'table_name' is not defined"); - flb_az_li_ctx_destroy(ctx); - return NULL; - } - - /* Allocate and set auth url */ - ctx->auth_url = flb_sds_create_size(sizeof(FLB_AZ_LI_AUTH_URL_TMPLT) - 1 + - flb_sds_len(ctx->tenant_id)); - if (!ctx->auth_url) { - flb_errno(); - flb_az_li_ctx_destroy(ctx); - return NULL; - } - flb_sds_snprintf(&ctx->auth_url, flb_sds_alloc(ctx->auth_url), - FLB_AZ_LI_AUTH_URL_TMPLT, ctx->tenant_id); - - /* Allocate and set dce full url */ - ctx->dce_u_url = flb_sds_create_size(sizeof(FLB_AZ_LI_DCE_URL_TMPLT) - 1 + - flb_sds_len(ctx->dce_url) + - flb_sds_len(ctx->dcr_id) + - flb_sds_len(ctx->table_name)); - if (!ctx->dce_u_url) { - flb_errno(); - flb_az_li_ctx_destroy(ctx); - return NULL; - } - flb_sds_snprintf(&ctx->dce_u_url, flb_sds_alloc(ctx->dce_u_url), - FLB_AZ_LI_DCE_URL_TMPLT, ctx->dce_url, - ctx->dcr_id, ctx->table_name); - - /* Initialize the auth mutex */ - pthread_mutex_init(&ctx->token_mutex, NULL); - - /* Create oauth2 context */ - ctx->u_auth = flb_oauth2_create(config, ctx->auth_url, - FLB_AZ_LI_TOKEN_TIMEOUT); - if (!ctx->u_auth) { - flb_plg_error(ins, "cannot create oauth2 context"); - flb_az_li_ctx_destroy(ctx); - return NULL; - } - - /* Create upstream context for Log Ingsetion endpoint */ - ctx->u_dce = flb_upstream_create_url(config, ctx->dce_url, - FLB_AZ_LI_TLS_MODE, ins->tls); - if (!ctx->u_dce) { - flb_plg_error(ins, "upstream creation failed"); - flb_az_li_ctx_destroy(ctx); - return NULL; - } - flb_output_upstream_set(ctx->u_dce, ins); - - flb_plg_info(ins, "dce_url='%s', dcr='%s', table='%s', stream='Custom-%s'", - ctx->dce_url, ctx->dcr_id, ctx->table_name, ctx->table_name); - - return ctx; -} - -/* Free the context and created memory */ -int flb_az_li_ctx_destroy(struct flb_az_li *ctx) -{ - if (!ctx) { - return -1; - } - - if (ctx->auth_url) { - flb_sds_destroy(ctx->auth_url); - } - - if (ctx->dce_u_url) { - flb_sds_destroy(ctx->dce_u_url); - } - - if (ctx->u_auth) { - flb_oauth2_destroy(ctx->u_auth); - } - - if (ctx->u_dce) { - flb_upstream_destroy(ctx->u_dce); - } - flb_free(ctx); - - return 0; -} diff --git a/fluent-bit/plugins/out_azure_logs_ingestion/azure_logs_ingestion_conf.h b/fluent-bit/plugins/out_azure_logs_ingestion/azure_logs_ingestion_conf.h deleted file mode 100644 index 3886f75bc..000000000 --- a/fluent-bit/plugins/out_azure_logs_ingestion/azure_logs_ingestion_conf.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_AZURE_LOGS_INGESTION_CONF_H -#define FLB_OUT_AZURE_LOGS_INGESTION_CONF_H - -#include "azure_logs_ingestion.h" - -struct flb_az_li* flb_az_li_ctx_create(struct flb_output_instance *ins, - struct flb_config *config); -int flb_az_li_ctx_destroy(struct flb_az_li *ctx); - -#endif diff --git a/fluent-bit/plugins/out_bigquery/CMakeLists.txt b/fluent-bit/plugins/out_bigquery/CMakeLists.txt deleted file mode 100644 index 042b71bec..000000000 --- a/fluent-bit/plugins/out_bigquery/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(src - bigquery_conf.c - bigquery.c - ) - -FLB_PLUGIN(out_bigquery "${src}" "") diff --git a/fluent-bit/plugins/out_bigquery/bigquery.c b/fluent-bit/plugins/out_bigquery/bigquery.c deleted file mode 100644 index ab5b4657f..000000000 --- a/fluent-bit/plugins/out_bigquery/bigquery.c +++ /dev/null @@ -1,1159 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "bigquery.h" -#include "bigquery_conf.h" - -// TODO: The following code is copied from the Stackdriver plugin and should be -// factored into common library functions. - -/* - * Base64 Encoding in JWT must: - * - * - remove any trailing padding '=' character - * - replace '+' with '-' - * - replace '/' with '_' - * - * ref: https://www.rfc-editor.org/rfc/rfc7515.txt Appendix C - */ -int bigquery_jwt_base64_url_encode(unsigned char *out_buf, size_t out_size, - unsigned char *in_buf, size_t in_size, - size_t *olen) - -{ - int i; - size_t len; - int result; - - /* do normal base64 encoding */ - result = flb_base64_encode((unsigned char *) out_buf, out_size - 1, - &len, in_buf, in_size); - if (result != 0) { - return -1; - } - - /* Replace '+' and '/' characters */ - for (i = 0; i < len && out_buf[i] != '='; i++) { - if (out_buf[i] == '+') { - out_buf[i] = '-'; - } - else if (out_buf[i] == '/') { - out_buf[i] = '_'; - } - } - - /* Now 'i' becomes the new length */ - *olen = i; - return 0; -} - -static int bigquery_jwt_encode(struct flb_bigquery *ctx, - char *payload, char *secret, - char **out_signature, size_t *out_size) -{ - int ret; - int len; - int buf_size; - size_t olen; - char *buf; - char *sigd; - char *headers = "{\"alg\": \"RS256\", \"typ\": \"JWT\"}"; - unsigned char sha256_buf[32] = {0}; - flb_sds_t out; - unsigned char sig[256] = {0}; - size_t sig_len; - - buf_size = (strlen(payload) + strlen(secret)) * 2; - buf = flb_malloc(buf_size); - if (!buf) { - flb_errno(); - return -1; - } - - /* Encode header */ - len = strlen(headers); - ret = flb_base64_encode((unsigned char *) buf, buf_size - 1, - &olen, (unsigned char *) headers, len); - if (ret != 0) { - flb_free(buf); - - return ret; - } - - /* Create buffer to store JWT */ - out = flb_sds_create_size(2048); - if (!out) { - flb_errno(); - flb_free(buf); - return -1; - } - - /* Append header */ - out = flb_sds_cat(out, buf, olen); - out = flb_sds_cat(out, ".", 1); - - /* Encode Payload */ - len = strlen(payload); - bigquery_jwt_base64_url_encode((unsigned char *) buf, buf_size, - (unsigned char *) payload, len, &olen); - - /* Append Payload */ - out = flb_sds_cat(out, buf, olen); - - /* do sha256() of base64(header).base64(payload) */ - ret = flb_hash_simple(FLB_HASH_SHA256, - (unsigned char *) out, flb_sds_len(out), - sha256_buf, sizeof(sha256_buf)); - - if (ret != FLB_CRYPTO_SUCCESS) { - flb_plg_error(ctx->ins, "error hashing token"); - flb_free(buf); - flb_sds_destroy(out); - return -1; - } - - /* In mbedTLS cert length must include the null byte */ - len = strlen(secret) + 1; - - sig_len = sizeof(sig); - - ret = flb_crypto_sign_simple(FLB_CRYPTO_PRIVATE_KEY, - FLB_CRYPTO_PADDING_PKCS1, - FLB_HASH_SHA256, - (unsigned char *) secret, len, - sha256_buf, sizeof(sha256_buf), - sig, &sig_len); - - if (ret != FLB_CRYPTO_SUCCESS) { - flb_plg_error(ctx->ins, "error creating RSA context"); - flb_free(buf); - flb_sds_destroy(out); - return -1; - } - - sigd = flb_malloc(2048); - if (!sigd) { - flb_errno(); - flb_free(buf); - flb_sds_destroy(out); - return -1; - } - - bigquery_jwt_base64_url_encode((unsigned char *) sigd, 2048, sig, 256, &olen); - - out = flb_sds_cat(out, ".", 1); - out = flb_sds_cat(out, sigd, olen); - - *out_signature = out; - *out_size = flb_sds_len(out); - - flb_free(buf); - flb_free(sigd); - - return 0; -} - -/* Create a new oauth2 context and get a oauth2 token */ -static int bigquery_get_oauth2_token(struct flb_bigquery *ctx) -{ - int ret; - char *token; - char *sig_data; - size_t sig_size; - time_t issued; - time_t expires; - char payload[1024]; - - /* Clear any previous oauth2 payload content */ - flb_oauth2_payload_clear(ctx->o); - - /* JWT encode for oauth2 */ - issued = time(NULL); - expires = issued + FLB_BIGQUERY_TOKEN_REFRESH; - - snprintf(payload, sizeof(payload) - 1, - "{\"iss\": \"%s\", \"scope\": \"%s\", " - "\"aud\": \"%s\", \"exp\": %lu, \"iat\": %lu}", - ctx->oauth_credentials->client_email, FLB_BIGQUERY_SCOPE, - FLB_BIGQUERY_AUTH_URL, - expires, issued); - - /* Compose JWT signature */ - ret = bigquery_jwt_encode(ctx, payload, ctx->oauth_credentials->private_key, - &sig_data, &sig_size); - if (ret != 0) { - flb_plg_error(ctx->ins, "JWT signature generation failed"); - return -1; - } - - flb_plg_debug(ctx->ins, "JWT signature:\n%s", sig_data); - - ret = flb_oauth2_payload_append(ctx->o, - "grant_type", -1, - "urn%3Aietf%3Aparams%3Aoauth%3A" - "grant-type%3Ajwt-bearer", -1); - if (ret == -1) { - flb_plg_error(ctx->ins, "error appending oauth2 params"); - flb_sds_destroy(sig_data); - return -1; - } - - ret = flb_oauth2_payload_append(ctx->o, - "assertion", -1, - sig_data, sig_size); - if (ret == -1) { - flb_plg_error(ctx->ins, "error appending oauth2 params"); - flb_sds_destroy(sig_data); - return -1; - } - flb_sds_destroy(sig_data); - - /* Retrieve access token */ - token = flb_oauth2_token_get(ctx->o); - if (!token) { - flb_plg_error(ctx->ins, "error retrieving oauth2 access token"); - return -1; - } - - return 0; -} - -static flb_sds_t add_aws_signature(struct flb_http_client *c, struct flb_bigquery *ctx) { - flb_sds_t signature; - - flb_plg_debug(ctx->ins, "Signing the request with AWS SigV4 using IMDS credentials"); - - signature = flb_signv4_do(c, FLB_TRUE, FLB_TRUE, time(NULL), - ctx->aws_region, "sts", - 0, NULL, ctx->aws_provider); - if (!signature) { - flb_plg_error(ctx->ins, "Could not sign the request with AWS SigV4"); - return NULL; - } - - return signature; -} - -static inline int to_encode_path(char c) -{ - if ((c >= 48 && c <= 57) || /* 0-9 */ - (c >= 65 && c <= 90) || /* A-Z */ - (c >= 97 && c <= 122) || /* a-z */ - (c == '-' || c == '_' || c == '.' || c == '~' || c == '/')) { - return FLB_FALSE; - } - - return FLB_TRUE; -} - -static flb_sds_t uri_encode(const char *uri, size_t len) -{ - int i; - flb_sds_t buf = NULL; - flb_sds_t tmp = NULL; - - buf = flb_sds_create_size(len * 2); - if (!buf) { - flb_error("[uri_encode] cannot allocate buffer for URI encoding"); - return NULL; - } - - for (i = 0; i < len; i++) { - if (to_encode_path(uri[i]) == FLB_TRUE) { - tmp = flb_sds_printf(&buf, "%%%02X", (unsigned char) *(uri + i)); - if (!tmp) { - flb_error("[uri_encode] error formatting special character"); - flb_sds_destroy(buf); - return NULL; - } - continue; - } - - /* Direct assignment, just copy the character */ - if (buf) { - tmp = flb_sds_cat(buf, uri + i, 1); - if (!tmp) { - flb_error("[uri_encode] error composing outgoing buffer"); - flb_sds_destroy(buf); - return NULL; - } - buf = tmp; - } - } - - return buf; -} - -/* https://cloud.google.com/iam/docs/using-workload-identity-federation */ -static int bigquery_exchange_aws_creds_for_google_oauth(struct flb_bigquery *ctx) -{ - struct flb_connection *aws_sts_conn; - struct flb_connection *google_sts_conn = NULL; - struct flb_connection *google_gen_access_token_conn = NULL; - struct flb_http_client *aws_sts_c = NULL; - struct flb_http_client *google_sts_c = NULL; - struct flb_http_client *google_gen_access_token_c = NULL; - int google_sts_ret; - int google_gen_access_token_ret; - size_t b_sent_google_sts; - size_t b_sent_google_gen_access_token; - flb_sds_t signature = NULL; - flb_sds_t sigv4_amz_date = NULL; - flb_sds_t sigv4_amz_sec_token = NULL; - flb_sds_t aws_gci_url = NULL; - flb_sds_t aws_gci_goog_target_resource = NULL; - flb_sds_t aws_gci_token = NULL; - flb_sds_t aws_gci_token_encoded = NULL; - flb_sds_t google_sts_token = NULL; - flb_sds_t google_gen_access_token_body = NULL; - flb_sds_t google_gen_access_token_url = NULL; - flb_sds_t google_federated_token = NULL; - flb_sds_t google_auth_header = NULL; - - if (ctx->sa_token) { - flb_sds_destroy(ctx->sa_token); - ctx->sa_token = NULL; - } - - /* Sign an AWS STS request with AWS SigV4 signature */ - aws_sts_conn = flb_upstream_conn_get(ctx->aws_sts_upstream); - if (!aws_sts_conn) { - flb_plg_error(ctx->ins, "Failed to get upstream connection for AWS STS"); - goto error; - } - - aws_sts_c = flb_http_client(aws_sts_conn, FLB_HTTP_POST, FLB_BIGQUERY_AWS_STS_ENDPOINT, - NULL, 0, NULL, 0, NULL, 0); - if (!aws_sts_c) { - flb_plg_error(ctx->ins, "Failed to create HTTP client for AWS STS"); - goto error; - } - - signature = add_aws_signature(aws_sts_c, ctx); - if (!signature) { - flb_plg_error(ctx->ins, "Failed to sign AWS STS request"); - goto error; - } - - sigv4_amz_date = flb_sds_create(flb_kv_get_key_value("x-amz-date", &aws_sts_c->headers)); - if (!sigv4_amz_date) { - flb_plg_error(ctx->ins, "Failed to extract `x-amz-date` header from AWS STS signed request"); - goto error; - } - - sigv4_amz_sec_token = flb_sds_create(flb_kv_get_key_value("x-amz-security-token", &aws_sts_c->headers)); - if (!sigv4_amz_sec_token) { - flb_plg_error(ctx->ins, "Failed to extract `x-amz-security-token` header from AWS STS signed request"); - goto error; - } - - /* Create an AWS GetCallerIdentity token */ - - /* AWS STS endpoint URL */ - aws_gci_url = flb_sds_create_size(128); - aws_gci_url = flb_sds_printf(&aws_gci_url, - "https://%s%s", - ctx->aws_sts_endpoint, - FLB_BIGQUERY_AWS_STS_ENDPOINT); - - /* x-goog-cloud-target-resource header */ - aws_gci_goog_target_resource = flb_sds_create_size(128); - aws_gci_goog_target_resource = flb_sds_printf(&aws_gci_goog_target_resource, - FLB_BIGQUERY_GOOGLE_CLOUD_TARGET_RESOURCE, - ctx->project_number, ctx->pool_id, ctx->provider_id); - - aws_gci_token = flb_sds_create_size(2048); - aws_gci_token = flb_sds_printf( - &aws_gci_token, - "{\"url\":\"%s\",\"method\":\"POST\",\"headers\":[" - "{\"key\":\"Authorization\",\"value\":\"%s\"}," - "{\"key\":\"host\",\"value\":\"%s\"}," - "{\"key\":\"x-amz-date\",\"value\":\"%s\"}," - "{\"key\":\"x-goog-cloud-target-resource\",\"value\":\"%s\"}," - "{\"key\":\"x-amz-security-token\",\"value\":\"%s\"}" - "]}", - aws_gci_url, - signature, - ctx->aws_sts_endpoint, - sigv4_amz_date, - aws_gci_goog_target_resource, - sigv4_amz_sec_token); - - aws_gci_token_encoded = uri_encode(aws_gci_token, flb_sds_len(aws_gci_token)); - if (!aws_gci_token_encoded) { - flb_plg_error(ctx->ins, "Failed to encode GetCallerIdentity token"); - goto error; - } - - /* To exchange the AWS credential for a federated access token, - * we need to pass the AWS GetCallerIdentity token to the Google Security Token Service's token() method */ - google_sts_token = flb_sds_create_size(2048); - google_sts_token = flb_sds_printf( - &google_sts_token, - "{\"audience\":\"%s\"," - "\"grantType\":\"%s\"," - "\"requestedTokenType\":\"%s\"," - "\"scope\":\"%s\"," - "\"subjectTokenType\":\"%s\"," - "\"subjectToken\":\"%s\"}", - aws_gci_goog_target_resource, - FLB_BIGQUERY_GOOGLE_STS_TOKEN_GRANT_TYPE, - FLB_BIGQUERY_GOOGLE_STS_TOKEN_REQUESTED_TOKEN_TYPE, - FLB_BIGQUERY_GOOGLE_STS_TOKEN_SCOPE, - FLB_BIGQUERY_GOOGLE_STS_TOKEN_SUBJECT_TOKEN_TYPE, - aws_gci_token_encoded); - - google_sts_conn = flb_upstream_conn_get(ctx->google_sts_upstream); - if (!google_sts_conn) { - flb_plg_error(ctx->ins, "Google STS connection setup failed"); - goto error; - } - - google_sts_c = flb_http_client(google_sts_conn, FLB_HTTP_POST, FLB_BIGQUERY_GOOGLE_CLOUD_TOKEN_ENDPOINT, - google_sts_token, flb_sds_len(google_sts_token), - NULL, 0, NULL, 0); - - google_sts_ret = flb_http_do(google_sts_c, &b_sent_google_sts); - if (google_sts_ret != 0) { - flb_plg_error(ctx->ins, "Google STS token request http_do=%i", google_sts_ret); - goto error; - } - - if (google_sts_c->resp.status != 200) { - flb_plg_error(ctx->ins, "Google STS token response status: %i, payload:\n%s", - google_sts_c->resp.status, google_sts_c->resp.payload); - goto error; - } - - /* To exchange the federated token for a service account access token, - * we need to call the Google Service Account Credentials API generateAccessToken() method */ - google_federated_token = flb_json_get_val(google_sts_c->resp.payload, - google_sts_c->resp.payload_size, - "access_token"); - if (!google_federated_token) { - flb_plg_error(ctx->ins, "Failed to extract Google federated access token from STS token() response"); - goto error; - } - - google_gen_access_token_conn = flb_upstream_conn_get(ctx->google_iam_upstream); - if (!google_gen_access_token_conn) { - flb_plg_error(ctx->ins, "Google Service Account Credentials API connection setup failed"); - goto error; - } - - google_gen_access_token_url = flb_sds_create_size(256); - google_gen_access_token_url = flb_sds_printf(&google_gen_access_token_url, - FLB_BIGQUERY_GOOGLE_GEN_ACCESS_TOKEN_URL, - ctx->google_service_account); - - google_gen_access_token_body = flb_sds_create(FLB_BIGQUERY_GOOGLE_GEN_ACCESS_TOKEN_REQUEST_BODY); - - google_gen_access_token_c = flb_http_client(google_gen_access_token_conn, FLB_HTTP_POST, google_gen_access_token_url, - google_gen_access_token_body, flb_sds_len(google_gen_access_token_body), - NULL, 0, NULL, 0); - - google_auth_header = flb_sds_create_size(2048 + 7); - google_auth_header = flb_sds_printf(&google_auth_header, "%s%s", - "Bearer ", google_federated_token); - - flb_http_add_header(google_gen_access_token_c, "Authorization", 13, - google_auth_header, flb_sds_len(google_auth_header)); - - flb_http_add_header(google_gen_access_token_c, "Content-Type", 12, - "application/json; charset=utf-8", 31); - - google_gen_access_token_ret = flb_http_do(google_gen_access_token_c, &b_sent_google_gen_access_token); - if (google_gen_access_token_ret != 0) { - flb_plg_error(ctx->ins, "Google Service Account Credentials API generateAccessToken() request http_do=%i", - google_gen_access_token_ret); - goto error; - } - - if (google_gen_access_token_c->resp.status != 200) { - flb_plg_error(ctx->ins, "Google Service Account Credentials API generateAccessToken() response " - "status: %i, payload:\n%s", - google_gen_access_token_c->resp.status, google_gen_access_token_c->resp.payload); - goto error; - } - - ctx->sa_token = flb_json_get_val(google_gen_access_token_c->resp.payload, - google_gen_access_token_c->resp.payload_size, - "accessToken"); - if (!ctx->sa_token) { - flb_plg_error(ctx->ins, "Failed to extract Google OAuth token " - "from Service Account Credentials API generateAccessToken() response"); - goto error; - } - - ctx->sa_token_expiry = time(NULL) + FLB_BIGQUERY_TOKEN_REFRESH; - - flb_sds_destroy(signature); - flb_sds_destroy(sigv4_amz_date); - flb_sds_destroy(sigv4_amz_sec_token); - flb_sds_destroy(aws_gci_url); - flb_sds_destroy(aws_gci_goog_target_resource); - flb_sds_destroy(aws_gci_token); - flb_sds_destroy(aws_gci_token_encoded); - flb_sds_destroy(google_sts_token); - flb_sds_destroy(google_gen_access_token_body); - flb_sds_destroy(google_gen_access_token_url); - flb_sds_destroy(google_federated_token); - flb_sds_destroy(google_auth_header); - - flb_http_client_destroy(aws_sts_c); - flb_http_client_destroy(google_sts_c); - flb_http_client_destroy(google_gen_access_token_c); - - flb_upstream_conn_release(aws_sts_conn); - flb_upstream_conn_release(google_sts_conn); - flb_upstream_conn_release(google_gen_access_token_conn); - - flb_plg_info(ctx->ins, "Retrieved Google service account OAuth token via Identity Federation"); - - return 0; - -error: - flb_sds_destroy(signature); - flb_sds_destroy(sigv4_amz_date); - flb_sds_destroy(sigv4_amz_sec_token); - flb_sds_destroy(aws_gci_url); - flb_sds_destroy(aws_gci_goog_target_resource); - flb_sds_destroy(aws_gci_token); - flb_sds_destroy(aws_gci_token_encoded); - flb_sds_destroy(google_sts_token); - flb_sds_destroy(google_gen_access_token_body); - flb_sds_destroy(google_gen_access_token_url); - flb_sds_destroy(google_federated_token); - flb_sds_destroy(google_auth_header); - - if (aws_sts_c) { - flb_http_client_destroy(aws_sts_c); - } - - if (google_sts_c) { - flb_http_client_destroy(google_sts_c); - } - - if (google_gen_access_token_c) { - flb_http_client_destroy(google_gen_access_token_c); - } - - if (aws_sts_conn) { - flb_upstream_conn_release(aws_sts_conn); - } - - if (google_sts_conn) { - flb_upstream_conn_release(google_sts_conn); - } - - if (google_gen_access_token_conn) { - flb_upstream_conn_release(google_gen_access_token_conn); - } - - return -1; -} - -static int flb_bigquery_google_token_expired(time_t expiry) -{ - time_t now; - - now = time(NULL); - if (expiry <= now) { - return FLB_TRUE; - } - - return FLB_FALSE; -} - -static flb_sds_t get_google_service_account_token(struct flb_bigquery *ctx) { - int ret = 0; - flb_sds_t output; - flb_plg_trace(ctx->ins, "Getting Google service account token"); - - if (!ctx->sa_token) { - flb_plg_trace(ctx->ins, "Acquiring new token"); - ret = bigquery_exchange_aws_creds_for_google_oauth(ctx); - } - else if (flb_bigquery_google_token_expired(ctx->sa_token_expiry) == FLB_TRUE) { - flb_plg_trace(ctx->ins, "Replacing expired token"); - ret = bigquery_exchange_aws_creds_for_google_oauth(ctx); - } - - if (ret != 0) { - return NULL; - } - - output = flb_sds_create_size(2048 + 7); - output = flb_sds_printf(&output, "%s%s", "Bearer ", ctx->sa_token); - return output; -} - -static flb_sds_t get_google_token(struct flb_bigquery *ctx) -{ - int ret = 0; - flb_sds_t output = NULL; - - if (pthread_mutex_lock(&ctx->token_mutex)){ - flb_plg_error(ctx->ins, "error locking mutex"); - return NULL; - } - - if (flb_oauth2_token_expired(ctx->o) == FLB_TRUE) { - ret = bigquery_get_oauth2_token(ctx); - } - - /* Copy string to prevent race conditions (get_oauth2 can free the string) */ - if (ret == 0) { - output = flb_sds_create(ctx->o->token_type); - flb_sds_printf(&output, " %s", ctx->o->access_token); - } - - if (pthread_mutex_unlock(&ctx->token_mutex)){ - flb_plg_error(ctx->ins, "error unlocking mutex"); - if (output) { - flb_sds_destroy(output); - } - return NULL; - } - - return output; -} - -static int cb_bigquery_init(struct flb_output_instance *ins, - struct flb_config *config, void *data) -{ - char *token; - int io_flags = FLB_IO_TLS; - struct flb_bigquery *ctx; - - /* Create config context */ - ctx = flb_bigquery_conf_create(ins, config); - if (!ctx) { - flb_plg_error(ins, "configuration failed"); - return -1; - } - - flb_output_set_context(ins, ctx); - - /* Network mode IPv6 */ - if (ins->host.ipv6 == FLB_TRUE) { - io_flags |= FLB_IO_IPV6; - } - - /* Create mutex for acquiring oauth tokens (they are shared across flush coroutines) */ - pthread_mutex_init(&ctx->token_mutex, NULL); - - /* - * Create upstream context for BigQuery Streaming Inserts - * (no oauth2 service) - */ - ctx->u = flb_upstream_create_url(config, FLB_BIGQUERY_URL_BASE, - io_flags, ins->tls); - if (!ctx->u) { - flb_plg_error(ctx->ins, "upstream creation failed"); - return -1; - } - - if (ctx->has_identity_federation) { - /* Configure AWS IMDS */ - ctx->aws_tls = flb_tls_create(FLB_TLS_CLIENT_MODE, - FLB_TRUE, - ins->tls_debug, - ins->tls_vhost, - ins->tls_ca_path, - ins->tls_ca_file, - ins->tls_crt_file, - ins->tls_key_file, - ins->tls_key_passwd); - - if (!ctx->aws_tls) { - flb_plg_error(ctx->ins, "Failed to create TLS context"); - flb_bigquery_conf_destroy(ctx); - return -1; - } - - ctx->aws_provider = flb_standard_chain_provider_create(config, - ctx->aws_tls, - NULL, - NULL, - NULL, - flb_aws_client_generator(), - NULL); - - if (!ctx->aws_provider) { - flb_plg_error(ctx->ins, "Failed to create AWS Credential Provider"); - flb_bigquery_conf_destroy(ctx); - return -1; - } - - /* initialize credentials in sync mode */ - ctx->aws_provider->provider_vtable->sync(ctx->aws_provider); - ctx->aws_provider->provider_vtable->init(ctx->aws_provider); - - /* set back to async */ - ctx->aws_provider->provider_vtable->async(ctx->aws_provider); - ctx->aws_provider->provider_vtable->upstream_set(ctx->aws_provider, ctx->ins); - - /* Configure AWS STS */ - ctx->aws_sts_tls = flb_tls_create(FLB_TLS_CLIENT_MODE, - FLB_TRUE, - ins->tls_debug, - ins->tls_vhost, - ins->tls_ca_path, - ins->tls_ca_file, - ins->tls_crt_file, - ins->tls_key_file, - ins->tls_key_passwd); - - if (!ctx->aws_sts_tls) { - flb_plg_error(ctx->ins, "Failed to create TLS context"); - flb_bigquery_conf_destroy(ctx); - return -1; - } - - ctx->aws_sts_upstream = flb_upstream_create(config, - ctx->aws_sts_endpoint, - 443, - io_flags, - ctx->aws_sts_tls); - - if (!ctx->aws_sts_upstream) { - flb_plg_error(ctx->ins, "AWS STS upstream creation failed"); - flb_bigquery_conf_destroy(ctx); - return -1; - } - - ctx->aws_sts_upstream->base.net.keepalive = FLB_FALSE; - - /* Configure Google STS */ - ctx->google_sts_tls = flb_tls_create(FLB_TLS_CLIENT_MODE, - FLB_TRUE, - ins->tls_debug, - ins->tls_vhost, - ins->tls_ca_path, - ins->tls_ca_file, - ins->tls_crt_file, - ins->tls_key_file, - ins->tls_key_passwd); - - if (!ctx->google_sts_tls) { - flb_plg_error(ctx->ins, "Failed to create TLS context"); - flb_bigquery_conf_destroy(ctx); - return -1; - } - - ctx->google_sts_upstream = flb_upstream_create_url(config, - FLB_BIGQUERY_GOOGLE_STS_URL, - io_flags, - ctx->google_sts_tls); - - if (!ctx->google_sts_upstream) { - flb_plg_error(ctx->ins, "Google STS upstream creation failed"); - flb_bigquery_conf_destroy(ctx); - return -1; - } - - /* Configure Google IAM */ - ctx->google_iam_tls = flb_tls_create(FLB_TLS_CLIENT_MODE, - FLB_TRUE, - ins->tls_debug, - ins->tls_vhost, - ins->tls_ca_path, - ins->tls_ca_file, - ins->tls_crt_file, - ins->tls_key_file, - ins->tls_key_passwd); - - if (!ctx->google_iam_tls) { - flb_plg_error(ctx->ins, "Failed to create TLS context"); - flb_bigquery_conf_destroy(ctx); - return -1; - } - - ctx->google_iam_upstream = flb_upstream_create_url(config, - FLB_BIGQUERY_GOOGLE_IAM_URL, - io_flags, - ctx->google_iam_tls); - - if (!ctx->google_iam_upstream) { - flb_plg_error(ctx->ins, "Google IAM upstream creation failed"); - flb_bigquery_conf_destroy(ctx); - return -1; - } - - /* Remove async flag from upstream */ - flb_stream_disable_async_mode(&ctx->aws_sts_upstream->base); - flb_stream_disable_async_mode(&ctx->google_sts_upstream->base); - flb_stream_disable_async_mode(&ctx->google_iam_upstream->base); - } - - /* Create oauth2 context */ - ctx->o = flb_oauth2_create(ctx->config, FLB_BIGQUERY_AUTH_URL, 3000); - if (!ctx->o) { - flb_plg_error(ctx->ins, "cannot create oauth2 context"); - return -1; - } - flb_output_upstream_set(ctx->u, ins); - - /* Get or renew the OAuth2 token */ - if (ctx->has_identity_federation) { - token = get_google_service_account_token(ctx); - } - else { - token = get_google_token(ctx); - } - - if (!token) { - flb_plg_warn(ctx->ins, "token retrieval failed"); - } - else { - flb_sds_destroy(token); - } - - return 0; -} - -static int bigquery_format(const void *data, size_t bytes, - const char *tag, size_t tag_len, - char **out_data, size_t *out_size, - struct flb_bigquery *ctx) -{ - int array_size = 0; - flb_sds_t out_buf; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - int ret; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return -1; - } - - array_size = flb_mp_count(data, bytes); - - /* Create temporary msgpack buffer */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - /* - * Pack root map (kind & rows): - * - * { - * "kind": "bigquery#tableDataInsertAllRequest", - * "skipInvalidRows": boolean, - * "ignoreUnknownValues": boolean, - * "rows": [] - * } - */ - msgpack_pack_map(&mp_pck, 4); - - msgpack_pack_str(&mp_pck, 4); - msgpack_pack_str_body(&mp_pck, "kind", 4); - - msgpack_pack_str(&mp_pck, 34); - msgpack_pack_str_body(&mp_pck, "bigquery#tableDataInsertAllRequest", 34); - - msgpack_pack_str(&mp_pck, 15); - msgpack_pack_str_body(&mp_pck, "skipInvalidRows", 15); - - if (ctx->skip_invalid_rows) { - msgpack_pack_true(&mp_pck); - } - else { - msgpack_pack_false(&mp_pck); - } - - msgpack_pack_str(&mp_pck, 19); - msgpack_pack_str_body(&mp_pck, "ignoreUnknownValues", 19); - - if (ctx->ignore_unknown_values) { - msgpack_pack_true(&mp_pck); - } - else { - msgpack_pack_false(&mp_pck); - } - - msgpack_pack_str(&mp_pck, 4); - msgpack_pack_str_body(&mp_pck, "rows", 4); - - /* Append entries */ - msgpack_pack_array(&mp_pck, array_size); - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - /* - * Pack entry - * - * { - * "json": {...} - * } - * - * For now, we don't support the insertId that's required for duplicate detection. - */ - msgpack_pack_map(&mp_pck, 1); - - /* json */ - msgpack_pack_str(&mp_pck, 4); - msgpack_pack_str_body(&mp_pck, "json", 4); - msgpack_pack_object(&mp_pck, *log_event.body); - } - - /* Convert from msgpack to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); - - flb_log_event_decoder_destroy(&log_decoder); - msgpack_sbuffer_destroy(&mp_sbuf); - - if (!out_buf) { - flb_plg_error(ctx->ins, "error formatting JSON payload"); - return -1; - } - - *out_data = out_buf; - *out_size = flb_sds_len(out_buf); - - return 0; -} - -static void cb_bigquery_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - (void) i_ins; - (void) config; - int ret; - int ret_code = FLB_RETRY; - size_t b_sent; - flb_sds_t token; - flb_sds_t payload_buf; - size_t payload_size; - struct flb_bigquery *ctx = out_context; - struct flb_connection *u_conn; - struct flb_http_client *c; - - flb_plg_trace(ctx->ins, "flushing bytes %zu", event_chunk->size); - - /* Get upstream connection */ - u_conn = flb_upstream_conn_get(ctx->u); - if (!u_conn) { - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Get or renew Token */ - if (ctx->has_identity_federation) { - token = get_google_service_account_token(ctx); - } - else { - token = get_google_token(ctx); - } - - if (!token) { - flb_plg_error(ctx->ins, "cannot retrieve oauth2 token"); - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Reformat msgpack to bigquery JSON payload */ - ret = bigquery_format(event_chunk->data, event_chunk->size, - event_chunk->tag, flb_sds_len(event_chunk->tag), - &payload_buf, &payload_size, ctx); - if (ret != 0) { - flb_upstream_conn_release(u_conn); - flb_sds_destroy(token); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Compose HTTP Client request */ - c = flb_http_client(u_conn, FLB_HTTP_POST, ctx->uri, - payload_buf, payload_size, NULL, 0, NULL, 0); - if (!c) { - flb_plg_error(ctx->ins, "cannot create HTTP client context"); - flb_upstream_conn_release(u_conn); - flb_sds_destroy(token); - flb_sds_destroy(payload_buf); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - flb_http_buffer_size(c, 4192); - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - flb_http_add_header(c, "Content-Type", 12, "application/json", 16); - - /* Compose and append Authorization header */ - flb_http_add_header(c, "Authorization", 13, token, flb_sds_len(token)); - - /* Send HTTP request */ - ret = flb_http_do(c, &b_sent); - - /* validate response */ - if (ret != 0) { - flb_plg_warn(ctx->ins, "http_do=%i", ret); - ret_code = FLB_RETRY; - } - else { - /* The request was issued successfully, validate the 'error' field */ - flb_plg_debug(ctx->ins, "HTTP Status=%i", c->resp.status); - if (c->resp.status == 200) { - ret_code = FLB_OK; - } - else { - if (c->resp.payload && c->resp.payload_size > 0) { - /* we got an error */ - flb_plg_warn(ctx->ins, "response\n%s", c->resp.payload); - } - ret_code = FLB_RETRY; - } - } - - /* Cleanup */ - flb_sds_destroy(payload_buf); - flb_sds_destroy(token); - flb_http_client_destroy(c); - flb_upstream_conn_release(u_conn); - - /* Done */ - FLB_OUTPUT_RETURN(ret_code); -} - -static int cb_bigquery_exit(void *data, struct flb_config *config) -{ - struct flb_bigquery *ctx = data; - - if (!ctx) { - return -1; - } - - if (ctx->u) { - flb_upstream_destroy(ctx->u); - } - - flb_bigquery_conf_destroy(ctx); - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "google_service_credentials", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_bigquery, credentials_file), - "Set the path for the google service credentials file" - }, - { - FLB_CONFIG_MAP_BOOL, "enable_identity_federation", "false", - 0, FLB_TRUE, offsetof(struct flb_bigquery, has_identity_federation), - "Enable identity federation" - }, - { - FLB_CONFIG_MAP_STR, "aws_region", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_bigquery, aws_region), - "Enable identity federation" - }, - { - FLB_CONFIG_MAP_STR, "project_number", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_bigquery, project_number), - "Set project number" - }, - { - FLB_CONFIG_MAP_STR, "pool_id", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_bigquery, pool_id), - "Set the pool id" - }, - { - FLB_CONFIG_MAP_STR, "provider_id", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_bigquery, provider_id), - "Set the provider id" - }, - { - FLB_CONFIG_MAP_STR, "google_service_account", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_bigquery, google_service_account), - "Set the google service account" - }, - // set in flb_bigquery_oauth_credentials - { - FLB_CONFIG_MAP_STR, "service_account_email", (char *)NULL, - 0, FLB_FALSE, 0, - "Set the service account email" - }, - // set in flb_bigquery_oauth_credentials - { - FLB_CONFIG_MAP_STR, "service_account_secret", (char *)NULL, - 0, FLB_FALSE, 0, - "Set the service account secret" - }, - { - FLB_CONFIG_MAP_STR, "project_id", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_bigquery, project_id), - "Set the project id" - }, - { - FLB_CONFIG_MAP_STR, "dataset_id", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_bigquery, dataset_id), - "Set the dataset id" - }, - { - FLB_CONFIG_MAP_STR, "table_id", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_bigquery, table_id), - "Set the table id" - }, - { - FLB_CONFIG_MAP_BOOL, "skip_invalid_rows", "false", - 0, FLB_TRUE, offsetof(struct flb_bigquery, skip_invalid_rows), - "Enable skipping of invalid rows", - }, - { - FLB_CONFIG_MAP_BOOL, "ignore_unknown_values", "false", - 0, FLB_TRUE, offsetof(struct flb_bigquery, ignore_unknown_values), - "Enable ignoring unknown value", - }, - /* EOF */ - {0} -}; - -struct flb_output_plugin out_bigquery_plugin = { - .name = "bigquery", - .description = "Send events to BigQuery via streaming insert", - .cb_init = cb_bigquery_init, - .cb_flush = cb_bigquery_flush, - .cb_exit = cb_bigquery_exit, - .config_map = config_map, - /* Plugin flags */ - .flags = FLB_OUTPUT_NET | FLB_IO_TLS, -}; diff --git a/fluent-bit/plugins/out_bigquery/bigquery.h b/fluent-bit/plugins/out_bigquery/bigquery.h deleted file mode 100644 index c48d9ba41..000000000 --- a/fluent-bit/plugins/out_bigquery/bigquery.h +++ /dev/null @@ -1,132 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_BIGQUERY -#define FLB_OUT_BIGQUERY - -#include -#include -#include -#include - -/* refresh token every 50 minutes */ -#define FLB_BIGQUERY_TOKEN_REFRESH 3000 - -/* BigQuery streaming inserts oauth scope */ -#define FLB_BIGQUERY_SCOPE "https://www.googleapis.com/auth/bigquery.insertdata" - -/* BigQuery authorization URL */ -#define FLB_BIGQUERY_AUTH_URL "https://oauth2.googleapis.com/token" - -#define FLB_BIGQUERY_RESOURCE_TEMPLATE "/bigquery/v2/projects/%s/datasets/%s/tables/%s/insertAll" -#define FLB_BIGQUERY_URL_BASE "https://www.googleapis.com" - -#define FLB_BIGQUERY_GOOGLE_STS_URL "https://sts.googleapis.com" -#define FLB_BIGQUERY_GOOGLE_IAM_URL "https://iamcredentials.googleapis.com" -#define FLB_BIGQUERY_AWS_STS_ENDPOINT "/?Action=GetCallerIdentity&Version=2011-06-15" - -#define FLB_BIGQUERY_GOOGLE_CLOUD_TARGET_RESOURCE \ - "//iam.googleapis.com/projects/%s/locations/global/workloadIdentityPools/%s/providers/%s" - -#define FLB_BIGQUERY_GOOGLE_STS_TOKEN_GRANT_TYPE "urn:ietf:params:oauth:grant-type:token-exchange" -#define FLB_BIGQUERY_GOOGLE_STS_TOKEN_REQUESTED_TOKEN_TYPE "urn:ietf:params:oauth:token-type:access_token" -#define FLB_BIGQUERY_GOOGLE_STS_TOKEN_SCOPE "https://www.googleapis.com/auth/cloud-platform" -#define FLB_BIGQUERY_GOOGLE_STS_TOKEN_SUBJECT_TOKEN_TYPE "urn:ietf:params:aws:token-type:aws4_request" -#define FLB_BIGQUERY_GOOGLE_CLOUD_TOKEN_ENDPOINT "/v1/token" - -#define FLB_BIGQUERY_GOOGLE_GEN_ACCESS_TOKEN_REQUEST_BODY \ - "{\"scope\": [\"https://www.googleapis.com/auth/cloud-platform\"]}" - -#define FLB_BIGQUERY_GOOGLE_GEN_ACCESS_TOKEN_URL \ - "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/%s:generateAccessToken" - -struct flb_bigquery_oauth_credentials { - /* parsed credentials file */ - flb_sds_t type; - flb_sds_t project_id; - flb_sds_t private_key_id; - flb_sds_t private_key; - flb_sds_t client_email; - flb_sds_t client_id; - flb_sds_t auth_uri; - flb_sds_t token_uri; -}; - -struct flb_bigquery { - /* credentials */ - flb_sds_t credentials_file; - - struct flb_bigquery_oauth_credentials *oauth_credentials; - - /* Workload Identity Federation */ - int has_identity_federation; - flb_sds_t project_number; - flb_sds_t pool_id; - flb_sds_t provider_id; - flb_sds_t aws_region; - flb_sds_t google_service_account; - - /* AWS IMDS */ - struct flb_tls *aws_tls; - struct flb_aws_provider *aws_provider; - - /* AWS STS */ - flb_sds_t aws_sts_endpoint; - struct flb_tls *aws_sts_tls; - struct flb_upstream *aws_sts_upstream; - - /* Google STS API */ - struct flb_tls *google_sts_tls; - struct flb_upstream *google_sts_upstream; - - /* Google Service Account Credentials API */ - struct flb_tls *google_iam_tls; - struct flb_upstream *google_iam_upstream; - - /* Google OAuth access token for service account, that was exchanged for AWS credentials */ - flb_sds_t sa_token; - time_t sa_token_expiry; - - /* bigquery configuration */ - flb_sds_t project_id; - flb_sds_t dataset_id; - flb_sds_t table_id; - - int skip_invalid_rows; - int ignore_unknown_values; - - flb_sds_t uri; - - /* oauth2 context */ - struct flb_oauth2 *o; - - /* mutex for acquiring oauth tokens */ - pthread_mutex_t token_mutex; - - /* Upstream connection to the backend server */ - struct flb_upstream *u; - - /* Fluent Bit context */ - struct flb_config *config; - - /* Plugin output instance reference */ - struct flb_output_instance *ins; -}; - -#endif diff --git a/fluent-bit/plugins/out_bigquery/bigquery_conf.c b/fluent-bit/plugins/out_bigquery/bigquery_conf.c deleted file mode 100644 index a7855d92f..000000000 --- a/fluent-bit/plugins/out_bigquery/bigquery_conf.c +++ /dev/null @@ -1,435 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "bigquery.h" -#include "bigquery_conf.h" - - -static inline int key_cmp(char *str, int len, char *cmp) { - - if (strlen(cmp) != len) { - return -1; - } - - return strncasecmp(str, cmp, len); -} - -static int flb_bigquery_read_credentials_file(struct flb_bigquery *ctx, - char *creds, - struct flb_bigquery_oauth_credentials *ctx_creds) -{ - int i; - int ret; - int len; - int key_len; - int val_len; - int tok_size = 32; - char *buf; - char *key; - char *val; - flb_sds_t tmp; - struct stat st; - jsmn_parser parser; - jsmntok_t *t; - jsmntok_t *tokens; - - /* Validate credentials path */ - ret = stat(creds, &st); - if (ret == -1) { - flb_errno(); - flb_plg_error(ctx->ins, "cannot open credentials file: %s", - creds); - return -1; - } - - if (!S_ISREG(st.st_mode) && !S_ISLNK(st.st_mode)) { - flb_plg_error(ctx->ins, "credentials file " - "is not a valid file: %s", creds); - return -1; - } - - /* Read file content */ - buf = mk_file_to_buffer(creds); - if (!buf) { - flb_plg_error(ctx->ins, "error reading credentials file: %s", - creds); - return -1; - } - - /* Parse content */ - jsmn_init(&parser); - tokens = flb_calloc(1, sizeof(jsmntok_t) * tok_size); - if (!tokens) { - flb_errno(); - flb_free(buf); - return -1; - } - - ret = jsmn_parse(&parser, buf, st.st_size, tokens, tok_size); - if (ret <= 0) { - flb_plg_error(ctx->ins, "invalid JSON credentials file: %s", - creds); - flb_free(buf); - flb_free(tokens); - return -1; - } - - t = &tokens[0]; - if (t->type != JSMN_OBJECT) { - flb_plg_error(ctx->ins, "invalid JSON map on file: %s", - creds); - flb_free(buf); - flb_free(tokens); - return -1; - } - - /* Parse JSON tokens */ - for (i = 1; i < ret; i++) { - t = &tokens[i]; - if (t->type != JSMN_STRING) { - continue; - } - - if (t->start == -1 || t->end == -1 || (t->start == 0 && t->end == 0)){ - break; - } - - /* Key */ - key = buf + t->start; - key_len = (t->end - t->start); - - /* Value */ - i++; - t = &tokens[i]; - val = buf + t->start; - val_len = (t->end - t->start); - - if (key_cmp(key, key_len, "type") == 0) { - ctx_creds->type = flb_sds_create_len(val, val_len); - } - else if (key_cmp(key, key_len, "project_id") == 0) { - ctx_creds->project_id = flb_sds_create_len(val, val_len); - } - else if (key_cmp(key, key_len, "private_key_id") == 0) { - ctx_creds->private_key_id = flb_sds_create_len(val, val_len); - } - else if (key_cmp(key, key_len, "private_key") == 0) { - tmp = flb_sds_create_len(val, val_len); - if (tmp) { - /* Unescape private key */ - len = flb_sds_len(tmp); - ctx_creds->private_key = flb_sds_create_size(len); - flb_unescape_string(tmp, len, - &ctx_creds->private_key); - flb_sds_destroy(tmp); - } - } - else if (key_cmp(key, key_len, "client_email") == 0) { - ctx_creds->client_email = flb_sds_create_len(val, val_len); - } - else if (key_cmp(key, key_len, "client_id") == 0) { - ctx_creds->client_id = flb_sds_create_len(val, val_len); - } - else if (key_cmp(key, key_len, "auth_uri") == 0) { - ctx_creds->auth_uri = flb_sds_create_len(val, val_len); - } - else if (key_cmp(key, key_len, "token_uri") == 0) { - ctx_creds->token_uri = flb_sds_create_len(val, val_len); - } - } - - flb_free(buf); - flb_free(tokens); - - return 0; -} - - -struct flb_bigquery *flb_bigquery_conf_create(struct flb_output_instance *ins, - struct flb_config *config) -{ - int ret; - const char *tmp; - char *tmp_aws_region; - struct flb_bigquery *ctx; - struct flb_bigquery_oauth_credentials *creds; - - /* Allocate config context */ - ctx = flb_calloc(1, sizeof(struct flb_bigquery)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - ctx->config = config; - - ret = flb_output_config_map_set(ins, (void *)ctx); - if (ret == -1) { - flb_plg_error(ins, "unable to load configuration"); - flb_free(ctx); - return NULL; - } - - /* Lookup credentials file */ - creds = flb_calloc(1, sizeof(struct flb_bigquery_oauth_credentials)); - if (!creds) { - flb_errno(); - flb_free(ctx); - return NULL; - } - ctx->oauth_credentials = creds; - - if (ctx->credentials_file == NULL) { - tmp = getenv("GOOGLE_SERVICE_CREDENTIALS"); - if (tmp) { - ctx->credentials_file = flb_sds_create(tmp); - } - } - - if (ctx->credentials_file && ctx->has_identity_federation) { - flb_plg_error(ctx->ins, "Either `google_service_credentials` or `enable_identity_federation` should be set"); - return NULL; - } - - if (ctx->aws_region) { - tmp_aws_region = flb_aws_endpoint("sts", ctx->aws_region); - if (!tmp_aws_region) { - flb_plg_error(ctx->ins, "Could not create AWS STS regional endpoint"); - return NULL; - } - ctx->aws_sts_endpoint = flb_sds_create(tmp_aws_region); - flb_free(tmp_aws_region); - } - - if (ctx->has_identity_federation) { - if (!ctx->aws_region) { - flb_plg_error(ctx->ins, "`aws_region` is required when `enable_identity_federation` is true"); - return NULL; - } - - if (!ctx->project_number) { - flb_plg_error(ctx->ins, "`project_number` is required when `enable_identity_federation` is true"); - return NULL; - } - - if (!ctx->pool_id) { - flb_plg_error(ctx->ins, "`pool_id` is required when `enable_identity_federation` is true"); - return NULL; - } - - if (!ctx->provider_id) { - flb_plg_error(ctx->ins, "`provider_id` is required when `enable_identity_federation` is true"); - return NULL; - } - - if (!ctx->google_service_account) { - flb_plg_error(ctx->ins, "`google_service_account` is required when `enable_identity_federation` is true"); - return NULL; - } - } - - if (ctx->credentials_file) { - ret = flb_bigquery_read_credentials_file(ctx, - ctx->credentials_file, - ctx->oauth_credentials); - if (ret != 0) { - flb_bigquery_conf_destroy(ctx); - return NULL; - } - } - else if (!ctx->credentials_file && !ctx->has_identity_federation) { - /* - * If no credentials file has been defined, do manual lookup of the - * client email and the private key. - */ - - /* Service Account Email */ - tmp = flb_output_get_property("service_account_email", ins); - if (tmp) { - creds->client_email = flb_sds_create(tmp); - } - else { - tmp = getenv("SERVICE_ACCOUNT_EMAIL"); - if (tmp) { - creds->client_email = flb_sds_create(tmp); - } - } - - /* Service Account Secret */ - tmp = flb_output_get_property("service_account_secret", ins); - if (tmp) { - creds->private_key = flb_sds_create(tmp); - } - else { - tmp = getenv("SERVICE_ACCOUNT_SECRET"); - if (tmp) { - creds->private_key = flb_sds_create(tmp); - } - } - - if (!creds->client_email) { - flb_plg_error(ctx->ins, "service_account_email/client_email is not defined"); - flb_bigquery_conf_destroy(ctx); - return NULL; - } - - if (!creds->private_key) { - flb_plg_error(ctx->ins, "service_account_secret/private_key is not defined"); - flb_bigquery_conf_destroy(ctx); - return NULL; - } - } - - /* config: 'project_id' */ - if (ctx->project_id == NULL) { - if (creds->project_id) { - /* flb_config_map_destroy uses the pointer within the config_map struct to - * free the value so if we assign it here it is safe to free later with the - * creds struct. If we do not we will leak here. - */ - ctx->project_id = creds->project_id; - if (!ctx->project_id) { - flb_plg_error(ctx->ins, - "failed extracting 'project_id' from credentials."); - flb_bigquery_conf_destroy(ctx); - return NULL; - } - } - else { - flb_plg_error(ctx->ins, - "no 'project_id' configured or present in credentials."); - flb_bigquery_conf_destroy(ctx); - return NULL; - } - } - - /* config: 'dataset_id' */ - if (ctx->dataset_id == NULL) { - flb_plg_error(ctx->ins, "property 'dataset_id' is not defined"); - flb_bigquery_conf_destroy(ctx); - return NULL; - } - - /* config: 'table_id' */ - if (ctx->table_id == NULL) { - flb_plg_error(ctx->ins, "property 'table_id' is not defined"); - flb_bigquery_conf_destroy(ctx); - return NULL; - } - - /* Create the target URI */ - ctx->uri = flb_sds_create_size(sizeof(FLB_BIGQUERY_RESOURCE_TEMPLATE)-6 + - flb_sds_len(ctx->project_id) + - flb_sds_len(ctx->dataset_id) + - flb_sds_len(ctx->table_id)); - if (!ctx->uri) { - flb_errno(); - flb_bigquery_conf_destroy(ctx); - return NULL; - } - ctx->uri = flb_sds_printf(&ctx->uri, FLB_BIGQUERY_RESOURCE_TEMPLATE, - ctx->project_id, ctx->dataset_id, ctx->table_id); - - flb_plg_info(ctx->ins, "project='%s' dataset='%s' table='%s'", - ctx->project_id, ctx->dataset_id, ctx->table_id); - - return ctx; -} - - -int flb_bigquery_oauth_credentials_destroy(struct flb_bigquery_oauth_credentials *creds) -{ - if (!creds) { - return -1; - } - flb_sds_destroy(creds->type); - flb_sds_destroy(creds->project_id); - flb_sds_destroy(creds->private_key_id); - flb_sds_destroy(creds->private_key); - flb_sds_destroy(creds->client_email); - flb_sds_destroy(creds->client_id); - flb_sds_destroy(creds->auth_uri); - flb_sds_destroy(creds->token_uri); - - flb_free(creds); - - return 0; -} - -int flb_bigquery_conf_destroy(struct flb_bigquery *ctx) -{ - if (!ctx) { - return -1; - } - - flb_bigquery_oauth_credentials_destroy(ctx->oauth_credentials); - - if (ctx->aws_sts_upstream) { - flb_upstream_destroy(ctx->aws_sts_upstream); - } - - if (ctx->google_sts_upstream) { - flb_upstream_destroy(ctx->google_sts_upstream); - } - - if (ctx->google_iam_upstream) { - flb_upstream_destroy(ctx->google_iam_upstream); - } - - if (ctx->aws_provider) { - flb_aws_provider_destroy(ctx->aws_provider); - } - - if (ctx->aws_tls) { - flb_tls_destroy(ctx->aws_tls); - } - - if (ctx->aws_sts_tls) { - flb_tls_destroy(ctx->aws_sts_tls); - } - - if (ctx->google_sts_tls) { - flb_tls_destroy(ctx->google_sts_tls); - } - - if (ctx->google_iam_tls) { - flb_tls_destroy(ctx->google_iam_tls); - } - - flb_sds_destroy(ctx->aws_sts_endpoint); - flb_sds_destroy(ctx->sa_token); - flb_sds_destroy(ctx->uri); - - if (ctx->o) { - flb_oauth2_destroy(ctx->o); - } - - flb_free(ctx); - return 0; -} diff --git a/fluent-bit/plugins/out_bigquery/bigquery_conf.h b/fluent-bit/plugins/out_bigquery/bigquery_conf.h deleted file mode 100644 index f06a86607..000000000 --- a/fluent-bit/plugins/out_bigquery/bigquery_conf.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_BIGQUERY_CONF_H -#define FLB_OUT_BIGQUERY_CONF_H - -#include "bigquery.h" - -struct flb_bigquery *flb_bigquery_conf_create(struct flb_output_instance *ins, - struct flb_config *config); -int flb_bigquery_conf_destroy(struct flb_bigquery *ctx); - -#endif diff --git a/fluent-bit/plugins/out_calyptia/CMakeLists.txt b/fluent-bit/plugins/out_calyptia/CMakeLists.txt deleted file mode 100644 index 064c4b835..000000000 --- a/fluent-bit/plugins/out_calyptia/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - calyptia.c) - -FLB_PLUGIN(out_calyptia "${src}" "") diff --git a/fluent-bit/plugins/out_calyptia/calyptia.c b/fluent-bit/plugins/out_calyptia/calyptia.c deleted file mode 100644 index 19811dcc9..000000000 --- a/fluent-bit/plugins/out_calyptia/calyptia.c +++ /dev/null @@ -1,1025 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "calyptia.h" - -#include -#include - -flb_sds_t custom_calyptia_pipeline_config_get(struct flb_config *ctx); - -static int get_io_flags(struct flb_output_instance *ins) -{ - int flags = 0; - - if (ins->use_tls) { - flags = FLB_IO_TLS; - } - else { - flags = FLB_IO_TCP; - } - - return flags; -} - -static int config_add_labels(struct flb_output_instance *ins, - struct flb_calyptia *ctx) -{ - struct mk_list *head; - struct flb_config_map_val *mv; - struct flb_slist_entry *k = NULL; - struct flb_slist_entry *v = NULL; - struct flb_kv *kv; - - if (!ctx->add_labels || mk_list_size(ctx->add_labels) == 0) { - return 0; - } - - /* iterate all 'add_label' definitions */ - flb_config_map_foreach(head, mv, ctx->add_labels) { - if (mk_list_size(mv->val.list) != 2) { - flb_plg_error(ins, "'add_label' expects a key and a value, " - "e.g: 'add_label version 1.8.x'"); - return -1; - } - - k = mk_list_entry_first(mv->val.list, struct flb_slist_entry, _head); - v = mk_list_entry_last(mv->val.list, struct flb_slist_entry, _head); - - kv = flb_kv_item_create(&ctx->kv_labels, k->str, v->str); - if (!kv) { - flb_plg_error(ins, "could not append label %s=%s\n", k->str, v->str); - return -1; - } - } - - return 0; -} - -static void append_labels(struct flb_calyptia *ctx, struct cmt *cmt) -{ - struct flb_kv *kv; - struct mk_list *head; - - mk_list_foreach(head, &ctx->kv_labels) { - kv = mk_list_entry(head, struct flb_kv, _head); - cmt_label_add(cmt, kv->key, kv->val); - } -} - -static void pack_str(msgpack_packer *mp_pck, char *str) -{ - int len; - - len = strlen(str); - msgpack_pack_str(mp_pck, len); - msgpack_pack_str_body(mp_pck, str, len); -} - -static void pack_env(struct flb_env *env, char *prefix, char *key, - struct flb_mp_map_header *h, - msgpack_packer *mp_pck) -{ - int len = 0; - char *val; - - /* prefix set in the key, if set, adjust the key name */ - if (prefix) { - len = strlen(prefix); - } - - val = (char *) flb_env_get(env, key); - if (val) { - flb_mp_map_header_append(h); - pack_str(mp_pck, key + len); - pack_str(mp_pck, val); - } -} - -static void pack_env_metadata(struct flb_env *env, - struct flb_mp_map_header *mh, msgpack_packer *mp_pck) -{ - char *tmp; - struct flb_mp_map_header h; - struct flb_mp_map_header meta; - - /* Metadata */ - flb_mp_map_header_append(mh); - pack_str(mp_pck, "metadata"); - - flb_mp_map_header_init(&meta, mp_pck); - - /* Kubernetes */ - tmp = (char *) flb_env_get(env, "k8s"); - if (tmp && strcasecmp(tmp, "enabled") == 0) { - flb_mp_map_header_append(&meta); - pack_str(mp_pck, "k8s"); - - /* adding k8s map */ - flb_mp_map_header_init(&h, mp_pck); - - pack_env(env, "k8s.", "k8s.namespace", &h, mp_pck); - pack_env(env, "k8s.", "k8s.pod_name", &h, mp_pck); - pack_env(env, "k8s.", "k8s.node_name", &h, mp_pck); - - flb_mp_map_header_end(&h); - } - - /* AWS */ - tmp = (char *) flb_env_get(env, "aws"); - if (tmp && strcasecmp(tmp, "enabled") == 0) { - flb_mp_map_header_append(&meta); - pack_str(mp_pck, "aws"); - - /* adding aws map */ - flb_mp_map_header_init(&h, mp_pck); - - pack_env(env, "aws.", "aws.az", &h, mp_pck); - pack_env(env, "aws.", "aws.ec2_instance_id", &h, mp_pck); - pack_env(env, "aws.", "aws.ec2_instance_type", &h, mp_pck); - pack_env(env, "aws.", "aws.private_ip", &h, mp_pck); - pack_env(env, "aws.", "aws.vpc_id", &h, mp_pck); - pack_env(env, "aws.", "aws.ami_id", &h, mp_pck); - pack_env(env, "aws.", "aws.account_id", &h, mp_pck); - pack_env(env, "aws.", "aws.hostname", &h, mp_pck); - - flb_mp_map_header_end(&h); - } - flb_mp_map_header_end(&meta); -} - -static flb_sds_t get_agent_metadata(struct flb_calyptia *ctx) -{ - int len; - char *host; - flb_sds_t conf; - flb_sds_t meta; - struct flb_mp_map_header mh; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - struct flb_config *config = ctx->config; - - /* init msgpack */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - /* pack map */ - flb_mp_map_header_init(&mh, &mp_pck); - - host = (char *) flb_env_get(ctx->env, "HOSTNAME"); - if (!host) { - host = "unknown"; - } - len = strlen(host); - - /* name */ - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 4); - msgpack_pack_str_body(&mp_pck, "name", 4); - msgpack_pack_str(&mp_pck, len); - msgpack_pack_str_body(&mp_pck, host, len); - - /* type */ - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 4); - msgpack_pack_str_body(&mp_pck, "type", 4); - msgpack_pack_str(&mp_pck, 9); - msgpack_pack_str_body(&mp_pck, "fluentbit", 9); - - /* rawConfig */ - conf = custom_calyptia_pipeline_config_get(ctx->config); - if (conf) { - flb_mp_map_header_append(&mh); - len = flb_sds_len(conf); - msgpack_pack_str(&mp_pck, 9); - msgpack_pack_str_body(&mp_pck, "rawConfig", 9); - msgpack_pack_str(&mp_pck, len); - msgpack_pack_str_body(&mp_pck, conf, len); - } - flb_sds_destroy(conf); - - /* version */ - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 7); - msgpack_pack_str_body(&mp_pck, "version", 7); - len = strlen(FLB_VERSION_STR); - msgpack_pack_str(&mp_pck, len); - msgpack_pack_str_body(&mp_pck, FLB_VERSION_STR, len); - - /* edition */ - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 7); - msgpack_pack_str_body(&mp_pck, "edition", 7); - msgpack_pack_str(&mp_pck, 9); - msgpack_pack_str_body(&mp_pck, "community", 9); - - /* machineID */ - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 9); - msgpack_pack_str_body(&mp_pck, "machineID", 9); - len = flb_sds_len(ctx->machine_id); - msgpack_pack_str(&mp_pck, len); - msgpack_pack_str_body(&mp_pck, ctx->machine_id, len); - - /* fleetID */ - if (ctx->fleet_id) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 7); - msgpack_pack_str_body(&mp_pck, "fleetID", 7); - len = flb_sds_len(ctx->fleet_id); - msgpack_pack_str(&mp_pck, len); - msgpack_pack_str_body(&mp_pck, ctx->fleet_id, len); - } - - /* pack environment metadata */ - pack_env_metadata(config->env, &mh, &mp_pck); - - /* finalize */ - flb_mp_map_header_end(&mh); - - /* convert to json */ - meta = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); - msgpack_sbuffer_destroy(&mp_sbuf); - - return meta; -} - -static int calyptia_http_do(struct flb_calyptia *ctx, struct flb_http_client *c, - int type) -{ - int ret; - size_t b_sent; - - /* append headers */ - if (type == CALYPTIA_ACTION_REGISTER) { - flb_http_add_header(c, - CALYPTIA_H_CTYPE, sizeof(CALYPTIA_H_CTYPE) - 1, - CALYPTIA_H_CTYPE_JSON, sizeof(CALYPTIA_H_CTYPE_JSON) - 1); - - flb_http_add_header(c, - CALYPTIA_H_PROJECT, sizeof(CALYPTIA_H_PROJECT) - 1, - ctx->api_key, flb_sds_len(ctx->api_key)); - } - else if (type == CALYPTIA_ACTION_PATCH) { - flb_http_add_header(c, - CALYPTIA_H_CTYPE, sizeof(CALYPTIA_H_CTYPE) - 1, - CALYPTIA_H_CTYPE_JSON, sizeof(CALYPTIA_H_CTYPE_JSON) - 1); - - flb_http_add_header(c, - CALYPTIA_H_AGENT_TOKEN, - sizeof(CALYPTIA_H_AGENT_TOKEN) - 1, - ctx->agent_token, flb_sds_len(ctx->agent_token)); - } - else if (type == CALYPTIA_ACTION_METRICS) { - flb_http_add_header(c, - CALYPTIA_H_CTYPE, sizeof(CALYPTIA_H_CTYPE) - 1, - CALYPTIA_H_CTYPE_MSGPACK, - sizeof(CALYPTIA_H_CTYPE_MSGPACK) - 1); - - flb_http_add_header(c, - CALYPTIA_H_AGENT_TOKEN, - sizeof(CALYPTIA_H_AGENT_TOKEN) - 1, - ctx->agent_token, flb_sds_len(ctx->agent_token)); - } -#ifdef FLB_HAVE_CHUNK_TRACE - else if (type == CALYPTIA_ACTION_TRACE) { - flb_http_add_header(c, - CALYPTIA_H_CTYPE, sizeof(CALYPTIA_H_CTYPE) - 1, - CALYPTIA_H_CTYPE_JSON, sizeof(CALYPTIA_H_CTYPE_JSON) - 1); - - flb_http_add_header(c, - CALYPTIA_H_AGENT_TOKEN, - sizeof(CALYPTIA_H_AGENT_TOKEN) - 1, - ctx->agent_token, flb_sds_len(ctx->agent_token)); - } -#endif - - /* Map debug callbacks */ - flb_http_client_debug(c, ctx->ins->callback); - - /* Perform HTTP request */ - ret = flb_http_do(c, &b_sent); - if (ret != 0) { - flb_plg_warn(ctx->ins, "http_do=%i", ret); - return FLB_RETRY; - } - - if (c->resp.status != 200 && c->resp.status != 201 && c->resp.status != 204) { - if (c->resp.payload_size > 0) { - flb_plg_warn(ctx->ins, "http_status=%i:\n%s", - c->resp.status, c->resp.payload); - } - else { - flb_plg_warn(ctx->ins, "http_status=%i", c->resp.status); - } - - /* invalid metrics */ - if (c->resp.status == 422) { - return FLB_ERROR; - } - return FLB_RETRY;; - } - - return FLB_OK; -} - -static flb_sds_t get_agent_info(char *buf, size_t size, char *k) -{ - int i; - int ret; - int type; - int len; - char *out_buf; - flb_sds_t v = NULL; - size_t off = 0; - size_t out_size; - msgpack_unpacked result; - msgpack_object root; - msgpack_object key; - msgpack_object val; - - len = strlen(k); - - ret = flb_pack_json(buf, size, &out_buf, &out_size, &type, NULL); - if (ret != 0) { - return NULL; - } - - msgpack_unpacked_init(&result); - ret = msgpack_unpack_next(&result, out_buf, out_size, &off); - if (ret != MSGPACK_UNPACK_SUCCESS) { - flb_free(out_buf); - msgpack_unpacked_destroy(&result); - return NULL; - } - - root = result.data; - if (root.type != MSGPACK_OBJECT_MAP) { - flb_free(out_buf); - msgpack_unpacked_destroy(&result); - return NULL; - } - - for (i = 0; i < root.via.map.size; i++) { - key = root.via.map.ptr[i].key; - val = root.via.map.ptr[i].val; - - if (key.type != MSGPACK_OBJECT_STR || val.type != MSGPACK_OBJECT_STR) { - continue; - } - - if (key.via.str.size != len) { - continue; - } - - if (strncmp(key.via.str.ptr, k, len) == 0) { - v = flb_sds_create_len(val.via.str.ptr, val.via.str.size); - break; - } - } - - flb_free(out_buf); - msgpack_unpacked_destroy(&result); - return v; -} - -/* Set the session content */ -static int store_session_set(struct flb_calyptia *ctx, char *buf, size_t size) -{ - int ret; - int type; - char *mp_buf; - size_t mp_size; - - /* remove any previous session file */ - if (ctx->fs_file) { - flb_fstore_file_delete(ctx->fs, ctx->fs_file); - } - - /* create session file */ - ctx->fs_file = flb_fstore_file_create(ctx->fs, ctx->fs_stream, - CALYPTIA_SESSION_FILE, 1024); - if (!ctx->fs_file) { - flb_plg_error(ctx->ins, "could not create new session file"); - return -1; - } - - /* store meta */ - flb_fstore_file_meta_set(ctx->fs, ctx->fs_file, - FLB_VERSION_STR "\n", sizeof(FLB_VERSION_STR) - 1); - - /* encode */ - ret = flb_pack_json(buf, size, &mp_buf, &mp_size, &type, NULL); - if (ret < 0) { - flb_plg_error(ctx->ins, "could not encode session information"); - return -1; - } - - /* store content */ - ret = flb_fstore_file_append(ctx->fs_file, mp_buf, mp_size); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not store session information"); - flb_free(mp_buf); - return -1; - } - - flb_free(mp_buf); - return 0; -} - -static int store_session_get(struct flb_calyptia *ctx, - void **out_buf, size_t *out_size) -{ - int ret; - void *buf; - size_t size; - flb_sds_t json; - - ret = flb_fstore_file_content_copy(ctx->fs, ctx->fs_file, - &buf, &size); - - if (size == 0) { - return -1; - } - - /* decode */ - json = flb_msgpack_raw_to_json_sds(buf, size); - flb_free(buf); - if (!json) { - return -1; - } - - *out_buf = json; - *out_size = flb_sds_len(json); - - return ret; -} - -static int store_init(struct flb_calyptia *ctx) -{ - int ret; - struct flb_fstore *fs; - struct flb_fstore_file *fsf; - void *buf; - size_t size; - - /* store context */ - fs = flb_fstore_create(ctx->store_path, FLB_FSTORE_FS); - if (!fs) { - flb_plg_error(ctx->ins, - "could not initialize 'store_path': %s", - ctx->store_path); - return -1; - } - ctx->fs = fs; - - /* stream */ - ctx->fs_stream = flb_fstore_stream_create(ctx->fs, "calyptia"); - if (!ctx->fs_stream) { - flb_plg_error(ctx->ins, "could not create storage stream"); - return -1; - } - - /* lookup any previous file */ - fsf = flb_fstore_file_get(ctx->fs, ctx->fs_stream, CALYPTIA_SESSION_FILE, - sizeof(CALYPTIA_SESSION_FILE) - 1); - if (!fsf) { - flb_plg_debug(ctx->ins, "no session file was found"); - return 0; - } - ctx->fs_file = fsf; - - /* retrieve session info */ - ret = store_session_get(ctx, &buf, &size); - if (ret == 0) { - /* agent id */ - ctx->agent_id = get_agent_info(buf, size, "id"); - - /* agent token */ - ctx->agent_token = get_agent_info(buf, size, "token"); - - if (ctx->agent_id && ctx->agent_token) { - flb_plg_info(ctx->ins, "session setup OK"); - } - else { - if (ctx->agent_id) { - flb_sds_destroy(ctx->agent_id); - } - if (ctx->agent_token) { - flb_sds_destroy(ctx->agent_token); - } - } - flb_sds_destroy(buf); - } - - return 0; -} - -/* Agent creation is perform on initialization using a sync upstream connection */ -static int api_agent_create(struct flb_config *config, struct flb_calyptia *ctx) -{ - int ret; - int flb_ret; - int flags; - int action = CALYPTIA_ACTION_REGISTER; - char uri[1024]; - flb_sds_t meta; - struct flb_upstream *u; - struct flb_connection *u_conn; - struct flb_http_client *c; - - /* Meta */ - meta = get_agent_metadata(ctx); - if (!meta) { - flb_plg_error(ctx->ins, "could not retrieve metadata"); - return -1; - } - - /* Upstream */ - flags = get_io_flags(ctx->ins); - u = flb_upstream_create(ctx->config, - ctx->cloud_host, ctx->cloud_port, - flags, ctx->ins->tls); - if (!u) { - flb_plg_error(ctx->ins, - "could not create upstream connection on 'agent create'"); - flb_sds_destroy(meta); - return -1; - } - - /* Make it synchronous */ - flb_stream_disable_async_mode(&u->base); - - /* Get upstream connection */ - u_conn = flb_upstream_conn_get(u); - if (!u_conn) { - flb_upstream_destroy(u); - flb_sds_destroy(meta); - return -1; - } - - if (ctx->agent_id && ctx->agent_token) { - /* Patch */ - action = CALYPTIA_ACTION_PATCH; - snprintf(uri, sizeof(uri) - 1, CALYPTIA_ENDPOINT_PATCH, ctx->agent_id); - c = flb_http_client(u_conn, FLB_HTTP_PATCH, uri, - meta, flb_sds_len(meta), NULL, 0, NULL, 0); - } - else { - /* Create */ - action = CALYPTIA_ACTION_REGISTER; - c = flb_http_client(u_conn, FLB_HTTP_POST, CALYPTIA_ENDPOINT_CREATE, - meta, flb_sds_len(meta), NULL, 0, NULL, 0); - } - - if (!c) { - flb_upstream_conn_release(u_conn); - flb_upstream_destroy(u); - return -1; - } - - /* perform request */ - flb_ret = calyptia_http_do(ctx, c, action); - if (flb_ret == FLB_OK && - (c->resp.status == 200 || c->resp.status == 201 || c->resp.status == 204)) { - if (c->resp.payload_size > 0) { - if (action == CALYPTIA_ACTION_REGISTER) { - /* agent id */ - ctx->agent_id = get_agent_info(c->resp.payload, - c->resp.payload_size, - "id"); - - /* agent token */ - ctx->agent_token = get_agent_info(c->resp.payload, - c->resp.payload_size, - "token"); - - if (ctx->agent_id && ctx->agent_token) { - flb_plg_info(ctx->ins, "connected to Calyptia, agent_id='%s'", - ctx->agent_id); - - if (ctx->store_path && ctx->fs) { - ret = store_session_set(ctx, - c->resp.payload, - c->resp.payload_size); - if (ret == -1) { - flb_plg_warn(ctx->ins, - "could not store Calyptia session"); - } - } - } - } - } - - if (action == CALYPTIA_ACTION_PATCH) { - flb_plg_info(ctx->ins, "known agent registration successful"); - } - } - - /* release resources */ - flb_sds_destroy(meta); - flb_http_client_destroy(c); - flb_upstream_conn_release(u_conn); - flb_upstream_destroy(u); - - return flb_ret; -} - -static struct flb_calyptia *config_init(struct flb_output_instance *ins, - struct flb_config *config) -{ - int ret; - int flags; - struct flb_calyptia *ctx; - - /* Calyptia plugin context */ - ctx = flb_calloc(1, sizeof(struct flb_calyptia)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - ctx->config = config; - flb_kv_init(&ctx->kv_labels); - - /* Load the config map */ - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return NULL; - } - - /* api_key */ - if (!ctx->api_key) { - flb_plg_error(ctx->ins, "configuration 'api_key' is missing"); - flb_free(ctx); - return NULL; - } - - /* parse 'add_label' */ - ret = config_add_labels(ins, ctx); - if (ret == -1) { - return NULL; - } - - /* env reader */ - ctx->env = flb_env_create(); - - /* Set context */ - flb_output_set_context(ins, ctx); - - /* Initialize optional storage */ - if (ctx->store_path) { - ret = store_init(ctx); - if (ret == -1) { - return NULL; - } - } - - /* the machine-id is provided by custom calyptia, which invokes this plugin. */ - if (!ctx->machine_id) { - flb_plg_error(ctx->ins, "machine_id has not been set"); - return NULL; - } - - flb_plg_debug(ctx->ins, "machine_id=%s", ctx->machine_id); - - /* Upstream */ - flags = get_io_flags(ctx->ins); - ctx->u = flb_upstream_create(ctx->config, - ctx->cloud_host, ctx->cloud_port, - flags, ctx->ins->tls); - if (!ctx->u) { - return NULL; - } - - /* Set instance flags into upstream */ - flb_output_upstream_set(ctx->u, ins); - - return ctx; -} - -static int cb_calyptia_init(struct flb_output_instance *ins, - struct flb_config *config, void *data) -{ - int ret; - struct flb_calyptia *ctx; - (void) data; - - /* create config context */ - ctx = config_init(ins, config); - if (!ctx) { - flb_plg_error(ins, "could not initialize configuration"); - return -1; - } - - /* - * This plugin instance uses the HTTP client interface, let's register - * it debugging callbacks. - */ - flb_output_set_http_debug_callbacks(ins); - - /* register/update agent */ - ret = api_agent_create(config, ctx); - if (ret != FLB_OK) { - flb_plg_error(ctx->ins, "agent registration failed"); - return -1; - } - - /* metrics endpoint */ - ctx->metrics_endpoint = flb_sds_create_size(256); - flb_sds_printf(&ctx->metrics_endpoint, CALYPTIA_ENDPOINT_METRICS, - ctx->agent_id); - -#ifdef FLB_HAVE_CHUNK_TRACE - ctx->trace_endpoint = flb_sds_create_size(256); - flb_sds_printf(&ctx->trace_endpoint, CALYPTIA_ENDPOINT_TRACE, - ctx->pipeline_id); -#endif /* FLB_HAVE_CHUNK_TRACE */ - return 0; -} - -static void debug_payload(struct flb_calyptia *ctx, void *data, size_t bytes) -{ - int ret; - size_t off = 0; - struct cmt *cmt; - cfl_sds_t out; - - ret = cmt_decode_msgpack_create(&cmt, (char *) data, bytes, &off); - if (ret != CMT_DECODE_MSGPACK_SUCCESS) { - flb_plg_warn(ctx->ins, "could not unpack debug payload"); - return; - } - - out = cmt_encode_text_create(cmt); - flb_plg_info(ctx->ins, "debug payload:\n%s", out); - cmt_encode_text_destroy(out); - cmt_destroy(cmt); -} - -static void cb_calyptia_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - int ret; - size_t off = 0; - size_t out_size = 0; - char *out_buf = NULL; - -/* used to create records for reporting traces to the cloud. */ -#ifdef FLB_HAVE_CHUNK_TRACE - flb_sds_t json; -#endif /* FLB_HAVE_CHUNK_TRACE */ - - struct flb_connection *u_conn; - struct flb_http_client *c; - struct flb_calyptia *ctx = out_context; - struct cmt *cmt; - (void) i_ins; - (void) config; - - /* Get upstream connection */ - u_conn = flb_upstream_conn_get(ctx->u); - if (!u_conn) { - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - if (event_chunk->type == FLB_EVENT_TYPE_METRICS) { - /* if we have labels append them */ - if (ctx->add_labels && mk_list_size(ctx->add_labels) > 0) { - ret = cmt_decode_msgpack_create(&cmt, - (char *) event_chunk->data, - event_chunk->size, - &off); - if (ret != CMT_DECODE_MSGPACK_SUCCESS) { - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_ERROR); - } - - /* append labels set by config */ - append_labels(ctx, cmt); - - /* encode back to msgpack */ - ret = cmt_encode_msgpack_create(cmt, &out_buf, &out_size); - if (ret != 0) { - cmt_destroy(cmt); - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_ERROR); - } - cmt_destroy(cmt); - } - else { - out_buf = (char *) event_chunk->data; - out_size = event_chunk->size; - } - - /* Compose HTTP Client request */ - c = flb_http_client(u_conn, FLB_HTTP_POST, ctx->metrics_endpoint, - out_buf, out_size, NULL, 0, NULL, 0); - if (!c) { - if (out_buf != event_chunk->data) { - cmt_encode_msgpack_destroy(out_buf); - } - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* perform request: 'ret' might be FLB_OK, FLB_ERROR or FLB_RETRY */ - ret = calyptia_http_do(ctx, c, CALYPTIA_ACTION_METRICS); - if (ret == FLB_OK) { - flb_plg_debug(ctx->ins, "metrics delivered OK"); - } - else if (ret == FLB_ERROR) { - flb_plg_error(ctx->ins, "could not deliver metrics"); - debug_payload(ctx, out_buf, out_size); - } - - if (out_buf != event_chunk->data) { - cmt_encode_msgpack_destroy(out_buf); - } - } - -#ifdef FLB_HAVE_CHUNK_TRACE - if (event_chunk->type == (FLB_EVENT_TYPE_LOGS | FLB_EVENT_TYPE_HAS_TRACE)) { - json = flb_pack_msgpack_to_json_format(event_chunk->data, - event_chunk->size, - FLB_PACK_JSON_FORMAT_STREAM, - FLB_PACK_JSON_DATE_DOUBLE, - NULL); - if (json == NULL) { - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - out_buf = (char *)json; - out_size = flb_sds_len(json); - - if (flb_sds_printf(&ctx->metrics_endpoint, CALYPTIA_ENDPOINT_METRICS, - ctx->agent_id) == NULL) { - flb_upstream_conn_release(u_conn); - flb_sds_destroy(json); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - c = flb_http_client(u_conn, FLB_HTTP_POST, ctx->trace_endpoint, - out_buf, out_size, NULL, 0, NULL, 0); - if (!c) { - flb_upstream_conn_release(u_conn); - flb_sds_destroy(json); - flb_sds_destroy(ctx->metrics_endpoint); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* perform request: 'ret' might be FLB_OK, FLB_ERROR or FLB_RETRY */ - ret = calyptia_http_do(ctx, c, CALYPTIA_ACTION_TRACE); - if (ret == FLB_OK) { - flb_plg_debug(ctx->ins, "trace delivered OK"); - } - else if (ret == FLB_ERROR) { - flb_plg_error(ctx->ins, "could not deliver trace"); - debug_payload(ctx, out_buf, out_size); - } - flb_sds_destroy(json); - } -#endif /* FLB_HAVE_CHUNK_TRACE */ - - flb_upstream_conn_release(u_conn); - flb_http_client_destroy(c); - FLB_OUTPUT_RETURN(ret); -} - -static int cb_calyptia_exit(void *data, struct flb_config *config) -{ - struct flb_calyptia *ctx = data; - - if (!ctx) { - return 0; - } - - if (ctx->u) { - flb_upstream_destroy(ctx->u); - } - - if (ctx->agent_id) { - flb_sds_destroy(ctx->agent_id); - } - - if (ctx->agent_token) { - flb_sds_destroy(ctx->agent_token); - } - - if (ctx->env) { - flb_env_destroy(ctx->env); - } - - if (ctx->metrics_endpoint) { - flb_sds_destroy(ctx->metrics_endpoint); - } - -#ifdef FLB_HAVE_CHUNK_TRACE - if (ctx->trace_endpoint) { - flb_sds_destroy(ctx->trace_endpoint); - } -#endif /* FLB_HAVE_CHUNK_TRACE */ - - if (ctx->fs) { - flb_fstore_destroy(ctx->fs); - } - - flb_kv_release(&ctx->kv_labels); - flb_free(ctx); - - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "cloud_host", CALYPTIA_HOST, - 0, FLB_TRUE, offsetof(struct flb_calyptia, cloud_host), - "", - }, - - { - FLB_CONFIG_MAP_INT, "cloud_port", CALYPTIA_PORT, - 0, FLB_TRUE, offsetof(struct flb_calyptia, cloud_port), - "", - }, - - { - FLB_CONFIG_MAP_STR, "api_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_calyptia, api_key), - "Calyptia Cloud API Key." - }, - { - FLB_CONFIG_MAP_STR, "machine_id", NULL, - 0, FLB_TRUE, offsetof(struct flb_calyptia, machine_id), - "Custom machine_id to be used when registering agent" - }, - { - FLB_CONFIG_MAP_STR, "fleet_id", NULL, - 0, FLB_TRUE, offsetof(struct flb_calyptia, fleet_id), - "Fleet ID for identifying as part of a managed fleet" - }, - - { - FLB_CONFIG_MAP_STR, "store_path", NULL, - 0, FLB_TRUE, offsetof(struct flb_calyptia, store_path), - "" - }, - - { - FLB_CONFIG_MAP_SLIST_1, "add_label", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct flb_calyptia, add_labels), - "Label to append to the generated metric." - }, - -#ifdef FLB_HAVE_CHUNK_TRACE - { - FLB_CONFIG_MAP_STR, "pipeline_id", NULL, - 0, FLB_TRUE, offsetof(struct flb_calyptia, pipeline_id), - "Pipeline ID for calyptia core traces." - }, -#endif - - /* EOF */ - {0} -}; - -struct flb_output_plugin out_calyptia_plugin = { - .name = "calyptia", - .description = "Calyptia Cloud", - .cb_init = cb_calyptia_init, - .cb_flush = cb_calyptia_flush, - .cb_exit = cb_calyptia_exit, - .config_map = config_map, - .flags = FLB_OUTPUT_NET | FLB_OUTPUT_PRIVATE | FLB_IO_OPT_TLS, - .event_type = FLB_OUTPUT_METRICS -}; diff --git a/fluent-bit/plugins/out_calyptia/calyptia.h b/fluent-bit/plugins/out_calyptia/calyptia.h deleted file mode 100644 index db640ff10..000000000 --- a/fluent-bit/plugins/out_calyptia/calyptia.h +++ /dev/null @@ -1,85 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_CALYPTIA_H -#define FLB_OUT_CALYPTIA_H - -#include -#include -#include -#include - -/* End point */ -#define CALYPTIA_HOST "cloud-api.calyptia.com" -#define CALYPTIA_PORT "443" - -/* HTTP action types */ -#define CALYPTIA_ACTION_REGISTER 0 -#define CALYPTIA_ACTION_PATCH 1 -#define CALYPTIA_ACTION_METRICS 2 -#define CALYPTIA_ACTION_TRACE 3 - -/* Endpoints */ -#define CALYPTIA_ENDPOINT_CREATE "/v1/agents" -#define CALYPTIA_ENDPOINT_PATCH "/v1/agents/%s" -#define CALYPTIA_ENDPOINT_METRICS "/v1/agents/%s/metrics" -#define CALYPTIA_ENDPOINT_TRACE "/v1/traces/%s" - -/* Storage */ -#define CALYPTIA_SESSION_FILE "session.CALYPTIA" - -/* Headers */ -#define CALYPTIA_H_PROJECT "X-Project-Token" -#define CALYPTIA_H_AGENT_TOKEN "X-Agent-Token" -#define CALYPTIA_H_CTYPE "Content-Type" -#define CALYPTIA_H_CTYPE_JSON "application/json" -#define CALYPTIA_H_CTYPE_MSGPACK "application/x-msgpack" - -struct flb_calyptia { - /* config map */ - int cloud_port; - flb_sds_t api_key; - flb_sds_t cloud_host; - flb_sds_t store_path; - - /* config reader for 'add_label' */ - struct mk_list *add_labels; - - /* internal */ - flb_sds_t agent_id; - flb_sds_t agent_token; - flb_sds_t machine_id; /* machine-id */ - flb_sds_t fleet_id; /* fleet-id */ - flb_sds_t metrics_endpoint; /* metrics endpoint */ - struct flb_fstore *fs; /* fstore ctx */ - struct flb_fstore_stream *fs_stream; /* fstore stream */ - struct flb_fstore_file *fs_file; /* fstore session file */ - struct flb_env *env; /* environment */ - struct flb_upstream *u; /* upstream connection */ - struct mk_list kv_labels; /* parsed add_labels */ - struct flb_output_instance *ins; /* plugin instance */ - struct flb_config *config; /* Fluent Bit context */ -/* used for reporting chunk trace records to calyptia cloud. */ -#ifdef FLB_HAVE_CHUNK_TRACE - flb_sds_t trace_endpoint; - flb_sds_t pipeline_id; -#endif /* FLB_HAVE_CHUNK_TRACE */ -}; - -#endif diff --git a/fluent-bit/plugins/out_chronicle/CMakeLists.txt b/fluent-bit/plugins/out_chronicle/CMakeLists.txt deleted file mode 100644 index ca9180305..000000000 --- a/fluent-bit/plugins/out_chronicle/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(src - chronicle_conf.c - chronicle.c - ) - -FLB_PLUGIN(out_chronicle "${src}" "") diff --git a/fluent-bit/plugins/out_chronicle/chronicle.c b/fluent-bit/plugins/out_chronicle/chronicle.c deleted file mode 100644 index 479dd8035..000000000 --- a/fluent-bit/plugins/out_chronicle/chronicle.c +++ /dev/null @@ -1,962 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "chronicle.h" -#include "chronicle_conf.h" - -// TODO: The following code is copied from the Stackdriver plugin and should be -// factored into common library functions. - -/* - * Base64 Encoding in JWT must: - * - * - remove any trailing padding '=' character - * - replace '+' with '-' - * - replace '/' with '_' - * - * ref: https://www.rfc-editor.org/rfc/rfc7515.txt Appendix C - */ -int chronicle_jwt_base64_url_encode(unsigned char *out_buf, size_t out_size, - unsigned char *in_buf, size_t in_size, - size_t *olen) - -{ - int i; - size_t len; - int result; - - /* do normal base64 encoding */ - result = flb_base64_encode((unsigned char *) out_buf, out_size - 1, - &len, in_buf, in_size); - if (result != 0) { - return -1; - } - - /* Replace '+' and '/' characters */ - for (i = 0; i < len && out_buf[i] != '='; i++) { - if (out_buf[i] == '+') { - out_buf[i] = '-'; - } - else if (out_buf[i] == '/') { - out_buf[i] = '_'; - } - } - - /* Now 'i' becomes the new length */ - *olen = i; - return 0; -} - -static int chronicle_jwt_encode(struct flb_chronicle *ctx, - char *payload, char *secret, - char **out_signature, size_t *out_size) -{ - int ret; - int len; - int buf_size; - size_t olen; - char *buf; - char *sigd; - char *headers = "{\"alg\": \"RS256\", \"typ\": \"JWT\"}"; - unsigned char sha256_buf[32] = {0}; - flb_sds_t out; - unsigned char sig[256] = {0}; - size_t sig_len; - - buf_size = (strlen(payload) + strlen(secret)) * 2; - buf = flb_malloc(buf_size); - if (!buf) { - flb_errno(); - return -1; - } - - /* Encode header */ - len = strlen(headers); - ret = flb_base64_encode((unsigned char *) buf, buf_size - 1, - &olen, (unsigned char *) headers, len); - if (ret != 0) { - flb_free(buf); - - return ret; - } - - /* Create buffer to store JWT */ - out = flb_sds_create_size(2048); - if (!out) { - flb_errno(); - flb_free(buf); - return -1; - } - - /* Append header */ - flb_sds_cat(out, buf, olen); - flb_sds_cat(out, ".", 1); - - /* Encode Payload */ - len = strlen(payload); - chronicle_jwt_base64_url_encode((unsigned char *) buf, buf_size, - (unsigned char *) payload, len, &olen); - - /* Append Payload */ - flb_sds_cat(out, buf, olen); - - /* do sha256() of base64(header).base64(payload) */ - ret = flb_hash_simple(FLB_HASH_SHA256, - (unsigned char *) out, flb_sds_len(out), - sha256_buf, sizeof(sha256_buf)); - - if (ret != FLB_CRYPTO_SUCCESS) { - flb_plg_error(ctx->ins, "error hashing token"); - flb_free(buf); - flb_sds_destroy(out); - return -1; - } - - len = strlen(secret); - sig_len = sizeof(sig); - - ret = flb_crypto_sign_simple(FLB_CRYPTO_PRIVATE_KEY, - FLB_CRYPTO_PADDING_PKCS1, - FLB_HASH_SHA256, - (unsigned char *) secret, len, - sha256_buf, sizeof(sha256_buf), - sig, &sig_len); - - if (ret != FLB_CRYPTO_SUCCESS) { - flb_plg_error(ctx->ins, "error creating RSA context"); - flb_free(buf); - flb_sds_destroy(out); - return -1; - } - - sigd = flb_malloc(2048); - if (!sigd) { - flb_errno(); - flb_free(buf); - flb_sds_destroy(out); - return -1; - } - - chronicle_jwt_base64_url_encode((unsigned char *) sigd, 2048, sig, 256, &olen); - - flb_sds_cat(out, ".", 1); - flb_sds_cat(out, sigd, olen); - - *out_signature = out; - *out_size = flb_sds_len(out); - - flb_free(buf); - flb_free(sigd); - - return 0; -} - -/* Create a new oauth2 context and get a oauth2 token */ -static int chronicle_get_oauth2_token(struct flb_chronicle *ctx) -{ - int ret; - char *token; - char *sig_data; - size_t sig_size; - time_t issued; - time_t expires; - char payload[1024]; - - /* Clear any previous oauth2 payload content */ - flb_oauth2_payload_clear(ctx->o); - - /* JWT encode for oauth2 */ - issued = time(NULL); - expires = issued + FLB_CHRONICLE_TOKEN_REFRESH; - - snprintf(payload, sizeof(payload) - 1, - "{\"iss\": \"%s\", \"scope\": \"%s\", " - "\"aud\": \"%s\", \"exp\": %lu, \"iat\": %lu}", - ctx->oauth_credentials->client_email, FLB_CHRONICLE_SCOPE, - FLB_CHRONICLE_AUTH_URL, - expires, issued); - - /* Compose JWT signature */ - ret = chronicle_jwt_encode(ctx, payload, ctx->oauth_credentials->private_key, - &sig_data, &sig_size); - if (ret != 0) { - flb_plg_error(ctx->ins, "JWT signature generation failed"); - return -1; - } - - flb_plg_debug(ctx->ins, "JWT signature:\n%s", sig_data); - - ret = flb_oauth2_payload_append(ctx->o, - "grant_type", -1, - "urn%3Aietf%3Aparams%3Aoauth%3A" - "grant-type%3Ajwt-bearer", -1); - if (ret == -1) { - flb_plg_error(ctx->ins, "error appending oauth2 params"); - flb_sds_destroy(sig_data); - return -1; - } - - ret = flb_oauth2_payload_append(ctx->o, - "assertion", -1, - sig_data, sig_size); - if (ret == -1) { - flb_plg_error(ctx->ins, "error appending oauth2 params"); - flb_sds_destroy(sig_data); - return -1; - } - flb_sds_destroy(sig_data); - - /* Retrieve access token */ - token = flb_oauth2_token_get(ctx->o); - if (!token) { - flb_plg_error(ctx->ins, "error retrieving oauth2 access token"); - return -1; - } - - return 0; -} - -static flb_sds_t get_google_token(struct flb_chronicle *ctx) -{ - int ret = 0; - flb_sds_t output = NULL; - - if (pthread_mutex_lock(&ctx->token_mutex)){ - flb_plg_error(ctx->ins, "error locking mutex"); - return NULL; - } - - if (flb_oauth2_token_expired(ctx->o) == FLB_TRUE) { - ret = chronicle_get_oauth2_token(ctx); - } - - /* Copy string to prevent race conditions (get_oauth2 can free the string) */ - if (ret == 0) { - output = flb_sds_create(ctx->o->token_type); - flb_sds_printf(&output, " %s", ctx->o->access_token); - } - - if (pthread_mutex_unlock(&ctx->token_mutex)){ - flb_plg_error(ctx->ins, "error unlocking mutex"); - if (output) { - flb_sds_destroy(output); - } - return NULL; - } - - return output; -} - -static int validate_log_type(struct flb_chronicle *ctx, struct flb_config *config, - const char *body, size_t len) -{ - int ret = -1; - int root_type; - char *msgpack_buf = NULL; - size_t msgpack_size; - size_t off = 0; - msgpack_unpacked result; - int i, j, k; - msgpack_object key; - msgpack_object val; - msgpack_object root; - msgpack_object *array; - msgpack_object *supported_type; - int root_map_size; - int array_size = 0; - - - ret = flb_pack_json(body, len, - &msgpack_buf, &msgpack_size, - &root_type, NULL); - - if (ret != 0 || root_type != JSMN_OBJECT) { - flb_plg_error(ctx->ins, "json to msgpack conversion error"); - } - - ret = -1; - msgpack_unpacked_init(&result); - while (msgpack_unpack_next(&result, msgpack_buf, msgpack_size, &off) == MSGPACK_UNPACK_SUCCESS) { - if (result.data.type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "Invalid log_type payload"); - ret = -2; - - goto cleanup; - } - - root = result.data; - root_map_size = root.via.map.size; - - for (i = 0; i < root_map_size; i++) { - key = root.via.map.ptr[i].key; - val = root.via.map.ptr[i].val; - - if (val.type != MSGPACK_OBJECT_ARRAY) { - flb_plg_error(ctx->ins, "Invalid inner array type of log_type payload"); - ret = -2; - - goto cleanup; - } - - array = val.via.array.ptr; - array_size = val.via.array.size; - - for (j = 0; j < array_size; j++) { - supported_type = &array[j]; - - if (supported_type->type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "Invalid inner maps of log_type payload"); - ret = -2; - - continue; - } - - for (k = 0; k < supported_type->via.map.size; k++) { - key = supported_type->via.map.ptr[k].key; - val = supported_type->via.map.ptr[k].val; - - if (strncmp("logType", key.via.str.ptr, key.via.str.size) == 0) { - if (strncmp(ctx->log_type, val.via.bin.ptr, val.via.str.size) == 0) { - ret = 0; - goto cleanup; - } - } - } - } - } - } - -cleanup: - msgpack_unpacked_destroy(&result); - - /* release 'out_buf' if it was allocated */ - if (msgpack_buf) { - flb_free(msgpack_buf); - } - - return ret; -} - -static int check_chronicle_log_type(struct flb_chronicle *ctx, struct flb_config *config) -{ - int ret; - size_t b_sent; - flb_sds_t token; - struct flb_connection *u_conn; - struct flb_http_client *c; - - /* Get upstream connection */ - u_conn = flb_upstream_conn_get(ctx->u); - if (!u_conn) { - return -1; - } - - /* Get or renew Token */ - token = get_google_token(ctx); - - if (!token) { - flb_plg_error(ctx->ins, "cannot retrieve oauth2 token"); - flb_upstream_conn_release(u_conn); - return -1; - } - - /* Compose HTTP Client request */ - c = flb_http_client(u_conn, FLB_HTTP_GET, FLB_CHRONICLE_LOG_TYPE_ENDPOINT, - NULL, 0, NULL, 0, NULL, 0); - if (!c) { - flb_plg_error(ctx->ins, "cannot create HTTP client context"); - flb_upstream_conn_release(u_conn); - flb_sds_destroy(token); - - return -1; - } - - /* Chronicle supported types are growing. Not to specify the read limit. */ - flb_http_buffer_size(c, 0); - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - flb_http_add_header(c, "Content-Type", 12, "application/json", 16); - - /* Compose and append Authorization header */ - flb_http_add_header(c, "Authorization", 13, token, flb_sds_len(token)); - - /* Send HTTP request */ - ret = flb_http_do(c, &b_sent); - - /* validate response */ - if (ret != 0) { - flb_plg_warn(ctx->ins, "http_do=%i", ret); - goto cleanup; - } - else { - /* The request was issued successfully, validate the 'error' field */ - flb_plg_debug(ctx->ins, "HTTP Status=%i", c->resp.status); - if (c->resp.status == 200) { - ret = validate_log_type(ctx, config, c->resp.payload, c->resp.payload_size); - if (ret != 0) { - flb_plg_error(ctx->ins, "Validate log_type is failed"); - goto cleanup; - } - } - else { - if (c->resp.payload && c->resp.payload_size > 0) { - /* we got an error */ - flb_plg_warn(ctx->ins, "response\n%s", c->resp.payload); - } - - goto cleanup; - } - } - -cleanup: - - /* Cleanup */ - flb_sds_destroy(token); - flb_http_client_destroy(c); - flb_upstream_conn_release(u_conn); - - return ret; -} - -static int cb_chronicle_init(struct flb_output_instance *ins, - struct flb_config *config, void *data) -{ - char *token; - int io_flags = FLB_IO_TLS; - struct flb_chronicle *ctx; - int ret; - - /* Create config context */ - ctx = flb_chronicle_conf_create(ins, config); - if (!ctx) { - flb_plg_error(ins, "configuration failed"); - return -1; - } - - flb_output_set_context(ins, ctx); - - /* Network mode IPv6 */ - if (ins->host.ipv6 == FLB_TRUE) { - io_flags |= FLB_IO_IPV6; - } - - /* Create mutex for acquiring oauth tokens (they are shared across flush coroutines) */ - pthread_mutex_init(&ctx->token_mutex, NULL); - - /* - * Create upstream context for Chronicle Streaming Inserts - * (no oauth2 service) - */ - ctx->u = flb_upstream_create_url(config, ctx->uri, - io_flags, ins->tls); - if (!ctx->u) { - flb_plg_error(ctx->ins, "upstream creation failed"); - return -1; - } - - /* Create oauth2 context */ - ctx->o = flb_oauth2_create(ctx->config, FLB_CHRONICLE_AUTH_URL, 3000); - if (!ctx->o) { - flb_plg_error(ctx->ins, "cannot create oauth2 context"); - return -1; - } - flb_output_upstream_set(ctx->u, ins); - - /* Get or renew the OAuth2 token */ - token = get_google_token(ctx); - - if (!token) { - flb_plg_warn(ctx->ins, "token retrieval failed"); - } - else { - flb_sds_destroy(token); - } - - ret = check_chronicle_log_type(ctx, config); - if (ret != 0) { - flb_plg_error(ctx->ins, "Validate log_type failed. '%s' is not supported. ret = %d", - ctx->log_type, ret); - return -1; - } - - return 0; -} - -static flb_sds_t flb_pack_msgpack_extract_log_key(void *out_context, uint64_t bytes, struct flb_log_event log_event) -{ - int i; - int map_size; - int check = FLB_FALSE; - int found = FLB_FALSE; - int log_key_missing = 0; - int ret; - struct flb_chronicle *ctx = out_context; - char *val_buf; - char *key_str = NULL; - size_t key_str_size = 0; - size_t msgpack_size = bytes + bytes / 4; - size_t val_offset = 0; - flb_sds_t out_buf; - msgpack_object map; - msgpack_object key; - msgpack_object val; - - /* Allocate buffer to store log_key contents */ - val_buf = flb_calloc(1, msgpack_size); - if (val_buf == NULL) { - flb_plg_error(ctx->ins, "Could not allocate enough " - "memory to read record"); - flb_errno(); - return NULL; - } - - /* Get the record/map */ - map = *log_event.body; - - if (map.type != MSGPACK_OBJECT_MAP) { - return NULL; - } - - map_size = map.via.map.size; - - /* Reset variables for found log_key and correct type */ - found = FLB_FALSE; - check = FLB_FALSE; - - /* Extract log_key from record and append to output buffer */ - for (i = 0; i < map_size; i++) { - key = map.via.map.ptr[i].key; - val = map.via.map.ptr[i].val; - - if (key.type == MSGPACK_OBJECT_BIN) { - key_str = (char *) key.via.bin.ptr; - key_str_size = key.via.bin.size; - check = FLB_TRUE; - } - if (key.type == MSGPACK_OBJECT_STR) { - key_str = (char *) key.via.str.ptr; - key_str_size = key.via.str.size; - check = FLB_TRUE; - } - - if (check == FLB_TRUE) { - if (strncmp(ctx->log_key, key_str, key_str_size) == 0) { - found = FLB_TRUE; - - /* - * Copy contents of value into buffer. Necessary to copy - * strings because flb_msgpack_to_json does not handle nested - * JSON gracefully and double escapes them. - */ - if (val.type == MSGPACK_OBJECT_BIN) { - memcpy(val_buf + val_offset, val.via.bin.ptr, val.via.bin.size); - val_offset += val.via.bin.size; - val_buf[val_offset] = '\0'; - val_offset++; - } - else if (val.type == MSGPACK_OBJECT_STR) { - memcpy(val_buf + val_offset, val.via.str.ptr, val.via.str.size); - val_offset += val.via.str.size; - val_buf[val_offset] = '\0'; - val_offset++; - } - else { - ret = flb_msgpack_to_json(val_buf + val_offset, - msgpack_size - val_offset, &val); - if (ret < 0) { - break; - } - val_offset += ret; - val_buf[val_offset] = '\0'; - val_offset++; - } - /* Exit early once log_key has been found for current record */ - break; - } - } - - /* If log_key was not found in the current record, mark log key as missing */ - if (found == FLB_FALSE) { - log_key_missing++; - } - } - - if (log_key_missing > 0) { - flb_plg_error(ctx->ins, "Could not find log_key '%s' in %d records", - ctx->log_key, log_key_missing); - } - - /* If nothing was read, destroy buffer */ - if (val_offset == 0) { - flb_free(val_buf); - return NULL; - } - val_buf[val_offset] = '\0'; - - /* Create output buffer to store contents */ - out_buf = flb_sds_create(val_buf); - if (out_buf == NULL) { - flb_plg_error(ctx->ins, "Error creating buffer to store log_key contents."); - flb_errno(); - } - flb_free(val_buf); - - return out_buf; -} - -static int chronicle_format(const void *data, size_t bytes, - const char *tag, size_t tag_len, - char **out_data, size_t *out_size, - struct flb_chronicle *ctx) -{ - int len; - int ret; - int array_size = 0; - size_t off = 0; - size_t last_off = 0; - size_t alloc_size = 0; - size_t s; - char time_formatted[255]; - /* Parameters for Timestamp */ - struct tm tm; - flb_sds_t out_buf; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - flb_sds_t log_text = NULL; - int log_text_size; - - /* Count number of records */ - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return -1; - } - - array_size = flb_mp_count(data, bytes); - - /* Create temporary msgpack buffer */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - /* - * Pack root map (unstructured log): - * see: https://cloud.google.com/chronicle/docs/reference/ingestion-api#request_body_2 - * { - * "customer_id": "c8c65bfa-5f2c-42d4-9189-64bb7b939f2c", - * "log_type": "BIND_DNS", - * "entries": [ - * { - * "log_text": "26-Feb-2019 13:35:02.187 client 10.120.20.32#4238: query: altostrat.com IN A + (203.0.113.102)", - * "ts_epoch_microseconds": 1551188102187000 - * }, - * { - * "log_text": "26-Feb-2019 13:37:04.523 client 10.50.100.33#1116: query: examplepetstore.com IN A + (203.0.113.102)", - * "ts_rfc3339": "2019-26-02T13:37:04.523-08:00" - * }, - * { - * "log_text": "26-Feb-2019 13:39:01.115 client 10.1.2.3#3333: query: www.example.com IN A + (203.0.113.102)" - * }, - * ] - * } - */ - msgpack_pack_map(&mp_pck, 3); - - msgpack_pack_str(&mp_pck, 11); - msgpack_pack_str_body(&mp_pck, "customer_id", 11); - - msgpack_pack_str(&mp_pck, strlen(ctx->customer_id)); - msgpack_pack_str_body(&mp_pck, ctx->customer_id, strlen(ctx->customer_id)); - - msgpack_pack_str(&mp_pck, 8); - msgpack_pack_str_body(&mp_pck, "log_type", 8); - - msgpack_pack_str(&mp_pck, strlen(ctx->log_type)); - msgpack_pack_str_body(&mp_pck, ctx->log_type, strlen(ctx->log_type)); - - msgpack_pack_str(&mp_pck, 7); - msgpack_pack_str_body(&mp_pck, "entries", 7); - - /* Append entries */ - msgpack_pack_array(&mp_pck, array_size); - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - off = log_decoder.offset; - alloc_size = (off - last_off) + 128; /* JSON is larger than msgpack */ - last_off = off; - - /* - * Pack entries - * - * { - * "log_text": {...}, - * "ts_rfc3339": "..." - * } - * - */ - msgpack_pack_map(&mp_pck, 2); - - /* log_text */ - msgpack_pack_str(&mp_pck, 8); - msgpack_pack_str_body(&mp_pck, "log_text", 8); - if (ctx->log_key != NULL) { - log_text = flb_pack_msgpack_extract_log_key(ctx, bytes, log_event); - log_text_size = flb_sds_len(log_text); - } - else { - log_text = flb_msgpack_to_json_str(alloc_size, log_event.body); - log_text_size = strlen(log_text); - } - - if (log_text == NULL) { - flb_plg_error(ctx->ins, "Could not marshal msgpack to output string"); - return -1; - } - msgpack_pack_str(&mp_pck, log_text_size); - msgpack_pack_str_body(&mp_pck, log_text, log_text_size); - - if (ctx->log_key != NULL) { - flb_sds_destroy(log_text); - } - else { - flb_free(log_text); - } - /* timestamp */ - msgpack_pack_str(&mp_pck, 10); - msgpack_pack_str_body(&mp_pck, "ts_rfc3339", 10); - - gmtime_r(&log_event.timestamp.tm.tv_sec, &tm); - s = strftime(time_formatted, sizeof(time_formatted) - 1, - FLB_STD_TIME_FMT, &tm); - len = snprintf(time_formatted + s, sizeof(time_formatted) - 1 - s, - ".%03" PRIu64 "Z", - (uint64_t) log_event.timestamp.tm.tv_nsec); - s += len; - - msgpack_pack_str(&mp_pck, s); - msgpack_pack_str_body(&mp_pck, time_formatted, s); - } - - /* Convert from msgpack to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); - - flb_log_event_decoder_destroy(&log_decoder); - msgpack_sbuffer_destroy(&mp_sbuf); - - if (!out_buf) { - flb_plg_error(ctx->ins, "error formatting JSON payload"); - return -1; - } - - *out_data = out_buf; - *out_size = flb_sds_len(out_buf); - - return 0; -} - -static void cb_chronicle_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - (void) i_ins; - (void) config; - int ret; - int ret_code = FLB_RETRY; - size_t b_sent; - flb_sds_t token; - flb_sds_t payload_buf; - size_t payload_size; - struct flb_chronicle *ctx = out_context; - struct flb_connection *u_conn; - struct flb_http_client *c; - - flb_plg_trace(ctx->ins, "flushing bytes %zu", event_chunk->size); - - /* Get upstream connection */ - u_conn = flb_upstream_conn_get(ctx->u); - if (!u_conn) { - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Get or renew Token */ - token = get_google_token(ctx); - - if (!token) { - flb_plg_error(ctx->ins, "cannot retrieve oauth2 token"); - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Reformat msgpack to chronicle JSON payload */ - ret = chronicle_format(event_chunk->data, event_chunk->size, - event_chunk->tag, flb_sds_len(event_chunk->tag), - &payload_buf, &payload_size, ctx); - if (ret != 0) { - flb_upstream_conn_release(u_conn); - flb_sds_destroy(token); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Compose HTTP Client request */ - c = flb_http_client(u_conn, FLB_HTTP_POST, ctx->endpoint, - payload_buf, payload_size, NULL, 0, NULL, 0); - if (!c) { - flb_plg_error(ctx->ins, "cannot create HTTP client context"); - flb_upstream_conn_release(u_conn); - flb_sds_destroy(token); - flb_sds_destroy(payload_buf); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - flb_http_buffer_size(c, 4192); - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - flb_http_add_header(c, "Content-Type", 12, "application/json", 16); - - /* Compose and append Authorization header */ - flb_http_add_header(c, "Authorization", 13, token, flb_sds_len(token)); - - /* Send HTTP request */ - ret = flb_http_do(c, &b_sent); - - /* validate response */ - if (ret != 0) { - flb_plg_warn(ctx->ins, "http_do=%i", ret); - ret_code = FLB_RETRY; - } - else { - /* The request was issued successfully, validate the 'error' field */ - flb_plg_debug(ctx->ins, "HTTP Status=%i", c->resp.status); - if (c->resp.status == 200) { - ret_code = FLB_OK; - } - else { - if (c->resp.payload && c->resp.payload_size > 0) { - /* we got an error */ - flb_plg_warn(ctx->ins, "response\n%s", c->resp.payload); - } - ret_code = FLB_RETRY; - } - } - - /* Cleanup */ - flb_sds_destroy(payload_buf); - flb_sds_destroy(token); - flb_http_client_destroy(c); - flb_upstream_conn_release(u_conn); - - /* Done */ - FLB_OUTPUT_RETURN(ret_code); -} - -static int cb_chronicle_exit(void *data, struct flb_config *config) -{ - struct flb_chronicle *ctx = data; - - if (!ctx) { - return -1; - } - - if (ctx->u) { - flb_upstream_destroy(ctx->u); - } - - flb_chronicle_conf_destroy(ctx); - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "google_service_credentials", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_chronicle, credentials_file), - "Set the path for the google service credentials file" - }, - // set in flb_chronicle_oauth_credentials - { - FLB_CONFIG_MAP_STR, "service_account_email", (char *)NULL, - 0, FLB_FALSE, 0, - "Set the service account email" - }, - // set in flb_chronicle_oauth_credentials - { - FLB_CONFIG_MAP_STR, "service_account_secret", (char *)NULL, - 0, FLB_FALSE, 0, - "Set the service account secret" - }, - { - FLB_CONFIG_MAP_STR, "project_id", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_chronicle, project_id), - "Set the project id" - }, - { - FLB_CONFIG_MAP_STR, "customer_id", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_chronicle, customer_id), - "Set the customer id" - }, - { - FLB_CONFIG_MAP_STR, "log_type", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_chronicle, log_type), - "Set the log type" - }, - { - FLB_CONFIG_MAP_STR, "region", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_chronicle, region), - "Set the region" - }, - { - FLB_CONFIG_MAP_STR, "log_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_chronicle, log_key), - "Set the log key" - }, - /* EOF */ - {0} -}; - -struct flb_output_plugin out_chronicle_plugin = { - .name = "chronicle", - .description = "Send logs to Google Chronicle as unstructured log", - .cb_init = cb_chronicle_init, - .cb_flush = cb_chronicle_flush, - .cb_exit = cb_chronicle_exit, - .config_map = config_map, - /* Plugin flags */ - .flags = FLB_OUTPUT_NET | FLB_IO_TLS, -}; diff --git a/fluent-bit/plugins/out_chronicle/chronicle.h b/fluent-bit/plugins/out_chronicle/chronicle.h deleted file mode 100644 index 0243223f0..000000000 --- a/fluent-bit/plugins/out_chronicle/chronicle.h +++ /dev/null @@ -1,96 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_CHRONICLE -#define FLB_OUT_CHRONICLE - -#include -#include -#include -#include - -/* refresh token every 50 minutes */ -#define FLB_CHRONICLE_TOKEN_REFRESH 3000 - -/* Timestamp format */ -#define FLB_STD_TIME_FMT "%Y-%m-%dT%H:%M:%S" - -/* Chronicle unstructureed logs oauth scope */ -#define FLB_CHRONICLE_SCOPE "https://www.googleapis.com/auth/malachite-ingestion" - -/* Chronicle authorization URL */ -#define FLB_CHRONICLE_AUTH_URL "https://oauth2.googleapis.com/token" - -#define FLB_CHRONICLE_UNSTRUCTURED_ENDPOINT "/v2/unstructuredlogentries:batchCreate" -#define FLB_CHRONICLE_LOG_TYPE_ENDPOINT "/v2/logtypes" -#define FLB_CHRONICLE_URL_BASE "https://malachiteingestion-pa.googleapis.com" -#define FLB_CHRONICLE_URL_BASE_EU "https://europe-malachiteingestion-pa.googleapis.com" -#define FLB_CHRONICLE_URL_BASE_UK "https://europe-west2-malachiteingestion-pa.googleapis.com" -#define FLB_CHRONICLE_URL_BASE_ASIA "https://asia-southeast1-malachiteingestion-pa.googleapis.com" - -struct flb_chronicle_oauth_credentials { - /* parsed credentials file */ - flb_sds_t type; - flb_sds_t project_id; - flb_sds_t private_key_id; - flb_sds_t private_key; - flb_sds_t client_email; - flb_sds_t client_id; - flb_sds_t auth_uri; - flb_sds_t token_uri; -}; - -struct flb_chronicle { - /* credentials */ - flb_sds_t credentials_file; - - struct flb_chronicle_oauth_credentials *oauth_credentials; - - /* chronicle configuration */ - flb_sds_t project_id; - flb_sds_t customer_id; - flb_sds_t log_type; - - flb_sds_t uri; - flb_sds_t health_uri; - flb_sds_t endpoint; - flb_sds_t region; - flb_sds_t log_key; - - int json_date_format; - flb_sds_t json_date_key; - flb_sds_t date_key; - - /* oauth2 context */ - struct flb_oauth2 *o; - - /* mutex for acquiring oauth tokens */ - pthread_mutex_t token_mutex; - - /* Upstream connection to the backend server */ - struct flb_upstream *u; - - /* Fluent Bit context */ - struct flb_config *config; - - /* Plugin output instance reference */ - struct flb_output_instance *ins; -}; - -#endif diff --git a/fluent-bit/plugins/out_chronicle/chronicle_conf.c b/fluent-bit/plugins/out_chronicle/chronicle_conf.c deleted file mode 100644 index 5d6cdf9b2..000000000 --- a/fluent-bit/plugins/out_chronicle/chronicle_conf.c +++ /dev/null @@ -1,421 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "chronicle.h" -#include "chronicle_conf.h" - - -static inline int key_cmp(char *str, int len, char *cmp) { - - if (strlen(cmp) != len) { - return -1; - } - - return strncasecmp(str, cmp, len); -} - -static int flb_chronicle_read_credentials_file(struct flb_chronicle *ctx, - char *creds, - struct flb_chronicle_oauth_credentials *ctx_creds) -{ - int i; - int ret; - int len; - int key_len; - int val_len; - int tok_size = 32; - char *buf; - char *key; - char *val; - flb_sds_t tmp; - struct stat st; - jsmn_parser parser; - jsmntok_t *t; - jsmntok_t *tokens; - - /* Validate credentials path */ - ret = stat(creds, &st); - if (ret == -1) { - flb_errno(); - flb_plg_error(ctx->ins, "cannot open credentials file: %s", - creds); - return -1; - } - - if (!S_ISREG(st.st_mode) && !S_ISLNK(st.st_mode)) { - flb_plg_error(ctx->ins, "credentials file " - "is not a valid file: %s", creds); - return -1; - } - - /* Read file content */ - buf = mk_file_to_buffer(creds); - if (!buf) { - flb_plg_error(ctx->ins, "error reading credentials file: %s", - creds); - return -1; - } - - /* Parse content */ - jsmn_init(&parser); - tokens = flb_calloc(1, sizeof(jsmntok_t) * tok_size); - if (!tokens) { - flb_errno(); - flb_free(buf); - return -1; - } - - ret = jsmn_parse(&parser, buf, st.st_size, tokens, tok_size); - if (ret <= 0) { - flb_plg_error(ctx->ins, "invalid JSON credentials file: %s", - creds); - flb_free(buf); - flb_free(tokens); - return -1; - } - - t = &tokens[0]; - if (t->type != JSMN_OBJECT) { - flb_plg_error(ctx->ins, "invalid JSON map on file: %s", - creds); - flb_free(buf); - flb_free(tokens); - return -1; - } - - /* Parse JSON tokens */ - for (i = 1; i < ret; i++) { - t = &tokens[i]; - if (t->type != JSMN_STRING) { - continue; - } - - if (t->start == -1 || t->end == -1 || (t->start == 0 && t->end == 0)){ - break; - } - - /* Key */ - key = buf + t->start; - key_len = (t->end - t->start); - - /* Value */ - i++; - t = &tokens[i]; - val = buf + t->start; - val_len = (t->end - t->start); - - if (key_cmp(key, key_len, "type") == 0) { - ctx_creds->type = flb_sds_create_len(val, val_len); - } - else if (key_cmp(key, key_len, "project_id") == 0) { - ctx_creds->project_id = flb_sds_create_len(val, val_len); - } - else if (key_cmp(key, key_len, "private_key_id") == 0) { - ctx_creds->private_key_id = flb_sds_create_len(val, val_len); - } - else if (key_cmp(key, key_len, "private_key") == 0) { - tmp = flb_sds_create_len(val, val_len); - if (tmp) { - /* Unescape private key */ - len = flb_sds_len(tmp); - ctx_creds->private_key = flb_sds_create_size(len); - flb_unescape_string(tmp, len, - &ctx_creds->private_key); - flb_sds_destroy(tmp); - } - } - else if (key_cmp(key, key_len, "client_email") == 0) { - ctx_creds->client_email = flb_sds_create_len(val, val_len); - } - else if (key_cmp(key, key_len, "client_id") == 0) { - ctx_creds->client_id = flb_sds_create_len(val, val_len); - } - else if (key_cmp(key, key_len, "auth_uri") == 0) { - ctx_creds->auth_uri = flb_sds_create_len(val, val_len); - } - else if (key_cmp(key, key_len, "token_uri") == 0) { - ctx_creds->token_uri = flb_sds_create_len(val, val_len); - } - } - - flb_free(buf); - flb_free(tokens); - - return 0; -} - - -struct flb_chronicle *flb_chronicle_conf_create(struct flb_output_instance *ins, - struct flb_config *config) -{ - int ret; - const char *tmp; - struct flb_chronicle *ctx; - struct flb_chronicle_oauth_credentials *creds; - - /* Allocate config context */ - ctx = flb_calloc(1, sizeof(struct flb_chronicle)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - ctx->config = config; - - ret = flb_output_config_map_set(ins, (void *)ctx); - if (ret == -1) { - flb_plg_error(ins, "unable to load configuration"); - flb_free(ctx); - return NULL; - } - - /* Lookup credentials file */ - creds = flb_calloc(1, sizeof(struct flb_chronicle_oauth_credentials)); - if (!creds) { - flb_errno(); - flb_free(ctx); - return NULL; - } - ctx->oauth_credentials = creds; - - if (ctx->credentials_file == NULL) { - tmp = getenv("GOOGLE_SERVICE_CREDENTIALS"); - if (tmp) { - ctx->credentials_file = flb_sds_create(tmp); - } - } - - if (ctx->credentials_file) { - ret = flb_chronicle_read_credentials_file(ctx, - ctx->credentials_file, - ctx->oauth_credentials); - if (ret != 0) { - flb_chronicle_conf_destroy(ctx); - return NULL; - } - } - else if (!ctx->credentials_file) { - /* - * If no credentials file has been defined, do manual lookup of the - * client email and the private key. - */ - - /* Service Account Email */ - tmp = flb_output_get_property("service_account_email", ins); - if (tmp) { - creds->client_email = flb_sds_create(tmp); - } - else { - tmp = getenv("SERVICE_ACCOUNT_EMAIL"); - if (tmp) { - creds->client_email = flb_sds_create(tmp); - } - } - - /* Service Account Secret */ - tmp = flb_output_get_property("service_account_secret", ins); - if (tmp) { - creds->private_key = flb_sds_create(tmp); - } - else { - tmp = getenv("SERVICE_ACCOUNT_SECRET"); - if (tmp) { - creds->private_key = flb_sds_create(tmp); - } - } - - if (!creds->client_email) { - flb_plg_error(ctx->ins, "service_account_email/client_email is not defined"); - flb_chronicle_conf_destroy(ctx); - return NULL; - } - - if (!creds->private_key) { - flb_plg_error(ctx->ins, "service_account_secret/private_key is not defined"); - flb_chronicle_conf_destroy(ctx); - return NULL; - } - } - - /* config: 'project_id' */ - if (ctx->project_id == NULL) { - if (creds->project_id) { - /* flb_config_map_destroy uses the pointer within the config_map struct to - * free the value so if we assign it here it is safe to free later with the - * creds struct. If we do not we will leak here. - */ - ctx->project_id = creds->project_id; - if (!ctx->project_id) { - flb_plg_error(ctx->ins, - "failed extracting 'project_id' from credentials."); - flb_chronicle_conf_destroy(ctx); - return NULL; - } - } - else { - flb_plg_error(ctx->ins, - "no 'project_id' configured or present in credentials."); - flb_chronicle_conf_destroy(ctx); - return NULL; - } - } - - /* config: 'customer_id' */ - if (ctx->customer_id == NULL) { - flb_plg_error(ctx->ins, "property 'customer_id' is not defined"); - flb_chronicle_conf_destroy(ctx); - return NULL; - } - - /* config: 'log_type' */ - if (ctx->log_type == NULL) { - flb_plg_error(ctx->ins, "property 'log_type' is not defined"); - flb_chronicle_conf_destroy(ctx); - return NULL; - } - - /* Date key */ - ctx->date_key = ctx->json_date_key; - tmp = flb_output_get_property("json_date_key", ins); - if (tmp) { - /* Just check if we have to disable it */ - if (flb_utils_bool(tmp) == FLB_FALSE) { - ctx->date_key = NULL; - } - } - - /* Date format for JSON output */ - ctx->json_date_format = FLB_PACK_JSON_DATE_ISO8601; - tmp = flb_output_get_property("json_date_format", ins); - if (tmp) { - ret = flb_pack_to_json_date_type(tmp); - if (ret == -1) { - flb_plg_error(ctx->ins, "invalid json_date_format '%s'. ", tmp); - return NULL; - } - else { - ctx->json_date_format = ret; - } - } - - /* Create the target endpoint URI */ - ctx->endpoint = flb_sds_create_size(sizeof(FLB_CHRONICLE_UNSTRUCTURED_ENDPOINT)); - if (!ctx->endpoint) { - flb_errno(); - flb_chronicle_conf_destroy(ctx); - return NULL; - } - ctx->endpoint = flb_sds_printf(&ctx->endpoint, FLB_CHRONICLE_UNSTRUCTURED_ENDPOINT); - - /* Create the base URI */ - if (ctx->region == NULL || strncasecmp(ctx->region, "US", 2) == 0) { - ctx->uri = flb_sds_create_size(sizeof(FLB_CHRONICLE_URL_BASE)); - if (!ctx->uri) { - flb_errno(); - flb_chronicle_conf_destroy(ctx); - return NULL; - } - ctx->uri = flb_sds_printf(&ctx->uri, FLB_CHRONICLE_URL_BASE); - } - else if (strncasecmp(ctx->region, "EU", 2) == 0){ - ctx->uri = flb_sds_create_size(sizeof(FLB_CHRONICLE_URL_BASE_EU)); - if (!ctx->uri) { - flb_errno(); - flb_chronicle_conf_destroy(ctx); - return NULL; - } - ctx->uri = flb_sds_printf(&ctx->uri, FLB_CHRONICLE_URL_BASE_EU); - } - else if (strncasecmp(ctx->region, "UK", 2) == 0) { - ctx->uri = flb_sds_create_size(sizeof(FLB_CHRONICLE_URL_BASE_UK)); - if (!ctx->uri) { - flb_errno(); - flb_chronicle_conf_destroy(ctx); - return NULL; - } - ctx->uri = flb_sds_printf(&ctx->uri, FLB_CHRONICLE_URL_BASE_UK); - } - else if (strncasecmp(ctx->region, "ASIA", 4) == 0) { - ctx->uri = flb_sds_create_size(sizeof(FLB_CHRONICLE_URL_BASE_ASIA)); - if (!ctx->uri) { - flb_errno(); - flb_chronicle_conf_destroy(ctx); - return NULL; - } - ctx->uri = flb_sds_printf(&ctx->uri, FLB_CHRONICLE_URL_BASE_ASIA); - } - else { - flb_plg_error(ctx->ins, "unsupported region"); - flb_chronicle_conf_destroy(ctx); - return NULL; - } - flb_plg_info(ctx->ins, "project='%s' custumer_id='%s' region='%s'", - ctx->project_id, ctx->customer_id, ctx->region); - - return ctx; -} - - -int flb_chronicle_oauth_credentials_destroy(struct flb_chronicle_oauth_credentials *creds) -{ - if (!creds) { - return -1; - } - flb_sds_destroy(creds->type); - flb_sds_destroy(creds->project_id); - flb_sds_destroy(creds->private_key_id); - flb_sds_destroy(creds->private_key); - flb_sds_destroy(creds->client_email); - flb_sds_destroy(creds->client_id); - flb_sds_destroy(creds->auth_uri); - flb_sds_destroy(creds->token_uri); - - flb_free(creds); - - return 0; -} - -int flb_chronicle_conf_destroy(struct flb_chronicle *ctx) -{ - if (!ctx) { - return -1; - } - - flb_chronicle_oauth_credentials_destroy(ctx->oauth_credentials); - - flb_sds_destroy(ctx->endpoint); - flb_sds_destroy(ctx->uri); - - if (ctx->o) { - flb_oauth2_destroy(ctx->o); - } - - flb_free(ctx); - return 0; -} diff --git a/fluent-bit/plugins/out_chronicle/chronicle_conf.h b/fluent-bit/plugins/out_chronicle/chronicle_conf.h deleted file mode 100644 index 76dcfb3d2..000000000 --- a/fluent-bit/plugins/out_chronicle/chronicle_conf.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_CHRONICLE_CONF_H -#define FLB_OUT_CHRONICLE_CONF_H - -#include "chronicle.h" - -struct flb_chronicle *flb_chronicle_conf_create(struct flb_output_instance *ins, - struct flb_config *config); -int flb_chronicle_conf_destroy(struct flb_chronicle *ctx); - -#endif diff --git a/fluent-bit/plugins/out_cloudwatch_logs/CMakeLists.txt b/fluent-bit/plugins/out_cloudwatch_logs/CMakeLists.txt deleted file mode 100644 index 9e48217aa..000000000 --- a/fluent-bit/plugins/out_cloudwatch_logs/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -set(src - cloudwatch_logs.c - cloudwatch_api.c) - -FLB_PLUGIN(out_cloudwatch_logs "${src}" "") diff --git a/fluent-bit/plugins/out_cloudwatch_logs/cloudwatch_api.c b/fluent-bit/plugins/out_cloudwatch_logs/cloudwatch_api.c deleted file mode 100644 index 8043968cf..000000000 --- a/fluent-bit/plugins/out_cloudwatch_logs/cloudwatch_api.c +++ /dev/null @@ -1,1564 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#ifndef FLB_SYSTEM_WINDOWS -#include -#endif - -#include "cloudwatch_api.h" - -#define ERR_CODE_ALREADY_EXISTS "ResourceAlreadyExistsException" -#define ERR_CODE_INVALID_SEQUENCE_TOKEN "InvalidSequenceTokenException" -#define ERR_CODE_NOT_FOUND "ResourceNotFoundException" -#define ERR_CODE_DATA_ALREADY_ACCEPTED "DataAlreadyAcceptedException" - -#define AMZN_REQUEST_ID_HEADER "x-amzn-RequestId" - -#define ONE_DAY_IN_MILLISECONDS 86400000 -#define FOUR_HOURS_IN_SECONDS 14400 - - -static struct flb_aws_header create_group_header = { - .key = "X-Amz-Target", - .key_len = 12, - .val = "Logs_20140328.CreateLogGroup", - .val_len = 28, -}; - -static struct flb_aws_header put_retention_policy_header = { - .key = "X-Amz-Target", - .key_len = 12, - .val = "Logs_20140328.PutRetentionPolicy", - .val_len = 32, -}; - -static struct flb_aws_header create_stream_header = { - .key = "X-Amz-Target", - .key_len = 12, - .val = "Logs_20140328.CreateLogStream", - .val_len = 29, -}; - -static struct flb_aws_header put_log_events_header[] = { - { - .key = "X-Amz-Target", - .key_len = 12, - .val = "Logs_20140328.PutLogEvents", - .val_len = 26, - }, - { - .key = "x-amzn-logs-format", - .key_len = 18, - .val = "", - .val_len = 0, - }, -}; - -int plugin_under_test() -{ - if (getenv("FLB_CLOUDWATCH_PLUGIN_UNDER_TEST") != NULL) { - return FLB_TRUE; - } - - return FLB_FALSE; -} - -char *mock_error_response(char *error_env_var) -{ - char *err_val = NULL; - char *error = NULL; - int len = 0; - - err_val = getenv(error_env_var); - if (err_val != NULL && strlen(err_val) > 0) { - error = flb_malloc(strlen(err_val) + sizeof(char)); - if (error == NULL) { - flb_errno(); - return NULL; - } - - len = strlen(err_val); - memcpy(error, err_val, len); - error[len] = '\0'; - return error; - } - - return NULL; -} - -struct flb_http_client *mock_http_call(char *error_env_var, char *api) -{ - /* create an http client so that we can set the response */ - struct flb_http_client *c = NULL; - char *error = mock_error_response(error_env_var); - - c = flb_calloc(1, sizeof(struct flb_http_client)); - if (!c) { - flb_errno(); - flb_free(error); - return NULL; - } - mk_list_init(&c->headers); - - if (error != NULL) { - c->resp.status = 400; - /* resp.data is freed on destroy, payload is supposed to reference it */ - c->resp.data = error; - c->resp.payload = c->resp.data; - c->resp.payload_size = strlen(error); - } - else { - c->resp.status = 200; - c->resp.payload = ""; - c->resp.payload_size = 0; - if (strcmp(api, "PutLogEvents") == 0) { - /* mocked success response */ - c->resp.payload = "{\"nextSequenceToken\": \"" - "49536701251539826331025683274032969384950891766572122113\"}"; - c->resp.payload_size = strlen(c->resp.payload); - } - else { - c->resp.payload = ""; - c->resp.payload_size = 0; - } - } - - return c; -} - -int compare_events(const void *a_arg, const void *b_arg) -{ - struct cw_event *r_a = (struct cw_event *) a_arg; - struct cw_event *r_b = (struct cw_event *) b_arg; - - if (r_a->timestamp < r_b->timestamp) { - return -1; - } - else if (r_a->timestamp == r_b->timestamp) { - return 0; - } - else { - return 1; - } -} - -static inline int try_to_write(char *buf, int *off, size_t left, - const char *str, size_t str_len) -{ - if (str_len <= 0){ - str_len = strlen(str); - } - if (left <= *off+str_len) { - return FLB_FALSE; - } - memcpy(buf+*off, str, str_len); - *off += str_len; - return FLB_TRUE; -} - -/* - * Writes the "header" for a put log events payload - */ -static int init_put_payload(struct flb_cloudwatch *ctx, struct cw_flush *buf, - struct log_stream *stream, int *offset) -{ - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - "{\"logGroupName\":\"", 17)) { - goto error; - } - - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - stream->group, 0)) { - goto error; - } - - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - "\",\"logStreamName\":\"", 19)) { - goto error; - } - - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - stream->name, 0)) { - goto error; - } - - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - "\",", 2)) { - goto error; - } - - if (stream->sequence_token) { - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - "\"sequenceToken\":\"", 17)) { - goto error; - } - - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - stream->sequence_token, 0)) { - goto error; - } - - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - "\",", 2)) { - goto error; - } - } - - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - "\"logEvents\":[", 13)) { - goto error; - } - - return 0; - -error: - return -1; -} - -/* - * Writes a log event to the output buffer - */ -static int write_event(struct flb_cloudwatch *ctx, struct cw_flush *buf, - struct cw_event *event, int *offset) -{ - char ts[50]; - - if (!snprintf(ts, 50, "%llu", event->timestamp)) { - goto error; - } - - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - "{\"timestamp\":", 13)) { - goto error; - } - - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - ts, 0)) { - goto error; - } - - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - ",\"message\":\"", 12)) { - goto error; - } - - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - event->json, event->len)) { - goto error; - } - - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - "\"}", 2)) { - goto error; - } - - return 0; - -error: - return -1; -} - -/* Terminates a PutLogEvents payload */ -static int end_put_payload(struct flb_cloudwatch *ctx, struct cw_flush *buf, - int *offset) -{ - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - "]}", 2)) { - return -1; - } - buf->out_buf[*offset] = '\0'; - - return 0; -} - -static unsigned long long stream_time_span(struct log_stream *stream, - struct cw_event *event) -{ - if (stream->oldest_event == 0 || stream->newest_event == 0) { - return 0; - } - - if (stream->oldest_event > event->timestamp) { - return stream->newest_event - event->timestamp; - } - else if (stream->newest_event < event->timestamp) { - return event->timestamp - stream->oldest_event; - } - - return stream->newest_event - stream->oldest_event; -} - -/* returns FLB_TRUE if time span is less than 24 hours, FLB_FALSE if greater */ -static int check_stream_time_span(struct log_stream *stream, - struct cw_event *event) -{ - unsigned long long span = stream_time_span(stream, event); - - if (span < ONE_DAY_IN_MILLISECONDS) { - return FLB_TRUE; - } - - return FLB_FALSE; -} - -/* sets the oldest_event and newest_event fields */ -static void set_stream_time_span(struct log_stream *stream, struct cw_event *event) -{ - if (stream->oldest_event == 0 || stream->oldest_event > event->timestamp) { - stream->oldest_event = event->timestamp; - } - - if (stream->newest_event == 0 || stream->newest_event < event->timestamp) { - stream->newest_event = event->timestamp; - } -} - -/* - * Truncate log if needed. If truncated, only `written` is modified - * returns FLB_TRUE if truncated - */ -static int truncate_log(const struct flb_cloudwatch *ctx, const char *log_buffer, - size_t *written) { - size_t trailing_backslash_count = 0; - - if (*written > MAX_EVENT_LEN) { - flb_plg_warn(ctx->ins, "[size=%zu] Truncating event which is larger than " - "max size allowed by CloudWatch", *written); - *written = MAX_EVENT_LEN; - - /* remove trailing unescaped backslash if inadvertently synthesized */ - while (trailing_backslash_count < *written && - log_buffer[(*written - 1) - trailing_backslash_count] == '\\') { - trailing_backslash_count++; - } - if (trailing_backslash_count % 2 == 1) { - /* odd number of trailing backslashes, remove unpaired backslash */ - (*written)--; - } - return FLB_TRUE; - } - return FLB_FALSE; -} - - -/* - * Processes the msgpack object - * -1 = failure, record not added - * 0 = success, record added - * 1 = we ran out of space, send and retry - * 2 = record could not be processed, discard it - * Returns 0 on success, -1 on general errors, - * and 1 if we ran out of space to write the event - * which means a send must occur - */ -int process_event(struct flb_cloudwatch *ctx, struct cw_flush *buf, - const msgpack_object *obj, struct flb_time *tms) -{ - size_t written; - int ret; - size_t size; - int offset = 0; - struct cw_event *event; - char *tmp_buf_ptr; - - tmp_buf_ptr = buf->tmp_buf + buf->tmp_buf_offset; - ret = flb_msgpack_to_json(tmp_buf_ptr, - buf->tmp_buf_size - buf->tmp_buf_offset, - obj); - if (ret <= 0) { - /* - * failure to write to buffer, - * which means we ran out of space, and must send the logs - */ - return 1; - } - written = (size_t) ret; - /* Discard empty messages (written == 2 means '""') */ - if (written <= 2) { - flb_plg_debug(ctx->ins, "Found empty log message"); - return 2; - } - - /* the json string must be escaped, unless the log_key option is used */ - if (ctx->log_key == NULL) { - /* - * check if event_buf is initialized and big enough - * If all chars need to be hex encoded (impossible), 6x space would be - * needed - */ - size = written * 6; - if (buf->event_buf == NULL || buf->event_buf_size < size) { - flb_free(buf->event_buf); - buf->event_buf = flb_malloc(size); - buf->event_buf_size = size; - if (buf->event_buf == NULL) { - flb_errno(); - return -1; - } - } - offset = 0; - if (!flb_utils_write_str(buf->event_buf, &offset, size, - tmp_buf_ptr, written)) { - return -1; - } - written = offset; - - tmp_buf_ptr = buf->tmp_buf + buf->tmp_buf_offset; - if ((buf->tmp_buf_size - buf->tmp_buf_offset) < written) { - /* not enough space, send logs */ - return 1; - } - - /* truncate log, if needed */ - truncate_log(ctx, buf->event_buf, &written); - - /* copy serialized json to tmp_buf */ - if (!strncpy(tmp_buf_ptr, buf->event_buf, written)) { - return -1; - } - } - else { - /* - * flb_msgpack_to_json will encase the value in quotes - * We don't want that for log_key, so we ignore the first - * and last character - */ - written -= 2; - tmp_buf_ptr++; /* pass over the opening quote */ - buf->tmp_buf_offset++; /* advance tmp_buf past opening quote */ - - /* truncate log, if needed */ - truncate_log(ctx, tmp_buf_ptr, &written); - } - - /* add log to events list */ - buf->tmp_buf_offset += written; - event = &buf->events[buf->event_index]; - event->json = tmp_buf_ptr; - event->len = written; - event->timestamp = (unsigned long long) (tms->tm.tv_sec * 1000ull + - tms->tm.tv_nsec/1000000); - - return 0; -} - -/* Resets or inits a cw_flush struct */ -void reset_flush_buf(struct flb_cloudwatch *ctx, struct cw_flush *buf) { - buf->event_index = 0; - buf->tmp_buf_offset = 0; - buf->event_index = 0; - buf->data_size = PUT_LOG_EVENTS_HEADER_LEN + PUT_LOG_EVENTS_FOOTER_LEN; - if (buf->current_stream != NULL) { - buf->data_size += strlen(buf->current_stream->name); - buf->data_size += strlen(buf->current_stream->group); - if (buf->current_stream->sequence_token) { - buf->data_size += strlen(buf->current_stream->sequence_token); - } - } -} - -/* sorts events, constructs a put payload, and then sends */ -int send_log_events(struct flb_cloudwatch *ctx, struct cw_flush *buf) { - int ret; - int offset; - int i; - struct cw_event *event; - - if (buf->event_index <= 0) { - return 0; - } - - /* events must be sorted by timestamp in a put payload */ - qsort(buf->events, buf->event_index, sizeof(struct cw_event), compare_events); - -retry: - buf->current_stream->newest_event = 0; - buf->current_stream->oldest_event = 0; - - offset = 0; - ret = init_put_payload(ctx, buf, buf->current_stream, &offset); - if (ret < 0) { - flb_plg_error(ctx->ins, "Failed to initialize PutLogEvents payload"); - return -1; - } - - for (i = 0; i < buf->event_index; i++) { - event = &buf->events[i]; - ret = write_event(ctx, buf, event, &offset); - if (ret < 0) { - flb_plg_error(ctx->ins, "Failed to write log event %d to " - "payload buffer", i); - return -1; - } - if (i != (buf->event_index - 1)) { - if (!try_to_write(buf->out_buf, &offset, buf->out_buf_size, - ",", 1)) { - flb_plg_error(ctx->ins, "Could not terminate log event with ','"); - return -1; - } - } - } - - ret = end_put_payload(ctx, buf, &offset); - if (ret < 0) { - flb_plg_error(ctx->ins, "Could not complete PutLogEvents payload"); - return -1; - } - - flb_plg_debug(ctx->ins, "cloudwatch:PutLogEvents: events=%d, payload=%d bytes", i, offset); - ret = put_log_events(ctx, buf, buf->current_stream, (size_t) offset); - if (ret < 0) { - flb_plg_error(ctx->ins, "Failed to send log events"); - return -1; - } - else if (ret > 0) { - goto retry; - } - - return 0; -} - - /* - * Processes the msgpack object, sends the current batch if needed - * -1 = failure, event not added - * 0 = success, event added - * 1 = event been skipped - * Returns 0 on success, -1 on general errors, - * and 1 if we found a empty event or a large event. - */ -int add_event(struct flb_cloudwatch *ctx, struct cw_flush *buf, - struct log_stream *stream, - const msgpack_object *obj, struct flb_time *tms) -{ - int ret; - struct cw_event *event; - int retry_add = FLB_FALSE; - int event_bytes = 0; - - if (buf->event_index > 0 && buf->current_stream != stream) { - /* we already have events for a different stream, send them first */ - retry_add = FLB_TRUE; - goto send; - } - -retry_add_event: - buf->current_stream = stream; - retry_add = FLB_FALSE; - if (buf->event_index == 0) { - /* init */ - reset_flush_buf(ctx, buf); - } - - ret = process_event(ctx, buf, obj, tms); - if (ret < 0) { - return -1; - } - else if (ret == 1) { - if (buf->event_index <= 0) { - /* somehow the record was larger than our entire request buffer */ - flb_plg_warn(ctx->ins, "Discarding massive log record"); - return 1; /* discard this record and return to caller */ - } - /* send logs and then retry the add */ - retry_add = FLB_TRUE; - goto send; - } - else if (ret == 2) { - /* - * discard this record and return to caller - * only happens for empty records in this plugin - */ - return 1; - } - - event = &buf->events[buf->event_index]; - event_bytes = event->len + PUT_LOG_EVENTS_PER_EVENT_LEN; - - if (check_stream_time_span(stream, event) == FLB_FALSE) { - /* do not send this event */ - retry_add = FLB_TRUE; - goto send; - } - - if ((buf->data_size + event_bytes) > PUT_LOG_EVENTS_PAYLOAD_SIZE) { - if (buf->event_index <= 0) { - /* somehow the record was larger than our entire request buffer */ - flb_plg_warn(ctx->ins, "Discarding massive log record"); - return 0; /* discard this record and return to caller */ - } - /* do not send this event */ - retry_add = FLB_TRUE; - goto send; - } - - buf->data_size += event_bytes; - set_stream_time_span(stream, event); - buf->event_index++; - - if (buf->event_index == MAX_EVENTS_PER_PUT) { - goto send; - } - - /* send is not needed yet, return to caller */ - return 0; - -send: - ret = send_log_events(ctx, buf); - reset_flush_buf(ctx, buf); - if (ret < 0) { - return -1; - } - - if (retry_add == FLB_TRUE) { - goto retry_add_event; - } - - return 0; -} - -int should_add_to_emf(struct flb_intermediate_metric *an_item) -{ - /* Valid for cpu plugin */ - if (strncmp(an_item->key.via.str.ptr, "cpu_", 4) == 0 - || strncmp(an_item->key.via.str.ptr, "user_p", 6) == 0 - || strncmp(an_item->key.via.str.ptr, "system_p", 8) == 0) { - return 1; - } - - /* Valid for mem plugin */ - if (strncmp(an_item->key.via.str.ptr, "Mem.total", 9) == 0 - || strncmp(an_item->key.via.str.ptr, "Mem.used", 8) == 0 - || strncmp(an_item->key.via.str.ptr, "Mem.free", 8) == 0 - || strncmp(an_item->key.via.str.ptr, "Swap.total", 10) == 0 - || strncmp(an_item->key.via.str.ptr, "Swap.used", 9) == 0 - || strncmp(an_item->key.via.str.ptr, "Swap.free", 9) == 0) { - return 1; - } - - return 0; -} - -int pack_emf_payload(struct flb_cloudwatch *ctx, - struct mk_list *flb_intermediate_metrics, - const char *input_plugin, - struct flb_time tms, - msgpack_sbuffer *mp_sbuf, - msgpack_unpacked *mp_result, - msgpack_object *emf_payload) -{ - int total_items = mk_list_size(flb_intermediate_metrics) + 1; - - struct mk_list *metric_temp; - struct mk_list *metric_head; - struct flb_intermediate_metric *an_item; - msgpack_unpack_return mp_ret; - - /* Serialize values into the buffer using msgpack_sbuffer_write */ - msgpack_packer mp_pck; - msgpack_packer_init(&mp_pck, mp_sbuf, msgpack_sbuffer_write); - msgpack_pack_map(&mp_pck, total_items); - - /* Pack the _aws map */ - msgpack_pack_str(&mp_pck, 4); - msgpack_pack_str_body(&mp_pck, "_aws", 4); - - msgpack_pack_map(&mp_pck, 2); - - msgpack_pack_str(&mp_pck, 9); - msgpack_pack_str_body(&mp_pck, "Timestamp", 9); - msgpack_pack_long_long(&mp_pck, tms.tm.tv_sec * 1000L); - - msgpack_pack_str(&mp_pck, 17); - msgpack_pack_str_body(&mp_pck, "CloudWatchMetrics", 17); - msgpack_pack_array(&mp_pck, 1); - - msgpack_pack_map(&mp_pck, 3); - - msgpack_pack_str(&mp_pck, 9); - msgpack_pack_str_body(&mp_pck, "Namespace", 9); - - if (ctx->metric_namespace) { - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->metric_namespace)); - msgpack_pack_str_body(&mp_pck, ctx->metric_namespace, - flb_sds_len(ctx->metric_namespace)); - } - else { - msgpack_pack_str(&mp_pck, 18); - msgpack_pack_str_body(&mp_pck, "fluent-bit-metrics", 18); - } - - msgpack_pack_str(&mp_pck, 10); - msgpack_pack_str_body(&mp_pck, "Dimensions", 10); - - struct mk_list *head, *inner_head; - struct flb_split_entry *dimension_list, *entry; - struct mk_list *csv_values; - if (ctx->metric_dimensions) { - msgpack_pack_array(&mp_pck, mk_list_size(ctx->metric_dimensions)); - - mk_list_foreach(head, ctx->metric_dimensions) { - dimension_list = mk_list_entry(head, struct flb_split_entry, _head); - csv_values = flb_utils_split(dimension_list->value, ',', 256); - msgpack_pack_array(&mp_pck, mk_list_size(csv_values)); - - mk_list_foreach(inner_head, csv_values) { - entry = mk_list_entry(inner_head, struct flb_split_entry, _head); - msgpack_pack_str(&mp_pck, entry->len); - msgpack_pack_str_body(&mp_pck, entry->value, entry->len); - } - flb_utils_split_free(csv_values); - } - } - else { - msgpack_pack_array(&mp_pck, 0); - } - - msgpack_pack_str(&mp_pck, 7); - msgpack_pack_str_body(&mp_pck, "Metrics", 7); - - if (strcmp(input_plugin, "cpu") == 0) { - msgpack_pack_array(&mp_pck, 3); - } - else if (strcmp(input_plugin, "mem") == 0) { - msgpack_pack_array(&mp_pck, 6); - } - else { - msgpack_pack_array(&mp_pck, 0); - } - - mk_list_foreach_safe(metric_head, metric_temp, flb_intermediate_metrics) { - an_item = mk_list_entry(metric_head, struct flb_intermediate_metric, _head); - if (should_add_to_emf(an_item) == 1) { - msgpack_pack_map(&mp_pck, 2); - msgpack_pack_str(&mp_pck, 4); - msgpack_pack_str_body(&mp_pck, "Name", 4); - msgpack_pack_object(&mp_pck, an_item->key); - msgpack_pack_str(&mp_pck, 4); - msgpack_pack_str_body(&mp_pck, "Unit", 4); - msgpack_pack_str(&mp_pck, strlen(an_item->metric_unit)); - msgpack_pack_str_body(&mp_pck, an_item->metric_unit, - strlen(an_item->metric_unit)); - } - } - - /* Pack the metric values for each record */ - mk_list_foreach_safe(metric_head, metric_temp, flb_intermediate_metrics) { - an_item = mk_list_entry(metric_head, struct flb_intermediate_metric, _head); - msgpack_pack_object(&mp_pck, an_item->key); - msgpack_pack_object(&mp_pck, an_item->value); - } - - /* - * Deserialize the buffer into msgpack_object instance. - */ - - mp_ret = msgpack_unpack_next(mp_result, mp_sbuf->data, mp_sbuf->size, NULL); - - if (mp_ret != MSGPACK_UNPACK_SUCCESS) { - flb_plg_error(ctx->ins, "msgpack_unpack returned non-success value %i", mp_ret); - return -1; - } - - *emf_payload = mp_result->data; - return 0; -} - -/* - * Main routine- processes msgpack and sends in batches which ignore the empty ones - * return value is the number of events processed and send. - */ -int process_and_send(struct flb_cloudwatch *ctx, const char *input_plugin, - struct cw_flush *buf, flb_sds_t tag, - const char *data, size_t bytes) -{ - int i = 0; - size_t map_size; - msgpack_object map; - msgpack_object_kv *kv; - msgpack_object key; - msgpack_object val; - msgpack_unpacked mp_emf_result; - msgpack_object emf_payload; - /* msgpack::sbuffer is a simple buffer implementation. */ - msgpack_sbuffer mp_sbuf; - - struct log_stream *stream; - - char *key_str = NULL; - size_t key_str_size = 0; - int j; - int ret; - int check = FLB_FALSE; - int found = FLB_FALSE; - - /* Added for EMF support */ - struct flb_intermediate_metric *metric; - struct mk_list *tmp; - struct mk_list *head; - struct flb_intermediate_metric *an_item; - - int intermediate_metric_type; - char *intermediate_metric_unit; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return -1; - } - - if (strncmp(input_plugin, "cpu", 3) == 0) { - intermediate_metric_type = GAUGE; - intermediate_metric_unit = PERCENT; - } - else if (strncmp(input_plugin, "mem", 3) == 0) { - intermediate_metric_type = GAUGE; - intermediate_metric_unit = BYTES; - } - - /* unpack msgpack */ - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - - /* Get the record/map */ - map = *log_event.body; - map_size = map.via.map.size; - - stream = get_log_stream(ctx, tag, map); - if (!stream) { - flb_plg_debug(ctx->ins, "Couldn't determine log group & stream for record with tag %s", tag); - goto error; - } - - if (ctx->log_key) { - key_str = NULL; - key_str_size = 0; - check = FLB_FALSE; - found = FLB_FALSE; - - kv = map.via.map.ptr; - - for(j=0; j < map_size; j++) { - key = (kv+j)->key; - if (key.type == MSGPACK_OBJECT_BIN) { - key_str = (char *) key.via.bin.ptr; - key_str_size = key.via.bin.size; - check = FLB_TRUE; - } - if (key.type == MSGPACK_OBJECT_STR) { - key_str = (char *) key.via.str.ptr; - key_str_size = key.via.str.size; - check = FLB_TRUE; - } - - if (check == FLB_TRUE) { - if (strncmp(ctx->log_key, key_str, key_str_size) == 0) { - found = FLB_TRUE; - val = (kv+j)->val; - ret = add_event(ctx, buf, stream, &val, - &log_event.timestamp); - if (ret < 0 ) { - goto error; - } - } - } - - } - if (found == FLB_FALSE) { - flb_plg_error(ctx->ins, "Could not find log_key '%s' in record", - ctx->log_key); - } - - if (ret == 0) { - i++; - } - - continue; - } - - if (strncmp(input_plugin, "cpu", 3) == 0 - || strncmp(input_plugin, "mem", 3) == 0) { - /* Added for EMF support: Construct a list */ - struct mk_list flb_intermediate_metrics; - mk_list_init(&flb_intermediate_metrics); - - kv = map.via.map.ptr; - - /* - * Iterate through the record map, extract intermediate metric data, - * and add to the list. - */ - for (i = 0; i < map_size; i++) { - metric = flb_calloc(1, sizeof(struct flb_intermediate_metric)); - if (!metric) { - goto error; - } - - metric->key = (kv + i)->key; - metric->value = (kv + i)->val; - metric->metric_type = intermediate_metric_type; - metric->metric_unit = intermediate_metric_unit; - metric->timestamp = log_event.timestamp; - - mk_list_add(&metric->_head, &flb_intermediate_metrics); - - } - - /* The msgpack object is only valid during the lifetime of the - * sbuffer & the unpacked result. - */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_unpacked_init(&mp_emf_result); - - ret = pack_emf_payload(ctx, - &flb_intermediate_metrics, - input_plugin, - log_event.timestamp, - &mp_sbuf, - &mp_emf_result, - &emf_payload); - - /* free the intermediate metric list */ - - mk_list_foreach_safe(head, tmp, &flb_intermediate_metrics) { - an_item = mk_list_entry(head, struct flb_intermediate_metric, _head); - mk_list_del(&an_item->_head); - flb_free(an_item); - } - - if (ret != 0) { - flb_plg_error(ctx->ins, "Failed to convert EMF metrics to msgpack object. ret=%i", ret); - msgpack_unpacked_destroy(&mp_emf_result); - msgpack_sbuffer_destroy(&mp_sbuf); - goto error; - } - ret = add_event(ctx, buf, stream, &emf_payload, - &log_event.timestamp); - - msgpack_unpacked_destroy(&mp_emf_result); - msgpack_sbuffer_destroy(&mp_sbuf); - - } else { - ret = add_event(ctx, buf, stream, &map, - &log_event.timestamp); - } - - if (ret < 0 ) { - goto error; - } - - if (ret == 0) { - i++; - } - } - flb_log_event_decoder_destroy(&log_decoder); - - /* send any remaining events */ - ret = send_log_events(ctx, buf); - reset_flush_buf(ctx, buf); - if (ret < 0) { - return -1; - } - - /* return number of events */ - return i; - -error: - flb_log_event_decoder_destroy(&log_decoder); - - return -1; -} - -struct log_stream *get_or_create_log_stream(struct flb_cloudwatch *ctx, - flb_sds_t stream_name, - flb_sds_t group_name) -{ - int ret; - struct log_stream *new_stream; - struct log_stream *stream; - struct mk_list *tmp; - struct mk_list *head; - time_t now; - - /* check if the stream already exists */ - now = time(NULL); - mk_list_foreach_safe(head, tmp, &ctx->streams) { - stream = mk_list_entry(head, struct log_stream, _head); - if (strcmp(stream_name, stream->name) == 0 && strcmp(group_name, stream->group) == 0) { - return stream; - } - else { - /* check if stream is expired, if so, clean it up */ - if (stream->expiration < now) { - mk_list_del(&stream->_head); - log_stream_destroy(stream); - } - } - } - - /* create the new stream */ - new_stream = flb_calloc(1, sizeof(struct log_stream)); - if (!new_stream) { - flb_errno(); - return NULL; - } - new_stream->name = flb_sds_create(stream_name); - if (new_stream->name == NULL) { - flb_errno(); - return NULL; - } - new_stream->group = flb_sds_create(group_name); - if (new_stream->group == NULL) { - flb_errno(); - return NULL; - } - - ret = create_log_stream(ctx, new_stream, FLB_TRUE); - if (ret < 0) { - log_stream_destroy(new_stream); - return NULL; - } - new_stream->expiration = time(NULL) + FOUR_HOURS_IN_SECONDS; - - mk_list_add(&new_stream->_head, &ctx->streams); - return new_stream; -} - -struct log_stream *get_log_stream(struct flb_cloudwatch *ctx, flb_sds_t tag, - const msgpack_object map) -{ - flb_sds_t group_name = NULL; - flb_sds_t stream_name = NULL; - flb_sds_t tmp_s = NULL; - int free_group = FLB_FALSE; - int free_stream = FLB_FALSE; - struct log_stream *stream; - - /* templates take priority */ - if (ctx->ra_stream) { - stream_name = flb_ra_translate_check(ctx->ra_stream, tag, flb_sds_len(tag), - map, NULL, FLB_TRUE); - } - - if (ctx->ra_group) { - group_name = flb_ra_translate_check(ctx->ra_group, tag, flb_sds_len(tag), - map, NULL, FLB_TRUE); - } - - if (stream_name == NULL) { - if (ctx->stream_name) { - stream_name = ctx->stream_name; - } else { - free_stream = FLB_TRUE; - /* use log_stream_prefix */ - stream_name = flb_sds_create(ctx->log_stream_prefix); - if (!stream_name) { - flb_errno(); - if (group_name) { - flb_sds_destroy(group_name); - } - return NULL; - } - - tmp_s = flb_sds_cat(stream_name, tag, flb_sds_len(tag)); - if (!tmp_s) { - flb_errno(); - flb_sds_destroy(stream_name); - if (group_name) { - flb_sds_destroy(group_name); - } - return NULL; - } - stream_name = tmp_s; - } - } else { - free_stream = FLB_TRUE; - } - - if (group_name == NULL) { - group_name = ctx->group_name; - } else { - free_group = FLB_TRUE; - } - - flb_plg_debug(ctx->ins, "Using stream=%s, group=%s", stream_name, group_name); - - stream = get_or_create_log_stream(ctx, stream_name, group_name); - - if (free_group == FLB_TRUE) { - flb_sds_destroy(group_name); - } - if (free_stream == FLB_TRUE) { - flb_sds_destroy(stream_name); - } - return stream; -} - - -static int set_log_group_retention(struct flb_cloudwatch *ctx, struct log_stream *stream) -{ - if (ctx->log_retention_days <= 0) { - /* no need to set */ - return 0; - } - - struct flb_http_client *c = NULL; - struct flb_aws_client *cw_client; - flb_sds_t body; - flb_sds_t tmp; - flb_sds_t error; - - flb_plg_info(ctx->ins, "Setting retention policy on log group %s to %dd", stream->group, ctx->log_retention_days); - - body = flb_sds_create_size(68 + strlen(stream->group)); - if (!body) { - flb_sds_destroy(body); - flb_errno(); - return -1; - } - - /* construct CreateLogGroup request body */ - tmp = flb_sds_printf(&body, "{\"logGroupName\":\"%s\",\"retentionInDays\":%d}", stream->group, ctx->log_retention_days); - if (!tmp) { - flb_sds_destroy(body); - flb_errno(); - return -1; - } - body = tmp; - - if (plugin_under_test() == FLB_TRUE) { - c = mock_http_call("TEST_PUT_RETENTION_POLICY_ERROR", "PutRetentionPolicy"); - } - else { - cw_client = ctx->cw_client; - c = cw_client->client_vtable->request(cw_client, FLB_HTTP_POST, - "/", body, strlen(body), - &put_retention_policy_header, 1); - } - - if (c) { - flb_plg_debug(ctx->ins, "PutRetentionPolicy http status=%d", c->resp.status); - - if (c->resp.status == 200) { - /* success */ - flb_plg_info(ctx->ins, "Set retention policy to %d", ctx->log_retention_days); - flb_sds_destroy(body); - flb_http_client_destroy(c); - return 0; - } - - /* Check error */ - if (c->resp.payload_size > 0) { - error = flb_aws_error(c->resp.payload, c->resp.payload_size); - if (error != NULL) { - /* some other error occurred; notify user */ - flb_aws_print_error(c->resp.payload, c->resp.payload_size, - "PutRetentionPolicy", ctx->ins); - flb_sds_destroy(error); - } - else { - /* error can not be parsed, print raw response to debug */ - flb_plg_debug(ctx->ins, "Raw response: %s", c->resp.payload); - } - } - } - - flb_plg_error(ctx->ins, "Failed to putRetentionPolicy"); - if (c) { - flb_http_client_destroy(c); - } - flb_sds_destroy(body); - - return -1; -} - -int create_log_group(struct flb_cloudwatch *ctx, struct log_stream *stream) -{ - struct flb_http_client *c = NULL; - struct flb_aws_client *cw_client; - flb_sds_t body; - flb_sds_t tmp; - flb_sds_t error; - int ret; - - flb_plg_info(ctx->ins, "Creating log group %s", stream->group); - - body = flb_sds_create_size(25 + strlen(stream->group)); - if (!body) { - flb_sds_destroy(body); - flb_errno(); - return -1; - } - - /* construct CreateLogGroup request body */ - tmp = flb_sds_printf(&body, "{\"logGroupName\":\"%s\"}", stream->group); - if (!tmp) { - flb_sds_destroy(body); - flb_errno(); - return -1; - } - body = tmp; - - if (plugin_under_test() == FLB_TRUE) { - c = mock_http_call("TEST_CREATE_LOG_GROUP_ERROR", "CreateLogGroup"); - } - else { - cw_client = ctx->cw_client; - c = cw_client->client_vtable->request(cw_client, FLB_HTTP_POST, - "/", body, strlen(body), - &create_group_header, 1); - } - - if (c) { - flb_plg_debug(ctx->ins, "CreateLogGroup http status=%d", c->resp.status); - - if (c->resp.status == 200) { - /* success */ - flb_plg_info(ctx->ins, "Created log group %s", stream->group); - flb_sds_destroy(body); - flb_http_client_destroy(c); - ret = set_log_group_retention(ctx, stream); - return ret; - } - - /* Check error */ - if (c->resp.payload_size > 0) { - error = flb_aws_error(c->resp.payload, c->resp.payload_size); - if (error != NULL) { - if (strcmp(error, ERR_CODE_ALREADY_EXISTS) == 0) { - flb_plg_info(ctx->ins, "Log Group %s already exists", - stream->group); - flb_sds_destroy(body); - flb_sds_destroy(error); - flb_http_client_destroy(c); - ret = set_log_group_retention(ctx, stream); - return ret; - } - /* some other error occurred; notify user */ - flb_aws_print_error(c->resp.payload, c->resp.payload_size, - "CreateLogGroup", ctx->ins); - flb_sds_destroy(error); - } - else { - /* error can not be parsed, print raw response to debug */ - flb_plg_debug(ctx->ins, "Raw response: %s", c->resp.payload); - } - } - } - - flb_plg_error(ctx->ins, "Failed to create log group"); - if (c) { - flb_http_client_destroy(c); - } - flb_sds_destroy(body); - return -1; -} - -int create_log_stream(struct flb_cloudwatch *ctx, struct log_stream *stream, - int can_retry) -{ - - struct flb_http_client *c = NULL; - struct flb_aws_client *cw_client; - flb_sds_t body; - flb_sds_t tmp; - flb_sds_t error; - int ret; - - flb_plg_info(ctx->ins, "Creating log stream %s in log group %s", - stream->name, stream->group); - - body = flb_sds_create_size(50 + strlen(stream->group) + - strlen(stream->name)); - if (!body) { - flb_sds_destroy(body); - flb_errno(); - return -1; - } - - /* construct CreateLogStream request body */ - tmp = flb_sds_printf(&body, - "{\"logGroupName\":\"%s\",\"logStreamName\":\"%s\"}", - stream->group, - stream->name); - if (!tmp) { - flb_sds_destroy(body); - flb_errno(); - return -1; - } - body = tmp; - - cw_client = ctx->cw_client; - if (plugin_under_test() == FLB_TRUE) { - c = mock_http_call("TEST_CREATE_LOG_STREAM_ERROR", "CreateLogStream"); - } - else { - c = cw_client->client_vtable->request(cw_client, FLB_HTTP_POST, - "/", body, strlen(body), - &create_stream_header, 1); - } - - if (c) { - flb_plg_debug(ctx->ins,"CreateLogStream http status=%d", - c->resp.status); - - if (c->resp.status == 200) { - /* success */ - flb_plg_info(ctx->ins, "Created log stream %s", stream->name); - flb_sds_destroy(body); - flb_http_client_destroy(c); - return 0; - } - - /* Check error */ - if (c->resp.payload_size > 0) { - error = flb_aws_error(c->resp.payload, c->resp.payload_size); - if (error != NULL) { - if (strcmp(error, ERR_CODE_ALREADY_EXISTS) == 0) { - flb_plg_info(ctx->ins, "Log Stream %s already exists", - stream->name); - flb_sds_destroy(body); - flb_sds_destroy(error); - flb_http_client_destroy(c); - return 0; - } - - if (strcmp(error, ERR_CODE_NOT_FOUND) == 0) { - flb_sds_destroy(body); - flb_sds_destroy(error); - flb_http_client_destroy(c); - - if (ctx->create_group == FLB_TRUE) { - flb_plg_info(ctx->ins, "Log Group %s not found. Will attempt to create it.", - stream->group); - ret = create_log_group(ctx, stream); - if (ret < 0) { - return -1; - } else { - if (can_retry == FLB_TRUE) { - /* retry stream creation */ - return create_log_stream(ctx, stream, FLB_FALSE); - } else { - /* we failed to create the stream */ - return -1; - } - } - } else { - flb_plg_error(ctx->ins, "Log Group %s not found and `auto_create_group` disabled.", - stream->group); - } - return -1; - } - /* some other error occurred; notify user */ - flb_aws_print_error(c->resp.payload, c->resp.payload_size, - "CreateLogStream", ctx->ins); - flb_sds_destroy(error); - } - else { - /* error can not be parsed, print raw response to debug */ - flb_plg_debug(ctx->ins, "Raw response: %s", c->resp.payload); - } - } - } - - flb_plg_error(ctx->ins, "Failed to create log stream"); - if (c) { - flb_http_client_destroy(c); - } - flb_sds_destroy(body); - return -1; -} - -/* - * Returns -1 on failure, 0 on success, and 1 for a sequence token error, - * which means the caller can retry. - */ -int put_log_events(struct flb_cloudwatch *ctx, struct cw_flush *buf, - struct log_stream *stream, size_t payload_size) -{ - - struct flb_http_client *c = NULL; - struct flb_aws_client *cw_client; - flb_sds_t tmp; - flb_sds_t error; - int num_headers = 1; - int retry = FLB_TRUE; - - flb_plg_debug(ctx->ins, "Sending log events to log stream %s", stream->name); - - /* stream is being used, update expiration */ - stream->expiration = time(NULL) + FOUR_HOURS_IN_SECONDS; - - if (ctx->log_format != NULL) { - put_log_events_header[1].val = (char *) ctx->log_format; - put_log_events_header[1].val_len = strlen(ctx->log_format); - num_headers = 2; - } - -retry_request: - if (plugin_under_test() == FLB_TRUE) { - c = mock_http_call("TEST_PUT_LOG_EVENTS_ERROR", "PutLogEvents"); - } - else { - cw_client = ctx->cw_client; - c = cw_client->client_vtable->request(cw_client, FLB_HTTP_POST, - "/", buf->out_buf, payload_size, - put_log_events_header, num_headers); - } - - if (c) { - flb_plg_debug(ctx->ins, "PutLogEvents http status=%d", c->resp.status); - - if (c->resp.status == 200) { - if (c->resp.data == NULL || c->resp.data_len == 0 || strstr(c->resp.data, AMZN_REQUEST_ID_HEADER) == NULL) { - /* code was 200, but response is invalid, treat as failure */ - if (c->resp.data != NULL) { - flb_plg_debug(ctx->ins, "Could not find sequence token in " - "response: response body is empty: full data: `%.*s`", c->resp.data_len, c->resp.data); - } - flb_http_client_destroy(c); - - if (retry == FLB_TRUE) { - flb_plg_debug(ctx->ins, "issuing immediate retry for invalid response"); - retry = FLB_FALSE; - goto retry_request; - } - flb_plg_error(ctx->ins, "Recieved code 200 but response was invalid, %s header not found", - AMZN_REQUEST_ID_HEADER); - return -1; - } - - - /* success */ - if (c->resp.payload_size > 0) { - flb_plg_debug(ctx->ins, "Sent events to %s", stream->name); - tmp = flb_json_get_val(c->resp.payload, c->resp.payload_size, - "nextSequenceToken"); - if (tmp) { - if (stream->sequence_token != NULL) { - flb_sds_destroy(stream->sequence_token); - } - stream->sequence_token = tmp; - - flb_http_client_destroy(c); - return 0; - } - else { - flb_plg_error(ctx->ins, "Could not find sequence token in " - "response: %s", c->resp.payload); - } - } - - flb_http_client_destroy(c); - return 0; - } - - /* Check error */ - if (c->resp.payload_size > 0) { - error = flb_aws_error(c->resp.payload, c->resp.payload_size); - if (error != NULL) { - if (strcmp(error, ERR_CODE_INVALID_SEQUENCE_TOKEN) == 0) { - /* - * This case will happen when we do not know the correct - * sequence token; we can find it in the error response - * and retry. - */ - flb_plg_debug(ctx->ins, "Sequence token was invalid, " - "will retry"); - tmp = flb_json_get_val(c->resp.payload, c->resp.payload_size, - "expectedSequenceToken"); - if (tmp) { - if (stream->sequence_token != NULL) { - flb_sds_destroy(stream->sequence_token); - } - stream->sequence_token = tmp; - flb_sds_destroy(error); - flb_http_client_destroy(c); - /* tell the caller to retry */ - return 1; - } - } else if (strcmp(error, ERR_CODE_DATA_ALREADY_ACCEPTED) == 0) { - /* not sure what causes this but it counts as success */ - flb_plg_info(ctx->ins, "Got %s, a previous retry must have succeeded asychronously", ERR_CODE_DATA_ALREADY_ACCEPTED); - flb_sds_destroy(error); - flb_http_client_destroy(c); - /* success */ - return 0; - } - /* some other error occurred; notify user */ - flb_aws_print_error(c->resp.payload, c->resp.payload_size, - "PutLogEvents", ctx->ins); - flb_sds_destroy(error); - } - else { - /* error could not be parsed, print raw response to debug */ - flb_plg_debug(ctx->ins, "Raw response: %s", c->resp.payload); - } - } - } - - flb_plg_error(ctx->ins, "Failed to send log events"); - if (c) { - flb_http_client_destroy(c); - } - return -1; -} - - -void cw_flush_destroy(struct cw_flush *buf) -{ - if (buf) { - flb_free(buf->tmp_buf); - flb_free(buf->out_buf); - flb_free(buf->events); - flb_free(buf->event_buf); - flb_free(buf); - } -} diff --git a/fluent-bit/plugins/out_cloudwatch_logs/cloudwatch_api.h b/fluent-bit/plugins/out_cloudwatch_logs/cloudwatch_api.h deleted file mode 100644 index 99919055b..000000000 --- a/fluent-bit/plugins/out_cloudwatch_logs/cloudwatch_api.h +++ /dev/null @@ -1,57 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_CLOUDWATCH_API -#define FLB_OUT_CLOUDWATCH_API - -/* - * The CloudWatch API documents that the maximum payload is 1,048,576 bytes - * For reasons that are under investigation, using that number in this plugin - * leads to API errors. No issues have been seen setting it to 1,000,000 bytes. - */ -#define PUT_LOG_EVENTS_PAYLOAD_SIZE 1048576 -#define MAX_EVENTS_PER_PUT 10000 - -/* number of characters needed to 'start' a PutLogEvents payload */ -#define PUT_LOG_EVENTS_HEADER_LEN 72 -/* number of characters needed per event in a PutLogEvents payload */ -#define PUT_LOG_EVENTS_PER_EVENT_LEN 42 -/* number of characters needed to 'end' a PutLogEvents payload */ -#define PUT_LOG_EVENTS_FOOTER_LEN 4 - -/* 256KiB minus 26 bytes for the event */ -#define MAX_EVENT_LEN 262118 - -#include "cloudwatch_logs.h" - -void cw_flush_destroy(struct cw_flush *buf); - -int process_and_send(struct flb_cloudwatch *ctx, const char *input_plugin, - struct cw_flush *buf, flb_sds_t tag, - const char *data, size_t bytes); -int create_log_stream(struct flb_cloudwatch *ctx, struct log_stream *stream, int can_retry); -struct log_stream *get_log_stream(struct flb_cloudwatch *ctx, flb_sds_t tag, - const msgpack_object map); -int put_log_events(struct flb_cloudwatch *ctx, struct cw_flush *buf, - struct log_stream *stream, - size_t payload_size); -int create_log_group(struct flb_cloudwatch *ctx, struct log_stream *stream); -int compare_events(const void *a_arg, const void *b_arg); - -#endif diff --git a/fluent-bit/plugins/out_cloudwatch_logs/cloudwatch_logs.c b/fluent-bit/plugins/out_cloudwatch_logs/cloudwatch_logs.c deleted file mode 100644 index f6aef2240..000000000 --- a/fluent-bit/plugins/out_cloudwatch_logs/cloudwatch_logs.c +++ /dev/null @@ -1,670 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "cloudwatch_logs.h" -#include "cloudwatch_api.h" - -static struct flb_aws_header content_type_header = { - .key = "Content-Type", - .key_len = 12, - .val = "application/x-amz-json-1.1", - .val_len = 26, -}; - -static int cb_cloudwatch_init(struct flb_output_instance *ins, - struct flb_config *config, void *data) -{ - const char *tmp; - char *session_name = NULL; - struct flb_cloudwatch *ctx = NULL; - struct cw_flush *buf = NULL; - int ret; - flb_sds_t tmp_sds = NULL; - (void) config; - (void) data; - - ctx = flb_calloc(1, sizeof(struct flb_cloudwatch)); - if (!ctx) { - flb_errno(); - return -1; - } - - mk_list_init(&ctx->streams); - - ctx->ins = ins; - - /* Populate context with config map defaults and incoming properties */ - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_plg_error(ctx->ins, "configuration error"); - goto error; - } - - tmp = flb_output_get_property("log_group_name", ins); - if (tmp) { - ctx->log_group = tmp; - ctx->group_name = flb_sds_create(tmp); - if (!ctx->group_name) { - flb_plg_error(ctx->ins, "Could not create log group context property"); - goto error; - } - } else { - flb_plg_error(ctx->ins, "'log_group_name' is a required field"); - goto error; - } - - tmp = flb_output_get_property("log_stream_name", ins); - if (tmp) { - ctx->log_stream_name = tmp; - ctx->stream_name = flb_sds_create(tmp); - if (!ctx->stream_name) { - flb_plg_error(ctx->ins, "Could not create log group context property"); - goto error; - } - } - - tmp = flb_output_get_property("log_stream_prefix", ins); - if (tmp) { - ctx->log_stream_prefix = tmp; - } - - if (!ctx->log_stream_name && !ctx->log_stream_prefix) { - flb_plg_error(ctx->ins, "Either 'log_stream_name' or 'log_stream_prefix'" - " is required"); - goto error; - } - - if (ctx->log_stream_name && ctx->log_stream_prefix) { - flb_plg_error(ctx->ins, "Either 'log_stream_name' or 'log_stream_prefix'" - " is required"); - goto error; - } - - tmp = flb_output_get_property("log_group_template", ins); - if (tmp) { - ctx->ra_group = flb_ra_create((char *) tmp, FLB_FALSE); - if (ctx->ra_group == NULL) { - flb_plg_error(ctx->ins, "Could not parse `log_group_template`"); - goto error; - } - } - - tmp = flb_output_get_property("log_stream_template", ins); - if (tmp) { - ctx->ra_stream = flb_ra_create((char *) tmp, FLB_FALSE); - if (ctx->ra_stream == NULL) { - flb_plg_error(ctx->ins, "Could not parse `log_stream_template`"); - goto error; - } - } - - tmp = flb_output_get_property("log_format", ins); - if (tmp) { - ctx->log_format = tmp; - } - - tmp = flb_output_get_property("endpoint", ins); - if (tmp) { - ctx->custom_endpoint = FLB_TRUE; - ctx->endpoint = removeProtocol((char *) tmp, "https://"); - } - else { - ctx->custom_endpoint = FLB_FALSE; - } - - tmp = flb_output_get_property("log_key", ins); - if (tmp) { - ctx->log_key = tmp; - } - - tmp = flb_output_get_property("extra_user_agent", ins); - if (tmp) { - ctx->extra_user_agent = tmp; - } - - tmp = flb_output_get_property("region", ins); - if (tmp) { - ctx->region = tmp; - } else { - flb_plg_error(ctx->ins, "'region' is a required field"); - goto error; - } - - tmp = flb_output_get_property("metric_namespace", ins); - if (tmp) - { - flb_plg_info(ctx->ins, "Metric Namespace=%s", tmp); - ctx->metric_namespace = flb_sds_create(tmp); - } - - tmp = flb_output_get_property("metric_dimensions", ins); - if (tmp) - { - flb_plg_info(ctx->ins, "Metric Dimensions=%s", tmp); - ctx->metric_dimensions = flb_utils_split(tmp, ';', 256); - } - - ctx->create_group = FLB_FALSE; - tmp = flb_output_get_property("auto_create_group", ins); - if (tmp) { - ctx->create_group = flb_utils_bool(tmp); - } - - ctx->retry_requests = FLB_TRUE; - tmp = flb_output_get_property("auto_retry_requests", ins); - /* native plugins use On/Off as bool, the old Go plugin used true/false */ - if (tmp && (strcasecmp(tmp, "Off") == 0 || strcasecmp(tmp, "false") == 0)) { - ctx->retry_requests = FLB_FALSE; - } - - ctx->log_retention_days = 0; - tmp = flb_output_get_property("log_retention_days", ins); - if (tmp) { - ctx->log_retention_days = atoi(tmp); - } - - tmp = flb_output_get_property("role_arn", ins); - if (tmp) { - ctx->role_arn = tmp; - } - - tmp = flb_output_get_property("sts_endpoint", ins); - if (tmp) { - ctx->sts_endpoint = (char *) tmp; - } - - /* one tls instance for provider, one for cw client */ - ctx->cred_tls = flb_tls_create(FLB_TLS_CLIENT_MODE, - FLB_TRUE, - ins->tls_debug, - ins->tls_vhost, - ins->tls_ca_path, - ins->tls_ca_file, - ins->tls_crt_file, - ins->tls_key_file, - ins->tls_key_passwd); - - if (!ctx->cred_tls) { - flb_plg_error(ctx->ins, "Failed to create tls context"); - goto error; - } - - ctx->client_tls = flb_tls_create(FLB_TLS_CLIENT_MODE, - ins->tls_verify, - ins->tls_debug, - ins->tls_vhost, - ins->tls_ca_path, - ins->tls_ca_file, - ins->tls_crt_file, - ins->tls_key_file, - ins->tls_key_passwd); - if (!ctx->client_tls) { - flb_plg_error(ctx->ins, "Failed to create tls context"); - goto error; - } - - ctx->aws_provider = flb_standard_chain_provider_create(config, - ctx->cred_tls, - (char *) ctx->region, - (char *) ctx->sts_endpoint, - NULL, - flb_aws_client_generator(), - ctx->profile); - if (!ctx->aws_provider) { - flb_plg_error(ctx->ins, "Failed to create AWS Credential Provider"); - goto error; - } - - if(ctx->role_arn) { - /* set up sts assume role provider */ - session_name = flb_sts_session_name(); - if (!session_name) { - flb_plg_error(ctx->ins, - "Failed to generate random STS session name"); - goto error; - } - - /* STS provider needs yet another separate TLS instance */ - ctx->sts_tls = flb_tls_create(FLB_TLS_CLIENT_MODE, - FLB_TRUE, - ins->tls_debug, - ins->tls_vhost, - ins->tls_ca_path, - ins->tls_ca_file, - ins->tls_crt_file, - ins->tls_key_file, - ins->tls_key_passwd); - if (!ctx->sts_tls) { - flb_errno(); - goto error; - } - - ctx->base_aws_provider = ctx->aws_provider; - - ctx->aws_provider = flb_sts_provider_create(config, - ctx->sts_tls, - ctx->base_aws_provider, - (char *) ctx->external_id, - (char *) ctx->role_arn, - session_name, - (char *) ctx->region, - (char *) ctx->sts_endpoint, - NULL, - flb_aws_client_generator()); - if (!ctx->aws_provider) { - flb_plg_error(ctx->ins, - "Failed to create AWS STS Credential Provider"); - goto error; - } - /* session name can freed after provider is created */ - flb_free(session_name); - session_name = NULL; - } - - /* initialize credentials and set to sync mode */ - ctx->aws_provider->provider_vtable->sync(ctx->aws_provider); - ctx->aws_provider->provider_vtable->init(ctx->aws_provider); - ctx->aws_provider->provider_vtable->upstream_set(ctx->aws_provider, ctx->ins); - - if (ctx->endpoint == NULL) { - ctx->endpoint = flb_aws_endpoint("logs", (char *) ctx->region); - if (!ctx->endpoint) { - goto error; - } - } - - struct flb_aws_client_generator *generator = flb_aws_client_generator(); - ctx->cw_client = generator->create(); - if (!ctx->cw_client) { - goto error; - } - ctx->cw_client->name = "cw_client"; - ctx->cw_client->has_auth = FLB_TRUE; - ctx->cw_client->provider = ctx->aws_provider; - ctx->cw_client->region = (char *) ctx->region; - ctx->cw_client->service = "logs"; - ctx->cw_client->port = (ins->host.port != 0) ? ins->host.port : 443; - ctx->cw_client->flags = (ins->use_tls) ? FLB_IO_TLS : FLB_IO_TCP; - ctx->cw_client->proxy = NULL; - ctx->cw_client->static_headers = &content_type_header; - ctx->cw_client->static_headers_len = 1; - tmp_sds = flb_sds_create(ctx->extra_user_agent); - if (!tmp_sds) { - flb_errno(); - goto error; - } - ctx->cw_client->extra_user_agent = tmp_sds; - ctx->cw_client->retry_requests = ctx->retry_requests; - - struct flb_upstream *upstream = flb_upstream_create(config, ctx->endpoint, - ctx->cw_client->port, - ctx->cw_client->flags, - ctx->client_tls); - if (!upstream) { - flb_plg_error(ctx->ins, "Connection initialization error"); - goto error; - } - - ctx->cw_client->upstream = upstream; - flb_output_upstream_set(upstream, ctx->ins); - ctx->cw_client->host = ctx->endpoint; - - /* alloc the payload/processing buffer */ - buf = flb_calloc(1, sizeof(struct cw_flush)); - if (!buf) { - flb_errno(); - goto error; - } - - buf->out_buf = flb_malloc(PUT_LOG_EVENTS_PAYLOAD_SIZE); - if (!buf->out_buf) { - flb_errno(); - cw_flush_destroy(buf); - goto error; - } - buf->out_buf_size = PUT_LOG_EVENTS_PAYLOAD_SIZE; - - buf->tmp_buf = flb_malloc(sizeof(char) * PUT_LOG_EVENTS_PAYLOAD_SIZE); - if (!buf->tmp_buf) { - flb_errno(); - cw_flush_destroy(buf); - goto error; - } - buf->tmp_buf_size = PUT_LOG_EVENTS_PAYLOAD_SIZE; - - buf->events = flb_malloc(sizeof(struct cw_event) * MAX_EVENTS_PER_PUT); - if (!buf->events) { - flb_errno(); - cw_flush_destroy(buf); - goto error; - } - buf->events_capacity = MAX_EVENTS_PER_PUT; - - ctx->buf = buf; - - - /* Export context */ - flb_output_set_context(ins, ctx); - - return 0; - -error: - flb_free(session_name); - flb_plg_error(ctx->ins, "Initialization failed"); - flb_cloudwatch_ctx_destroy(ctx); - return -1; -} - -static void cb_cloudwatch_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - struct flb_cloudwatch *ctx = out_context; - int event_count; - (void) i_ins; - (void) config; - - event_count = process_and_send(ctx, i_ins->p->name, ctx->buf, event_chunk->tag, - event_chunk->data, event_chunk->size); - if (event_count < 0) { - flb_plg_error(ctx->ins, "Failed to send events"); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - // TODO: this msg is innaccurate if events are skipped - flb_plg_debug(ctx->ins, "Sent %d events to CloudWatch", event_count); - - FLB_OUTPUT_RETURN(FLB_OK); -} - -void flb_cloudwatch_ctx_destroy(struct flb_cloudwatch *ctx) -{ - struct log_stream *stream; - struct mk_list *tmp; - struct mk_list *head; - - if (ctx != NULL) { - if (ctx->base_aws_provider) { - flb_aws_provider_destroy(ctx->base_aws_provider); - } - - if (ctx->buf) { - cw_flush_destroy(ctx->buf); - } - - if (ctx->aws_provider) { - flb_aws_provider_destroy(ctx->aws_provider); - } - - if (ctx->cred_tls) { - flb_tls_destroy(ctx->cred_tls); - } - - if (ctx->sts_tls) { - flb_tls_destroy(ctx->sts_tls); - } - - if (ctx->client_tls) { - flb_tls_destroy(ctx->client_tls); - } - - if (ctx->cw_client) { - flb_aws_client_destroy(ctx->cw_client); - } - - if (ctx->custom_endpoint == FLB_FALSE) { - flb_free(ctx->endpoint); - } - - if (ctx->ra_group) { - flb_ra_destroy(ctx->ra_group); - } - - if (ctx->ra_stream) { - flb_ra_destroy(ctx->ra_stream); - } - - if (ctx->group_name) { - flb_sds_destroy(ctx->group_name); - } - - if (ctx->stream_name) { - flb_sds_destroy(ctx->stream_name); - } - - mk_list_foreach_safe(head, tmp, &ctx->streams) { - stream = mk_list_entry(head, struct log_stream, _head); - mk_list_del(&stream->_head); - log_stream_destroy(stream); - } - flb_free(ctx); - } -} - -static int cb_cloudwatch_exit(void *data, struct flb_config *config) -{ - struct flb_cloudwatch *ctx = data; - - flb_cloudwatch_ctx_destroy(ctx); - return 0; -} - -void log_stream_destroy(struct log_stream *stream) -{ - if (stream) { - if (stream->name) { - flb_sds_destroy(stream->name); - } - if (stream->sequence_token) { - flb_sds_destroy(stream->sequence_token); - } - if (stream->group) { - flb_sds_destroy(stream->group); - } - flb_free(stream); - } -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "region", NULL, - 0, FLB_FALSE, 0, - "The AWS region to send logs to" - }, - - { - FLB_CONFIG_MAP_STR, "log_group_name", NULL, - 0, FLB_FALSE, 0, - "CloudWatch Log Group Name" - }, - - { - FLB_CONFIG_MAP_STR, "log_stream_name", NULL, - 0, FLB_FALSE, 0, - "CloudWatch Log Stream Name; not compatible with `log_stream_prefix`" - }, - - { - FLB_CONFIG_MAP_STR, "log_stream_prefix", NULL, - 0, FLB_FALSE, 0, - "Prefix for CloudWatch Log Stream Name; the tag is appended to the prefix" - " to form the stream name" - }, - - { - FLB_CONFIG_MAP_STR, "log_group_template", NULL, - 0, FLB_FALSE, 0, - "Template for CW Log Group name using record accessor syntax. " - "Plugin falls back to the log_group_name configured if needed." - }, - - { - FLB_CONFIG_MAP_STR, "log_stream_template", NULL, - 0, FLB_FALSE, 0, - "Template for CW Log Stream name using record accessor syntax. " - "Plugin falls back to the log_stream_name or log_stream_prefix configured if needed." - }, - - { - FLB_CONFIG_MAP_STR, "log_key", NULL, - 0, FLB_FALSE, 0, - "By default, the whole log record will be sent to CloudWatch. " - "If you specify a key name with this option, then only the value of " - "that key will be sent to CloudWatch. For example, if you are using " - "the Fluentd Docker log driver, you can specify log_key log and only " - "the log message will be sent to CloudWatch." - }, - - { - FLB_CONFIG_MAP_STR, "extra_user_agent", NULL, - 0, FLB_FALSE, 0, - "This option appends a string to the default user agent. " - "AWS asks that you not manually set this field yourself, " - "it is reserved for use in our vended configurations, " - "for example, EKS Container Insights." - }, - - { - FLB_CONFIG_MAP_STR, "log_format", NULL, - 0, FLB_FALSE, 0, - "An optional parameter that can be used to tell CloudWatch the format " - "of the data. A value of json/emf enables CloudWatch to extract custom " - "metrics embedded in a JSON payload." - }, - - { - FLB_CONFIG_MAP_STR, "role_arn", NULL, - 0, FLB_FALSE, 0, - "ARN of an IAM role to assume (ex. for cross account access)." - }, - - { - FLB_CONFIG_MAP_BOOL, "auto_create_group", "false", - 0, FLB_FALSE, 0, - "Automatically create the log group (log streams will always automatically" - " be created)" - }, - - { - FLB_CONFIG_MAP_BOOL, "auto_retry_requests", "true", - 0, FLB_FALSE, 0, - "Immediately retry failed requests to AWS services once. This option " - "does not affect the normal Fluent Bit retry mechanism with backoff. " - "Instead, it enables an immediate retry with no delay for networking " - "errors, which may help improve throughput when there are transient/random " - "networking issues." - }, - - { - FLB_CONFIG_MAP_INT, "log_retention_days", "0", - 0, FLB_FALSE, 0, - "If set to a number greater than zero, and newly create log group's " - "retention policy is set to this many days. " - "Valid values are: [1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, 3653]" - }, - - { - FLB_CONFIG_MAP_STR, "endpoint", NULL, - 0, FLB_FALSE, 0, - "Specify a custom endpoint for the CloudWatch Logs API" - }, - - { - FLB_CONFIG_MAP_STR, "sts_endpoint", NULL, - 0, FLB_FALSE, 0, - "Specify a custom endpoint for the STS API, can be used with the role_arn parameter" - }, - - { - FLB_CONFIG_MAP_STR, "external_id", NULL, - 0, FLB_TRUE, offsetof(struct flb_cloudwatch, external_id), - "Specify an external ID for the STS API, can be used with the role_arn parameter if your role " - "requires an external ID." - }, - - { - FLB_CONFIG_MAP_STR, "metric_namespace", NULL, - 0, FLB_FALSE, 0, - "Metric namespace for CloudWatch EMF logs" - }, - - { - FLB_CONFIG_MAP_STR, "metric_dimensions", NULL, - 0, FLB_FALSE, 0, - "Metric dimensions is a list of lists. If you have only one list of " - "dimensions, put the values as a comma seperated string. If you want to put " - "list of lists, use the list as semicolon seperated strings. If your value " - "is 'd1,d2;d3', we will consider it as [[d1, d2],[d3]]." - }, - - { - FLB_CONFIG_MAP_STR, "profile", NULL, - 0, FLB_TRUE, offsetof(struct flb_cloudwatch, profile), - "AWS Profile name. AWS Profiles can be configured with AWS CLI and are usually stored in " - "$HOME/.aws/ directory." - }, - - /* EOF */ - {0} -}; - -/* Plugin registration */ -struct flb_output_plugin out_cloudwatch_logs_plugin = { - .name = "cloudwatch_logs", - .description = "Send logs to Amazon CloudWatch", - .cb_init = cb_cloudwatch_init, - .cb_flush = cb_cloudwatch_flush, - .cb_exit = cb_cloudwatch_exit, - - /* - * Allow cloudwatch to use async network stack synchronously by opting into - * FLB_OUTPUT_SYNCHRONOUS synchronous task scheduler - */ - .flags = FLB_OUTPUT_SYNCHRONOUS, - .workers = 1, - - /* Configuration */ - .config_map = config_map, -}; diff --git a/fluent-bit/plugins/out_cloudwatch_logs/cloudwatch_logs.h b/fluent-bit/plugins/out_cloudwatch_logs/cloudwatch_logs.h deleted file mode 100644 index 7fe8bf0b7..000000000 --- a/fluent-bit/plugins/out_cloudwatch_logs/cloudwatch_logs.h +++ /dev/null @@ -1,158 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_CLOUDWATCH_LOGS_H -#define FLB_OUT_CLOUDWATCH_LOGS_H - -#include -#include -#include -#include -#include -#include - -#include -#include - -/* buffers used for each flush */ -struct cw_flush { - /* temporary buffer for storing the serialized event messages */ - char *tmp_buf; - size_t tmp_buf_size; - /* current index of tmp_buf */ - size_t tmp_buf_offset; - - /* projected final size of the payload for this flush */ - size_t data_size; - - /* log events- each of these has a pointer to their message in tmp_buf */ - struct cw_event *events; - int events_capacity; - /* current event */ - int event_index; - - /* the payload of the API request */ - char *out_buf; - size_t out_buf_size; - - /* buffer used to temporarily hold an event during processing */ - char *event_buf; - size_t event_buf_size; - - /* current log stream that we are sending records too */ - struct log_stream *current_stream; -}; - -struct cw_event { - char *json; - size_t len; - // TODO: re-usable in kinesis streams plugin if we make it timespec instead - // uint64_t? - unsigned long long timestamp; -}; - -struct log_stream { - flb_sds_t name; - flb_sds_t group; - flb_sds_t sequence_token; - /* - * log streams in CloudWatch do not expire; but our internal representations - * of them are periodically cleaned up if they have been unused for too long - */ - time_t expiration; - - /* - * Used to track the "time span" of a single PutLogEvents payload - * Which can not exceed 24 hours. - */ - unsigned long long oldest_event; - unsigned long long newest_event; - - struct mk_list _head; -}; - -void log_stream_destroy(struct log_stream *stream); - -struct flb_cloudwatch { - /* - * TLS instances can not be re-used. So we have one for: - * - Base cred provider (needed for EKS provider) - * - STS Assume role provider - * - The CloudWatch Logs client for this plugin - */ - struct flb_tls *cred_tls; - struct flb_tls *sts_tls; - struct flb_tls *client_tls; - struct flb_aws_provider *aws_provider; - struct flb_aws_provider *base_aws_provider; - struct flb_aws_client *cw_client; - - /* configuration options */ - const char *log_stream_name; - const char *log_stream_prefix; - const char *log_group; - const char *region; - const char *sts_endpoint; - const char *log_format; - const char *role_arn; - const char *log_key; - const char *extra_user_agent; - const char *external_id; - const char *profile; - int custom_endpoint; - /* Should the plugin create the log group */ - int create_group; - - flb_sds_t group_name; - flb_sds_t stream_name; - - /* Should requests to AWS services be retried */ - int retry_requests; - - /* If set to a number greater than zero, and newly create log group's retention policy is set to this many days. */ - int log_retention_days; - - /* must be freed on shutdown if custom_endpoint is not set */ - char *endpoint; - - /* templates */ - struct flb_record_accessor *ra_group; - struct flb_record_accessor *ra_stream; - - /* stores log streams we're putting to */ - struct mk_list streams; - - /* buffers for data processing and request payload */ - struct cw_flush *buf; - /* The namespace to use for the metric */ - flb_sds_t metric_namespace; - - /* Metric dimensions is a list of lists. If you have only one list of - dimensions, put the values as a comma seperated string. If you want to put - list of lists, use the list as semicolon seperated strings. If your value - is 'd1,d2;d3', we will consider it as [[d1, d2],[d3]]*/ - struct mk_list *metric_dimensions; - - /* Plugin output instance reference */ - struct flb_output_instance *ins; -}; - -void flb_cloudwatch_ctx_destroy(struct flb_cloudwatch *ctx); - -#endif diff --git a/fluent-bit/plugins/out_counter/CMakeLists.txt b/fluent-bit/plugins/out_counter/CMakeLists.txt deleted file mode 100644 index 63f205699..000000000 --- a/fluent-bit/plugins/out_counter/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - counter.c) - -FLB_PLUGIN(out_counter "${src}" "") diff --git a/fluent-bit/plugins/out_counter/counter.c b/fluent-bit/plugins/out_counter/counter.c deleted file mode 100644 index 812b96782..000000000 --- a/fluent-bit/plugins/out_counter/counter.c +++ /dev/null @@ -1,106 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include -#include -#include - -struct flb_counter_ctx { - uint64_t total; -}; - -static int cb_counter_init(struct flb_output_instance *ins, - struct flb_config *config, - void *data) -{ - (void) ins; - (void) config; - (void) data; - struct flb_counter_ctx *ctx; - - ctx = flb_malloc(sizeof(struct flb_counter_ctx)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->total = 0; - flb_output_set_context(ins, ctx); - if (flb_output_config_map_set(ins, (void *)ctx) == -1) { - flb_plg_error(ins, "unable to load configuration"); - flb_free(ctx); - return -1; - } - - return 0; -} - -static void cb_counter_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - (void) i_ins; - (void) out_context; - (void) config; - size_t cnt; - struct flb_counter_ctx *ctx = out_context; - struct flb_time tm; - - /* Count number of parent items */ - cnt = flb_mp_count(event_chunk->data, event_chunk->size); - ctx->total += cnt; - - flb_time_get(&tm); - printf("%f,%lu (total = %"PRIu64")\n", flb_time_to_double(&tm), cnt, - ctx->total); - - FLB_OUTPUT_RETURN(FLB_OK); -} - -static int cb_counter_exit(void *data, struct flb_config *config) -{ - struct flb_counter_ctx *ctx = data; - - if (!ctx) { - return 0; - } - - flb_free(ctx); - return 0; -} - -static struct flb_config_map config_map[] = { - /* EOF */ - {0} -}; - -struct flb_output_plugin out_counter_plugin = { - .name = "counter", - .description = "Records counter", - .cb_init = cb_counter_init, - .cb_flush = cb_counter_flush, - .cb_exit = cb_counter_exit, - .config_map = config_map, - .flags = 0, -}; diff --git a/fluent-bit/plugins/out_datadog/CMakeLists.txt b/fluent-bit/plugins/out_datadog/CMakeLists.txt deleted file mode 100644 index 6c32b3961..000000000 --- a/fluent-bit/plugins/out_datadog/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(src - datadog.c - datadog_conf.c - datadog_remap.c) - -FLB_PLUGIN(out_datadog "${src}" "") diff --git a/fluent-bit/plugins/out_datadog/datadog.c b/fluent-bit/plugins/out_datadog/datadog.c deleted file mode 100644 index 082ab0fac..000000000 --- a/fluent-bit/plugins/out_datadog/datadog.c +++ /dev/null @@ -1,568 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "datadog.h" -#include "datadog_conf.h" -#include "datadog_remap.h" - -static int cb_datadog_init(struct flb_output_instance *ins, - struct flb_config *config, void *data) -{ - struct flb_out_datadog *ctx = NULL; - (void) data; - - ctx = flb_datadog_conf_create(ins, config); - if (!ctx) { - return -1; - } - - /* Set the plugin context */ - flb_output_set_context(ins, ctx); - return 0; -} - -static int64_t timestamp_format(const struct flb_time* tms) { - int64_t timestamp = 0; - - /* Format the time, use milliseconds precision not nanoseconds */ - timestamp = tms->tm.tv_sec * 1000; - timestamp += tms->tm.tv_nsec / 1000000; - - /* round up if necessary */ - if (tms->tm.tv_nsec % 1000000 >= 500000) { - ++timestamp; - } - return timestamp; -} - -static void dd_msgpack_pack_key_value_str(msgpack_packer* mp_pck, - const char *key, size_t key_size, - const char *val, size_t val_size) -{ - msgpack_pack_str(mp_pck, key_size); - msgpack_pack_str_body(mp_pck, key, key_size); - msgpack_pack_str(mp_pck, val_size); - msgpack_pack_str_body(mp_pck,val, val_size); -} - -static int dd_compare_msgpack_obj_key_with_str(const msgpack_object obj, const char *key, size_t key_size) { - - if (obj.via.str.size == key_size && memcmp(obj.via.str.ptr,key, key_size) == 0) { - return FLB_TRUE; - } - - return FLB_FALSE; -} - -static int datadog_format(struct flb_config *config, - struct flb_input_instance *ins, - void *plugin_context, - void *flush_ctx, - int event_type, - const char *tag, int tag_len, - const void *data, size_t bytes, - void **out_data, size_t *out_size) -{ - int i; - int ind; - int byte_cnt = 64; - int remap_cnt; - int ret; - /* for msgpack global structs */ - size_t array_size = 0; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - /* for sub msgpack objs */ - int map_size; - int64_t timestamp; - msgpack_object map; - msgpack_object k; - msgpack_object v; - struct flb_out_datadog *ctx = plugin_context; - struct flb_event_chunk *event_chunk; - - /* output buffer */ - flb_sds_t out_buf; - flb_sds_t remapped_tags = NULL; - flb_sds_t tmp = NULL; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - /* in normal flush callback we have the event_chunk set as flush context - * so we don't need to calculate the event len. - * But in test mode the formatter won't get the event_chunk as flush_ctx - */ - if (flush_ctx != NULL) { - event_chunk = flush_ctx; - array_size = event_chunk->total_events; - } else { - array_size = flb_mp_count(data, bytes); - } - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return -1; - } - - /* Create temporary msgpack buffer */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - /* Prepare array for all entries */ - msgpack_pack_array(&mp_pck, array_size); - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - timestamp = timestamp_format(&log_event.timestamp); - - map = *log_event.body; - map_size = map.via.map.size; - - /* - * msgpack requires knowing/allocating exact map size in advance, so we need to - * loop through the map twice. First time here to count how many attr we can - * remap to tags, and second time later where we actually perform the remapping. - */ - remap_cnt = 0, byte_cnt = ctx->dd_tags ? flb_sds_len(ctx->dd_tags) : 0; - if (ctx->remap) { - for (i = 0; i < map_size; i++) { - if (dd_attr_need_remapping(map.via.map.ptr[i].key, - map.via.map.ptr[i].val) >= 0) { - remap_cnt++; - /* - * here we also *estimated* the size of buffer needed to hold the - * remapped tags. We can't know the size for sure until we do the - * remapping, the estimation here is just for efficiency, so that - * appending tags won't cause repeated resizing/copying - */ - byte_cnt += 2 * (map.via.map.ptr[i].key.via.str.size + - map.via.map.ptr[i].val.via.str.size); - } - } - - if (!remapped_tags) { - remapped_tags = flb_sds_create_size(byte_cnt); - if (!remapped_tags) { - flb_errno(); - msgpack_sbuffer_destroy(&mp_sbuf); - flb_log_event_decoder_destroy(&log_decoder); - return -1; - } - } - else if (flb_sds_len(remapped_tags) < byte_cnt) { - tmp = flb_sds_increase(remapped_tags, byte_cnt - flb_sds_len(remapped_tags)); - if (!tmp) { - flb_errno(); - flb_sds_destroy(remapped_tags); - msgpack_sbuffer_destroy(&mp_sbuf); - flb_log_event_decoder_destroy(&log_decoder); - return -1; - } - remapped_tags = tmp; - } - - /* - * we reuse this buffer across messages, which means we have to clear it - * for each message flb_sds doesn't have a clear function, so we copy a - * empty string to achieve the same effect - */ - remapped_tags = flb_sds_copy(remapped_tags, "", 0); - } - - /* - * build new object(map) with additional space for datadog entries for those - * remapped attributes, we need to remove them from the map. Note: If there were - * no dd_tags specified AND there will be remapped attributes, we need to add 1 - * to account for the new presense of the dd_tags - */ - if (remap_cnt && (ctx->dd_tags == NULL)) { - msgpack_pack_map(&mp_pck, - ctx->nb_additional_entries + map_size + 1 - remap_cnt); - } - else { - msgpack_pack_map(&mp_pck, ctx->nb_additional_entries + map_size - remap_cnt); - } - - /* timestamp */ - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->json_date_key)); - msgpack_pack_str_body(&mp_pck, - ctx->json_date_key, - flb_sds_len(ctx->json_date_key)); - msgpack_pack_int64(&mp_pck, timestamp); - - /* include_tag_key */ - if (ctx->include_tag_key == FLB_TRUE) { - dd_msgpack_pack_key_value_str(&mp_pck, - ctx->tag_key, flb_sds_len(ctx->tag_key), - tag, tag_len); - } - - /* dd_source */ - if (ctx->dd_source != NULL) { - dd_msgpack_pack_key_value_str(&mp_pck, - FLB_DATADOG_DD_SOURCE_KEY, - sizeof(FLB_DATADOG_DD_SOURCE_KEY) -1, - ctx->dd_source, flb_sds_len(ctx->dd_source)); - } - - /* dd_service */ - if (ctx->dd_service != NULL) { - dd_msgpack_pack_key_value_str(&mp_pck, - FLB_DATADOG_DD_SERVICE_KEY, - sizeof(FLB_DATADOG_DD_SERVICE_KEY) -1, - ctx->dd_service, flb_sds_len(ctx->dd_service)); - } - - /* Append initial object k/v */ - ind = 0; - for (i = 0; i < map_size; i++) { - k = map.via.map.ptr[i].key; - v = map.via.map.ptr[i].val; - - /* - * actually perform the remapping here. For matched attr, we remap and - * append them to remapped_tags buffer, then skip the rest of processing - * (so they won't be packed as attr) - */ - if (ctx->remap && (ind = dd_attr_need_remapping(k, v)) >=0 ) { - ret = remapping[ind].remap_to_tag(remapping[ind].remap_tag_name, v, - &remapped_tags); - if (ret < 0) { - flb_plg_error(ctx->ins, "Failed to remap tag: %s, skipping", remapping[ind].remap_tag_name); - } - continue; - } - - /* Mapping between input keys to specific datadog keys */ - if (ctx->dd_message_key != NULL && - dd_compare_msgpack_obj_key_with_str(k, ctx->dd_message_key, - flb_sds_len(ctx->dd_message_key)) == FLB_TRUE) { - msgpack_pack_str(&mp_pck, sizeof(FLB_DATADOG_DD_MESSAGE_KEY)-1); - msgpack_pack_str_body(&mp_pck, FLB_DATADOG_DD_MESSAGE_KEY, - sizeof(FLB_DATADOG_DD_MESSAGE_KEY)-1); - } - else { - msgpack_pack_object(&mp_pck, k); - } - - msgpack_pack_object(&mp_pck, v); - } - - /* here we concatenate ctx->dd_tags and remapped_tags, depending on their presence */ - if (remap_cnt) { - if (ctx->dd_tags != NULL) { - tmp = flb_sds_cat(remapped_tags, FLB_DATADOG_TAG_SEPERATOR, - strlen(FLB_DATADOG_TAG_SEPERATOR)); - if (!tmp) { - flb_errno(); - flb_sds_destroy(remapped_tags); - msgpack_sbuffer_destroy(&mp_sbuf); - flb_log_event_decoder_destroy(&log_decoder); - return -1; - } - remapped_tags = tmp; - flb_sds_cat(remapped_tags, ctx->dd_tags, strlen(ctx->dd_tags)); - if (!tmp) { - flb_errno(); - flb_sds_destroy(remapped_tags); - msgpack_sbuffer_destroy(&mp_sbuf); - flb_log_event_decoder_destroy(&log_decoder); - return -1; - } - remapped_tags = tmp; - } - dd_msgpack_pack_key_value_str(&mp_pck, - FLB_DATADOG_DD_TAGS_KEY, - sizeof(FLB_DATADOG_DD_TAGS_KEY) -1, - remapped_tags, flb_sds_len(remapped_tags)); - } - else if (ctx->dd_tags != NULL) { - dd_msgpack_pack_key_value_str(&mp_pck, - FLB_DATADOG_DD_TAGS_KEY, - sizeof(FLB_DATADOG_DD_TAGS_KEY) -1, - ctx->dd_tags, flb_sds_len(ctx->dd_tags)); - } - } - - /* Convert from msgpack to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); - msgpack_sbuffer_destroy(&mp_sbuf); - - if (!out_buf) { - flb_plg_error(ctx->ins, "error formatting JSON payload"); - if (remapped_tags) { - flb_sds_destroy(remapped_tags); - } - flb_log_event_decoder_destroy(&log_decoder); - return -1; - } - - *out_data = out_buf; - *out_size = flb_sds_len(out_buf); - - /* Cleanup */ - flb_log_event_decoder_destroy(&log_decoder); - - if (remapped_tags) { - flb_sds_destroy(remapped_tags); - } - - return 0; -} - -static void cb_datadog_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - struct flb_out_datadog *ctx = out_context; - struct flb_connection *upstream_conn; - struct flb_http_client *client; - void *out_buf; - size_t out_size; - flb_sds_t payload_buf; - size_t payload_size = 0; - void *final_payload_buf = NULL; - size_t final_payload_size = 0; - size_t b_sent; - int ret = FLB_ERROR; - int compressed = FLB_FALSE; - - /* Get upstream connection */ - upstream_conn = flb_upstream_conn_get(ctx->upstream); - if (!upstream_conn) { - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Convert input data into a Datadog JSON payload */ - ret = datadog_format(config, i_ins, - ctx, NULL, - event_chunk->type, - event_chunk->tag, flb_sds_len(event_chunk->tag), - event_chunk->data, event_chunk->size, - &out_buf, &out_size); - if (ret == -1) { - flb_upstream_conn_release(upstream_conn); - FLB_OUTPUT_RETURN(FLB_ERROR); - } - - payload_buf = (flb_sds_t) out_buf; - payload_size = out_size; - - /* Should we compress the payload ? */ - if (ctx->compress_gzip == FLB_TRUE) { - ret = flb_gzip_compress((void *) payload_buf, payload_size, - &final_payload_buf, &final_payload_size); - if (ret == -1) { - flb_error("[out_http] cannot gzip payload, disabling compression"); - } else { - compressed = FLB_TRUE; - } - } else { - final_payload_buf = payload_buf; - final_payload_size = payload_size; - } - - /* Create HTTP client context */ - client = flb_http_client(upstream_conn, FLB_HTTP_POST, ctx->uri, - final_payload_buf, final_payload_size, - ctx->host, ctx->port, - ctx->proxy, 0); - if (!client) { - flb_upstream_conn_release(upstream_conn); - FLB_OUTPUT_RETURN(FLB_ERROR); - } - - /* Add the required headers to the URI */ - flb_http_add_header(client, "User-Agent", 10, "Fluent-Bit", 10); - flb_http_add_header(client, FLB_DATADOG_API_HDR, sizeof(FLB_DATADOG_API_HDR) - 1, ctx->api_key, flb_sds_len(ctx->api_key)); - flb_http_add_header(client, FLB_DATADOG_ORIGIN_HDR, sizeof(FLB_DATADOG_ORIGIN_HDR) - 1, "Fluent-Bit", 10); - flb_http_add_header(client, FLB_DATADOG_ORIGIN_VERSION_HDR, sizeof(FLB_DATADOG_ORIGIN_VERSION_HDR) - 1, FLB_VERSION_STR, sizeof(FLB_VERSION_STR) - 1); - flb_http_add_header(client, - FLB_DATADOG_CONTENT_TYPE, sizeof(FLB_DATADOG_CONTENT_TYPE) - 1, - FLB_DATADOG_MIME_JSON, sizeof(FLB_DATADOG_MIME_JSON) - 1); - - /* Content Encoding: gzip */ - if (compressed == FLB_TRUE) { - flb_http_set_content_encoding_gzip(client); - } - /* TODO: Append other headers if needed*/ - - /* finaly send the query */ - ret = flb_http_do(client, &b_sent); - if (ret == 0) { - if (client->resp.status < 200 || client->resp.status > 205) { - flb_plg_error(ctx->ins, "%s%s:%i HTTP status=%i", - ctx->scheme, ctx->host, ctx->port, - client->resp.status); - ret = FLB_RETRY; - } - else { - if (client->resp.payload) { - flb_plg_debug(ctx->ins, "%s%s, port=%i, HTTP status=%i payload=%s", - ctx->scheme, ctx->host, ctx->port, - client->resp.status, client->resp.payload); - } - else { - flb_plg_debug(ctx->ins, "%s%s, port=%i, HTTP status=%i", - ctx->scheme, ctx->host, ctx->port, - client->resp.status); - } - ret = FLB_OK; - } - } - else { - flb_plg_error(ctx->ins, "could not flush records to %s:%i (http_do=%i)", - ctx->host, ctx->port, ret); - ret = FLB_RETRY; - } - - /* - * If the final_payload_buf buffer is different than payload_buf, means - * we generated a different payload and must be freed. - */ - if (final_payload_buf != payload_buf) { - flb_free(final_payload_buf); - } - /* Destroy HTTP client context */ - flb_sds_destroy(payload_buf); - flb_http_client_destroy(client); - flb_upstream_conn_release(upstream_conn); - - FLB_OUTPUT_RETURN(ret); -} - - -static int cb_datadog_exit(void *data, struct flb_config *config) -{ - struct flb_out_datadog *ctx = data; - - if (!ctx) { - return 0; - } - - flb_datadog_conf_destroy(ctx); - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "compress", "false", - 0, FLB_FALSE, 0, - "compresses the payload in GZIP format, " - "Datadog supports and recommends setting this to 'gzip'." - }, - { - FLB_CONFIG_MAP_STR, "apikey", NULL, - 0, FLB_TRUE, offsetof(struct flb_out_datadog, api_key), - "Datadog API key" - }, - { - FLB_CONFIG_MAP_STR, "dd_service", NULL, - 0, FLB_TRUE, offsetof(struct flb_out_datadog, dd_service), - "The human readable name for your service generating the logs " - "- the name of your application or database." - }, - { - FLB_CONFIG_MAP_STR, "dd_source", NULL, - 0, FLB_TRUE, offsetof(struct flb_out_datadog, dd_source), - "A human readable name for the underlying technology of your service. " - "For example, 'postgres' or 'nginx'." - }, - { - FLB_CONFIG_MAP_STR, "dd_tags", NULL, - 0, FLB_TRUE, offsetof(struct flb_out_datadog, dd_tags), - "The tags you want to assign to your logs in Datadog." - }, - - { - FLB_CONFIG_MAP_STR, "proxy", NULL, - 0, FLB_TRUE, offsetof(struct flb_out_datadog, proxy), - "Specify an HTTP Proxy. The expected format of this value is http://host:port. " - "Note that https is not supported yet." - }, - { - FLB_CONFIG_MAP_BOOL, "include_tag_key", "false", - 0, FLB_TRUE, offsetof(struct flb_out_datadog, include_tag_key), - "If enabled, tag is appended to output. " - "The key name is used 'tag_key' property." - }, - { - FLB_CONFIG_MAP_STR, "tag_key", FLB_DATADOG_DEFAULT_TAG_KEY, - 0, FLB_TRUE, offsetof(struct flb_out_datadog, tag_key), - "The key name of tag. If 'include_tag_key' is false, " - "This property is ignored" - }, - { - FLB_CONFIG_MAP_STR, "dd_message_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_out_datadog, dd_message_key), - "By default, the plugin searches for the key 'log' " - "and remap the value to the key 'message'. " - "If the property is set, the plugin will search the property name key." - }, - { - FLB_CONFIG_MAP_STR, "provider", NULL, - 0, FLB_FALSE, 0, - "To activate the remapping, specify configuration flag provider with value 'ecs'" - }, - { - FLB_CONFIG_MAP_STR, "json_date_key", FLB_DATADOG_DEFAULT_TIME_KEY, - 0, FLB_TRUE, offsetof(struct flb_out_datadog, json_date_key), - "Date key name for output." - }, - - /* EOF */ - {0} -}; - -struct flb_output_plugin out_datadog_plugin = { - .name = "datadog", - .description = "Send events to DataDog HTTP Event Collector", - .cb_init = cb_datadog_init, - .cb_flush = cb_datadog_flush, - .cb_exit = cb_datadog_exit, - - /* Test */ - .test_formatter.callback = datadog_format, - - /* Config map */ - .config_map = config_map, - - /* Plugin flags */ - .flags = FLB_OUTPUT_NET | FLB_IO_OPT_TLS, -}; diff --git a/fluent-bit/plugins/out_datadog/datadog.h b/fluent-bit/plugins/out_datadog/datadog.h deleted file mode 100644 index 1ca2d6f05..000000000 --- a/fluent-bit/plugins/out_datadog/datadog.h +++ /dev/null @@ -1,81 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_DATADOG_H -#define FLB_OUT_DATADOG_H - -#include -#include - -#define FLB_DATADOG_DEFAULT_HOST "http-intake.logs.datadoghq.com" -#define FLB_DATADOG_DEFAULT_PORT 443 -#define FLB_DATADOG_DEFAULT_TIME_KEY "timestamp" -#define FLB_DATADOG_DEFAULT_TAG_KEY "tagkey" -#define FLB_DATADOG_DD_SOURCE_KEY "ddsource" -#define FLB_DATADOG_DD_SERVICE_KEY "service" -#define FLB_DATADOG_DD_TAGS_KEY "ddtags" -#define FLB_DATADOG_DD_MESSAGE_KEY "message" -#define FLB_DATADOG_DD_LOG_KEY "log" - -#define FLB_DATADOG_REMAP_PROVIDER "ecs" -#define FLB_DATADOG_TAG_SEPERATOR "," - -#define FLB_DATADOG_API_HDR "DD-API-KEY" -#define FLB_DATADOG_ORIGIN_HDR "DD-EVP-ORIGIN" -#define FLB_DATADOG_ORIGIN_VERSION_HDR "DD-EVP-ORIGIN-VERSION" - -#define FLB_DATADOG_CONTENT_TYPE "Content-Type" -#define FLB_DATADOG_MIME_JSON "application/json" - -struct flb_out_datadog { - - /* Proxy */ - flb_sds_t proxy; - char *proxy_host; - int proxy_port; - - /* Configuration */ - flb_sds_t scheme; - flb_sds_t host; - int port; - flb_sds_t uri; - flb_sds_t api_key; - int include_tag_key; - flb_sds_t tag_key; - bool remap; - - /* final result */ - flb_sds_t json_date_key; - int nb_additional_entries; - flb_sds_t dd_source; - flb_sds_t dd_service; - flb_sds_t dd_tags; - flb_sds_t dd_message_key; - - /* Compression mode (gzip) */ - int compress_gzip; - - /* Upstream connection to the backend server */ - struct flb_upstream *upstream; - - /* Plugin instance reference */ - struct flb_output_instance *ins; -}; - -#endif // FLB_OUT_DATADOG_H diff --git a/fluent-bit/plugins/out_datadog/datadog_conf.c b/fluent-bit/plugins/out_datadog/datadog_conf.c deleted file mode 100644 index 68377386c..000000000 --- a/fluent-bit/plugins/out_datadog/datadog_conf.c +++ /dev/null @@ -1,223 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include "datadog.h" -#include "datadog_conf.h" - -struct flb_out_datadog *flb_datadog_conf_create(struct flb_output_instance *ins, - struct flb_config *config) -{ - struct flb_out_datadog *ctx = NULL; - int io_flags = 0; - struct flb_upstream *upstream; - const char *api_key; - const char *tmp; - flb_sds_t tmp_sds; - - int ret; - char *protocol = NULL; - char *host = NULL; - char *port = NULL; - char *uri = NULL; - - /* Start resource creation */ - ctx = flb_calloc(1, sizeof(struct flb_out_datadog)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - ctx->nb_additional_entries = 0; - - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_plg_error(ins, "flb_output_config_map_set failed"); - flb_free(ctx); - return NULL; - } - - tmp = flb_output_get_property("proxy", ins); - if (tmp) { - ret = flb_utils_url_split(tmp, &protocol, &host, &port, &uri); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not parse proxy parameter: '%s'", tmp); - flb_datadog_conf_destroy(ctx); - return NULL; - } - - ctx->proxy_host = host; - ctx->proxy_port = atoi(port); - flb_free(protocol); - flb_free(port); - flb_free(uri); - } - - /* use TLS ? */ - if (ins->use_tls == FLB_TRUE) { - io_flags = FLB_IO_TLS; - tmp_sds = flb_sds_create("https://"); - } - else { - io_flags = FLB_IO_TCP; - tmp_sds = flb_sds_create("http://"); - } - if (!tmp_sds) { - flb_errno(); - flb_datadog_conf_destroy(ctx); - return NULL; - } - ctx->scheme = tmp_sds; - flb_plg_debug(ctx->ins, "scheme: %s", ctx->scheme); - - /* configure URI */ - api_key = flb_output_get_property("apikey", ins); - if (api_key == NULL) { - flb_plg_error(ctx->ins, "no ApiKey configuration key defined"); - flb_datadog_conf_destroy(ctx); - return NULL; - } - - /* Tag Key */ - if (ctx->include_tag_key == FLB_TRUE) { - ctx->nb_additional_entries++; - } - - tmp = flb_output_get_property("dd_source", ins); - if (tmp) { - ctx->nb_additional_entries++; - } - - tmp = flb_output_get_property("dd_service", ins); - if (tmp) { - ctx->nb_additional_entries++; - } - - tmp = flb_output_get_property("dd_tags", ins); - if (tmp) { - ctx->nb_additional_entries++; - } - - tmp = flb_output_get_property("provider", ins); - ctx->remap = tmp && (strlen(tmp) == strlen(FLB_DATADOG_REMAP_PROVIDER)) && \ - (strncmp(tmp, FLB_DATADOG_REMAP_PROVIDER, strlen(tmp)) == 0); - - ctx->uri = flb_sds_create("/api/v2/logs"); - if (!ctx->uri) { - flb_plg_error(ctx->ins, "error on uri generation"); - flb_datadog_conf_destroy(ctx); - return NULL; - } - - flb_plg_debug(ctx->ins, "uri: %s", ctx->uri); - - /* Get network configuration */ - if (!ins->host.name) { - tmp_sds = flb_sds_create(FLB_DATADOG_DEFAULT_HOST); - } - else { - tmp_sds = flb_sds_create(ins->host.name); - } - if (!tmp_sds) { - flb_errno(); - flb_datadog_conf_destroy(ctx); - return NULL; - } - ctx->host = tmp_sds; - flb_plg_debug(ctx->ins, "host: %s", ctx->host); - - if (ins->host.port != 0) { - ctx->port = ins->host.port; - } - if (ctx->port == 0) { - ctx->port = FLB_DATADOG_DEFAULT_PORT; - if (ins->use_tls == FLB_FALSE) { - ctx->port = 80; - } - } - flb_plg_debug(ctx->ins, "port: %i", ctx->port); - - /* Date tag for JSON output */ - ctx->nb_additional_entries++; - flb_plg_debug(ctx->ins, "json_date_key: %s", ctx->json_date_key); - - /* Compress (gzip) */ - tmp = flb_output_get_property("compress", ins); - ctx->compress_gzip = FLB_FALSE; - if (tmp) { - if (strcasecmp(tmp, "gzip") == 0) { - ctx->compress_gzip = FLB_TRUE; - } - } - flb_plg_debug(ctx->ins, "compress_gzip: %i", ctx->compress_gzip); - - /* Prepare an upstream handler */ - if (ctx->proxy) { - flb_plg_trace(ctx->ins, "[out_datadog] Upstream Proxy=%s:%i", - ctx->proxy_host, ctx->proxy_port); - upstream = flb_upstream_create(config, - ctx->proxy_host, - ctx->proxy_port, - io_flags, - ins->tls); - } - else { - upstream = flb_upstream_create(config, ctx->host, ctx->port, io_flags, ins->tls); - } - - if (!upstream) { - flb_plg_error(ctx->ins, "cannot create Upstream context"); - flb_datadog_conf_destroy(ctx); - return NULL; - } - ctx->upstream = upstream; - flb_output_upstream_set(ctx->upstream, ins); - - return ctx; -} - -int flb_datadog_conf_destroy(struct flb_out_datadog *ctx) -{ - if (!ctx) { - return -1; - } - - if (ctx->proxy_host) { - flb_free(ctx->proxy_host); - } - if (ctx->scheme) { - flb_sds_destroy(ctx->scheme); - } - if (ctx->host) { - flb_sds_destroy(ctx->host); - } - if (ctx->uri) { - flb_sds_destroy(ctx->uri); - } - if (ctx->upstream) { - flb_upstream_destroy(ctx->upstream); - } - flb_free(ctx); - - return 0; -} diff --git a/fluent-bit/plugins/out_datadog/datadog_conf.h b/fluent-bit/plugins/out_datadog/datadog_conf.h deleted file mode 100644 index 057a5e5f2..000000000 --- a/fluent-bit/plugins/out_datadog/datadog_conf.h +++ /dev/null @@ -1,33 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_DATADOG_CONF_H -#define FLB_OUT_DATADOG_CONF_H - -#include -#include - -#include "datadog.h" - -struct flb_out_datadog *flb_datadog_conf_create(struct flb_output_instance *ins, - struct flb_config *config); - -int flb_datadog_conf_destroy(struct flb_out_datadog *ctx); - -#endif // FLB_OUT_DATADOG_CONF_H diff --git a/fluent-bit/plugins/out_datadog/datadog_remap.c b/fluent-bit/plugins/out_datadog/datadog_remap.c deleted file mode 100644 index 7599a8f80..000000000 --- a/fluent-bit/plugins/out_datadog/datadog_remap.c +++ /dev/null @@ -1,277 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include "datadog.h" -#include "datadog_remap.h" - -const char *ECS_ARN_PREFIX = "arn:aws:ecs:"; -const char *ECS_CLUSTER_PREFIX = "cluster/"; -const char *ECS_TASK_PREFIX = "task/"; - -static int dd_remap_append_kv_to_ddtags(const char *key, - const char *val, size_t val_len, flb_sds_t *dd_tags_buf) -{ - flb_sds_t tmp; - - if (flb_sds_len(*dd_tags_buf) != 0) { - tmp = flb_sds_cat(*dd_tags_buf, FLB_DATADOG_TAG_SEPERATOR, strlen(FLB_DATADOG_TAG_SEPERATOR)); - if (!tmp) { - flb_errno(); - return -1; - } - *dd_tags_buf = tmp; - } - - tmp = flb_sds_cat(*dd_tags_buf, key, strlen(key)); - if (!tmp) { - flb_errno(); - return -1; - } - *dd_tags_buf = tmp; - - tmp = flb_sds_cat(*dd_tags_buf, ":", 1); - if (!tmp) { - flb_errno(); - return -1; - } - *dd_tags_buf = tmp; - - tmp = flb_sds_cat(*dd_tags_buf, val, val_len); - if (!tmp) { - flb_errno(); - return -1; - } - *dd_tags_buf = tmp; - - return 0; -} - -/* default remapping: just move the key/val pair under dd_tags */ -static int dd_remap_move_to_tags(const char *tag_name, - msgpack_object attr_value, flb_sds_t *dd_tags_buf) -{ - return dd_remap_append_kv_to_ddtags(tag_name, attr_value.via.str.ptr, - attr_value.via.str.size, dd_tags_buf); -} - -/* remapping function for container_name */ -static int dd_remap_container_name(const char *tag_name, - msgpack_object attr_value, flb_sds_t *dd_tags_buf) -{ - /* remove the first / if present */ - unsigned int adjust; - flb_sds_t buf = NULL; - int ret; - - adjust = attr_value.via.str.ptr[0] == '/' ? 1 : 0; - buf = flb_sds_create_len(attr_value.via.str.ptr + adjust, - attr_value.via.str.size - adjust); - if (!buf) { - flb_errno(); - return -1; - } - ret = dd_remap_append_kv_to_ddtags(tag_name, buf, strlen(buf), dd_tags_buf); - flb_sds_destroy(buf); - if (ret < 0) { - return -1; - } - - return 0; -} - -/* remapping function for ecs_cluster */ -static int dd_remap_ecs_cluster(const char *tag_name, - msgpack_object attr_value, flb_sds_t *dd_tags_buf) -{ - flb_sds_t buf = NULL; - char *cluster_name; - int ret; - - buf = flb_sds_create_len(attr_value.via.str.ptr, attr_value.via.str.size); - if (!buf) { - flb_errno(); - return -1; - } - cluster_name = strstr(buf, ECS_CLUSTER_PREFIX); - - if (cluster_name != NULL) { - cluster_name += strlen(ECS_CLUSTER_PREFIX); - ret = dd_remap_append_kv_to_ddtags(tag_name, cluster_name, strlen(cluster_name), dd_tags_buf); - if (ret < 0) { - flb_sds_destroy(buf); - return -1; - } - } - else { - /* - * here the input is invalid: not in form of "XXXXXXcluster/"cluster-name - * we preverse the original value under tag "cluster_name". - */ - ret = dd_remap_append_kv_to_ddtags(tag_name, buf, strlen(buf), dd_tags_buf); - if (ret < 0) { - flb_sds_destroy(buf); - return -1; - } - } - flb_sds_destroy(buf); - return 0; -} - -/* remapping function for ecs_task_definition */ -static int dd_remap_ecs_task_definition(const char *tag_name, - msgpack_object attr_value, flb_sds_t *dd_tags_buf) -{ - flb_sds_t buf = NULL; - char *split; - int ret; - - buf = flb_sds_create_len(attr_value.via.str.ptr, attr_value.via.str.size); - if (!buf) { - flb_errno(); - return -1; - } - split = strchr(buf, ':'); - - if (split != NULL) { - ret = dd_remap_append_kv_to_ddtags("task_family", buf, split-buf, dd_tags_buf); - if (ret < 0) { - flb_sds_destroy(buf); - return -1; - } - ret = dd_remap_append_kv_to_ddtags("task_version", split+1, strlen(split+1), dd_tags_buf); - if (ret < 0) { - flb_sds_destroy(buf); - return -1; - } - } - else { - /* - * here the input is invalid: not in form of task_name:task_version - * we preverse the original value under tag "ecs_task_definition". - */ - ret = dd_remap_append_kv_to_ddtags(tag_name, buf, strlen(buf), dd_tags_buf); - if (ret < 0) { - flb_sds_destroy(buf); - return -1; - } - } - flb_sds_destroy(buf); - return 0; -} - -/* remapping function for ecs_task_arn */ -static int dd_remap_ecs_task_arn(const char *tag_name, - msgpack_object attr_value, flb_sds_t *dd_tags_buf) -{ - flb_sds_t buf; - char *remain; - char *split; - char *task_arn; - int ret; - - buf = flb_sds_create_len(attr_value.via.str.ptr, attr_value.via.str.size); - if (!buf) { - flb_errno(); - return -1; - } - - /* - * if the input is invalid, not in the form of "arn:aws:ecs:region:XXXX" - * then we won't add the "region" in the dd_tags. - */ - if ((strlen(buf) > strlen(ECS_ARN_PREFIX)) && - (strncmp(buf, ECS_ARN_PREFIX, strlen(ECS_ARN_PREFIX)) == 0)) { - - remain = buf + strlen(ECS_ARN_PREFIX); - split = strchr(remain, ':'); - - if (split != NULL) { - ret = dd_remap_append_kv_to_ddtags("region", remain, split-remain, dd_tags_buf); - if (ret < 0) { - flb_sds_destroy(buf); - return -1; - } - } - } - - task_arn = strstr(buf, ECS_TASK_PREFIX); - if (task_arn != NULL) { - /* parse out the task_arn */ - task_arn += strlen(ECS_TASK_PREFIX); - ret = dd_remap_append_kv_to_ddtags(tag_name, task_arn, strlen(task_arn), dd_tags_buf); - } - else { - /* - * if the input is invalid, not in the form of "XXXXXXXXtask/"task-arn - * then we preverse the original value under tag "task_arn". - */ - ret = dd_remap_append_kv_to_ddtags(tag_name, buf, strlen(buf), dd_tags_buf); - } - flb_sds_destroy(buf); - if (ret < 0) { - return -1; - } - - return 0; -} - -/* - * Statically defines the set of remappings rules in the form of - * 1) original attr name 2) remapped tag name 3) remapping functions - * The remapping functions assume the input is valid, and will always - * produce one or more tags to be added in dd_tags. - */ -const struct dd_attr_tag_remapping remapping[] = { - {"container_id", "container_id", dd_remap_move_to_tags}, - {"container_name", "container_name", dd_remap_container_name}, - {"container_image", "container_image", dd_remap_move_to_tags}, - {"ecs_cluster", "cluster_name", dd_remap_ecs_cluster}, - {"ecs_task_definition", "ecs_task_definition", dd_remap_ecs_task_definition}, - {"ecs_task_arn", "task_arn", dd_remap_ecs_task_arn} -}; - -/* - * Check against dd_attr_tag_remapping to see if a given attributes key/val - * pair need remapping. The key has to match origin_attr_name, and the val - * has to be of type string and non-empty. - * return value is the index of the remapping rule in dd_attr_tag_remapping, - * or -1 if no need to remap - */ -int dd_attr_need_remapping(const msgpack_object key, const msgpack_object val) -{ - int i; - - if ((val.type != MSGPACK_OBJECT_STR) || (val.via.str.size == 0)) { - return -1; - } - - for (i = 0; i < sizeof(remapping) / sizeof(struct dd_attr_tag_remapping); i++) { - if ((key.via.str.size == strlen(remapping[i].origin_attr_name) && - memcmp(key.via.str.ptr, - remapping[i].origin_attr_name, key.via.str.size) == 0)) { - return i; - } - } - - return -1; -} diff --git a/fluent-bit/plugins/out_datadog/datadog_remap.h b/fluent-bit/plugins/out_datadog/datadog_remap.h deleted file mode 100644 index f7061b0f2..000000000 --- a/fluent-bit/plugins/out_datadog/datadog_remap.h +++ /dev/null @@ -1,37 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_DATADOG_REMAP_H -#define FLB_OUT_DATADOG_REMAP_H - -#include "datadog.h" - -typedef int (*dd_attr_remap_to_tag_fn)(const char*, msgpack_object, flb_sds_t*); - -struct dd_attr_tag_remapping { - char* origin_attr_name; /* original attribute name */ - char* remap_tag_name; /* tag name to remap to */ - dd_attr_remap_to_tag_fn remap_to_tag; /* remapping function */ -}; - -extern const struct dd_attr_tag_remapping remapping[]; - -int dd_attr_need_remapping(const msgpack_object key, const msgpack_object val); - -#endif // FLB_OUT_DATADOG_REMAP_H diff --git a/fluent-bit/plugins/out_es/CMakeLists.txt b/fluent-bit/plugins/out_es/CMakeLists.txt deleted file mode 100644 index 4fad4f27c..000000000 --- a/fluent-bit/plugins/out_es/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -set(src - es_bulk.c - es_conf.c - es.c - murmur3.c) - -FLB_PLUGIN(out_es "${src}" "mk_core") -target_link_libraries(flb-plugin-out_es) diff --git a/fluent-bit/plugins/out_es/es.c b/fluent-bit/plugins/out_es/es.c deleted file mode 100644 index db2bcee5b..000000000 --- a/fluent-bit/plugins/out_es/es.c +++ /dev/null @@ -1,1230 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "es.h" -#include "es_conf.h" -#include "es_bulk.h" -#include "murmur3.h" - -struct flb_output_plugin out_es_plugin; - -static int es_pack_array_content(msgpack_packer *tmp_pck, - msgpack_object array, - struct flb_elasticsearch *ctx); - -#ifdef FLB_HAVE_AWS -static flb_sds_t add_aws_auth(struct flb_http_client *c, - struct flb_elasticsearch *ctx) -{ - flb_sds_t signature = NULL; - int ret; - - flb_plg_debug(ctx->ins, "Signing request with AWS Sigv4"); - - /* Amazon OpenSearch Sigv4 does not allow the host header to include the port */ - ret = flb_http_strip_port_from_host(c); - if (ret < 0) { - flb_plg_error(ctx->ins, "could not strip port from host for sigv4"); - return NULL; - } - - /* AWS Fluent Bit user agent */ - flb_http_add_header(c, "User-Agent", 10, "aws-fluent-bit-plugin", 21); - - signature = flb_signv4_do(c, FLB_TRUE, FLB_TRUE, time(NULL), - ctx->aws_region, ctx->aws_service_name, - S3_MODE_SIGNED_PAYLOAD, ctx->aws_unsigned_headers, - ctx->aws_provider); - if (!signature) { - flb_plg_error(ctx->ins, "could not sign request with sigv4"); - return NULL; - } - return signature; -} -#endif /* FLB_HAVE_AWS */ - -static int es_pack_map_content(msgpack_packer *tmp_pck, - msgpack_object map, - struct flb_elasticsearch *ctx) -{ - int i; - char *ptr_key = NULL; - char buf_key[256]; - msgpack_object *k; - msgpack_object *v; - - for (i = 0; i < map.via.map.size; i++) { - k = &map.via.map.ptr[i].key; - v = &map.via.map.ptr[i].val; - ptr_key = NULL; - - /* Store key */ - const char *key_ptr = NULL; - size_t key_size = 0; - - if (k->type == MSGPACK_OBJECT_BIN) { - key_ptr = k->via.bin.ptr; - key_size = k->via.bin.size; - } - else if (k->type == MSGPACK_OBJECT_STR) { - key_ptr = k->via.str.ptr; - key_size = k->via.str.size; - } - - if (key_size < (sizeof(buf_key) - 1)) { - memcpy(buf_key, key_ptr, key_size); - buf_key[key_size] = '\0'; - ptr_key = buf_key; - } - else { - /* Long map keys have a performance penalty */ - ptr_key = flb_malloc(key_size + 1); - if (!ptr_key) { - flb_errno(); - return -1; - } - - memcpy(ptr_key, key_ptr, key_size); - ptr_key[key_size] = '\0'; - } - - /* - * Sanitize key name, Elastic Search 2.x don't allow dots - * in field names: - * - * https://goo.gl/R5NMTr - */ - if (ctx->replace_dots == FLB_TRUE) { - char *p = ptr_key; - char *end = ptr_key + key_size; - while (p != end) { - if (*p == '.') *p = '_'; - p++; - } - } - - /* Append the key */ - msgpack_pack_str(tmp_pck, key_size); - msgpack_pack_str_body(tmp_pck, ptr_key, key_size); - - /* Release temporary key if was allocated */ - if (ptr_key && ptr_key != buf_key) { - flb_free(ptr_key); - } - ptr_key = NULL; - - /* - * The value can be any data type, if it's a map we need to - * sanitize to avoid dots. - */ - if (v->type == MSGPACK_OBJECT_MAP) { - msgpack_pack_map(tmp_pck, v->via.map.size); - es_pack_map_content(tmp_pck, *v, ctx); - } - /* - * The value can be any data type, if it's an array we need to - * pass it to es_pack_array_content. - */ - else if (v->type == MSGPACK_OBJECT_ARRAY) { - msgpack_pack_array(tmp_pck, v->via.array.size); - es_pack_array_content(tmp_pck, *v, ctx); - } - else { - msgpack_pack_object(tmp_pck, *v); - } - } - return 0; -} - -/* - * Iterate through the array and sanitize elements. - * Mutual recursion with es_pack_map_content. - */ -static int es_pack_array_content(msgpack_packer *tmp_pck, - msgpack_object array, - struct flb_elasticsearch *ctx) -{ - int i; - msgpack_object *e; - - for (i = 0; i < array.via.array.size; i++) { - e = &array.via.array.ptr[i]; - if (e->type == MSGPACK_OBJECT_MAP) - { - msgpack_pack_map(tmp_pck, e->via.map.size); - es_pack_map_content(tmp_pck, *e, ctx); - } - else if (e->type == MSGPACK_OBJECT_ARRAY) - { - msgpack_pack_array(tmp_pck, e->via.array.size); - es_pack_array_content(tmp_pck, *e, ctx); - } - else - { - msgpack_pack_object(tmp_pck, *e); - } - } - return 0; -} - -/* - * Get _id value from incoming record. - * If it successed, return the value as flb_sds_t. - * If it failed, return NULL. -*/ -static flb_sds_t es_get_id_value(struct flb_elasticsearch *ctx, - msgpack_object *map) -{ - struct flb_ra_value *rval = NULL; - flb_sds_t tmp_str; - rval = flb_ra_get_value_object(ctx->ra_id_key, *map); - if (rval == NULL) { - flb_plg_warn(ctx->ins, "the value of %s is missing", - ctx->id_key); - return NULL; - } - else if(rval->o.type != MSGPACK_OBJECT_STR) { - flb_plg_warn(ctx->ins, "the value of %s is not string", - ctx->id_key); - flb_ra_key_value_destroy(rval); - return NULL; - } - - tmp_str = flb_sds_create_len(rval->o.via.str.ptr, - rval->o.via.str.size); - if (tmp_str == NULL) { - flb_plg_warn(ctx->ins, "cannot create ID string from record"); - flb_ra_key_value_destroy(rval); - return NULL; - } - flb_ra_key_value_destroy(rval); - return tmp_str; -} - -static int compose_index_header(struct flb_elasticsearch *ctx, - int es_index_custom_len, - char *logstash_index, size_t logstash_index_size, - char *separator_str, - struct tm *tm) -{ - int ret; - int len; - char *p; - size_t s; - - /* Compose Index header */ - if (es_index_custom_len > 0) { - p = logstash_index + es_index_custom_len; - } else { - p = logstash_index + flb_sds_len(ctx->logstash_prefix); - } - len = p - logstash_index; - ret = snprintf(p, logstash_index_size - len, "%s", - separator_str); - if (ret > logstash_index_size - len) { - /* exceed limit */ - return -1; - } - p += strlen(separator_str); - len += strlen(separator_str); - - s = strftime(p, logstash_index_size - len, - ctx->logstash_dateformat, tm); - if (s==0) { - /* exceed limit */ - return -1; - } - p += s; - *p++ = '\0'; - - return 0; -} - -/* - * Convert the internal Fluent Bit data representation to the required - * one by Elasticsearch. - * - * 'Sadly' this process involves to convert from Msgpack to JSON. - */ -static int elasticsearch_format(struct flb_config *config, - struct flb_input_instance *ins, - void *plugin_context, - void *flush_ctx, - int event_type, - const char *tag, int tag_len, - const void *data, size_t bytes, - void **out_data, size_t *out_size) -{ - int ret; - int len; - int map_size; - int index_len = 0; - size_t s = 0; - size_t off = 0; - size_t off_prev = 0; - char *es_index; - char logstash_index[256]; - char time_formatted[256]; - char index_formatted[256]; - char es_uuid[37]; - flb_sds_t out_buf; - size_t out_buf_len = 0; - flb_sds_t tmp_buf; - flb_sds_t id_key_str = NULL; - // msgpack_unpacked result; - // msgpack_object root; - msgpack_object map; - // msgpack_object *obj; - flb_sds_t j_index; - struct es_bulk *bulk; - struct tm tm; - struct flb_time tms; - msgpack_sbuffer tmp_sbuf; - msgpack_packer tmp_pck; - uint16_t hash[8]; - int es_index_custom_len; - struct flb_elasticsearch *ctx = plugin_context; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - j_index = flb_sds_create_size(ES_BULK_HEADER); - if (j_index == NULL) { - flb_errno(); - return -1; - } - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - flb_sds_destroy(j_index); - - return -1; - } - - /* Create the bulk composer */ - bulk = es_bulk_create(bytes); - if (!bulk) { - flb_log_event_decoder_destroy(&log_decoder); - flb_sds_destroy(j_index); - return -1; - } - - /* Copy logstash prefix if logstash format is enabled */ - if (ctx->logstash_format == FLB_TRUE) { - strncpy(logstash_index, ctx->logstash_prefix, sizeof(logstash_index)); - logstash_index[sizeof(logstash_index) - 1] = '\0'; - } - - /* - * If logstash format and id generation are disabled, pre-generate - * the index line for all records. - * - * The header stored in 'j_index' will be used for the all records on - * this payload. - */ - if (ctx->logstash_format == FLB_FALSE && ctx->generate_id == FLB_FALSE) { - flb_time_get(&tms); - gmtime_r(&tms.tm.tv_sec, &tm); - strftime(index_formatted, sizeof(index_formatted) - 1, - ctx->index, &tm); - es_index = index_formatted; - if (ctx->suppress_type_name) { - index_len = flb_sds_snprintf(&j_index, - flb_sds_alloc(j_index), - ES_BULK_INDEX_FMT_WITHOUT_TYPE, - ctx->es_action, - es_index); - } - else { - index_len = flb_sds_snprintf(&j_index, - flb_sds_alloc(j_index), - ES_BULK_INDEX_FMT, - ctx->es_action, - es_index, ctx->type); - } - } - - /* - * Some broken clients may have time drift up to year 1970 - * this will generate corresponding index in Elasticsearch - * in order to prevent generating millions of indexes - * we can set to always use current time for index generation - */ - if (ctx->current_time_index == FLB_TRUE) { - flb_time_get(&tms); - } - - /* Iterate each record and do further formatting */ - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - - /* Only pop time from record if current_time_index is disabled */ - if (ctx->current_time_index == FLB_FALSE) { - flb_time_copy(&tms, &log_event.timestamp); - } - - map = *log_event.body; - map_size = map.via.map.size; - - es_index_custom_len = 0; - if (ctx->logstash_prefix_key) { - flb_sds_t v = flb_ra_translate(ctx->ra_prefix_key, - (char *) tag, tag_len, - map, NULL); - if (v) { - len = flb_sds_len(v); - if (len > 128) { - len = 128; - memcpy(logstash_index, v, 128); - } - else { - memcpy(logstash_index, v, len); - } - es_index_custom_len = len; - flb_sds_destroy(v); - } - } - - /* Create temporary msgpack buffer */ - msgpack_sbuffer_init(&tmp_sbuf); - msgpack_packer_init(&tmp_pck, &tmp_sbuf, msgpack_sbuffer_write); - - if (ctx->include_tag_key == FLB_TRUE) { - map_size++; - } - - /* Set the new map size */ - msgpack_pack_map(&tmp_pck, map_size + 1); - - /* Append the time key */ - msgpack_pack_str(&tmp_pck, flb_sds_len(ctx->time_key)); - msgpack_pack_str_body(&tmp_pck, ctx->time_key, flb_sds_len(ctx->time_key)); - - /* Format the time */ - gmtime_r(&tms.tm.tv_sec, &tm); - s = strftime(time_formatted, sizeof(time_formatted) - 1, - ctx->time_key_format, &tm); - if (ctx->time_key_nanos) { - len = snprintf(time_formatted + s, sizeof(time_formatted) - 1 - s, - ".%09" PRIu64 "Z", (uint64_t) tms.tm.tv_nsec); - } else { - len = snprintf(time_formatted + s, sizeof(time_formatted) - 1 - s, - ".%03" PRIu64 "Z", - (uint64_t) tms.tm.tv_nsec / 1000000); - } - - s += len; - msgpack_pack_str(&tmp_pck, s); - msgpack_pack_str_body(&tmp_pck, time_formatted, s); - - es_index = ctx->index; - if (ctx->logstash_format == FLB_TRUE) { - ret = compose_index_header(ctx, es_index_custom_len, - &logstash_index[0], sizeof(logstash_index), - ctx->logstash_prefix_separator, &tm); - if (ret < 0) { - /* retry with default separator */ - compose_index_header(ctx, es_index_custom_len, - &logstash_index[0], sizeof(logstash_index), - "-", &tm); - } - - es_index = logstash_index; - if (ctx->generate_id == FLB_FALSE) { - if (ctx->suppress_type_name) { - index_len = flb_sds_snprintf(&j_index, - flb_sds_alloc(j_index), - ES_BULK_INDEX_FMT_WITHOUT_TYPE, - ctx->es_action, - es_index); - } - else { - index_len = flb_sds_snprintf(&j_index, - flb_sds_alloc(j_index), - ES_BULK_INDEX_FMT, - ctx->es_action, - es_index, ctx->type); - } - } - } - else if (ctx->current_time_index == FLB_TRUE) { - /* Make sure we handle index time format for index */ - strftime(index_formatted, sizeof(index_formatted) - 1, - ctx->index, &tm); - es_index = index_formatted; - } - - /* Tag Key */ - if (ctx->include_tag_key == FLB_TRUE) { - msgpack_pack_str(&tmp_pck, flb_sds_len(ctx->tag_key)); - msgpack_pack_str_body(&tmp_pck, ctx->tag_key, flb_sds_len(ctx->tag_key)); - msgpack_pack_str(&tmp_pck, tag_len); - msgpack_pack_str_body(&tmp_pck, tag, tag_len); - } - - /* - * The map_content routine iterate over each Key/Value pair found in - * the map and do some sanitization for the key names. - * - * Elasticsearch have a restriction that key names cannot contain - * a dot; if some dot is found, it's replaced with an underscore. - */ - ret = es_pack_map_content(&tmp_pck, map, ctx); - if (ret == -1) { - flb_log_event_decoder_destroy(&log_decoder); - msgpack_sbuffer_destroy(&tmp_sbuf); - es_bulk_destroy(bulk); - flb_sds_destroy(j_index); - return -1; - } - - if (ctx->generate_id == FLB_TRUE) { - MurmurHash3_x64_128(tmp_sbuf.data, tmp_sbuf.size, 42, hash); - snprintf(es_uuid, sizeof(es_uuid), - "%04x%04x-%04x-%04x-%04x-%04x%04x%04x", - hash[0], hash[1], hash[2], hash[3], - hash[4], hash[5], hash[6], hash[7]); - if (ctx->suppress_type_name) { - index_len = flb_sds_snprintf(&j_index, - flb_sds_alloc(j_index), - ES_BULK_INDEX_FMT_ID_WITHOUT_TYPE, - ctx->es_action, - es_index, es_uuid); - } - else { - index_len = flb_sds_snprintf(&j_index, - flb_sds_alloc(j_index), - ES_BULK_INDEX_FMT_ID, - ctx->es_action, - es_index, ctx->type, es_uuid); - } - } - if (ctx->ra_id_key) { - id_key_str = es_get_id_value(ctx ,&map); - if (id_key_str) { - if (ctx->suppress_type_name) { - index_len = flb_sds_snprintf(&j_index, - flb_sds_alloc(j_index), - ES_BULK_INDEX_FMT_ID_WITHOUT_TYPE, - ctx->es_action, - es_index, id_key_str); - } - else { - index_len = flb_sds_snprintf(&j_index, - flb_sds_alloc(j_index), - ES_BULK_INDEX_FMT_ID, - ctx->es_action, - es_index, ctx->type, id_key_str); - } - flb_sds_destroy(id_key_str); - id_key_str = NULL; - } - } - - /* Convert msgpack to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(tmp_sbuf.data, tmp_sbuf.size); - msgpack_sbuffer_destroy(&tmp_sbuf); - if (!out_buf) { - flb_log_event_decoder_destroy(&log_decoder); - es_bulk_destroy(bulk); - flb_sds_destroy(j_index); - return -1; - } - - out_buf_len = flb_sds_len(out_buf); - if (strcasecmp(ctx->write_operation, FLB_ES_WRITE_OP_UPDATE) == 0) { - tmp_buf = out_buf; - out_buf = flb_sds_create_len(NULL, out_buf_len = out_buf_len + sizeof(ES_BULK_UPDATE_OP_BODY) - 2); - out_buf_len = snprintf(out_buf, out_buf_len, ES_BULK_UPDATE_OP_BODY, tmp_buf); - flb_sds_destroy(tmp_buf); - } - else if (strcasecmp(ctx->write_operation, FLB_ES_WRITE_OP_UPSERT) == 0) { - tmp_buf = out_buf; - out_buf = flb_sds_create_len(NULL, out_buf_len = out_buf_len + sizeof(ES_BULK_UPSERT_OP_BODY) - 2); - out_buf_len = snprintf(out_buf, out_buf_len, ES_BULK_UPSERT_OP_BODY, tmp_buf); - flb_sds_destroy(tmp_buf); - } - - ret = es_bulk_append(bulk, j_index, index_len, - out_buf, out_buf_len, - bytes, off_prev); - flb_sds_destroy(out_buf); - - off_prev = off; - if (ret == -1) { - /* We likely ran out of memory, abort here */ - flb_log_event_decoder_destroy(&log_decoder); - *out_size = 0; - es_bulk_destroy(bulk); - flb_sds_destroy(j_index); - return -1; - } - } - flb_log_event_decoder_destroy(&log_decoder); - - /* Set outgoing data */ - *out_data = bulk->ptr; - *out_size = bulk->len; - - /* - * Note: we don't destroy the bulk as we need to keep the allocated - * buffer with the data. Instead we just release the bulk context and - * return the bulk->ptr buffer - */ - flb_free(bulk); - if (ctx->trace_output) { - fwrite(*out_data, 1, *out_size, stdout); - fflush(stdout); - } - flb_sds_destroy(j_index); - return 0; -} - -static int cb_es_init(struct flb_output_instance *ins, - struct flb_config *config, - void *data) -{ - struct flb_elasticsearch *ctx; - - ctx = flb_es_conf_create(ins, config); - if (!ctx) { - flb_plg_error(ins, "cannot initialize plugin"); - return -1; - } - - flb_plg_debug(ctx->ins, "host=%s port=%i uri=%s index=%s type=%s", - ins->host.name, ins->host.port, ctx->uri, - ctx->index, ctx->type); - - flb_output_set_context(ins, ctx); - - /* - * This plugin instance uses the HTTP client interface, let's register - * it debugging callbacks. - */ - flb_output_set_http_debug_callbacks(ins); - - return 0; -} - -static int elasticsearch_error_check(struct flb_elasticsearch *ctx, - struct flb_http_client *c) -{ - int i, j, k; - int ret; - int check = FLB_FALSE; - int root_type; - char *out_buf; - size_t off = 0; - size_t out_size; - msgpack_unpacked result; - msgpack_object root; - msgpack_object key; - msgpack_object val; - msgpack_object item; - msgpack_object item_key; - msgpack_object item_val; - - /* - * Check if our payload is complete: there is such situations where - * the Elasticsearch HTTP response body is bigger than the HTTP client - * buffer so payload can be incomplete. - */ - /* Convert JSON payload to msgpack */ - ret = flb_pack_json(c->resp.payload, c->resp.payload_size, - &out_buf, &out_size, &root_type, NULL); - if (ret == -1) { - /* Is this an incomplete HTTP Request ? */ - if (c->resp.payload_size <= 0) { - return FLB_TRUE; - } - - /* Lookup error field */ - if (strstr(c->resp.payload, "\"errors\":false,\"items\":[")) { - return FLB_FALSE; - } - - flb_plg_error(ctx->ins, "could not pack/validate JSON response\n%s", - c->resp.payload); - return FLB_TRUE; - } - - /* Lookup error field */ - msgpack_unpacked_init(&result); - ret = msgpack_unpack_next(&result, out_buf, out_size, &off); - if (ret != MSGPACK_UNPACK_SUCCESS) { - flb_plg_error(ctx->ins, "Cannot unpack response to find error\n%s", - c->resp.payload); - return FLB_TRUE; - } - - root = result.data; - if (root.type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "unexpected payload type=%i", - root.type); - check = FLB_TRUE; - goto done; - } - - for (i = 0; i < root.via.map.size; i++) { - key = root.via.map.ptr[i].key; - if (key.type != MSGPACK_OBJECT_STR) { - flb_plg_error(ctx->ins, "unexpected key type=%i", - key.type); - check = FLB_TRUE; - goto done; - } - - if (key.via.str.size == 6 && strncmp(key.via.str.ptr, "errors", 6) == 0) { - val = root.via.map.ptr[i].val; - if (val.type != MSGPACK_OBJECT_BOOLEAN) { - flb_plg_error(ctx->ins, "unexpected 'error' value type=%i", - val.type); - check = FLB_TRUE; - goto done; - } - - /* If error == false, we are OK (no errors = FLB_FALSE) */ - if (!val.via.boolean) { - /* no errors */ - check = FLB_FALSE; - goto done; - } - } - else if (key.via.str.size == 5 && strncmp(key.via.str.ptr, "items", 5) == 0) { - val = root.via.map.ptr[i].val; - if (val.type != MSGPACK_OBJECT_ARRAY) { - flb_plg_error(ctx->ins, "unexpected 'items' value type=%i", - val.type); - check = FLB_TRUE; - goto done; - } - - for (j = 0; j < val.via.array.size; j++) { - item = val.via.array.ptr[j]; - if (item.type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "unexpected 'item' outer value type=%i", - item.type); - check = FLB_TRUE; - goto done; - } - - if (item.via.map.size != 1) { - flb_plg_error(ctx->ins, "unexpected 'item' size=%i", - item.via.map.size); - check = FLB_TRUE; - goto done; - } - - item = item.via.map.ptr[0].val; - if (item.type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "unexpected 'item' inner value type=%i", - item.type); - check = FLB_TRUE; - goto done; - } - - for (k = 0; k < item.via.map.size; k++) { - item_key = item.via.map.ptr[k].key; - if (item_key.type != MSGPACK_OBJECT_STR) { - flb_plg_error(ctx->ins, "unexpected key type=%i", - item_key.type); - check = FLB_TRUE; - goto done; - } - - if (item_key.via.str.size == 6 && strncmp(item_key.via.str.ptr, "status", 6) == 0) { - item_val = item.via.map.ptr[k].val; - - if (item_val.type != MSGPACK_OBJECT_POSITIVE_INTEGER) { - flb_plg_error(ctx->ins, "unexpected 'status' value type=%i", - item_val.type); - check = FLB_TRUE; - goto done; - } - /* Check for errors other than version conflict (document already exists) */ - if (item_val.via.i64 != 409) { - check = FLB_TRUE; - goto done; - } - } - } - } - } - } - - done: - flb_free(out_buf); - msgpack_unpacked_destroy(&result); - return check; -} - -static void cb_es_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *ins, void *out_context, - struct flb_config *config) -{ - int ret; - size_t pack_size; - char *pack; - void *out_buf; - size_t out_size; - size_t b_sent; - struct flb_elasticsearch *ctx = out_context; - struct flb_connection *u_conn; - struct flb_http_client *c; - flb_sds_t signature = NULL; - int compressed = FLB_FALSE; - - /* Get upstream connection */ - u_conn = flb_upstream_conn_get(ctx->u); - if (!u_conn) { - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Convert format */ - ret = elasticsearch_format(config, ins, - ctx, NULL, - event_chunk->type, - event_chunk->tag, flb_sds_len(event_chunk->tag), - event_chunk->data, event_chunk->size, - &out_buf, &out_size); - if (ret != 0) { - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_ERROR); - } - - pack = (char *) out_buf; - pack_size = out_size; - - /* Should we compress the payload ? */ - if (ctx->compress_gzip == FLB_TRUE) { - ret = flb_gzip_compress((void *) pack, pack_size, - &out_buf, &out_size); - if (ret == -1) { - flb_plg_error(ctx->ins, - "cannot gzip payload, disabling compression"); - } - else { - compressed = FLB_TRUE; - } - - /* - * The payload buffer is different than pack, means we must be free it. - */ - if (out_buf != pack) { - flb_free(pack); - } - - pack = (char *) out_buf; - pack_size = out_size; - } - - /* Compose HTTP Client request */ - c = flb_http_client(u_conn, FLB_HTTP_POST, ctx->uri, - pack, pack_size, NULL, 0, NULL, 0); - - flb_http_buffer_size(c, ctx->buffer_size); - -#ifndef FLB_HAVE_AWS - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); -#endif - - flb_http_add_header(c, "Content-Type", 12, "application/x-ndjson", 20); - - if (ctx->http_user && ctx->http_passwd) { - flb_http_basic_auth(c, ctx->http_user, ctx->http_passwd); - } - else if (ctx->cloud_user && ctx->cloud_passwd) { - flb_http_basic_auth(c, ctx->cloud_user, ctx->cloud_passwd); - } - -#ifdef FLB_HAVE_AWS - if (ctx->has_aws_auth == FLB_TRUE) { - signature = add_aws_auth(c, ctx); - if (!signature) { - goto retry; - } - } - else { - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - } -#endif - - /* Content Encoding: gzip */ - if (compressed == FLB_TRUE) { - flb_http_set_content_encoding_gzip(c); - } - - /* Map debug callbacks */ - flb_http_client_debug(c, ctx->ins->callback); - - ret = flb_http_do(c, &b_sent); - if (ret != 0) { - flb_plg_warn(ctx->ins, "http_do=%i URI=%s", ret, ctx->uri); - goto retry; - } - else { - /* The request was issued successfully, validate the 'error' field */ - flb_plg_debug(ctx->ins, "HTTP Status=%i URI=%s", c->resp.status, ctx->uri); - if (c->resp.status != 200 && c->resp.status != 201) { - if (c->resp.payload_size > 0) { - flb_plg_error(ctx->ins, "HTTP status=%i URI=%s, response:\n%s\n", - c->resp.status, ctx->uri, c->resp.payload); - } - else { - flb_plg_error(ctx->ins, "HTTP status=%i URI=%s", - c->resp.status, ctx->uri); - } - goto retry; - } - - if (c->resp.payload_size > 0) { - /* - * Elasticsearch payload should be JSON, we convert it to msgpack - * and lookup the 'error' field. - */ - ret = elasticsearch_error_check(ctx, c); - if (ret == FLB_TRUE) { - /* we got an error */ - if (ctx->trace_error) { - /* - * If trace_error is set, trace the actual - * response from Elasticsearch explaining the problem. - * Trace_Output can be used to see the request. - */ - if (pack_size < 4000) { - flb_plg_debug(ctx->ins, "error caused by: Input\n%.*s\n", - (int) pack_size, pack); - } - if (c->resp.payload_size < 4000) { - flb_plg_error(ctx->ins, "error: Output\n%s", - c->resp.payload); - } else { - /* - * We must use fwrite since the flb_log functions - * will truncate data at 4KB - */ - fwrite(c->resp.payload, 1, c->resp.payload_size, stderr); - fflush(stderr); - } - } - goto retry; - } - else { - flb_plg_debug(ctx->ins, "Elasticsearch response\n%s", - c->resp.payload); - } - } - else { - goto retry; - } - } - - /* Cleanup */ - flb_http_client_destroy(c); - flb_free(pack); - flb_upstream_conn_release(u_conn); - if (signature) { - flb_sds_destroy(signature); - } - FLB_OUTPUT_RETURN(FLB_OK); - - /* Issue a retry */ - retry: - flb_http_client_destroy(c); - flb_free(pack); - - if (out_buf != pack) { - flb_free(out_buf); - } - - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_RETRY); -} - -static int cb_es_exit(void *data, struct flb_config *config) -{ - struct flb_elasticsearch *ctx = data; - - flb_es_conf_destroy(ctx); - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "index", FLB_ES_DEFAULT_INDEX, - 0, FLB_TRUE, offsetof(struct flb_elasticsearch, index), - "Set an index name" - }, - { - FLB_CONFIG_MAP_STR, "type", FLB_ES_DEFAULT_TYPE, - 0, FLB_TRUE, offsetof(struct flb_elasticsearch, type), - "Set the document type property" - }, - { - FLB_CONFIG_MAP_BOOL, "suppress_type_name", "false", - 0, FLB_TRUE, offsetof(struct flb_elasticsearch, suppress_type_name), - "If true, mapping types is removed. (for v7.0.0 or later)" - }, - - /* HTTP Authentication */ - { - FLB_CONFIG_MAP_STR, "http_user", NULL, - 0, FLB_TRUE, offsetof(struct flb_elasticsearch, http_user), - "Optional username credential for Elastic X-Pack access" - }, - { - FLB_CONFIG_MAP_STR, "http_passwd", "", - 0, FLB_TRUE, offsetof(struct flb_elasticsearch, http_passwd), - "Password for user defined in HTTP_User" - }, - - /* HTTP Compression */ - { - FLB_CONFIG_MAP_STR, "compress", NULL, - 0, FLB_FALSE, 0, - "Set payload compression mechanism. Option available is 'gzip'" - }, - - /* Cloud Authentication */ - { - FLB_CONFIG_MAP_STR, "cloud_id", NULL, - 0, FLB_FALSE, 0, - "Elastic cloud ID of the cluster to connect to" - }, - { - FLB_CONFIG_MAP_STR, "cloud_auth", NULL, - 0, FLB_FALSE, 0, - "Elastic cloud authentication credentials" - }, - - /* AWS Authentication */ -#ifdef FLB_HAVE_AWS - { - FLB_CONFIG_MAP_BOOL, "aws_auth", "false", - 0, FLB_TRUE, offsetof(struct flb_elasticsearch, has_aws_auth), - "Enable AWS Sigv4 Authentication" - }, - { - FLB_CONFIG_MAP_STR, "aws_region", NULL, - 0, FLB_TRUE, offsetof(struct flb_elasticsearch, aws_region), - "AWS Region of your Amazon OpenSearch Service cluster" - }, - { - FLB_CONFIG_MAP_STR, "aws_sts_endpoint", NULL, - 0, FLB_TRUE, offsetof(struct flb_elasticsearch, aws_sts_endpoint), - "Custom endpoint for the AWS STS API, used with the AWS_Role_ARN option" - }, - { - FLB_CONFIG_MAP_STR, "aws_role_arn", NULL, - 0, FLB_FALSE, 0, - "AWS IAM Role to assume to put records to your Amazon OpenSearch cluster" - }, - { - FLB_CONFIG_MAP_STR, "aws_external_id", NULL, - 0, FLB_FALSE, 0, - "External ID for the AWS IAM Role specified with `aws_role_arn`" - }, - { - FLB_CONFIG_MAP_STR, "aws_service_name", "es", - 0, FLB_TRUE, offsetof(struct flb_elasticsearch, aws_service_name), - "AWS Service Name" - }, - { - FLB_CONFIG_MAP_STR, "aws_profile", NULL, - 0, FLB_TRUE, offsetof(struct flb_elasticsearch, aws_profile), - "AWS Profile name. AWS Profiles can be configured with AWS CLI and are usually stored in " - "$HOME/.aws/ directory." - }, -#endif - - /* Logstash compatibility */ - { - FLB_CONFIG_MAP_BOOL, "logstash_format", "false", - 0, FLB_TRUE, offsetof(struct flb_elasticsearch, logstash_format), - "Enable Logstash format compatibility" - }, - { - FLB_CONFIG_MAP_STR, "logstash_prefix", FLB_ES_DEFAULT_PREFIX, - 0, FLB_TRUE, offsetof(struct flb_elasticsearch, logstash_prefix), - "When Logstash_Format is enabled, the Index name is composed using a prefix " - "and the date, e.g: If Logstash_Prefix is equals to 'mydata' your index will " - "become 'mydata-YYYY.MM.DD'. The last string appended belongs to the date " - "when the data is being generated" - }, - { - FLB_CONFIG_MAP_STR, "logstash_prefix_separator", "-", - 0, FLB_TRUE, offsetof(struct flb_elasticsearch, logstash_prefix_separator), - "Set a separator between logstash_prefix and date." - }, - { - FLB_CONFIG_MAP_STR, "logstash_prefix_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_elasticsearch, logstash_prefix_key), - "When included: the value in the record that belongs to the key will be looked " - "up and over-write the Logstash_Prefix for index generation. If the key/value " - "is not found in the record then the Logstash_Prefix option will act as a " - "fallback. Nested keys are supported through record accessor pattern" - }, - { - FLB_CONFIG_MAP_STR, "logstash_dateformat", FLB_ES_DEFAULT_TIME_FMT, - 0, FLB_TRUE, offsetof(struct flb_elasticsearch, logstash_dateformat), - "Time format (based on strftime) to generate the second part of the Index name" - }, - - /* Custom Time and Tag keys */ - { - FLB_CONFIG_MAP_STR, "time_key", FLB_ES_DEFAULT_TIME_KEY, - 0, FLB_TRUE, offsetof(struct flb_elasticsearch, time_key), - "When Logstash_Format is enabled, each record will get a new timestamp field. " - "The Time_Key property defines the name of that field" - }, - { - FLB_CONFIG_MAP_STR, "time_key_format", FLB_ES_DEFAULT_TIME_KEYF, - 0, FLB_TRUE, offsetof(struct flb_elasticsearch, time_key_format), - "When Logstash_Format is enabled, this property defines the format of the " - "timestamp" - }, - { - FLB_CONFIG_MAP_BOOL, "time_key_nanos", "false", - 0, FLB_TRUE, offsetof(struct flb_elasticsearch, time_key_nanos), - "When Logstash_Format is enabled, enabling this property sends nanosecond " - "precision timestamps" - }, - { - FLB_CONFIG_MAP_BOOL, "include_tag_key", "false", - 0, FLB_TRUE, offsetof(struct flb_elasticsearch, include_tag_key), - "When enabled, it append the Tag name to the record" - }, - { - FLB_CONFIG_MAP_STR, "tag_key", FLB_ES_DEFAULT_TAG_KEY, - 0, FLB_TRUE, offsetof(struct flb_elasticsearch, tag_key), - "When Include_Tag_Key is enabled, this property defines the key name for the tag" - }, - { - FLB_CONFIG_MAP_SIZE, "buffer_size", FLB_ES_DEFAULT_HTTP_MAX, - 0, FLB_TRUE, offsetof(struct flb_elasticsearch, buffer_size), - "Specify the buffer size used to read the response from the Elasticsearch HTTP " - "service. This option is useful for debugging purposes where is required to read " - "full responses, note that response size grows depending of the number of records " - "inserted. To set an unlimited amount of memory set this value to 'false', " - "otherwise the value must be according to the Unit Size specification" - }, - - /* Elasticsearch specifics */ - { - FLB_CONFIG_MAP_STR, "path", NULL, - 0, FLB_FALSE, 0, - "Elasticsearch accepts new data on HTTP query path '/_bulk'. But it is also " - "possible to serve Elasticsearch behind a reverse proxy on a subpath. This " - "option defines such path on the fluent-bit side. It simply adds a path " - "prefix in the indexing HTTP POST URI" - }, - { - FLB_CONFIG_MAP_STR, "pipeline", NULL, - 0, FLB_FALSE, 0, - "Newer versions of Elasticsearch allows to setup filters called pipelines. " - "This option allows to define which pipeline the database should use. For " - "performance reasons is strongly suggested to do parsing and filtering on " - "Fluent Bit side, avoid pipelines" - }, - { - FLB_CONFIG_MAP_BOOL, "generate_id", "false", - 0, FLB_TRUE, offsetof(struct flb_elasticsearch, generate_id), - "When enabled, generate _id for outgoing records. This prevents duplicate " - "records when retrying ES" - }, - { - FLB_CONFIG_MAP_STR, "write_operation", "create", - 0, FLB_TRUE, offsetof(struct flb_elasticsearch, write_operation), - "Operation to use to write in bulk requests" - }, - { - FLB_CONFIG_MAP_STR, "id_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_elasticsearch, id_key), - "If set, _id will be the value of the key from incoming record." - }, - { - FLB_CONFIG_MAP_BOOL, "replace_dots", "false", - 0, FLB_TRUE, offsetof(struct flb_elasticsearch, replace_dots), - "When enabled, replace field name dots with underscore, required by Elasticsearch " - "2.0-2.3." - }, - - { - FLB_CONFIG_MAP_BOOL, "current_time_index", "false", - 0, FLB_TRUE, offsetof(struct flb_elasticsearch, current_time_index), - "Use current time for index generation instead of message record" - }, - - /* Trace */ - { - FLB_CONFIG_MAP_BOOL, "trace_output", "false", - 0, FLB_TRUE, offsetof(struct flb_elasticsearch, trace_output), - "When enabled print the Elasticsearch API calls to stdout (for diag only)" - }, - { - FLB_CONFIG_MAP_BOOL, "trace_error", "false", - 0, FLB_TRUE, offsetof(struct flb_elasticsearch, trace_error), - "When enabled print the Elasticsearch exception to stderr (for diag only)" - }, - - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_output_plugin out_es_plugin = { - .name = "es", - .description = "Elasticsearch", - .cb_init = cb_es_init, - .cb_pre_run = NULL, - .cb_flush = cb_es_flush, - .cb_exit = cb_es_exit, - .workers = 2, - - /* Configuration */ - .config_map = config_map, - - /* Test */ - .test_formatter.callback = elasticsearch_format, - - /* Plugin flags */ - .flags = FLB_OUTPUT_NET | FLB_IO_OPT_TLS, -}; diff --git a/fluent-bit/plugins/out_es/es.h b/fluent-bit/plugins/out_es/es.h deleted file mode 100644 index 5d187049f..000000000 --- a/fluent-bit/plugins/out_es/es.h +++ /dev/null @@ -1,140 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_ES_H -#define FLB_OUT_ES_H - -#define FLB_ES_DEFAULT_HOST "127.0.0.1" -#define FLB_ES_DEFAULT_PORT 92000 -#define FLB_ES_DEFAULT_INDEX "fluent-bit" -#define FLB_ES_DEFAULT_TYPE "_doc" -#define FLB_ES_DEFAULT_PREFIX "logstash" -#define FLB_ES_DEFAULT_TIME_FMT "%Y.%m.%d" -#define FLB_ES_DEFAULT_TIME_KEY "@timestamp" -#define FLB_ES_DEFAULT_TIME_KEYF "%Y-%m-%dT%H:%M:%S" -#define FLB_ES_DEFAULT_TAG_KEY "flb-key" -#define FLB_ES_DEFAULT_HTTP_MAX "512k" -#define FLB_ES_DEFAULT_HTTPS_PORT 443 -#define FLB_ES_WRITE_OP_INDEX "index" -#define FLB_ES_WRITE_OP_CREATE "create" -#define FLB_ES_WRITE_OP_UPDATE "update" -#define FLB_ES_WRITE_OP_UPSERT "upsert" - -struct flb_elasticsearch { - /* Elasticsearch index (database) and type (table) */ - char *index; - char *type; - char suppress_type_name; - - /* HTTP Auth */ - char *http_user; - char *http_passwd; - - /* Elastic Cloud Auth */ - char *cloud_user; - char *cloud_passwd; - - /* AWS Auth */ -#ifdef FLB_HAVE_AWS - int has_aws_auth; - char *aws_region; - char *aws_sts_endpoint; - char *aws_profile; - struct flb_aws_provider *aws_provider; - struct flb_aws_provider *base_aws_provider; - /* tls instances can't be re-used; aws provider requires a separate one */ - struct flb_tls *aws_tls; - /* one for the standard chain provider, one for sts assume role */ - struct flb_tls *aws_sts_tls; - char *aws_session_name; - char *aws_service_name; - struct mk_list *aws_unsigned_headers; -#endif - - /* HTTP Client Setup */ - size_t buffer_size; - - /* - * If enabled, replace field name dots with underscore, required for - * Elasticsearch 2.0-2.3. - */ - int replace_dots; - - int trace_output; - int trace_error; - - /* - * Logstash compatibility options - * ============================== - */ - - /* enabled/disabled */ - int logstash_format; - int generate_id; - int current_time_index; - - /* prefix */ - flb_sds_t logstash_prefix; - flb_sds_t logstash_prefix_separator; - - /* prefix key */ - flb_sds_t logstash_prefix_key; - - /* date format */ - flb_sds_t logstash_dateformat; - - /* time key */ - flb_sds_t time_key; - - /* time key format */ - flb_sds_t time_key_format; - - /* time key nanoseconds */ - int time_key_nanos; - - - /* write operation */ - flb_sds_t write_operation; - /* write operation elasticsearch operation */ - flb_sds_t es_action; - - /* id_key */ - flb_sds_t id_key; - struct flb_record_accessor *ra_id_key; - - /* include_tag_key */ - int include_tag_key; - flb_sds_t tag_key; - - /* Elasticsearch HTTP API */ - char uri[256]; - - struct flb_record_accessor *ra_prefix_key; - - /* Compression mode (gzip) */ - int compress_gzip; - - /* Upstream connection to the backend server */ - struct flb_upstream *u; - - /* Plugin output instance reference */ - struct flb_output_instance *ins; -}; - -#endif diff --git a/fluent-bit/plugins/out_es/es_bulk.c b/fluent-bit/plugins/out_es/es_bulk.c deleted file mode 100644 index 221f45ebd..000000000 --- a/fluent-bit/plugins/out_es/es_bulk.c +++ /dev/null @@ -1,113 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include -#include "es_bulk.h" - -struct es_bulk *es_bulk_create(size_t estimated_size) -{ - struct es_bulk *b; - - if (estimated_size < ES_BULK_CHUNK) { - estimated_size = ES_BULK_CHUNK; - } - - b = flb_malloc(sizeof(struct es_bulk)); - if (!b) { - perror("calloc"); - return NULL; - } - b->ptr = flb_malloc(estimated_size); - if (b->ptr == NULL) { - perror("malloc"); - flb_free(b); - return NULL; - } - - b->size = estimated_size; - b->len = 0; - - return b; -} - -void es_bulk_destroy(struct es_bulk *bulk) -{ - if (bulk->size > 0) { - flb_free(bulk->ptr); - } - flb_free(bulk); -} - -int es_bulk_append(struct es_bulk *bulk, char *index, int i_len, - char *json, size_t j_len, - size_t whole_size, size_t converted_size) -{ - int available; - int append_size; - int required; - int remaining_size; - char *ptr; - - required = i_len + j_len + ES_BULK_HEADER + 1; - available = (bulk->size - bulk->len); - - if (available < required) { - /* - * estimate a converted size of json - * calculate - * 1. rest of msgpack data size - * 2. ratio from bulk json size and processed msgpack size. - */ - append_size = required - available; - if (converted_size == 0) { - /* converted_size = 0 causes div/0 */ - flb_debug("[out_es] converted_size is 0"); - } else { - remaining_size = ceil((whole_size - converted_size) /* rest of size to convert */ - * ((double)bulk->size / converted_size)); /* = json size / msgpack size */ - append_size = fmax(append_size, remaining_size); - } - if (append_size < ES_BULK_CHUNK) { - /* append at least ES_BULK_CHUNK size */ - append_size = ES_BULK_CHUNK; - } - ptr = flb_realloc(bulk->ptr, bulk->size + append_size); - if (!ptr) { - flb_errno(); - return -1; - } - bulk->ptr = ptr; - bulk->size += append_size; - } - - memcpy(bulk->ptr + bulk->len, index, i_len); - bulk->len += i_len; - - memcpy(bulk->ptr + bulk->len, json, j_len); - bulk->len += j_len; - bulk->ptr[bulk->len] = '\n'; - bulk->len++; - - return 0; -}; diff --git a/fluent-bit/plugins/out_es/es_bulk.h b/fluent-bit/plugins/out_es/es_bulk.h deleted file mode 100644 index 7bb66dbbc..000000000 --- a/fluent-bit/plugins/out_es/es_bulk.h +++ /dev/null @@ -1,46 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_ES_BULK_H -#define FLB_OUT_ES_BULK_H - -#include - -#define ES_BULK_CHUNK 4096 /* Size of buffer chunks */ -#define ES_BULK_HEADER 165 /* ES Bulk API prefix line */ -#define ES_BULK_INDEX_FMT "{\"%s\":{\"_index\":\"%s\",\"_type\":\"%s\"}}\n" -#define ES_BULK_INDEX_FMT_ID "{\"%s\":{\"_index\":\"%s\",\"_type\":\"%s\",\"_id\":\"%s\"}}\n" -#define ES_BULK_INDEX_FMT_WITHOUT_TYPE "{\"%s\":{\"_index\":\"%s\"}}\n" -#define ES_BULK_INDEX_FMT_ID_WITHOUT_TYPE "{\"%s\":{\"_index\":\"%s\",\"_id\":\"%s\"}}\n" -#define ES_BULK_UPDATE_OP_BODY "{\"doc\":%s}" -#define ES_BULK_UPSERT_OP_BODY "{\"doc_as_upsert\":true,\"doc\":%s}" - -struct es_bulk { - char *ptr; - uint32_t len; - uint32_t size; -}; - -struct es_bulk *es_bulk_create(size_t estimated_size); -int es_bulk_append(struct es_bulk *bulk, char *index, int i_len, - char *json, size_t j_len, - size_t whole_size, size_t curr_size); -void es_bulk_destroy(struct es_bulk *bulk); - -#endif diff --git a/fluent-bit/plugins/out_es/es_conf.c b/fluent-bit/plugins/out_es/es_conf.c deleted file mode 100644 index 48c8c3e25..000000000 --- a/fluent-bit/plugins/out_es/es_conf.c +++ /dev/null @@ -1,537 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "es.h" -#include "es_conf.h" - -/* - * extract_cloud_host extracts the public hostname - * of a deployment from a Cloud ID string. - * - * The Cloud ID string has the format ":". - * Once decoded, the "base64_info" string has the format "$$" - * and the function returns "." token. - */ -static flb_sds_t extract_cloud_host(struct flb_elasticsearch *ctx, - const char *cloud_id) -{ - - char *colon; - char *region; - char *host; - char *port = NULL; - char buf[256] = {0}; - char cloud_host_buf[256] = {0}; - const char dollar[2] = "$"; - size_t len; - int ret; - - /* keep only part after first ":" */ - colon = strchr(cloud_id, ':'); - if (colon == NULL) { - return NULL; - } - colon++; - - /* decode base64 */ - ret = flb_base64_decode((unsigned char *)buf, sizeof(buf), &len, (unsigned char *)colon, strlen(colon)); - if (ret) { - flb_plg_error(ctx->ins, "cannot decode cloud_id"); - return NULL; - } - region = strtok(buf, dollar); - if (region == NULL) { - return NULL; - } - host = strtok(NULL, dollar); - if (host == NULL) { - return NULL; - } - - /* - * Some cloud id format is "$:$" . - * e.g. https://github.com/elastic/beats/blob/v8.4.1/libbeat/cloudid/cloudid_test.go#L60 - * - * It means the variable "host" can contains ':' and port number. - */ - colon = strchr(host, ':'); - if (colon != NULL) { - /* host contains host number */ - *colon = '\0'; /* remove port number from host */ - port = colon+1; - } - - strcpy(cloud_host_buf, host); - strcat(cloud_host_buf, "."); - strcat(cloud_host_buf, region); - if (port != NULL) { - strcat(cloud_host_buf, ":"); - strcat(cloud_host_buf, port); - } - return flb_sds_create(cloud_host_buf); -} - -/* - * set_cloud_credentials gets a cloud_auth - * and sets the context's cloud_user and cloud_passwd. - * Example: - * cloud_auth = elastic:ZXVyb3BxxxxxxZTA1Ng - * ----> - * cloud_user = elastic - * cloud_passwd = ZXVyb3BxxxxxxZTA1Ng - */ -static void set_cloud_credentials(struct flb_elasticsearch *ctx, - const char *cloud_auth) -{ - /* extract strings */ - int items = 0; - struct mk_list *toks; - struct mk_list *head; - struct flb_split_entry *entry; - toks = flb_utils_split((const char *)cloud_auth, ':', -1); - mk_list_foreach(head, toks) { - items++; - entry = mk_list_entry(head, struct flb_split_entry, _head); - if (items == 1) { - ctx->cloud_user = flb_strdup(entry->value); - } - if (items == 2) { - ctx->cloud_passwd = flb_strdup(entry->value); - } - } - flb_utils_split_free(toks); -} - -struct flb_elasticsearch *flb_es_conf_create(struct flb_output_instance *ins, - struct flb_config *config) -{ - int len; - int io_flags = 0; - ssize_t ret; - char *buf; - const char *tmp; - const char *path; -#ifdef FLB_HAVE_AWS - char *aws_role_arn = NULL; - char *aws_external_id = NULL; - char *aws_session_name = NULL; -#endif - char *cloud_port_char; - char *cloud_host = NULL; - int cloud_host_port = 0; - int cloud_port = FLB_ES_DEFAULT_HTTPS_PORT; - struct flb_uri *uri = ins->host.uri; - struct flb_uri_field *f_index = NULL; - struct flb_uri_field *f_type = NULL; - struct flb_upstream *upstream; - struct flb_elasticsearch *ctx; - - /* Allocate context */ - ctx = flb_calloc(1, sizeof(struct flb_elasticsearch)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - - if (uri) { - if (uri->count >= 2) { - f_index = flb_uri_get(uri, 0); - f_type = flb_uri_get(uri, 1); - } - } - - /* handle cloud_id */ - tmp = flb_output_get_property("cloud_id", ins); - if (tmp) { - cloud_host = extract_cloud_host(ctx, tmp); - if (cloud_host == NULL) { - flb_plg_error(ctx->ins, "cannot extract cloud_host"); - flb_es_conf_destroy(ctx); - return NULL; - } - flb_plg_debug(ctx->ins, "extracted cloud_host: '%s'", cloud_host); - - cloud_port_char = strchr(cloud_host, ':'); - - if (cloud_port_char == NULL) { - flb_plg_debug(ctx->ins, "cloud_host: '%s' does not contain a port: '%s'", cloud_host, cloud_host); - } - else { - cloud_port_char[0] = '\0'; - cloud_port_char = &cloud_port_char[1]; - flb_plg_debug(ctx->ins, "extracted cloud_port_char: '%s'", cloud_port_char); - cloud_host_port = (int) strtol(cloud_port_char, (char **) NULL, 10); - flb_plg_debug(ctx->ins, "converted cloud_port_char to port int: '%i'", cloud_host_port); - } - - if (cloud_host_port == 0) { - cloud_host_port = cloud_port; - } - - flb_plg_debug(ctx->ins, - "checked whether extracted port was null and set it to " - "default https port or not. Outcome: '%i' and cloud_host: '%s'.", - cloud_host_port, cloud_host); - - if (ins->host.name != NULL) { - flb_sds_destroy(ins->host.name); - } - - ins->host.name = cloud_host; - ins->host.port = cloud_host_port; - } - - /* Set default network configuration */ - flb_output_net_default("127.0.0.1", 9200, ins); - - /* Populate context with config map defaults and incoming properties */ - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_plg_error(ctx->ins, "configuration error"); - flb_es_conf_destroy(ctx); - return NULL; - } - - /* handle cloud_auth */ - tmp = flb_output_get_property("cloud_auth", ins); - if (tmp) { - set_cloud_credentials(ctx, tmp); - } - - /* use TLS ? */ - if (ins->use_tls == FLB_TRUE) { - io_flags = FLB_IO_TLS; - } - else { - io_flags = FLB_IO_TCP; - } - - if (ins->host.ipv6 == FLB_TRUE) { - io_flags |= FLB_IO_IPV6; - } - - /* Compress (gzip) */ - tmp = flb_output_get_property("compress", ins); - ctx->compress_gzip = FLB_FALSE; - if (tmp) { - if (strcasecmp(tmp, "gzip") == 0) { - ctx->compress_gzip = FLB_TRUE; - } - } - - /* Prepare an upstream handler */ - upstream = flb_upstream_create(config, - ins->host.name, - ins->host.port, - io_flags, - ins->tls); - if (!upstream) { - flb_plg_error(ctx->ins, "cannot create Upstream context"); - flb_es_conf_destroy(ctx); - return NULL; - } - ctx->u = upstream; - - /* Set instance flags into upstream */ - flb_output_upstream_set(ctx->u, ins); - - /* Set manual Index and Type */ - if (f_index) { - ctx->index = flb_strdup(f_index->value); /* FIXME */ - } - - if (f_type) { - ctx->type = flb_strdup(f_type->value); /* FIXME */ - } - - /* HTTP Payload (response) maximum buffer size (0 == unlimited) */ - if (ctx->buffer_size == -1) { - ctx->buffer_size = 0; - } - - /* Elasticsearch: Path */ - path = flb_output_get_property("path", ins); - if (!path) { - path = ""; - } - - /* Elasticsearch: Pipeline */ - tmp = flb_output_get_property("pipeline", ins); - if (tmp) { - snprintf(ctx->uri, sizeof(ctx->uri) - 1, "%s/_bulk/?pipeline=%s", path, tmp); - } - else { - snprintf(ctx->uri, sizeof(ctx->uri) - 1, "%s/_bulk", path); - } - - if (ctx->id_key) { - ctx->ra_id_key = flb_ra_create(ctx->id_key, FLB_FALSE); - if (ctx->ra_id_key == NULL) { - flb_plg_error(ins, "could not create record accessor for Id Key"); - } - if (ctx->generate_id == FLB_TRUE) { - flb_plg_warn(ins, "Generate_ID is ignored when ID_key is set"); - ctx->generate_id = FLB_FALSE; - } - } - - if (ctx->write_operation) { - if (strcasecmp(ctx->write_operation, FLB_ES_WRITE_OP_INDEX) == 0) { - ctx->es_action = flb_strdup(FLB_ES_WRITE_OP_INDEX); - } - else if (strcasecmp(ctx->write_operation, FLB_ES_WRITE_OP_CREATE) == 0) { - ctx->es_action = flb_strdup(FLB_ES_WRITE_OP_CREATE); - } - else if (strcasecmp(ctx->write_operation, FLB_ES_WRITE_OP_UPDATE) == 0 - || strcasecmp(ctx->write_operation, FLB_ES_WRITE_OP_UPSERT) == 0) { - ctx->es_action = flb_strdup(FLB_ES_WRITE_OP_UPDATE); - } - else { - flb_plg_error(ins, "wrong Write_Operation (should be one of index, create, update, upsert)"); - flb_es_conf_destroy(ctx); - return NULL; - } - if (strcasecmp(ctx->es_action, FLB_ES_WRITE_OP_UPDATE) == 0 - && !ctx->ra_id_key && ctx->generate_id == FLB_FALSE) { - flb_plg_error(ins, "Id_Key or Generate_Id must be set when Write_Operation update or upsert"); - flb_es_conf_destroy(ctx); - return NULL; - } - } - - if (ctx->logstash_prefix_key) { - if (ctx->logstash_prefix_key[0] != '$') { - len = flb_sds_len(ctx->logstash_prefix_key); - buf = flb_malloc(len + 2); - if (!buf) { - flb_errno(); - flb_es_conf_destroy(ctx); - return NULL; - } - buf[0] = '$'; - memcpy(buf + 1, ctx->logstash_prefix_key, len); - buf[len + 1] = '\0'; - - ctx->ra_prefix_key = flb_ra_create(buf, FLB_TRUE); - flb_free(buf); - } - else { - ctx->ra_prefix_key = flb_ra_create(ctx->logstash_prefix_key, FLB_TRUE); - } - - if (!ctx->ra_prefix_key) { - flb_plg_error(ins, "invalid logstash_prefix_key pattern '%s'", tmp); - flb_es_conf_destroy(ctx); - return NULL; - } - } - -#ifdef FLB_HAVE_AWS - /* AWS Auth Unsigned Headers */ - ctx->aws_unsigned_headers = flb_malloc(sizeof(struct mk_list)); - if (ret != 0) { - flb_es_conf_destroy(ctx); - } - flb_slist_create(ctx->aws_unsigned_headers); - ret = flb_slist_add(ctx->aws_unsigned_headers, "Content-Length"); - if (ret != 0) { - flb_es_conf_destroy(ctx); - return NULL; - } - - /* AWS Auth */ - ctx->has_aws_auth = FLB_FALSE; - tmp = flb_output_get_property("aws_auth", ins); - if (tmp) { - if (strncasecmp(tmp, "On", 2) == 0) { - ctx->has_aws_auth = FLB_TRUE; - flb_debug("[out_es] Enabled AWS Auth"); - - /* AWS provider needs a separate TLS instance */ - ctx->aws_tls = flb_tls_create(FLB_TLS_CLIENT_MODE, - FLB_TRUE, - ins->tls_debug, - ins->tls_vhost, - ins->tls_ca_path, - ins->tls_ca_file, - ins->tls_crt_file, - ins->tls_key_file, - ins->tls_key_passwd); - if (!ctx->aws_tls) { - flb_errno(); - flb_es_conf_destroy(ctx); - return NULL; - } - - tmp = flb_output_get_property("aws_region", ins); - if (!tmp) { - flb_error("[out_es] aws_auth enabled but aws_region not set"); - flb_es_conf_destroy(ctx); - return NULL; - } - ctx->aws_region = (char *) tmp; - - tmp = flb_output_get_property("aws_sts_endpoint", ins); - if (tmp) { - ctx->aws_sts_endpoint = (char *) tmp; - } - - ctx->aws_provider = flb_standard_chain_provider_create(config, - ctx->aws_tls, - ctx->aws_region, - ctx->aws_sts_endpoint, - NULL, - flb_aws_client_generator(), - ctx->aws_profile); - if (!ctx->aws_provider) { - flb_error("[out_es] Failed to create AWS Credential Provider"); - flb_es_conf_destroy(ctx); - return NULL; - } - - tmp = flb_output_get_property("aws_role_arn", ins); - if (tmp) { - /* Use the STS Provider */ - ctx->base_aws_provider = ctx->aws_provider; - aws_role_arn = (char *) tmp; - aws_external_id = NULL; - tmp = flb_output_get_property("aws_external_id", ins); - if (tmp) { - aws_external_id = (char *) tmp; - } - - aws_session_name = flb_sts_session_name(); - if (!aws_session_name) { - flb_error("[out_es] Failed to create aws iam role " - "session name"); - flb_es_conf_destroy(ctx); - return NULL; - } - - /* STS provider needs yet another separate TLS instance */ - ctx->aws_sts_tls = flb_tls_create(FLB_TLS_CLIENT_MODE, - FLB_TRUE, - ins->tls_debug, - ins->tls_vhost, - ins->tls_ca_path, - ins->tls_ca_file, - ins->tls_crt_file, - ins->tls_key_file, - ins->tls_key_passwd); - if (!ctx->aws_sts_tls) { - flb_errno(); - flb_es_conf_destroy(ctx); - return NULL; - } - - ctx->aws_provider = flb_sts_provider_create(config, - ctx->aws_sts_tls, - ctx-> - base_aws_provider, - aws_external_id, - aws_role_arn, - aws_session_name, - ctx->aws_region, - ctx->aws_sts_endpoint, - NULL, - flb_aws_client_generator()); - /* Session name can be freed once provider is created */ - flb_free(aws_session_name); - if (!ctx->aws_provider) { - flb_error("[out_es] Failed to create AWS STS Credential " - "Provider"); - flb_es_conf_destroy(ctx); - return NULL; - } - - } - - /* initialize credentials in sync mode */ - ctx->aws_provider->provider_vtable->sync(ctx->aws_provider); - ctx->aws_provider->provider_vtable->init(ctx->aws_provider); - /* set back to async */ - ctx->aws_provider->provider_vtable->async(ctx->aws_provider); - ctx->aws_provider->provider_vtable->upstream_set(ctx->aws_provider, ctx->ins); - } - } -#endif - - return ctx; -} - -int flb_es_conf_destroy(struct flb_elasticsearch *ctx) -{ - if (!ctx) { - return 0; - } - - if (ctx->u) { - flb_upstream_destroy(ctx->u); - } - if (ctx->ra_id_key) { - flb_ra_destroy(ctx->ra_id_key); - ctx->ra_id_key = NULL; - } - if (ctx->es_action) { - flb_free(ctx->es_action); - } - -#ifdef FLB_HAVE_AWS - if (ctx->base_aws_provider) { - flb_aws_provider_destroy(ctx->base_aws_provider); - } - - if (ctx->aws_provider) { - flb_aws_provider_destroy(ctx->aws_provider); - } - - if (ctx->aws_tls) { - flb_tls_destroy(ctx->aws_tls); - } - - if (ctx->aws_sts_tls) { - flb_tls_destroy(ctx->aws_sts_tls); - } - - if (ctx->aws_unsigned_headers) { - flb_slist_destroy(ctx->aws_unsigned_headers); - flb_free(ctx->aws_unsigned_headers); - } -#endif - - if (ctx->ra_prefix_key) { - flb_ra_destroy(ctx->ra_prefix_key); - } - - flb_free(ctx->cloud_passwd); - flb_free(ctx->cloud_user); - flb_free(ctx); - - return 0; -} diff --git a/fluent-bit/plugins/out_es/es_conf.h b/fluent-bit/plugins/out_es/es_conf.h deleted file mode 100644 index 3c421becf..000000000 --- a/fluent-bit/plugins/out_es/es_conf.h +++ /dev/null @@ -1,33 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_ES_CONF_H -#define FLB_OUT_ES_CONF_H - -#include -#include -#include - -#include "es.h" - -struct flb_elasticsearch *flb_es_conf_create(struct flb_output_instance *ins, - struct flb_config *config); -int flb_es_conf_destroy(struct flb_elasticsearch *ctx); - -#endif diff --git a/fluent-bit/plugins/out_es/murmur3.c b/fluent-bit/plugins/out_es/murmur3.c deleted file mode 100644 index 8fccb6de7..000000000 --- a/fluent-bit/plugins/out_es/murmur3.c +++ /dev/null @@ -1,314 +0,0 @@ -//----------------------------------------------------------------------------- -// MurmurHash3 was written by Austin Appleby, and is placed in the public -// domain. The author hereby disclaims copyright to this source code. - -// Note - The x86 and x64 versions do _not_ produce the same results, as the -// algorithms are optimized for their respective platforms. You can still -// compile and run any of them on any platform, but your performance with the -// non-native version will be less than optimal. - -#include "murmur3.h" - -//----------------------------------------------------------------------------- -// Platform-specific functions and macros - -#ifdef __GNUC__ -#define FORCE_INLINE __attribute__((always_inline)) inline -#else -#define FORCE_INLINE inline -#endif - -static FORCE_INLINE uint32_t rotl32 ( uint32_t x, int8_t r ) -{ - return (x << r) | (x >> (32 - r)); -} - -static FORCE_INLINE uint64_t rotl64 ( uint64_t x, int8_t r ) -{ - return (x << r) | (x >> (64 - r)); -} - -#define ROTL32(x,y) rotl32(x,y) -#define ROTL64(x,y) rotl64(x,y) - -#define BIG_CONSTANT(x) (x##LLU) - -//----------------------------------------------------------------------------- -// Block read - if your platform needs to do endian-swapping or can only -// handle aligned reads, do the conversion here - -#define getblock(p, i) (p[i]) - -//----------------------------------------------------------------------------- -// Finalization mix - force all bits of a hash block to avalanche - -static FORCE_INLINE uint32_t fmix32 ( uint32_t h ) -{ - h ^= h >> 16; - h *= 0x85ebca6b; - h ^= h >> 13; - h *= 0xc2b2ae35; - h ^= h >> 16; - - return h; -} - -//---------- - -static FORCE_INLINE uint64_t fmix64 ( uint64_t k ) -{ - k ^= k >> 33; - k *= BIG_CONSTANT(0xff51afd7ed558ccd); - k ^= k >> 33; - k *= BIG_CONSTANT(0xc4ceb9fe1a85ec53); - k ^= k >> 33; - - return k; -} - -//----------------------------------------------------------------------------- - -void MurmurHash3_x86_32 ( const void * key, int len, - uint32_t seed, void * out ) -{ - const uint8_t * data = (const uint8_t*)key; - const int nblocks = len / 4; - int i; - - uint32_t h1 = seed; - - uint32_t c1 = 0xcc9e2d51; - uint32_t c2 = 0x1b873593; - - //---------- - // body - - const uint32_t * blocks = (const uint32_t *)(data + nblocks*4); - - for(i = -nblocks; i; i++) - { - uint32_t k1 = getblock(blocks,i); - - k1 *= c1; - k1 = ROTL32(k1,15); - k1 *= c2; - - h1 ^= k1; - h1 = ROTL32(h1,13); - h1 = h1*5+0xe6546b64; - } - - //---------- - // tail - - const uint8_t * tail = (const uint8_t*)(data + nblocks*4); - - uint32_t k1 = 0; - - switch(len & 3) - { - case 3: k1 ^= tail[2] << 16; - case 2: k1 ^= tail[1] << 8; - case 1: k1 ^= tail[0]; - k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1; - }; - - //---------- - // finalization - - h1 ^= len; - - h1 = fmix32(h1); - - *(uint32_t*)out = h1; -} - -//----------------------------------------------------------------------------- - -void MurmurHash3_x86_128 ( const void * key, const int len, - uint32_t seed, void * out ) -{ - const uint8_t * data = (const uint8_t*)key; - const int nblocks = len / 16; - int i; - - uint32_t h1 = seed; - uint32_t h2 = seed; - uint32_t h3 = seed; - uint32_t h4 = seed; - - uint32_t c1 = 0x239b961b; - uint32_t c2 = 0xab0e9789; - uint32_t c3 = 0x38b34ae5; - uint32_t c4 = 0xa1e38b93; - - //---------- - // body - - const uint32_t * blocks = (const uint32_t *)(data + nblocks*16); - - for(i = -nblocks; i; i++) - { - uint32_t k1 = getblock(blocks,i*4+0); - uint32_t k2 = getblock(blocks,i*4+1); - uint32_t k3 = getblock(blocks,i*4+2); - uint32_t k4 = getblock(blocks,i*4+3); - - k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1; - - h1 = ROTL32(h1,19); h1 += h2; h1 = h1*5+0x561ccd1b; - - k2 *= c2; k2 = ROTL32(k2,16); k2 *= c3; h2 ^= k2; - - h2 = ROTL32(h2,17); h2 += h3; h2 = h2*5+0x0bcaa747; - - k3 *= c3; k3 = ROTL32(k3,17); k3 *= c4; h3 ^= k3; - - h3 = ROTL32(h3,15); h3 += h4; h3 = h3*5+0x96cd1c35; - - k4 *= c4; k4 = ROTL32(k4,18); k4 *= c1; h4 ^= k4; - - h4 = ROTL32(h4,13); h4 += h1; h4 = h4*5+0x32ac3b17; - } - - //---------- - // tail - - const uint8_t * tail = (const uint8_t*)(data + nblocks*16); - - uint32_t k1 = 0; - uint32_t k2 = 0; - uint32_t k3 = 0; - uint32_t k4 = 0; - - switch(len & 15) - { - case 15: k4 ^= tail[14] << 16; - case 14: k4 ^= tail[13] << 8; - case 13: k4 ^= tail[12] << 0; - k4 *= c4; k4 = ROTL32(k4,18); k4 *= c1; h4 ^= k4; - - case 12: k3 ^= tail[11] << 24; - case 11: k3 ^= tail[10] << 16; - case 10: k3 ^= tail[ 9] << 8; - case 9: k3 ^= tail[ 8] << 0; - k3 *= c3; k3 = ROTL32(k3,17); k3 *= c4; h3 ^= k3; - - case 8: k2 ^= tail[ 7] << 24; - case 7: k2 ^= tail[ 6] << 16; - case 6: k2 ^= tail[ 5] << 8; - case 5: k2 ^= tail[ 4] << 0; - k2 *= c2; k2 = ROTL32(k2,16); k2 *= c3; h2 ^= k2; - - case 4: k1 ^= tail[ 3] << 24; - case 3: k1 ^= tail[ 2] << 16; - case 2: k1 ^= tail[ 1] << 8; - case 1: k1 ^= tail[ 0] << 0; - k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1; - }; - - //---------- - // finalization - - h1 ^= len; h2 ^= len; h3 ^= len; h4 ^= len; - - h1 += h2; h1 += h3; h1 += h4; - h2 += h1; h3 += h1; h4 += h1; - - h1 = fmix32(h1); - h2 = fmix32(h2); - h3 = fmix32(h3); - h4 = fmix32(h4); - - h1 += h2; h1 += h3; h1 += h4; - h2 += h1; h3 += h1; h4 += h1; - - ((uint32_t*)out)[0] = h1; - ((uint32_t*)out)[1] = h2; - ((uint32_t*)out)[2] = h3; - ((uint32_t*)out)[3] = h4; -} - -//----------------------------------------------------------------------------- - -void MurmurHash3_x64_128 ( const void * key, const int len, - const uint32_t seed, void * out ) -{ - const uint8_t * data = (const uint8_t*)key; - const int nblocks = len / 16; - int i; - - uint64_t h1 = seed; - uint64_t h2 = seed; - - uint64_t c1 = BIG_CONSTANT(0x87c37b91114253d5); - uint64_t c2 = BIG_CONSTANT(0x4cf5ad432745937f); - - //---------- - // body - - const uint64_t * blocks = (const uint64_t *)(data); - - for(i = 0; i < nblocks; i++) - { - uint64_t k1 = getblock(blocks,i*2+0); - uint64_t k2 = getblock(blocks,i*2+1); - - k1 *= c1; k1 = ROTL64(k1,31); k1 *= c2; h1 ^= k1; - - h1 = ROTL64(h1,27); h1 += h2; h1 = h1*5+0x52dce729; - - k2 *= c2; k2 = ROTL64(k2,33); k2 *= c1; h2 ^= k2; - - h2 = ROTL64(h2,31); h2 += h1; h2 = h2*5+0x38495ab5; - } - - //---------- - // tail - - const uint8_t * tail = (const uint8_t*)(data + nblocks*16); - - uint64_t k1 = 0; - uint64_t k2 = 0; - - switch(len & 15) - { - case 15: k2 ^= (uint64_t)(tail[14]) << 48; - case 14: k2 ^= (uint64_t)(tail[13]) << 40; - case 13: k2 ^= (uint64_t)(tail[12]) << 32; - case 12: k2 ^= (uint64_t)(tail[11]) << 24; - case 11: k2 ^= (uint64_t)(tail[10]) << 16; - case 10: k2 ^= (uint64_t)(tail[ 9]) << 8; - case 9: k2 ^= (uint64_t)(tail[ 8]) << 0; - k2 *= c2; k2 = ROTL64(k2,33); k2 *= c1; h2 ^= k2; - - case 8: k1 ^= (uint64_t)(tail[ 7]) << 56; - case 7: k1 ^= (uint64_t)(tail[ 6]) << 48; - case 6: k1 ^= (uint64_t)(tail[ 5]) << 40; - case 5: k1 ^= (uint64_t)(tail[ 4]) << 32; - case 4: k1 ^= (uint64_t)(tail[ 3]) << 24; - case 3: k1 ^= (uint64_t)(tail[ 2]) << 16; - case 2: k1 ^= (uint64_t)(tail[ 1]) << 8; - case 1: k1 ^= (uint64_t)(tail[ 0]) << 0; - k1 *= c1; k1 = ROTL64(k1,31); k1 *= c2; h1 ^= k1; - }; - - //---------- - // finalization - - h1 ^= len; h2 ^= len; - - h1 += h2; - h2 += h1; - - h1 = fmix64(h1); - h2 = fmix64(h2); - - h1 += h2; - h2 += h1; - - ((uint64_t*)out)[0] = h1; - ((uint64_t*)out)[1] = h2; -} - -//----------------------------------------------------------------------------- diff --git a/fluent-bit/plugins/out_es/murmur3.h b/fluent-bit/plugins/out_es/murmur3.h deleted file mode 100644 index c85395a14..000000000 --- a/fluent-bit/plugins/out_es/murmur3.h +++ /dev/null @@ -1,29 +0,0 @@ -//----------------------------------------------------------------------------- -// MurmurHash3 was written by Austin Appleby, and is placed in the -// public domain. The author hereby disclaims copyright to this source -// code. - -#ifndef _MURMURHASH3_H_ -#define _MURMURHASH3_H_ - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -//----------------------------------------------------------------------------- - -void MurmurHash3_x86_32 (const void *key, int len, uint32_t seed, void *out); - -void MurmurHash3_x86_128(const void *key, int len, uint32_t seed, void *out); - -void MurmurHash3_x64_128(const void *key, int len, uint32_t seed, void *out); - -//----------------------------------------------------------------------------- - -#ifdef __cplusplus -} -#endif - -#endif // _MURMURHASH3_H_ \ No newline at end of file diff --git a/fluent-bit/plugins/out_exit/CMakeLists.txt b/fluent-bit/plugins/out_exit/CMakeLists.txt deleted file mode 100644 index d9b7168e2..000000000 --- a/fluent-bit/plugins/out_exit/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - exit.c) - -FLB_PLUGIN(out_exit "${src}" "") diff --git a/fluent-bit/plugins/out_exit/exit.c b/fluent-bit/plugins/out_exit/exit.c deleted file mode 100644 index dc3532b2e..000000000 --- a/fluent-bit/plugins/out_exit/exit.c +++ /dev/null @@ -1,108 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#define FLB_EXIT_FLUSH_COUNT "1" - -struct flb_exit { - int is_running; - int count; - - /* config */ - int flush_count; -}; - -static int cb_exit_init(struct flb_output_instance *ins, struct flb_config *config, - void *data) -{ - int ret; - (void) config; - (void) data; - struct flb_exit *ctx; - - ctx = flb_malloc(sizeof(struct flb_exit)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->count = 0; - ctx->is_running = FLB_TRUE; - - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return -1; - } - - flb_output_set_context(ins, ctx); - - return 0; -} - -static void cb_exit_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - (void) i_ins; - (void) out_context; - struct flb_exit *ctx = out_context; - - ctx->count++; - if (ctx->is_running == FLB_TRUE && ctx->count >= ctx->flush_count) { - flb_engine_exit(config); - ctx->is_running = FLB_FALSE; - } - - FLB_OUTPUT_RETURN(FLB_OK); -} - -static int cb_exit_exit(void *data, struct flb_config *config) -{ - struct flb_exit *ctx = data; - (void) config; - - flb_free(ctx); - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_INT, "flush_count", FLB_EXIT_FLUSH_COUNT, - 0, FLB_TRUE, offsetof(struct flb_exit, flush_count), - NULL - }, - - /* EOF */ - {0} -}; - -struct flb_output_plugin out_exit_plugin = { - .name = "exit", - .description = "Exit after a number of flushes (test purposes)", - .cb_init = cb_exit_init, - .cb_flush = cb_exit_flush, - .cb_exit = cb_exit_exit, - .config_map = config_map, - .flags = 0, -}; diff --git a/fluent-bit/plugins/out_file/CMakeLists.txt b/fluent-bit/plugins/out_file/CMakeLists.txt deleted file mode 100644 index 8db7675a4..000000000 --- a/fluent-bit/plugins/out_file/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - file.c) - -FLB_PLUGIN(out_file "${src}" "") diff --git a/fluent-bit/plugins/out_file/file.c b/fluent-bit/plugins/out_file/file.c deleted file mode 100644 index d5f8a036a..000000000 --- a/fluent-bit/plugins/out_file/file.c +++ /dev/null @@ -1,705 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#ifdef FLB_SYSTEM_WINDOWS -#include -#include -#endif - -#include "file.h" - -#ifdef FLB_SYSTEM_WINDOWS -#define NEWLINE "\r\n" -#define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR) -#else -#define NEWLINE "\n" -#endif - -struct flb_file_conf { - const char *out_path; - const char *out_file; - const char *delimiter; - const char *label_delimiter; - const char *template; - int format; - int csv_column_names; - int mkdir; - struct flb_output_instance *ins; -}; - -static char *check_delimiter(const char *str) -{ - if (str == NULL) { - return NULL; - } - - if (!strcasecmp(str, "\\t") || !strcasecmp(str, "tab")) { - return "\t"; - } - else if (!strcasecmp(str, "space")) { - return " "; - } - else if (!strcasecmp(str, "comma")) { - return ","; - } - - return NULL; -} - - -static int cb_file_init(struct flb_output_instance *ins, - struct flb_config *config, - void *data) -{ - int ret; - const char *tmp; - char *ret_str; - (void) config; - (void) data; - struct flb_file_conf *ctx; - - ctx = flb_calloc(1, sizeof(struct flb_file_conf)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->ins = ins; - ctx->format = FLB_OUT_FILE_FMT_JSON; /* default */ - ctx->delimiter = NULL; - ctx->label_delimiter = NULL; - ctx->template = NULL; - - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return -1; - } - - /* Optional, file format */ - tmp = flb_output_get_property("Format", ins); - if (tmp) { - if (!strcasecmp(tmp, "csv")) { - ctx->format = FLB_OUT_FILE_FMT_CSV; - ctx->delimiter = ","; - } - else if (!strcasecmp(tmp, "ltsv")) { - ctx->format = FLB_OUT_FILE_FMT_LTSV; - ctx->delimiter = "\t"; - ctx->label_delimiter = ":"; - } - else if (!strcasecmp(tmp, "plain")) { - ctx->format = FLB_OUT_FILE_FMT_PLAIN; - ctx->delimiter = NULL; - ctx->label_delimiter = NULL; - } - else if (!strcasecmp(tmp, "msgpack")) { - ctx->format = FLB_OUT_FILE_FMT_MSGPACK; - ctx->delimiter = NULL; - ctx->label_delimiter = NULL; - } - else if (!strcasecmp(tmp, "template")) { - ctx->format = FLB_OUT_FILE_FMT_TEMPLATE; - } - else if (!strcasecmp(tmp, "out_file")) { - /* for explicit setting */ - ctx->format = FLB_OUT_FILE_FMT_JSON; - } - else { - flb_plg_error(ctx->ins, "unknown format %s. abort.", tmp); - flb_free(ctx); - return -1; - } - } - - tmp = flb_output_get_property("delimiter", ins); - ret_str = check_delimiter(tmp); - if (ret_str != NULL) { - ctx->delimiter = ret_str; - } - - tmp = flb_output_get_property("label_delimiter", ins); - ret_str = check_delimiter(tmp); - if (ret_str != NULL) { - ctx->label_delimiter = ret_str; - } - - /* Set the context */ - flb_output_set_context(ins, ctx); - - return 0; -} - -static int csv_output(FILE *fp, int column_names, - struct flb_time *tm, msgpack_object *obj, - struct flb_file_conf *ctx) -{ - int i; - int map_size; - msgpack_object_kv *kv = NULL; - - if (obj->type == MSGPACK_OBJECT_MAP && obj->via.map.size > 0) { - kv = obj->via.map.ptr; - map_size = obj->via.map.size; - - if (column_names == FLB_TRUE) { - fprintf(fp, "timestamp%s", ctx->delimiter); - for (i = 0; i < map_size; i++) { - msgpack_object_print(fp, (kv+i)->key); - if (i + 1 < map_size) { - fprintf(fp, "%s", ctx->delimiter); - } - } - fprintf(fp, NEWLINE); - } - - fprintf(fp, "%lld.%.09ld%s", - (long long) tm->tm.tv_sec, tm->tm.tv_nsec, ctx->delimiter); - - for (i = 0; i < map_size - 1; i++) { - msgpack_object_print(fp, (kv+i)->val); - fprintf(fp, "%s", ctx->delimiter); - } - - msgpack_object_print(fp, (kv+(map_size-1))->val); - fprintf(fp, NEWLINE); - } - return 0; -} - -static int ltsv_output(FILE *fp, struct flb_time *tm, msgpack_object *obj, - struct flb_file_conf *ctx) -{ - msgpack_object_kv *kv = NULL; - int i; - int map_size; - - if (obj->type == MSGPACK_OBJECT_MAP && obj->via.map.size > 0) { - kv = obj->via.map.ptr; - map_size = obj->via.map.size; - fprintf(fp, "\"time\"%s%f%s", - ctx->label_delimiter, - flb_time_to_double(tm), - ctx->delimiter); - - for (i = 0; i < map_size - 1; i++) { - msgpack_object_print(fp, (kv+i)->key); - fprintf(fp, "%s", ctx->label_delimiter); - msgpack_object_print(fp, (kv+i)->val); - fprintf(fp, "%s", ctx->delimiter); - } - - msgpack_object_print(fp, (kv+(map_size-1))->key); - fprintf(fp, "%s", ctx->label_delimiter); - msgpack_object_print(fp, (kv+(map_size-1))->val); - fprintf(fp, NEWLINE); - } - return 0; -} - -static int template_output_write(struct flb_file_conf *ctx, - FILE *fp, struct flb_time *tm, msgpack_object *obj, - const char *key, int size) -{ - int i; - msgpack_object_kv *kv; - - /* - * Right now we treat "{time}" specially and fill the placeholder - * with the metadata timestamp (formatted as float). - */ - if (!strncmp(key, "time", size)) { - fprintf(fp, "%f", flb_time_to_double(tm)); - return 0; - } - - if (obj->type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "invalid object type (type=%i)", obj->type); - return -1; - } - - for (i = 0; i < obj->via.map.size; i++) { - kv = obj->via.map.ptr + i; - - if (size != kv->key.via.str.size) { - continue; - } - - if (!memcmp(key, kv->key.via.str.ptr, size)) { - if (kv->val.type == MSGPACK_OBJECT_STR) { - fwrite(kv->val.via.str.ptr, 1, kv->val.via.str.size, fp); - } - else { - msgpack_object_print(fp, kv->val); - } - return 0; - } - } - return -1; -} - -/* - * Python-like string templating for out_file. - * - * This accepts a format string like "my name is {name}" and fills - * placeholders using corresponding values in a record. - * - * e.g. {"name":"Tom"} => "my name is Tom" - */ -static int template_output(FILE *fp, struct flb_time *tm, msgpack_object *obj, - struct flb_file_conf *ctx) -{ - int i; - int len = strlen(ctx->template); - int keysize; - const char *key; - const char *pos; - const char *inbrace = NULL; /* points to the last open brace */ - - for (i = 0; i < len; i++) { - pos = ctx->template + i; - if (*pos == '{') { - if (inbrace) { - /* - * This means that we find another open brace inside - * braces (e.g. "{a{b}"). Ignore the previous one. - */ - fwrite(inbrace, 1, pos - inbrace, fp); - } - inbrace = pos; - } - else if (*pos == '}' && inbrace) { - key = inbrace + 1; - keysize = pos - inbrace - 1; - - if (template_output_write(ctx, fp, tm, obj, key, keysize)) { - fwrite(inbrace, 1, pos - inbrace + 1, fp); - } - inbrace = NULL; - } - else { - if (!inbrace) { - fputc(*pos, fp); - } - } - } - - /* Handle an unclosed brace like "{abc" */ - if (inbrace) { - fputs(inbrace, fp); - } - fputs(NEWLINE, fp); - return 0; -} - - -static int plain_output(FILE *fp, msgpack_object *obj, size_t alloc_size) -{ - char *buf; - - buf = flb_msgpack_to_json_str(alloc_size, obj); - if (buf) { - fprintf(fp, "%s" NEWLINE, - buf); - flb_free(buf); - } - return 0; -} - -static void print_metrics_text(struct flb_output_instance *ins, - FILE *fp, - const void *data, size_t bytes) -{ - int ret; - size_t off = 0; - cfl_sds_t text; - struct cmt *cmt = NULL; - - /* get cmetrics context */ - ret = cmt_decode_msgpack_create(&cmt, (char *) data, bytes, &off); - if (ret != 0) { - flb_plg_error(ins, "could not process metrics payload"); - return; - } - - /* convert to text representation */ - text = cmt_encode_text_create(cmt); - - /* destroy cmt context */ - cmt_destroy(cmt); - - fprintf(fp, "%s", text); - cmt_encode_text_destroy(text); -} - -static int mkpath(struct flb_output_instance *ins, const char *dir) -{ - struct stat st; - char *dup_dir = NULL; -#ifdef FLB_SYSTEM_MACOS - char *parent_dir = NULL; -#endif - - int ret; - - if (!dir) { - errno = EINVAL; - return -1; - } - - if (strlen(dir) == 0) { - errno = EINVAL; - return -1; - } - - if (stat(dir, &st) == 0) { - if (S_ISDIR (st.st_mode)) { - return 0; - } - flb_plg_error(ins, "%s is not a directory", dir); - errno = ENOTDIR; - return -1; - } - -#ifdef FLB_SYSTEM_WINDOWS - char path[MAX_PATH]; - - if (_fullpath(path, dir, MAX_PATH) == NULL) { - return -1; - } - - if (SHCreateDirectoryExA(NULL, path, NULL) != ERROR_SUCCESS) { - return -1; - } - return 0; -#elif FLB_SYSTEM_MACOS - dup_dir = strdup(dir); - if (!dup_dir) { - return -1; - } - - /* macOS's dirname(3) should return current directory when slash - * charachter is not included in passed string. - * And note that macOS's dirname(3) does not modify passed string. - */ - parent_dir = dirname(dup_dir); - if (stat(parent_dir, &st) == 0 && strncmp(parent_dir, ".", 1)) { - if (S_ISDIR (st.st_mode)) { - flb_plg_debug(ins, "creating directory %s", dup_dir); - ret = mkdir(dup_dir, 0755); - free(dup_dir); - return ret; - } - } - - ret = mkpath(ins, dirname(dup_dir)); - if (ret != 0) { - free(dup_dir); - return ret; - } - flb_plg_debug(ins, "creating directory %s", dup_dir); - ret = mkdir(dup_dir, 0755); - free(dup_dir); - return ret; -#else - dup_dir = strdup(dir); - if (!dup_dir) { - return -1; - } - ret = mkpath(ins, dirname(dup_dir)); - free(dup_dir); - if (ret != 0) { - return ret; - } - flb_plg_debug(ins, "creating directory %s", dir); - return mkdir(dir, 0755); -#endif -} - -static void cb_file_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *ins, - void *out_context, - struct flb_config *config) -{ - int ret; - int column_names; - FILE * fp; - size_t off = 0; - size_t last_off = 0; - size_t alloc_size = 0; - size_t total; - char out_file[PATH_MAX]; - char *buf; - long file_pos; - struct flb_file_conf *ctx = out_context; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - char* out_file_copy; - - (void) config; - - /* Set the right output file */ - if (ctx->out_path) { - if (ctx->out_file) { - snprintf(out_file, PATH_MAX - 1, "%s/%s", - ctx->out_path, ctx->out_file); - } - else { - snprintf(out_file, PATH_MAX - 1, "%s/%s", - ctx->out_path, event_chunk->tag); - } - } - else { - if (ctx->out_file) { - snprintf(out_file, PATH_MAX - 1, "%s", ctx->out_file); - } - else { - snprintf(out_file, PATH_MAX - 1, "%s", event_chunk->tag); - } - } - - /* Open output file with default name as the Tag */ - fp = fopen(out_file, "ab+"); - if (ctx->mkdir == FLB_TRUE && fp == NULL && errno == ENOENT) { - out_file_copy = strdup(out_file); - if (out_file_copy) { -#ifdef FLB_SYSTEM_WINDOWS - PathRemoveFileSpecA(out_file_copy); - ret = mkpath(ctx->ins, out_file_copy); -#else - ret = mkpath(ctx->ins, dirname(out_file_copy)); -#endif - free(out_file_copy); - if (ret == 0) { - fp = fopen(out_file, "ab+"); - } - } - } - if (fp == NULL) { - flb_errno(); - flb_plg_error(ctx->ins, "error opening: %s", out_file); - FLB_OUTPUT_RETURN(FLB_ERROR); - } - - /* - * Get current file stream position, we gather this in case 'csv' format - * needs to write the column names. - */ - file_pos = ftell(fp); - - /* Check if the event type is metrics, handle the payload differently */ - if (event_chunk->type == FLB_INPUT_METRICS) { - print_metrics_text(ctx->ins, fp, - event_chunk->data, event_chunk->size); - fclose(fp); - FLB_OUTPUT_RETURN(FLB_OK); - } - - /* - * Msgpack output format used to create unit tests files, useful for - * Fluent Bit developers. - */ - if (ctx->format == FLB_OUT_FILE_FMT_MSGPACK) { - off = 0; - total = 0; - - do { - ret = fwrite((char *) event_chunk->data + off, 1, - event_chunk->size - off, fp); - if (ret < 0) { - flb_errno(); - fclose(fp); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - total += ret; - } while (total < event_chunk->size); - - fclose(fp); - FLB_OUTPUT_RETURN(FLB_OK); - } - - ret = flb_log_event_decoder_init(&log_decoder, - (char *) event_chunk->data, - event_chunk->size); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - fclose(fp); - FLB_OUTPUT_RETURN(FLB_ERROR); - } - - /* - * Upon flush, for each array, lookup the time and the first field - * of the map to use as a data point. - */ - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - alloc_size = (off - last_off) + 128; /* JSON is larger than msgpack */ - last_off = off; - - switch (ctx->format){ - case FLB_OUT_FILE_FMT_JSON: - buf = flb_msgpack_to_json_str(alloc_size, log_event.body); - if (buf) { - fprintf(fp, "%s: [%"PRIu64".%09lu, %s]" NEWLINE, - event_chunk->tag, - log_event.timestamp.tm.tv_sec, log_event.timestamp.tm.tv_nsec, - buf); - flb_free(buf); - } - else { - flb_log_event_decoder_destroy(&log_decoder); - fclose(fp); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - break; - case FLB_OUT_FILE_FMT_CSV: - if (ctx->csv_column_names == FLB_TRUE && file_pos == 0) { - column_names = FLB_TRUE; - file_pos = 1; - } - else { - column_names = FLB_FALSE; - } - csv_output(fp, column_names, - &log_event.timestamp, - log_event.body, ctx); - break; - case FLB_OUT_FILE_FMT_LTSV: - ltsv_output(fp, - &log_event.timestamp, - log_event.body, ctx); - break; - case FLB_OUT_FILE_FMT_PLAIN: - plain_output(fp, log_event.body, alloc_size); - - break; - case FLB_OUT_FILE_FMT_TEMPLATE: - template_output(fp, - &log_event.timestamp, - log_event.body, ctx); - - break; - } - } - - flb_log_event_decoder_destroy(&log_decoder); - - fclose(fp); - - FLB_OUTPUT_RETURN(FLB_OK); -} - -static int cb_file_exit(void *data, struct flb_config *config) -{ - struct flb_file_conf *ctx = data; - - if (!ctx) { - return 0; - } - - flb_free(ctx); - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "path", NULL, - 0, FLB_TRUE, offsetof(struct flb_file_conf, out_path), - "Absolute path to store the files. This parameter is optional" - }, - - { - FLB_CONFIG_MAP_STR, "file", NULL, - 0, FLB_TRUE, offsetof(struct flb_file_conf, out_file), - "Name of the target file to write the records. If 'path' is specified, " - "the value is prefixed" - }, - - { - FLB_CONFIG_MAP_STR, "format", NULL, - 0, FLB_FALSE, 0, - "Specify the output data format, the available options are: plain (json), " - "csv, ltsv and template. If no value is set the outgoing data is formatted " - "using the tag and the record in json" - }, - - { - FLB_CONFIG_MAP_STR, "delimiter", NULL, - 0, FLB_FALSE, 0, - "Set a custom delimiter for the records" - }, - - { - FLB_CONFIG_MAP_STR, "label_delimiter", NULL, - 0, FLB_FALSE, 0, - "Set a custom label delimiter, to be used with 'ltsv' format" - }, - - { - FLB_CONFIG_MAP_STR, "template", "{time} {message}", - 0, FLB_TRUE, offsetof(struct flb_file_conf, template), - "Set a custom template format for the data" - }, - - { - FLB_CONFIG_MAP_BOOL, "csv_column_names", "false", - 0, FLB_TRUE, offsetof(struct flb_file_conf, csv_column_names), - "Add column names (keys) in the first line of the target file" - }, - - { - FLB_CONFIG_MAP_BOOL, "mkdir", "false", - 0, FLB_TRUE, offsetof(struct flb_file_conf, mkdir), - "Recursively create output directory if it does not exist. Permissions set to 0755" - }, - - /* EOF */ - {0} -}; - -struct flb_output_plugin out_file_plugin = { - .name = "file", - .description = "Generate log file", - .cb_init = cb_file_init, - .cb_flush = cb_file_flush, - .cb_exit = cb_file_exit, - .flags = 0, - .workers = 1, - .event_type = FLB_OUTPUT_LOGS | FLB_OUTPUT_METRICS, - .config_map = config_map, -}; diff --git a/fluent-bit/plugins/out_file/file.h b/fluent-bit/plugins/out_file/file.h deleted file mode 100644 index c04e51b5c..000000000 --- a/fluent-bit/plugins/out_file/file.h +++ /dev/null @@ -1,32 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_FILE -#define FLB_OUT_FILE - -enum { - FLB_OUT_FILE_FMT_JSON, - FLB_OUT_FILE_FMT_CSV, - FLB_OUT_FILE_FMT_LTSV, - FLB_OUT_FILE_FMT_PLAIN, - FLB_OUT_FILE_FMT_MSGPACK, - FLB_OUT_FILE_FMT_TEMPLATE, -}; - -#endif diff --git a/fluent-bit/plugins/out_flowcounter/CMakeLists.txt b/fluent-bit/plugins/out_flowcounter/CMakeLists.txt deleted file mode 100644 index 7699b196a..000000000 --- a/fluent-bit/plugins/out_flowcounter/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - out_flowcounter.c) - -FLB_PLUGIN(out_flowcounter "${src}" "") diff --git a/fluent-bit/plugins/out_flowcounter/out_flowcounter.c b/fluent-bit/plugins/out_flowcounter/out_flowcounter.c deleted file mode 100644 index 1ed14636f..000000000 --- a/fluent-bit/plugins/out_flowcounter/out_flowcounter.c +++ /dev/null @@ -1,297 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include - -#include "out_flowcounter.h" - -#include -#include -#include -#include - - -#define PLUGIN_NAME "out_flowcounter" - -static void count_initialized(struct flb_out_fcount_buffer* buf) -{ - buf->bytes = 0; - buf->counts = 0; -} - -static int time_is_valid(time_t t, struct flb_flowcounter *ctx) -{ - if (t < ctx->buf[ctx->index].until - ctx->tick) { - return FLB_FALSE; - } - return FLB_TRUE; -} - -static int configure(struct flb_flowcounter *ctx, - struct flb_output_instance *ins, - struct flb_config *config) -{ - int i; - time_t base = time(NULL); - const char* pval = NULL; - - /* default */ - ctx->unit = FLB_UNIT_MIN; - ctx->tick = 60; - - pval = flb_output_get_property("unit", ins); - if (pval != NULL) { - /* check unit of duration */ - if (!strcasecmp(pval, FLB_UNIT_SEC)) { - ctx->unit = FLB_UNIT_SEC; - ctx->tick = 1; - } - else if (!strcasecmp(pval, FLB_UNIT_HOUR)) { - ctx->unit = FLB_UNIT_HOUR; - ctx->tick = 3600; - } - else if(!strcasecmp(pval, FLB_UNIT_DAY)) { - ctx->unit = FLB_UNIT_DAY; - ctx->tick = 86400; - } - } - - flb_plg_debug(ctx->ins, "unit is \"%s\"", ctx->unit); - - /* initialize buffer */ - ctx->size = (config->flush / ctx->tick) + 1; - flb_plg_debug(ctx->ins, "buffer size=%d", ctx->size); - - ctx->index = 0; - ctx->buf = flb_malloc(sizeof(struct flb_out_fcount_buffer) * ctx->size); - if (!ctx->buf) { - flb_errno(); - return -1; - } - - for (i = 0; i < ctx->size; i++) { - ctx->buf[i].until = base + ctx->tick*i; - count_initialized(&ctx->buf[i]); - } - - return 0; -} - -static void output_fcount(FILE *f, struct flb_flowcounter *ctx, - struct flb_out_fcount_buffer *buf) -{ - fprintf(f, - "[%s] [%lu, {" - "\"counts\":%"PRIu64", " - "\"bytes\":%"PRIu64", " - "\"counts/%s\":%"PRIu64", " - "\"bytes/%s\":%"PRIu64" }" - "]\n", - PLUGIN_NAME, buf->until, - buf->counts, - buf->bytes, - ctx->unit, buf->counts/ctx->tick, - ctx->unit, buf->bytes/ctx->tick); - /* TODO filtering with tag? */ -} - -static void count_up(struct flb_log_event *log_event, - struct flb_out_fcount_buffer *ctx, uint64_t size) -{ - ctx->counts++; - ctx->bytes += size; - - /*TODO extract specific data from log_event */ -} - -static int out_fcount_init(struct flb_output_instance *ins, struct flb_config *config, - void *data) -{ - int ret; - (void) data; - - struct flb_flowcounter *ctx = NULL; - - ctx = flb_malloc(sizeof(struct flb_flowcounter)); - if (ctx == NULL) { - flb_errno(); - return -1; - } - ctx->ins = ins; - - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return -1; - } - - ret = configure(ctx, ins, config); - if (ret < 0) { - flb_free(ctx); - return -1; - } - - flb_output_set_context(ins, ctx); - - return 0; -} - -static struct flb_out_fcount_buffer* seek_buffer(time_t t, - struct flb_flowcounter *ctx) -{ - int i = ctx->index; - int32_t diff; - - while (1) { - diff = (int32_t) difftime(ctx->buf[i].until, t); - if (diff >= 0 && diff <= ctx->tick) { - return &ctx->buf[i]; - } - i++; - - if (i >= ctx->size) { - i = 0; - } - - if (i == ctx->index) { - break; - } - } - return NULL; -} - - - -static void out_fcount_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - struct flb_flowcounter *ctx = out_context; - struct flb_out_fcount_buffer *buf = NULL; - size_t off = 0; - time_t t; - uint64_t last_off = 0; - uint64_t byte_data = 0; - struct flb_time tm; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - int ret; - - (void) i_ins; - (void) config; - - ret = flb_log_event_decoder_init(&log_decoder, - (char *) event_chunk->data, - event_chunk->size); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - FLB_OUTPUT_RETURN(FLB_RETRY); - } - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - if (ctx->event_based) { - flb_time_copy(&tm, &log_event.timestamp); - } - else { - flb_time_get(&tm); - } - t = tm.tm.tv_sec; - if (time_is_valid(t, ctx) == FLB_FALSE) { - flb_plg_warn(ctx->ins, "out of range. Skip the record"); - continue; - } - - byte_data = (uint64_t)(off - last_off); - last_off = off; - - buf = seek_buffer(t, ctx); - - while (buf == NULL) { - /* flush buffer */ - output_fcount(stdout, ctx, &ctx->buf[ctx->index]); - count_initialized(&ctx->buf[ctx->index]); - ctx->buf[ctx->index].until += ctx->tick * ctx->size; - - ctx->index++; - if (ctx->index >= ctx->size) { - ctx->index = 0; - } - buf = seek_buffer(t, ctx); - } - - if (buf != NULL) { - count_up(&log_event, buf, byte_data); - } - } - - flb_log_event_decoder_destroy(&log_decoder); - - FLB_OUTPUT_RETURN(FLB_OK); -} - -static int out_fcount_exit(void *data, struct flb_config* config) -{ - struct flb_flowcounter *ctx = data; - - if (!ctx) { - return 0; - } - - flb_free(ctx->buf); - flb_free(ctx); - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "unit", NULL, - 0, FLB_FALSE, 0, - NULL - }, - { - FLB_CONFIG_MAP_BOOL, "event_based", "false", - 0, FLB_TRUE, offsetof(struct flb_flowcounter, event_based), - NULL - }, - - /* EOF */ - {0} -}; - -struct flb_output_plugin out_flowcounter_plugin = { - .name = "flowcounter", - .description = "FlowCounter", - .cb_init = out_fcount_init, - .cb_flush = out_fcount_flush, - .cb_exit = out_fcount_exit, - .config_map = config_map, - .flags = 0, -}; diff --git a/fluent-bit/plugins/out_flowcounter/out_flowcounter.h b/fluent-bit/plugins/out_flowcounter/out_flowcounter.h deleted file mode 100644 index de714cf9e..000000000 --- a/fluent-bit/plugins/out_flowcounter/out_flowcounter.h +++ /dev/null @@ -1,49 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_FLOWCOUNTER -#define FLB_OUT_FLOWCOUNTER - -#include -#include - -#define FLB_UNIT_SEC "second" -#define FLB_UNIT_MIN "minute" -#define FLB_UNIT_HOUR "hour" -#define FLB_UNIT_DAY "day" - -struct flb_out_fcount_buffer { - time_t until; - uint64_t counts; - uint64_t bytes; -}; - -struct flb_flowcounter { - char *unit; - int32_t tick; - int event_based; - - struct flb_out_fcount_buffer *buf; - int index; - int size; - - struct flb_output_instance *ins; -}; - -#endif diff --git a/fluent-bit/plugins/out_forward/CMakeLists.txt b/fluent-bit/plugins/out_forward/CMakeLists.txt deleted file mode 100644 index fd639c2eb..000000000 --- a/fluent-bit/plugins/out_forward/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(src - forward.c - forward_format.c - ) - -FLB_PLUGIN(out_forward "${src}" "") diff --git a/fluent-bit/plugins/out_forward/README.md b/fluent-bit/plugins/out_forward/README.md deleted file mode 100644 index f08b45619..000000000 --- a/fluent-bit/plugins/out_forward/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Fluentd Forward Protocol Implementation - -This plugin is based in Fluentd Forward Protocol Spec v1 available here: - -- https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1 - -The following Event modes are implemented: - -- Message Mode -- Forward Mode - -Depending of the configuration, the plugin will decide to go with Message Mode or Forward Mode. diff --git a/fluent-bit/plugins/out_forward/forward.c b/fluent-bit/plugins/out_forward/forward.c deleted file mode 100644 index 8cc2ca2cc..000000000 --- a/fluent-bit/plugins/out_forward/forward.c +++ /dev/null @@ -1,1832 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "forward.h" -#include "forward_format.h" - -#ifdef FLB_HAVE_UNIX_SOCKET -#include -#include -#endif - -#define SECURED_BY "Fluent Bit" - -pthread_once_t uds_connection_tls_slot_init_once_control = PTHREAD_ONCE_INIT; -FLB_TLS_DEFINE(struct flb_forward_uds_connection, uds_connection); - -void initialize_uds_connection_tls_slot() -{ - FLB_TLS_INIT(uds_connection); -} - -#ifdef FLB_HAVE_UNIX_SOCKET -static flb_sockfd_t forward_unix_connect(struct flb_forward_config *config, - struct flb_forward *ctx) -{ - flb_sockfd_t fd = -1; - struct sockaddr_un address; - - if (sizeof(address.sun_path) <= flb_sds_len(config->unix_path)) { - flb_plg_error(ctx->ins, "unix_path is too long"); - return -1; - } - - memset(&address, 0, sizeof(struct sockaddr_un)); - - fd = flb_net_socket_create(AF_UNIX, FLB_FALSE); - if (fd < 0) { - flb_plg_error(ctx->ins, "flb_net_socket_create error"); - return -1; - } - - address.sun_family = AF_UNIX; - strncpy(address.sun_path, config->unix_path, flb_sds_len(config->unix_path)); - - if(connect(fd, (const struct sockaddr*) &address, sizeof(address)) < 0) { - flb_errno(); - close(fd); - - return -1; - } - - return fd; -} - -static flb_sockfd_t forward_uds_get_conn(struct flb_forward_config *config, - struct flb_forward *ctx) -{ - struct flb_forward_uds_connection *connection_entry; - flb_sockfd_t connection; - - connection_entry = FLB_TLS_GET(uds_connection); - - /* We need to allow the code to try to get the value from the TLS - * regardless of if it's provided with a config and context because - * when we establish the connection we do have both of them but those - * are not passed along to the functions in charge of doing IO. - */ - - if (connection_entry == NULL) { - if (config == NULL || - ctx == NULL) { - return -1; - } - - connection_entry = flb_calloc(1, sizeof(struct flb_forward_uds_connection)); - - if (connection_entry == NULL) { - flb_errno(); - - return -1; - } - - connection = forward_unix_connect(config, ctx); - - if (connection == -1) { - flb_free(connection_entry); - - return -1; - } - - connection_entry->descriptor = connection; - - pthread_mutex_lock(&ctx->uds_connection_list_mutex); - - cfl_list_add(&connection_entry->_head, &ctx->uds_connection_list); - - pthread_mutex_unlock(&ctx->uds_connection_list_mutex); - - FLB_TLS_SET(uds_connection, connection_entry); - } - - return connection_entry->descriptor; -} - -static void forward_uds_drop_conn(struct flb_forward *ctx, - flb_sockfd_t connection) -{ - struct flb_forward_uds_connection *connection_entry; - - if (ctx != NULL) { - connection_entry = FLB_TLS_GET(uds_connection); - - if (connection_entry != NULL) { - pthread_mutex_lock(&ctx->uds_connection_list_mutex); - - if (connection == connection_entry->descriptor) { - close(connection); - - if (!cfl_list_entry_is_orphan(&connection_entry->_head)) { - cfl_list_del(&connection_entry->_head); - } - - free(connection_entry); - - FLB_TLS_SET(uds_connection, NULL); - } - - pthread_mutex_unlock(&ctx->uds_connection_list_mutex); - } - } -} - -static void forward_uds_drop_all(struct flb_forward *ctx) -{ - struct flb_forward_uds_connection *connection_entry; - struct cfl_list *head; - struct cfl_list *tmp; - - if (ctx != NULL) { - pthread_mutex_lock(&ctx->uds_connection_list_mutex); - - cfl_list_foreach_safe(head, tmp, &ctx->uds_connection_list) { - connection_entry = cfl_list_entry(head, - struct flb_forward_uds_connection, - _head); - - if (connection_entry->descriptor != -1) { - close(connection_entry->descriptor); - - connection_entry->descriptor = -1; - } - - if (!cfl_list_entry_is_orphan(&connection_entry->_head)) { - cfl_list_del(&connection_entry->_head); - } - - free(connection_entry); - } - - pthread_mutex_unlock(&ctx->uds_connection_list_mutex); - } -} - -/* In these functions forward_uds_get_conn - * should not return -1 because it should have been - * called earlier with a proper context and it should - * have saved a file descriptor to the TLS. - */ - -static int io_unix_write(struct flb_connection *unused, int deprecated_fd, const void* data, - size_t len, size_t *out_len) -{ - flb_sockfd_t uds_conn; - - uds_conn = forward_uds_get_conn(NULL, NULL); - - return flb_io_fd_write(uds_conn, data, len, out_len); -} - -static int io_unix_read(struct flb_connection *unused, int deprecated_fd, void* buf,size_t len) -{ - flb_sockfd_t uds_conn; - - uds_conn = forward_uds_get_conn(NULL, NULL); - - return flb_io_fd_read(uds_conn, buf, len); -} - -#else - -static flb_sockfd_t forward_uds_get_conn(struct flb_forward_config *config, - struct flb_forward *ctx) -{ - (void) config; - (void) ctx; - - return -1; -} - -static void forward_uds_drop_conn(struct flb_forward *ctx, - flb_sockfd_t connection) -{ - (void) ctx; - (void) connection; -} - -static void forward_uds_drop_all(struct flb_forward *ctx) -{ - (void) ctx; -} - -#endif - -#ifdef FLB_HAVE_TLS - -static int io_net_write(struct flb_connection *conn, int unused_fd, - const void* data, size_t len, size_t *out_len) -{ - return flb_io_net_write(conn, data, len, out_len); -} - -static int io_net_read(struct flb_connection *conn, int unused_fd, - void* buf, size_t len) -{ - return flb_io_net_read(conn, buf, len); -} - -static int secure_forward_init(struct flb_forward *ctx, - struct flb_forward_config *fc) -{ - return 0; -} - -#endif - -static inline void print_msgpack_status(struct flb_forward *ctx, - int ret, char *context) -{ - switch (ret) { - case MSGPACK_UNPACK_EXTRA_BYTES: - flb_plg_error(ctx->ins, "%s MSGPACK_UNPACK_EXTRA_BYTES", context); - break; - case MSGPACK_UNPACK_CONTINUE: - flb_plg_trace(ctx->ins, "%s MSGPACK_UNPACK_CONTINUE", context); - break; - case MSGPACK_UNPACK_PARSE_ERROR: - flb_plg_error(ctx->ins, "%s MSGPACK_UNPACK_PARSE_ERROR", context); - break; - case MSGPACK_UNPACK_NOMEM_ERROR: - flb_plg_error(ctx->ins, "%s MSGPACK_UNPACK_NOMEM_ERROR", context); - break; - } -} - -/* Read a secure forward msgpack message */ -static int secure_forward_read(struct flb_forward *ctx, - struct flb_connection *u_conn, - struct flb_forward_config *fc, - char *buf, size_t size, size_t *out_len) -{ - int ret; - size_t off; - size_t avail; - size_t buf_off = 0; - msgpack_unpacked result; - - msgpack_unpacked_init(&result); - while (1) { - avail = size - buf_off; - if (avail < 1) { - goto error; - } - - /* Read the message */ - ret = fc->io_read(u_conn, fc->unix_fd, buf + buf_off, size - buf_off); - if (ret <= 0) { - goto error; - } - buf_off += ret; - - /* Validate */ - off = 0; - ret = msgpack_unpack_next(&result, buf, buf_off, &off); - switch (ret) { - case MSGPACK_UNPACK_SUCCESS: - msgpack_unpacked_destroy(&result); - *out_len = buf_off; - return 0; - default: - print_msgpack_status(ctx, ret, "handshake"); - goto error; - }; - } - - error: - msgpack_unpacked_destroy(&result); - return -1; -} - -static void secure_forward_set_ping(struct flb_forward_ping *ping, - msgpack_object *map) -{ - int i; - msgpack_object key; - msgpack_object val; - const char *ptr; - int len; - - memset(ping, 0, sizeof(struct flb_forward_ping)); - ping->keepalive = 1; /* default, as per spec */ - - for (i = 0; i < map->via.map.size; i++) { - key = map->via.map.ptr[i].key; - val = map->via.map.ptr[i].val; - - ptr = key.via.str.ptr; - len = key.via.str.size; - - if (len == 5 && memcmp(ptr, "nonce", len) == 0) { - ping->nonce = val.via.bin.ptr; - ping->nonce_len = val.via.bin.size; - } - else if (len == 4 && memcmp(ptr, "auth", len) == 0) { - ping->auth = val.via.bin.ptr; - ping->auth_len = val.via.bin.size; - } - else if (len == 9 && memcmp(ptr, "keepalive", len) == 0) { - ping->keepalive = val.via.boolean; - } - } -} - -static int secure_forward_hash_shared_key(struct flb_forward_config *fc, - struct flb_forward_ping *ping, - char *buf, int buflen) -{ - size_t length_entries[4]; - unsigned char *data_entries[4]; - uint8_t hash[64]; - int result; - - if (buflen < 128) { - return -1; - } - - data_entries[0] = (unsigned char *) fc->shared_key_salt; - length_entries[0] = 16; - - data_entries[1] = (unsigned char *) fc->self_hostname; - length_entries[1] = strlen(fc->self_hostname); - - data_entries[2] = (unsigned char *) ping->nonce; - length_entries[2] = ping->nonce_len; - - data_entries[3] = (unsigned char *) fc->shared_key; - length_entries[3] = strlen(fc->shared_key); - - result = flb_hash_simple_batch(FLB_HASH_SHA512, - 4, - data_entries, - length_entries, - hash, - sizeof(hash)); - - if (result != FLB_CRYPTO_SUCCESS) { - return -1; - } - - flb_forward_format_bin_to_hex(hash, 64, buf); - - return 0; -} - -static int secure_forward_hash_password(struct flb_forward_config *fc, - struct flb_forward_ping *ping, - char *buf, int buflen) -{ - size_t length_entries[3]; - unsigned char *data_entries[3]; - uint8_t hash[64]; - int result; - - if (buflen < 128) { - return -1; - } - - data_entries[0] = (unsigned char *) ping->auth; - length_entries[0] = ping->auth_len; - - data_entries[1] = (unsigned char *) fc->username; - length_entries[1] = strlen(fc->username); - - data_entries[2] = (unsigned char *) fc->password; - length_entries[2] = strlen(fc->password); - - result = flb_hash_simple_batch(FLB_HASH_SHA512, - 3, - data_entries, - length_entries, - hash, - sizeof(hash)); - - if (result != FLB_CRYPTO_SUCCESS) { - return -1; - } - - flb_forward_format_bin_to_hex(hash, 64, buf); - - return 0; -} - -static int secure_forward_ping(struct flb_connection *u_conn, - msgpack_object map, - struct flb_forward_config *fc, - struct flb_forward *ctx) -{ - int ret; - size_t bytes_sent; - char shared_key_hexdigest[128]; - char password_hexdigest[128]; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - struct flb_forward_ping ping; - - secure_forward_set_ping(&ping, &map); - - if (ping.nonce == NULL) { - flb_plg_error(ctx->ins, "nonce not found"); - return -1; - } - - if (secure_forward_hash_shared_key(fc, &ping, shared_key_hexdigest, 128)) { - flb_plg_error(ctx->ins, "failed to hash shared_key"); - return -1; - } - - if (ping.auth != NULL) { - if (secure_forward_hash_password(fc, &ping, password_hexdigest, 128)) { - flb_plg_error(ctx->ins, "failed to hash password"); - return -1; - } - } - - /* Prepare outgoing msgpack PING */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - msgpack_pack_array(&mp_pck, 6); - - /* [0] PING */ - msgpack_pack_str(&mp_pck, 4); - msgpack_pack_str_body(&mp_pck, "PING", 4); - - /* [1] Hostname */ - msgpack_pack_str(&mp_pck, flb_sds_len(fc->self_hostname)); - msgpack_pack_str_body(&mp_pck, fc->self_hostname, - flb_sds_len(fc->self_hostname)); - - /* [2] Shared key salt */ - msgpack_pack_str(&mp_pck, 16); - msgpack_pack_str_body(&mp_pck, fc->shared_key_salt, 16); - - /* [3] Shared key in Hexdigest format */ - msgpack_pack_str(&mp_pck, 128); - msgpack_pack_str_body(&mp_pck, shared_key_hexdigest, 128); - - /* [4] Username and password (optional) */ - if (ping.auth != NULL) { - msgpack_pack_str(&mp_pck, strlen(fc->username)); - msgpack_pack_str_body(&mp_pck, fc->username, strlen(fc->username)); - msgpack_pack_str(&mp_pck, 128); - msgpack_pack_str_body(&mp_pck, password_hexdigest, 128); - } - else { - msgpack_pack_str(&mp_pck, 0); - msgpack_pack_str_body(&mp_pck, "", 0); - msgpack_pack_str(&mp_pck, 0); - msgpack_pack_str_body(&mp_pck, "", 0); - } - - ret = fc->io_write(u_conn, fc->unix_fd, mp_sbuf.data, mp_sbuf.size, &bytes_sent); - flb_plg_debug(ctx->ins, "PING sent: ret=%i bytes sent=%lu", ret, bytes_sent); - - msgpack_sbuffer_destroy(&mp_sbuf); - - if (ret > -1 && bytes_sent > 0) { - return 0; - } - - return -1; -} - -static int secure_forward_pong(struct flb_forward *ctx, char *buf, int buf_size) -{ - int ret; - char msg[32] = {0}; - size_t off = 0; - msgpack_unpacked result; - msgpack_object root; - msgpack_object o; - - msgpack_unpacked_init(&result); - ret = msgpack_unpack_next(&result, buf, buf_size, &off); - if (ret != MSGPACK_UNPACK_SUCCESS) { - return -1; - } - - root = result.data; - if (root.type != MSGPACK_OBJECT_ARRAY) { - goto error; - } - - if (root.via.array.size < 4) { - goto error; - } - - o = root.via.array.ptr[0]; - if (o.type != MSGPACK_OBJECT_STR) { - goto error; - } - - if (strncmp(o.via.str.ptr, "PONG", 4) != 0 || o.via.str.size != 4) { - goto error; - } - - o = root.via.array.ptr[1]; - if (o.type != MSGPACK_OBJECT_BOOLEAN) { - goto error; - } - - if (o.via.boolean) { - msgpack_unpacked_destroy(&result); - return 0; - } - else { - o = root.via.array.ptr[2]; - memcpy(msg, o.via.str.ptr, o.via.str.size); - flb_plg_error(ctx->ins, "failed authorization: %s", msg); - } - - error: - msgpack_unpacked_destroy(&result); - return -1; -} - -static int secure_forward_handshake(struct flb_connection *u_conn, - struct flb_forward_config *fc, - struct flb_forward *ctx) -{ - int ret; - char buf[1024]; - size_t out_len; - size_t off; - msgpack_unpacked result; - msgpack_object root; - msgpack_object o; - - /* Wait for server HELO */ - ret = secure_forward_read(ctx, u_conn, fc, buf, sizeof(buf) - 1, &out_len); - if (ret == -1) { - flb_plg_error(ctx->ins, "handshake error expecting HELO"); - return -1; - } - - /* Unpack message and validate */ - off = 0; - msgpack_unpacked_init(&result); - ret = msgpack_unpack_next(&result, buf, out_len, &off); - if (ret != MSGPACK_UNPACK_SUCCESS) { - print_msgpack_status(ctx, ret, "HELO"); - return -1; - } - - /* Parse HELO message */ - root = result.data; - if (root.via.array.size < 2) { - flb_plg_error(ctx->ins, "Invalid HELO message"); - msgpack_unpacked_destroy(&result); - return -1; - } - - o = root.via.array.ptr[0]; - if (o.type != MSGPACK_OBJECT_STR) { - flb_plg_error(ctx->ins, "Invalid HELO type message"); - msgpack_unpacked_destroy(&result); - return -1; - } - - if (strncmp(o.via.str.ptr, "HELO", 4) != 0 || o.via.str.size != 4) { - flb_plg_error(ctx->ins, "Invalid HELO content message"); - msgpack_unpacked_destroy(&result); - return -1; - } - - flb_plg_debug(ctx->ins, "protocol: received HELO"); - - /* Compose and send PING message */ - o = root.via.array.ptr[1]; - ret = secure_forward_ping(u_conn, o, fc, ctx); - if (ret == -1) { - flb_plg_error(ctx->ins, "Failed PING"); - msgpack_unpacked_destroy(&result); - return -1; - } - - /* Expect a PONG */ - ret = secure_forward_read(ctx, u_conn, fc, buf, sizeof(buf) - 1, &out_len); - if (ret == -1) { - flb_plg_error(ctx->ins, "handshake error expecting HELO"); - msgpack_unpacked_destroy(&result); - return -1; - } - - /* Process PONG */ - ret = secure_forward_pong(ctx, buf, out_len); - if (ret == -1) { - msgpack_unpacked_destroy(&result); - return -1; - } - - msgpack_unpacked_destroy(&result); - return 0; -} - -static int forward_read_ack(struct flb_forward *ctx, - struct flb_forward_config *fc, - struct flb_connection *u_conn, - char *chunk, int chunk_len) -{ - int ret; - int i; - size_t out_len; - size_t off; - const char *ack; - size_t ack_len; - msgpack_unpacked result; - msgpack_object root; - msgpack_object_map map; - msgpack_object key; - msgpack_object val; - char buf[512]; /* ack should never be bigger */ - - flb_plg_trace(ctx->ins, "wait ACK (%.*s)", chunk_len, chunk); - - /* Wait for server ACK */ - ret = secure_forward_read(ctx, u_conn, fc, buf, sizeof(buf) - 1, &out_len); - if (ret == -1) { - flb_plg_error(ctx->ins, "cannot get ack"); - return -1; - } - - /* Unpack message and validate */ - off = 0; - msgpack_unpacked_init(&result); - ret = msgpack_unpack_next(&result, buf, out_len, &off); - if (ret != MSGPACK_UNPACK_SUCCESS) { - print_msgpack_status(ctx, ret, "ACK"); - goto error; - } - - /* Parse ACK message */ - root = result.data; - if (root.type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "ACK response not MAP (type:%d)", root.type); - goto error; - } - - map = root.via.map; - ack = NULL; - /* Lookup ack field */ - for (i = 0; i < map.size; i++) { - key = map.ptr[i].key; - if (key.via.str.size == 3 && strncmp(key.via.str.ptr, "ack", 3) == 0) { - val = map.ptr[i].val; - ack_len = val.via.str.size; - ack = val.via.str.ptr; - break; - } - } - - if (!ack) { - flb_plg_error(ctx->ins, "ack: ack not found"); - goto error; - } - - if (ack_len != chunk_len) { - flb_plg_error(ctx->ins, - "ack: ack len does not match ack(%ld)(%.*s) chunk(%d)(%.*s)", - ack_len, (int) ack_len, ack, - chunk_len, (int) chunk_len, chunk); - goto error; - } - - if (strncmp(ack, chunk, ack_len) != 0) { - flb_plg_error(ctx->ins, "ACK: mismatch received=%s, expected=(%.*s)", - ack, chunk_len, chunk); - goto error; - } - - flb_plg_debug(ctx->ins, "protocol: received ACK %.*s", (int)ack_len, ack); - msgpack_unpacked_destroy(&result); - return 0; - - error: - msgpack_unpacked_destroy(&result); - return -1; -} - - -static int forward_config_init(struct flb_forward_config *fc, - struct flb_forward *ctx) -{ - if (fc->io_read == NULL || fc->io_write == NULL) { - flb_plg_error(ctx->ins, "io_read/io_write is NULL"); - return -1; - } - -#ifdef FLB_HAVE_TLS - /* Initialize Secure Forward mode */ - if (fc->secured == FLB_TRUE) { - secure_forward_init(ctx, fc); - } -#endif - - /* Generate the shared key salt */ - if (flb_random_bytes(fc->shared_key_salt, 16)) { - flb_plg_error(ctx->ins, "cannot generate shared key salt"); - return -1; - } - - mk_list_add(&fc->_head, &ctx->configs); - return 0; -} - -static flb_sds_t config_get_property(char *prop, - struct flb_upstream_node *node, - struct flb_forward *ctx) -{ - if (node) { - return (flb_sds_t) flb_upstream_node_get_property(prop, node); - } - else { - return (flb_sds_t) flb_output_get_property(prop, ctx->ins); - } -} - -static int config_set_properties(struct flb_upstream_node *node, - struct flb_forward_config *fc, - struct flb_forward *ctx) -{ - flb_sds_t tmp; - - /* Shared Key */ - tmp = config_get_property("empty_shared_key", node, ctx); - if (tmp && flb_utils_bool(tmp)) { - fc->empty_shared_key = FLB_TRUE; - } - else { - fc->empty_shared_key = FLB_FALSE; - } - - tmp = config_get_property("shared_key", node, ctx); - if (fc->empty_shared_key) { - fc->shared_key = flb_sds_create(""); - } - else if (tmp) { - fc->shared_key = flb_sds_create(tmp); - } - else { - fc->shared_key = NULL; - } - - tmp = config_get_property("username", node, ctx); - if (tmp) { - fc->username = tmp; - } - else { - fc->username = ""; - } - - tmp = config_get_property("password", node, ctx); - if (tmp) { - fc->password = tmp; - } - else { - fc->password = ""; - } - - /* Self Hostname */ - tmp = config_get_property("self_hostname", node, ctx); - if (tmp) { - fc->self_hostname = flb_sds_create(tmp); - } - else { - fc->self_hostname = flb_sds_create("localhost"); - } - - /* Backward compatible timing mode */ - tmp = config_get_property("time_as_integer", node, ctx); - if (tmp) { - fc->time_as_integer = flb_utils_bool(tmp); - } - else { - fc->time_as_integer = FLB_FALSE; - } - - /* send always options (with size) */ - tmp = config_get_property("send_options", node, ctx); - if (tmp) { - fc->send_options = flb_utils_bool(tmp); - } - - /* add_option -> extra_options: if the user has defined 'add_option' - * we need to enable the 'send_options' flag - */ - if (fc->extra_options && mk_list_size(fc->extra_options) > 0) { - fc->send_options = FLB_TRUE; - } - - /* require ack response (implies send_options) */ - tmp = config_get_property("require_ack_response", node, ctx); - if (tmp) { - fc->require_ack_response = flb_utils_bool(tmp); - if (fc->require_ack_response) { - fc->send_options = FLB_TRUE; - } - } - - /* Tag Overwrite */ - tmp = config_get_property("tag", node, ctx); - if (tmp) { - /* Set the tag */ - fc->tag = flb_sds_create(tmp); - if (!fc->tag) { - flb_plg_error(ctx->ins, "cannot allocate tag"); - return -1; - } - -#ifdef FLB_HAVE_RECORD_ACCESSOR - /* Record Accessor */ - fc->ra_tag = flb_ra_create(fc->tag, FLB_TRUE); - if (!fc->ra_tag) { - flb_plg_error(ctx->ins, "cannot create record accessor for tag: %s", - fc->tag); - return -1; - } - - /* Static record accessor ? (no dynamic values from map) */ - fc->ra_static = flb_ra_is_static(fc->ra_tag); -#endif - } - else { - fc->tag = NULL; - - } - - /* compress (implies send_options) */ - tmp = config_get_property("compress", node, ctx); - if (tmp) { - if (!strcasecmp(tmp, "text")) { - fc->compress = COMPRESS_NONE; - } - else if (!strcasecmp(tmp, "gzip")) { - fc->compress = COMPRESS_GZIP; - fc->send_options = FLB_TRUE; - } - else { - flb_plg_error(ctx->ins, "invalid compress mode: %s", tmp); - return -1; - } - } - else { - fc->compress = COMPRESS_NONE; - } - - if (fc->compress != COMPRESS_NONE && fc->time_as_integer == FLB_TRUE) { - flb_plg_error(ctx->ins, "compress mode %s is incompatible with " - "time_as_integer", tmp); - return -1; - } - -#ifdef FLB_HAVE_RECORD_ACCESSOR - if (fc->compress != COMPRESS_NONE && - (fc->ra_tag && fc->ra_static == FLB_FALSE) ) { - flb_plg_error(ctx->ins, "compress mode %s is incompatible with dynamic " - "tags", tmp); - return -1; - } -#endif - - return 0; -} - -static void forward_config_destroy(struct flb_forward_config *fc) -{ - flb_sds_destroy(fc->shared_key); - flb_sds_destroy(fc->self_hostname); - flb_sds_destroy(fc->tag); - -#ifdef FLB_HAVE_RECORD_ACCESSOR - if (fc->ra_tag) { - flb_ra_destroy(fc->ra_tag); - } -#endif - - flb_free(fc); -} - -/* Configure in HA mode */ -static int forward_config_ha(const char *upstream_file, - struct flb_forward *ctx, - struct flb_config *config) -{ - int ret; - struct mk_list *head; - struct flb_upstream_node *node; - struct flb_forward_config *fc = NULL; - - ctx->ha_mode = FLB_TRUE; - ctx->ha = flb_upstream_ha_from_file(upstream_file, config); - if (!ctx->ha) { - flb_plg_error(ctx->ins, "cannot load Upstream file"); - return -1; - } - - /* Iterate nodes and create a forward_config context */ - mk_list_foreach(head, &ctx->ha->nodes) { - node = mk_list_entry(head, struct flb_upstream_node, _head); - - /* create forward_config context */ - fc = flb_calloc(1, sizeof(struct flb_forward_config)); - if (!fc) { - flb_errno(); - flb_plg_error(ctx->ins, "failed config allocation"); - continue; - } - fc->unix_fd = -1; - fc->secured = FLB_FALSE; - fc->io_write = io_net_write; - fc->io_read = io_net_read; - - /* Is TLS enabled ? */ - if (node->tls_enabled == FLB_TRUE) { - fc->secured = FLB_TRUE; - } - - /* Read properties into 'fc' context */ - config_set_properties(node, fc, ctx); - - /* Initialize and validate forward_config context */ - ret = forward_config_init(fc, ctx); - if (ret == -1) { - if (fc) { - forward_config_destroy(fc); - } - return -1; - } - - /* Set our forward_config context into the node */ - flb_upstream_node_set_data(fc, node); - } - - flb_output_upstream_ha_set(ctx->ha, ctx->ins); - - return 0; -} - -static int forward_config_simple(struct flb_forward *ctx, - struct flb_output_instance *ins, - struct flb_config *config) -{ - int ret; - int io_flags; - struct flb_forward_config *fc = NULL; - struct flb_upstream *upstream; - - /* Set default network configuration if not set */ - flb_output_net_default("127.0.0.1", 24224, ins); - - /* Configuration context */ - fc = flb_calloc(1, sizeof(struct flb_forward_config)); - if (!fc) { - flb_errno(); - return -1; - } - fc->unix_fd = -1; - fc->secured = FLB_FALSE; - fc->io_write = NULL; - fc->io_read = NULL; - - /* Set default values */ - ret = flb_output_config_map_set(ins, fc); - if (ret == -1) { - flb_free(fc); - return -1; - } - - /* Check if TLS is enabled */ -#ifdef FLB_HAVE_TLS - if (ins->use_tls == FLB_TRUE) { - io_flags = FLB_IO_TLS; - fc->secured = FLB_TRUE; - } - else { - io_flags = FLB_IO_TCP; - } -#else - io_flags = FLB_IO_TCP; -#endif - - if (ins->host.ipv6 == FLB_TRUE) { - io_flags |= FLB_IO_IPV6; - } - - if (fc->unix_path) { -#ifdef FLB_HAVE_UNIX_SOCKET - /* In older versions if the UDS server was not up - * at this point fluent-bit would fail because it - * would not be able to establish the conntection. - * - * With the concurrency fixes we moved the connection - * to a later stage which will cause fluent-bit to - * properly launch but if the UDS server is not - * available at flush time then an error similar to - * the one we would get for a network based output - * plugin will be logged and FLB_RETRY will be returned. - */ - - fc->io_write = io_unix_write; - fc->io_read = io_unix_read; -#else - flb_plg_error(ctx->ins, "unix_path is not supported"); - flb_free(fc); - flb_free(ctx); - return -1; -#endif /* FLB_HAVE_UNIX_SOCKET */ - } - else { - /* Prepare an upstream handler */ - upstream = flb_upstream_create(config, - ins->host.name, - ins->host.port, - io_flags, ins->tls); - if (!upstream) { - flb_free(fc); - flb_free(ctx); - return -1; - } - fc->io_write = io_net_write; - fc->io_read = io_net_read; - ctx->u = upstream; - flb_output_upstream_set(ctx->u, ins); - } - /* Read properties into 'fc' context */ - config_set_properties(NULL, fc, ctx); - - /* Initialize and validate forward_config context */ - ret = forward_config_init(fc, ctx); - if (ret == -1) { - if (fc) { - forward_config_destroy(fc); - } - return -1; - } - - return 0; -} - -static int cb_forward_init(struct flb_output_instance *ins, - struct flb_config *config, void *data) -{ - int ret; - const char *tmp; - struct flb_forward *ctx; - (void) data; - - ctx = flb_calloc(1, sizeof(struct flb_forward)); - if (!ctx) { - flb_errno(); - return -1; - } - - ret = pthread_once(&uds_connection_tls_slot_init_once_control, - initialize_uds_connection_tls_slot); - - if (ret != 0) { - flb_errno(); - flb_free(ctx); - - return -1; - } - - ret = pthread_mutex_init(&ctx->uds_connection_list_mutex, NULL); - - if (ret != 0) { - flb_errno(); - flb_free(ctx); - - return -1; - } - - cfl_list_init(&ctx->uds_connection_list); - - ctx->ins = ins; - mk_list_init(&ctx->configs); - flb_output_set_context(ins, ctx); - - - /* Configure HA or simple mode ? */ - tmp = flb_output_get_property("upstream", ins); - if (tmp) { - ret = forward_config_ha(tmp, ctx, config); - } - else { - ret = forward_config_simple(ctx, ins, config); - } - - return ret; -} - -struct flb_forward_config *flb_forward_target(struct flb_forward *ctx, - struct flb_upstream_node **node) -{ - struct flb_forward_config *fc = NULL; - struct flb_upstream_node *f_node; - - if (ctx->ha_mode == FLB_TRUE) { - f_node = flb_upstream_ha_node_get(ctx->ha); - if (!f_node) { - return NULL; - } - - /* Get forward_config stored in node opaque data */ - fc = flb_upstream_node_get_data(f_node); - *node = f_node; - } - else { - fc = mk_list_entry_first(&ctx->configs, - struct flb_forward_config, - _head); - *node = NULL; - } - return fc; -} - -static int flush_message_mode(struct flb_forward *ctx, - struct flb_forward_config *fc, - struct flb_connection *u_conn, - char *buf, size_t size) -{ - int ret; - int ok = MSGPACK_UNPACK_SUCCESS; - size_t sent = 0; - size_t rec_size; - size_t pre = 0; - size_t off = 0; - msgpack_object root; - msgpack_object options; - msgpack_object chunk; - msgpack_unpacked result; - - /* If the sender requires 'ack' from the remote end-point */ - if (fc->require_ack_response) { - msgpack_unpacked_init(&result); - while (msgpack_unpack_next(&result, buf, size, &off) == ok) { - /* get the record size */ - rec_size = off - pre; - - /* write single message */ - ret = fc->io_write(u_conn,fc->unix_fd, - buf + pre, rec_size, &sent); - pre = off; - - if (ret == -1) { - /* - * FIXME: we might take advantage of 'flush_ctx' and store the - * message that failed it delivery, we could have retries but with - * the flush context. - */ - flb_plg_error(ctx->ins, "message_mode: error sending message"); - msgpack_unpacked_destroy(&result); - return FLB_RETRY; - } - - /* Sucessful delivery, now get message 'chunk' and wait for it */ - root = result.data; - options = root.via.array.ptr[3]; - chunk = options.via.map.ptr[0].val; - - /* Read ACK */ - ret = forward_read_ack(ctx, fc, u_conn, - (char *) chunk.via.str.ptr, chunk.via.str.size); - if (ret == -1) { - msgpack_unpacked_destroy(&result); - return FLB_RETRY; - } - } - - /* All good */ - msgpack_unpacked_destroy(&result); - return FLB_OK; - } - - /* Normal data write */ - ret = fc->io_write(u_conn, fc->unix_fd, buf, size, &sent); - if (ret == -1) { - flb_plg_error(ctx->ins, "message_mode: error sending data"); - return FLB_RETRY; - } - - return FLB_OK; -} - -/* pack payloads of cmetrics or ctraces with Fluentd compat format */ -static int pack_metricses_payload(msgpack_packer *mp_pck, const void *data, size_t bytes) { - int entries; - struct flb_time tm; - - /* Format with event stream format of entries: [[time, [{entries map}]]] */ - msgpack_pack_array(mp_pck, 1); - msgpack_pack_array(mp_pck, 2); - flb_time_get(&tm); - flb_time_append_to_msgpack(&tm, mp_pck, 0); - entries = flb_mp_count(data, bytes); - msgpack_pack_array(mp_pck, entries); - - return 0; -} - -#include -/* - * Forward Mode: this is the generic mechanism used in Fluent Bit, it takes - * advantage of the internal data representation and avoid re-formatting data, - * it only sends a msgpack header, pre-existent 'data' records and options. - * - * note: if the user has enabled time_as_integer (compat mode for Fluentd <= 0.12), - * the 'flush_forward_compat_mode' is used instead. - */ -static int flush_forward_mode(struct flb_forward *ctx, - struct flb_forward_config *fc, - struct flb_connection *u_conn, - int event_type, - const char *tag, int tag_len, - const void *data, size_t bytes, - char *opts_buf, size_t opts_size) -{ - int ret; - int entries; - int send_options; - size_t off = 0; - size_t bytes_sent; - msgpack_object root; - msgpack_object chunk; - msgpack_unpacked result; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - void *final_data; - size_t final_bytes; - char *transcoded_buffer; - size_t transcoded_length; - - transcoded_buffer = NULL; - transcoded_length = 0; - - /* Pack message header */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - send_options = fc->send_options; - if (event_type == FLB_EVENT_TYPE_METRICS || event_type == FLB_EVENT_TYPE_TRACES) { - send_options = FLB_TRUE; - } - msgpack_pack_array(&mp_pck, send_options ? 3 : 2); - - /* Tag */ - flb_forward_format_append_tag(ctx, fc, &mp_pck, NULL, tag, tag_len); - - if (!fc->fwd_retain_metadata && event_type == FLB_EVENT_TYPE_LOGS) { - ret = flb_forward_format_transcode(ctx, FLB_LOG_EVENT_FORMAT_FORWARD, - (char *) data, bytes, - &transcoded_buffer, - &transcoded_length); - - if (ret != 0) { - flb_plg_error(ctx->ins, "could not transcode entries"); - msgpack_sbuffer_destroy(&mp_sbuf); - return FLB_RETRY; - } - } - - if (fc->compress == COMPRESS_GZIP) { - /* When compress is set, we switch from using Forward mode to using - * CompressedPackedForward mode. - */ - - if (transcoded_buffer != NULL) { - ret = flb_gzip_compress((void *) transcoded_buffer, - transcoded_length, - &final_data, - &final_bytes); - } - else { - ret = flb_gzip_compress((void *) data, bytes, &final_data, &final_bytes); - } - - if (ret == -1) { - flb_plg_error(ctx->ins, "could not compress entries"); - msgpack_sbuffer_destroy(&mp_sbuf); - - if (transcoded_buffer != NULL) { - flb_free(transcoded_buffer); - } - - return FLB_RETRY; - } - - msgpack_pack_bin(&mp_pck, final_bytes); - } - else { - if (transcoded_buffer != NULL) { - final_data = (void *) transcoded_buffer; - final_bytes = transcoded_length; - } - else { - final_data = (void *) data; - final_bytes = bytes; - } - - if (event_type == FLB_EVENT_TYPE_LOGS) { - /* for log events we create an array for the serialized messages */ - entries = flb_mp_count(data, bytes); - msgpack_pack_array(&mp_pck, entries); - } - else { - /* FLB_EVENT_TYPE_METRICS and FLB_EVENT_TYPE_TRACES */ - if (fc->fluentd_compat) { - pack_metricses_payload(&mp_pck, data, bytes); - } - else { - msgpack_pack_bin(&mp_pck, final_bytes); - } - } - } - - /* Write message header */ - ret = fc->io_write(u_conn, fc->unix_fd, mp_sbuf.data, mp_sbuf.size, &bytes_sent); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not write forward header"); - msgpack_sbuffer_destroy(&mp_sbuf); - if (fc->compress == COMPRESS_GZIP) { - flb_free(final_data); - } - - if (transcoded_buffer != NULL) { - flb_free(transcoded_buffer); - } - - return FLB_RETRY; - } - msgpack_sbuffer_destroy(&mp_sbuf); - - /* Write msgpack content / entries */ - ret = fc->io_write(u_conn, fc->unix_fd, final_data, final_bytes, &bytes_sent); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not write forward entries"); - if (fc->compress == COMPRESS_GZIP) { - flb_free(final_data); - } - - if (transcoded_buffer != NULL) { - flb_free(transcoded_buffer); - } - - return FLB_RETRY; - } - - if (fc->compress == COMPRESS_GZIP) { - flb_free(final_data); - } - - if (transcoded_buffer != NULL) { - flb_free(transcoded_buffer); - } - - /* Write options */ - if (send_options == FLB_TRUE) { - ret = fc->io_write(u_conn, fc->unix_fd, opts_buf, opts_size, &bytes_sent); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not write forward options"); - return FLB_RETRY; - } - } - - /* If the sender requires 'ack' from the remote end-point */ - if (fc->require_ack_response) { - msgpack_unpacked_init(&result); - ret = msgpack_unpack_next(&result, opts_buf, opts_size, &off); - if (ret != MSGPACK_UNPACK_SUCCESS) { - msgpack_unpacked_destroy(&result); - return -1; - } - - /* Sucessful delivery, now get message 'chunk' and wait for it */ - root = result.data; - - /* 'chunk' is always in the first key of the map */ - chunk = root.via.map.ptr[0].val; - - /* Read ACK */ - ret = forward_read_ack(ctx, fc, u_conn, - (char *) chunk.via.str.ptr, chunk.via.str.size); - if (ret == -1) { - msgpack_unpacked_destroy(&result); - return FLB_RETRY; - } - - /* All good */ - msgpack_unpacked_destroy(&result); - return FLB_OK; - } - - return FLB_OK; -} - -/* - * Forward Mode Compat: data is packaged in Forward mode but the timestamps are - * integers (compat mode for Fluentd <= 0.12). - */ -static int flush_forward_compat_mode(struct flb_forward *ctx, - struct flb_forward_config *fc, - struct flb_connection *u_conn, - const char *tag, int tag_len, - const void *data, size_t bytes) -{ - int ret; - size_t off = 0; - size_t bytes_sent; - msgpack_object root; - msgpack_object chunk; - msgpack_object map; /* dummy parameter */ - msgpack_unpacked result; - - /* Write message header */ - ret = fc->io_write(u_conn, fc->unix_fd, data, bytes, &bytes_sent); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not write forward compat mode records"); - return FLB_RETRY; - } - - /* If the sender requires 'ack' from the remote end-point */ - if (fc->require_ack_response) { - msgpack_unpacked_init(&result); - ret = msgpack_unpack_next(&result, data, bytes, &off); - if (ret != MSGPACK_UNPACK_SUCCESS) { - msgpack_unpacked_destroy(&result); - return -1; - } - - /* Sucessful delivery, now get message 'chunk' and wait for it */ - root = result.data; - - map = root.via.array.ptr[2]; - - /* 'chunk' is always in the first key of the map */ - chunk = map.via.map.ptr[0].val; - - /* Read ACK */ - ret = forward_read_ack(ctx, fc, u_conn, - (char *) chunk.via.str.ptr, chunk.via.str.size); - if (ret == -1) { - msgpack_unpacked_destroy(&result); - return FLB_RETRY; - } - - /* All good */ - msgpack_unpacked_destroy(&result); - return FLB_OK; - } - - return FLB_OK; -} - -static void cb_forward_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - int ret = -1; - int mode; - msgpack_packer mp_pck; - msgpack_sbuffer mp_sbuf; - void *out_buf = NULL; - size_t out_size = 0; - struct flb_forward *ctx = out_context; - struct flb_forward_config *fc = NULL; - struct flb_connection *u_conn = NULL; - struct flb_upstream_node *node = NULL; - struct flb_forward_flush *flush_ctx; - flb_sockfd_t uds_conn; - - (void) i_ins; - (void) config; - - fc = flb_forward_target(ctx, &node); - if (!fc) { - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - flb_plg_debug(ctx->ins, "request %lu bytes to flush", - event_chunk->size); - - /* Initialize packager */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - /* - * Flush context: structure used to pass custom information to the - * formatter function. - */ - flush_ctx = flb_calloc(1, sizeof(struct flb_forward_flush)); - if (!flush_ctx) { - flb_errno(); - msgpack_sbuffer_destroy(&mp_sbuf); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - flush_ctx->fc = fc; - - /* Format the right payload and retrieve the 'forward mode' used */ - mode = flb_forward_format(config, i_ins, ctx, flush_ctx, - event_chunk->type, - event_chunk->tag, flb_sds_len(event_chunk->tag), - event_chunk->data, event_chunk->size, - &out_buf, &out_size); - - /* Get a TCP connection instance */ - if (fc->unix_path == NULL) { - if (ctx->ha_mode == FLB_TRUE) { - u_conn = flb_upstream_conn_get(node->u); - } - else { - u_conn = flb_upstream_conn_get(ctx->u); - } - - if (!u_conn) { - flb_plg_error(ctx->ins, "no upstream connections available"); - msgpack_sbuffer_destroy(&mp_sbuf); - if (fc->time_as_integer == FLB_TRUE) { - flb_free(out_buf); - } - flb_free(flush_ctx); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - uds_conn = -1; - } - else { - uds_conn = forward_uds_get_conn(fc, ctx); - - if (uds_conn == -1) { - flb_plg_error(ctx->ins, "no unix socket connection available"); - - msgpack_sbuffer_destroy(&mp_sbuf); - if (fc->time_as_integer == FLB_TRUE) { - flb_free(out_buf); - } - flb_free(flush_ctx); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* This is a hack, because the rest of the code is written to use - * the shared forward config unix_fd field so at this point we need - * to ensure that we either have a working connection or we can - * establish one regardless of not passing it along. - * - * Later on we will get the file descriptor from the TLS. - */ - } - - /* - * Shared Key: if ka_count > 0 it means the handshake has already been done lately - */ - if (fc->shared_key && u_conn->ka_count == 0) { - ret = secure_forward_handshake(u_conn, fc, ctx); - flb_plg_debug(ctx->ins, "handshake status = %i", ret); - if (ret == -1) { - if (u_conn) { - flb_upstream_conn_release(u_conn); - } - - if (uds_conn != -1) { - forward_uds_drop_conn(ctx, uds_conn); - } - - msgpack_sbuffer_destroy(&mp_sbuf); - if (fc->time_as_integer == FLB_TRUE) { - flb_free(out_buf); - } - flb_free(flush_ctx); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - } - - /* - * Note about the mode used for different type of events/messages: - * - * - Logs can be send either by using MODE_MESSAGE, MODE_FORWARD - * OR MODE_FORWARD_COMPAT. - * - * - Metrics and Traces uses MODE_FORWARD only. - */ - - if (mode == MODE_MESSAGE) { - ret = flush_message_mode(ctx, fc, u_conn, out_buf, out_size); - flb_free(out_buf); - } - else if (mode == MODE_FORWARD) { - ret = flush_forward_mode(ctx, fc, u_conn, - event_chunk->type, - event_chunk->tag, flb_sds_len(event_chunk->tag), - event_chunk->data, event_chunk->size, - out_buf, out_size); - flb_free(out_buf); - } - else if (mode == MODE_FORWARD_COMPAT) { - ret = flush_forward_compat_mode(ctx, fc, u_conn, - event_chunk->tag, - flb_sds_len(event_chunk->tag), - out_buf, out_size); - flb_free(out_buf); - } - - if (u_conn) { - flb_upstream_conn_release(u_conn); - } - - if (ret != FLB_OK) { - /* Since UDS connections have been used as permanent - * connections up to this point we only release the - * connection in case of error. - * - * There could be a logical error in here but what - * I think at the moment is, if something goes wrong - * we can just drop the connection and let the worker - * establish a new one the next time a flush happens. - */ - - if (uds_conn != -1) { - forward_uds_drop_conn(ctx, uds_conn); - } - } - - flb_free(flush_ctx); - FLB_OUTPUT_RETURN(ret); -} - -static int cb_forward_exit(void *data, struct flb_config *config) -{ - struct flb_forward *ctx = data; - struct flb_forward_config *fc; - struct mk_list *head; - struct mk_list *tmp; - (void) config; - - if (!ctx) { - return 0; - } - - /* Destroy forward_config contexts */ - mk_list_foreach_safe(head, tmp, &ctx->configs) { - fc = mk_list_entry(head, struct flb_forward_config, _head); - - mk_list_del(&fc->_head); - forward_config_destroy(fc); - } - - forward_uds_drop_all(ctx); - - if (ctx->ha_mode == FLB_TRUE) { - if (ctx->ha) { - flb_upstream_ha_destroy(ctx->ha); - } - } - else { - if (ctx->u) { - flb_upstream_destroy(ctx->u); - } - } - - pthread_mutex_destroy(&ctx->uds_connection_list_mutex); - - flb_free(ctx); - - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_BOOL, "time_as_integer", "false", - 0, FLB_TRUE, offsetof(struct flb_forward_config, time_as_integer), - "Set timestamp in integer format (compat mode for old Fluentd v0.12)" - }, - { - FLB_CONFIG_MAP_BOOL, "retain_metadata_in_forward_mode", "false", - 0, FLB_TRUE, offsetof(struct flb_forward_config, fwd_retain_metadata), - "Retain metadata when operating in forward mode" - }, - { - FLB_CONFIG_MAP_STR, "shared_key", NULL, - 0, FLB_FALSE, 0, - "Shared key for authentication" - }, - { - FLB_CONFIG_MAP_STR, "self_hostname", NULL, - 0, FLB_FALSE, 0, - "Hostname" - }, - { - FLB_CONFIG_MAP_BOOL, "empty_shared_key", "false", - 0, FLB_TRUE, offsetof(struct flb_forward_config, empty_shared_key), - "Set an empty shared key for authentication" - }, - { - FLB_CONFIG_MAP_BOOL, "send_options", "false", - 0, FLB_TRUE, offsetof(struct flb_forward_config, send_options), - "Send 'forward protocol options' to remote endpoint" - }, - { - FLB_CONFIG_MAP_BOOL, "require_ack_response", "false", - 0, FLB_TRUE, offsetof(struct flb_forward_config, require_ack_response), - "Require that remote endpoint confirms data reception" - }, - { - FLB_CONFIG_MAP_STR, "username", "", - 0, FLB_TRUE, offsetof(struct flb_forward_config, username), - "Username for authentication" - }, - { - FLB_CONFIG_MAP_STR, "password", "", - 0, FLB_TRUE, offsetof(struct flb_forward_config, password), - "Password for authentication" - }, - { - FLB_CONFIG_MAP_STR, "unix_path", NULL, - 0, FLB_TRUE, offsetof(struct flb_forward_config, unix_path), - "Path to unix socket. It is ignored when 'upstream' property is set" - }, - { - FLB_CONFIG_MAP_STR, "upstream", NULL, - 0, FLB_FALSE, 0, - "Path to 'upstream' configuration file (define multiple nodes)" - }, - { - FLB_CONFIG_MAP_STR, "tag", NULL, - 0, FLB_FALSE, 0, - "Set a custom Tag for the outgoing records" - }, - { - FLB_CONFIG_MAP_STR, "compress", NULL, - 0, FLB_FALSE, 0, - "Compression mode" - }, - { - FLB_CONFIG_MAP_BOOL, "fluentd_compat", "false", - 0, FLB_TRUE, offsetof(struct flb_forward_config, fluentd_compat), - "Send metrics and traces with Fluentd compatible format" - }, - - { - FLB_CONFIG_MAP_SLIST_2, "add_option", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct flb_forward_config, extra_options), - "Set an extra Forward protocol option. This is an advance feature, use it only for " - "very specific use-cases." - }, - - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_output_plugin out_forward_plugin = { - .name = "forward", - .description = "Forward (Fluentd protocol)", - - /* Callbacks */ - .cb_init = cb_forward_init, - .cb_pre_run = NULL, - .cb_flush = cb_forward_flush, - .cb_exit = cb_forward_exit, - .workers = 2, - - /* Config map validator */ - .config_map = config_map, - - /* Test */ - .test_formatter.callback = flb_forward_format, - - /* Flags */ - .flags = FLB_OUTPUT_NET | FLB_IO_OPT_TLS, - - /* Event types */ - .event_type = FLB_OUTPUT_LOGS | FLB_OUTPUT_METRICS | FLB_OUTPUT_TRACES -}; diff --git a/fluent-bit/plugins/out_forward/forward.h b/fluent-bit/plugins/out_forward/forward.h deleted file mode 100644 index 8e77e6e11..000000000 --- a/fluent-bit/plugins/out_forward/forward.h +++ /dev/null @@ -1,146 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_FORWARD -#define FLB_OUT_FORWARD - -#include -#include -#include -#include -#include -#include -#include - -/* - * Forward modes - * ============= - */ - -/* - * Message mode - * ------------ - * https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1#message-modes - */ -#define MODE_MESSAGE 0 - -/* - * Forward mode - * ------------ - * https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1#forward-mode - */ -#define MODE_FORWARD 1 - -/* - * Forward Compat: similar to MODE_FORWARD, but it sends the timestamps as unsigned - * integers for compatibility with very old versions of Fluentd that don't have timestamps - * with nanoseconds. This mode only applies for Logs. - */ -#define MODE_FORWARD_COMPAT 3 - -/* Compression options */ -#define COMPRESS_NONE 0 -#define COMPRESS_GZIP 1 - -/* - * Configuration: we put this separate from the main - * context so every Upstream Node can have it own configuration - * reference and pass it smoothly to the required caller. - * - * On simple mode (no HA), the structure is referenced - * by flb_forward->config. In HA mode the structure is referenced - * by the Upstream node context as an opaque data type. - */ -struct flb_forward_config { - int secured; /* Using Secure Forward mode ? */ - int compress; /* Using compression ? */ - int time_as_integer; /* Use backward compatible timestamp ? */ - int fluentd_compat; /* Use Fluentd compatible payload for - * metrics and ctraces */ - - /* add extra options to the Forward payload (advanced) */ - struct mk_list *extra_options; - - int fwd_retain_metadata; /* Do not drop metadata in forward mode */ - - /* config */ - flb_sds_t shared_key; /* shared key */ - flb_sds_t self_hostname; /* hostname used in certificate */ - flb_sds_t tag; /* Overwrite tag on forward */ - int empty_shared_key; /* use an empty string as shared key */ - int require_ack_response; /* Require acknowledge for "chunk" */ - int send_options; /* send options in messages */ - flb_sds_t unix_path; /* unix socket path */ - int unix_fd; - - const char *username; - const char *password; - - /* mbedTLS specifics */ - unsigned char shared_key_salt[16]; - -#ifdef FLB_HAVE_RECORD_ACCESSOR - struct flb_record_accessor *ra_tag; /* Tag Record accessor */ - int ra_static; /* Is the record accessor static ? */ -#endif - int (*io_write)(struct flb_connection* conn, int fd, const void* data, - size_t len, size_t *out_len); - int (*io_read)(struct flb_connection* conn, int fd, void* buf, size_t len); - struct mk_list _head; /* Link to list flb_forward->configs */ -}; - -struct flb_forward_uds_connection { - flb_sockfd_t descriptor; - struct cfl_list _head; /* Link to list flb_forward->uds_connnection_list */ -}; - -/* Plugin Context */ -struct flb_forward { - /* if HA mode is enabled */ - int ha_mode; /* High Availability mode enabled ? */ - char *ha_upstream; /* Upstream configuration file */ - struct flb_upstream_ha *ha; - - struct cfl_list uds_connection_list; - pthread_mutex_t uds_connection_list_mutex; - - /* Upstream handler and config context for single mode (no HA) */ - struct flb_upstream *u; - struct mk_list configs; - struct flb_output_instance *ins; -}; - -struct flb_forward_ping { - const char *nonce; - int nonce_len; - const char *auth; - int auth_len; - int keepalive; -}; - -/* Flush callback context */ -struct flb_forward_flush { - struct flb_forward_config *fc; - char checksum_hex[33]; -}; - -struct flb_forward_config *flb_forward_target(struct flb_forward *ctx, - struct flb_upstream_node **node); - -#endif diff --git a/fluent-bit/plugins/out_forward/forward_format.c b/fluent-bit/plugins/out_forward/forward_format.c deleted file mode 100644 index 48dedd862..000000000 --- a/fluent-bit/plugins/out_forward/forward_format.c +++ /dev/null @@ -1,640 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "forward.h" - -void flb_forward_format_bin_to_hex(uint8_t *buf, size_t len, char *out) -{ - int i; - static char map[] = "0123456789abcdef"; - - for (i = 0; i < len; i++) { - out[i * 2] = map[buf[i] >> 4]; - out[i * 2 + 1] = map[buf[i] & 0x0f]; - } -} - -int flb_forward_format_append_tag(struct flb_forward *ctx, - struct flb_forward_config *fc, - msgpack_packer *mp_pck, - msgpack_object *map, - const char *tag, int tag_len) -{ -#ifdef FLB_HAVE_RECORD_ACCESSOR - flb_sds_t tmp; - msgpack_object m; - - memset(&m, 0, sizeof(m)); - - if (!fc->ra_tag) { - msgpack_pack_str(mp_pck, tag_len); - msgpack_pack_str_body(mp_pck, tag, tag_len); - return 0; - } - - if (map) { - m = *map; - } - - /* Tag */ - tmp = flb_ra_translate(fc->ra_tag, (char *) tag, tag_len, m, NULL); - if (!tmp) { - flb_plg_warn(ctx->ins, "Tag translation failed, using default Tag"); - msgpack_pack_str(mp_pck, tag_len); - msgpack_pack_str_body(mp_pck, tag, tag_len); - } - else { - msgpack_pack_str(mp_pck, flb_sds_len(tmp)); - msgpack_pack_str_body(mp_pck, tmp, flb_sds_len(tmp)); - flb_sds_destroy(tmp); - } -#else - msgpack_pack_str(mp_pck, tag_len); - msgpack_pack_str_body(mp_pck, tag, tag_len); - -#endif - - return 0; -} - -static int append_options(struct flb_forward *ctx, - struct flb_forward_config *fc, - int event_type, - msgpack_packer *mp_pck, - int entries, void *data, size_t bytes, - msgpack_object *metadata, - char *out_chunk) -{ - char *chunk = NULL; - uint8_t checksum[64]; - int result; - struct mk_list *head; - struct flb_config_map_val *mv; - struct flb_mp_map_header mh; - struct flb_slist_entry *eopt_key; - struct flb_slist_entry *eopt_val; - - /* options is map, use the dynamic map type */ - flb_mp_map_header_init(&mh, mp_pck); - - if (fc->require_ack_response == FLB_TRUE) { - /* - * for ack we calculate sha512 of context, take 16 bytes, - * make 32 byte hex string of it - */ - result = flb_hash_simple(FLB_HASH_SHA512, - data, bytes, - checksum, sizeof(checksum)); - - if (result != FLB_CRYPTO_SUCCESS) { - return -1; - } - - flb_forward_format_bin_to_hex(checksum, 16, out_chunk); - - out_chunk[32] = '\0'; - chunk = (char *) out_chunk; - } - - /* "chunk": '' */ - if (chunk) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, 5); - msgpack_pack_str_body(mp_pck, "chunk", 5); - msgpack_pack_str(mp_pck, 32); - msgpack_pack_str_body(mp_pck, out_chunk, 32); - } - - /* "size": entries */ - if (entries > 0) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, 4); - msgpack_pack_str_body(mp_pck, "size", 4); - msgpack_pack_int64(mp_pck, entries); - } - - /* "compressed": "gzip" */ - if (entries > 0 && /* not message mode */ - fc->time_as_integer == FLB_FALSE && /* not compat mode */ - fc->compress == COMPRESS_GZIP) { - - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, 10); - msgpack_pack_str_body(mp_pck, "compressed", 10); - msgpack_pack_str(mp_pck, 4); - msgpack_pack_str_body(mp_pck, "gzip", 4); - } - - /* event type (FLB_EVENT_TYPE_LOGS, FLB_EVENT_TYPE_METRICS, FLB_EVENT_TYPE_TRACES) */ - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, 13); - msgpack_pack_str_body(mp_pck, "fluent_signal", 13); - msgpack_pack_int64(mp_pck, event_type); - - /* process 'extra_option(s)' */ - if (fc->extra_options) { - flb_config_map_foreach(head, mv, fc->extra_options) { - eopt_key = mk_list_entry_first(mv->val.list, struct flb_slist_entry, _head); - eopt_val = mk_list_entry_last(mv->val.list, struct flb_slist_entry, _head); - - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, flb_sds_len(eopt_key->str)); - msgpack_pack_str_body(mp_pck, eopt_key->str, flb_sds_len(eopt_key->str)); - msgpack_pack_str(mp_pck, flb_sds_len(eopt_val->str)); - msgpack_pack_str_body(mp_pck, eopt_val->str, flb_sds_len(eopt_val->str)); - } - } - - if (metadata != NULL && - metadata->type == MSGPACK_OBJECT_MAP && - metadata->via.map.size > 0) { - flb_mp_map_header_append(&mh); - msgpack_pack_str_with_body(mp_pck, "metadata", 8); - msgpack_pack_object(mp_pck, *metadata); - } - - flb_mp_map_header_end(&mh); - - flb_plg_debug(ctx->ins, - "send options records=%d chunk='%s'", - entries, out_chunk ? out_chunk : "NULL"); - return 0; -} - -#ifdef FLB_HAVE_RECORD_ACCESSOR -/* - * Forward Protocol: Message Mode - * ------------------------------ - * This mode is only used if the Tag is dynamically composed using some - * content of the records. - * - * [ - * "TAG", - * TIMESTAMP, - * RECORD/MAP, - * *OPTIONS* - * ] - * - */ -static int flb_forward_format_message_mode(struct flb_forward *ctx, - struct flb_forward_config *fc, - struct flb_forward_flush *ff, - const char *tag, int tag_len, - const void *data, size_t bytes, - void **out_buf, size_t *out_size) -{ - int entries = 0; - size_t pre = 0; - size_t off = 0; - size_t record_size; - char *chunk; - char chunk_buf[33]; - msgpack_packer mp_pck; - msgpack_sbuffer mp_sbuf; - struct flb_time tm; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - int ret; - - /* - * Our only reason to use Message Mode is because the user wants to generate - * dynamic Tags based on records content. - */ - if (!fc->ra_tag) { - return -1; - } - - /* - * if the case, we need to compose a new outgoing buffer instead - * of use the original one. - */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return -1; - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - flb_time_copy(&tm, &log_event.timestamp); - - /* Prepare main array: tag, timestamp and record/map */ - msgpack_pack_array(&mp_pck, 4); - - /* Generate dynamic Tag or use default one */ - flb_forward_format_append_tag(ctx, fc, &mp_pck, - log_event.body, - tag, tag_len); - - /* Pack timestamp */ - if (fc->time_as_integer == FLB_TRUE) { - flb_time_append_to_msgpack(&log_event.timestamp, - &mp_pck, - FLB_TIME_ETFMT_INT); - } - else { - flb_time_append_to_msgpack(&log_event.timestamp, - &mp_pck, - FLB_TIME_ETFMT_V1_FIXEXT); - } - - /* Pack records */ - msgpack_pack_object(&mp_pck, *log_event.body); - - record_size = off - pre; - - if (ff) { - chunk = ff->checksum_hex; - } - else { - chunk = chunk_buf; - } - - append_options(ctx, fc, FLB_EVENT_TYPE_LOGS, &mp_pck, 0, - (char *) data + pre, record_size, - log_event.metadata, - chunk); - - pre = off; - entries++; - } - - flb_log_event_decoder_destroy(&log_decoder); - - *out_buf = mp_sbuf.data; - *out_size = mp_sbuf.size; - - return entries; -} -#endif - -int flb_forward_format_transcode( - struct flb_forward *ctx, int format, - char *input_buffer, size_t input_length, - char **output_buffer, size_t *output_length) -{ - struct flb_log_event_encoder log_encoder; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - int result; - - result = flb_log_event_decoder_init(&log_decoder, input_buffer, input_length); - - if (result != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", result); - - return -1; - } - - result = flb_log_event_encoder_init(&log_encoder, format); - - if (result != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event encoder initialization error : %d", result); - - flb_log_event_decoder_destroy(&log_decoder); - - return -1; - } - - while ((result = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - - result = flb_log_event_encoder_begin_record(&log_encoder); - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_set_timestamp( - &log_encoder, &log_event.timestamp); - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_set_metadata_from_msgpack_object( - &log_encoder, - log_event.metadata); - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_set_body_from_msgpack_object( - &log_encoder, - log_event.body); - } - - if (result == FLB_EVENT_ENCODER_SUCCESS) { - result = flb_log_event_encoder_commit_record(&log_encoder); - } - } - - if (log_encoder.output_length > 0) { - *output_buffer = log_encoder.output_buffer; - *output_length = log_encoder.output_length; - - flb_log_event_encoder_claim_internal_buffer_ownership(&log_encoder); - - result = 0; - } - else { - flb_plg_error(ctx->ins, - "Log event encoder error : %d", result); - - result = -1; - } - - flb_log_event_decoder_destroy(&log_decoder); - flb_log_event_encoder_destroy(&log_encoder); - - return result; -} - -/* - * Forward Protocol: Forward Mode - * ------------------------------ - * In forward mode we don't format the serialized entries. We just compose - * the outgoing 'options'. - */ -static int flb_forward_format_forward_mode(struct flb_forward *ctx, - struct flb_forward_config *fc, - struct flb_forward_flush *ff, - int event_type, - const char *tag, int tag_len, - const void *data, size_t bytes, - void **out_buf, size_t *out_size) -{ - int result; - int entries = 0; - char *chunk; - char chunk_buf[33]; - msgpack_packer mp_pck; - msgpack_sbuffer mp_sbuf; - char *transcoded_buffer; - size_t transcoded_length; - - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - if (ff) { - chunk = ff->checksum_hex; - } - else { - chunk = chunk_buf; - } - - if (fc->send_options == FLB_TRUE || (event_type == FLB_EVENT_TYPE_METRICS || event_type == FLB_EVENT_TYPE_TRACES)) { - if (event_type == FLB_EVENT_TYPE_LOGS) { - entries = flb_mp_count(data, bytes); - } - else { - /* for non logs, we don't count the number of entries */ - entries = 0; - } - - if (!fc->fwd_retain_metadata && event_type == FLB_EVENT_TYPE_LOGS) { - result = flb_forward_format_transcode(ctx, FLB_LOG_EVENT_FORMAT_FORWARD, - (char *) data, bytes, - &transcoded_buffer, - &transcoded_length); - - if (result == 0) { - append_options(ctx, fc, event_type, &mp_pck, entries, - transcoded_buffer, - transcoded_length, - NULL, chunk); - - free(transcoded_buffer); - } - } - else { - append_options(ctx, fc, event_type, &mp_pck, entries, (char *) data, bytes, NULL, chunk); - } - } - - *out_buf = mp_sbuf.data; - *out_size = mp_sbuf.size; - - return 0; -} - -/* - * Forward Protocol: Forward Mode Compat (for Fluentd <= 0.12) - * ----------------------------------------------------------- - * Use Forward mode but format the timestamp as integers - * - * note: yes, the function name it's a big long... - */ -static int flb_forward_format_forward_compat_mode(struct flb_forward *ctx, - struct flb_forward_config *fc, - struct flb_forward_flush *ff, - const char *tag, int tag_len, - const void *data, size_t bytes, - void **out_buf, size_t *out_size) -{ - int entries = 0; - char *chunk; - char chunk_buf[33]; - msgpack_packer mp_pck; - msgpack_sbuffer mp_sbuf; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - int ret; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return -1; - } - - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - if (ff) { - chunk = ff->checksum_hex; - } - else { - chunk = chunk_buf; - } - - msgpack_pack_array(&mp_pck, fc->send_options ? 3 : 2); - - /* Tag */ - flb_forward_format_append_tag(ctx, fc, &mp_pck, - NULL, tag, tag_len); - - /* Entries */ - entries = flb_mp_count(data, bytes); - msgpack_pack_array(&mp_pck, entries); - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - msgpack_pack_array(&mp_pck, 2); - - /* Pack timestamp */ - if (fc->time_as_integer == FLB_TRUE) { - flb_time_append_to_msgpack(&log_event.timestamp, - &mp_pck, - FLB_TIME_ETFMT_INT); - } - else { - flb_time_append_to_msgpack(&log_event.timestamp, - &mp_pck, - FLB_TIME_ETFMT_V1_FIXEXT); - } - - /* Pack records */ - msgpack_pack_object(&mp_pck, *log_event.body); - } - - if (fc->send_options == FLB_TRUE) { - append_options(ctx, fc, FLB_EVENT_TYPE_LOGS, &mp_pck, entries, - (char *) data, bytes, NULL, chunk); - } - - flb_log_event_decoder_destroy(&log_decoder); - - *out_buf = mp_sbuf.data; - *out_size = mp_sbuf.size; - - return 0; -} - -int flb_forward_format(struct flb_config *config, - struct flb_input_instance *ins, - void *ins_ctx, - void *flush_ctx, - int event_type, - const char *tag, int tag_len, - const void *data, size_t bytes, - void **out_buf, size_t *out_size) -{ - int ret = 0; - int mode = MODE_FORWARD; - struct flb_upstream_node *node = NULL; - struct flb_forward_config *fc; - struct flb_forward_flush *ff = flush_ctx; - struct flb_forward *ctx = ins_ctx; - - if (!flush_ctx) { - fc = flb_forward_target(ctx, &node); - } - else { - fc = ff->fc; - } - - if (!fc) { - flb_plg_error(ctx->ins, "cannot get an Upstream single or HA node"); - return -1; - } - - if (event_type == FLB_EVENT_TYPE_METRICS) { - mode = MODE_FORWARD; - goto do_formatting; - } - else if (event_type == FLB_EVENT_TYPE_TRACES) { - mode = MODE_FORWARD; - goto do_formatting; - } - -#ifdef FLB_HAVE_RECORD_ACCESSOR - /* - * Based in the configuration, decide the preferred protocol mode - */ - if (fc->ra_tag && fc->ra_static == FLB_FALSE) { - /* - * Dynamic tag per records needs to include the Tag for every entry, - * if record accessor option has been enabled we jump into this - * mode. - */ - mode = MODE_MESSAGE; - } - else { -#endif - /* Forward Modes */ - if (fc->time_as_integer == FLB_FALSE) { - /* - * In forward mode we optimize in memory allocation and we reuse the - * original msgpack buffer. So we don't compose the outgoing buffer - * and just let the caller handle it. - */ - mode = MODE_FORWARD; - } - else if (fc->time_as_integer == FLB_TRUE) { - /* - * This option is similar to MODE_FORWARD but since we have to convert the - * timestamp to integer type, we need to format the buffer (in the previous - * case we avoid that step. - */ - mode = MODE_FORWARD_COMPAT; - } - -#ifdef FLB_HAVE_RECORD_ACCESSOR - } -#endif - - -do_formatting: - - /* Message Mode: the user needs custom Tags */ - if (mode == MODE_MESSAGE) { -#ifdef FLB_HAVE_RECORD_ACCESSOR - ret = flb_forward_format_message_mode(ctx, fc, ff, - tag, tag_len, - data, bytes, - out_buf, out_size); -#endif - } - else if (mode == MODE_FORWARD) { - ret = flb_forward_format_forward_mode(ctx, fc, ff, - event_type, - tag, tag_len, - data, bytes, - out_buf, out_size); - } - else if (mode == MODE_FORWARD_COMPAT) { - ret = flb_forward_format_forward_compat_mode(ctx, fc, ff, - tag, tag_len, - data, bytes, - out_buf, out_size); - } - - if (ret == -1) { - return -1; - } - - return mode; -} diff --git a/fluent-bit/plugins/out_forward/forward_format.h b/fluent-bit/plugins/out_forward/forward_format.h deleted file mode 100644 index bc6c47349..000000000 --- a/fluent-bit/plugins/out_forward/forward_format.h +++ /dev/null @@ -1,48 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_FORWARD_FORMAT_H -#define FLB_OUT_FORWARD_FORMAT_H - -#include -#include "forward.h" - -void flb_forward_format_bin_to_hex(uint8_t *buf, size_t len, char *out); - -int flb_forward_format_append_tag(struct flb_forward *ctx, - struct flb_forward_config *fc, - msgpack_packer *mp_pck, - msgpack_object *map, - const char *tag, int tag_len); - -int flb_forward_format(struct flb_config *config, - struct flb_input_instance *ins, - void *ins_ctx, - void *flush_ctx, - int event_type, - const char *tag, int tag_len, - const void *data, size_t bytes, - void **out_buf, size_t *out_size); - -int flb_forward_format_transcode( - struct flb_forward *ctx, int format, - char *input_buffer, size_t input_length, - char **output_buffer, size_t *output_length); - -#endif diff --git a/fluent-bit/plugins/out_gelf/CMakeLists.txt b/fluent-bit/plugins/out_gelf/CMakeLists.txt deleted file mode 100644 index 2de1e9987..000000000 --- a/fluent-bit/plugins/out_gelf/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -set(src - gelf.c - ) - -FLB_PLUGIN(out_gelf "${src}" "") diff --git a/fluent-bit/plugins/out_gelf/gelf.c b/fluent-bit/plugins/out_gelf/gelf.c deleted file mode 100644 index 6d7284641..000000000 --- a/fluent-bit/plugins/out_gelf/gelf.c +++ /dev/null @@ -1,556 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "gelf.h" - -#ifndef MSG_DONTWAIT - #define MSG_DONTWAIT 0 -#endif - -#ifndef MSG_NOSIGNAL - #define MSG_NOSIGNAL 0 -#endif - -/* - * Version 1.1 (11/2013) - * A GELF message is a GZIP’d or ZLIB’d JSON string with the following fields: - * version string (UTF-8) GELF spec version – “1.1”; MUST be set by client - * library. - * host string (UTF-8) the name of the host, source or application that sent - * this message; MUST be set by client library. - * short_message string (UTF-8) a short descriptive message; MUST be set by - * client library. - * full_message string (UTF-8) a long message that can i.e. contain a - * backtrace; optional. - * timestamp number Seconds since UNIX epoch with optional decimal places - * for milliseconds; SHOULD be set by client library. Will be set to NOW - * by server if absent. - * level number the level equal to the standard syslog levels; optional, - * default is 1 (ALERT). - * facility string (UTF-8) optional, deprecated. Send as additional field i - * instead. - * line number the line in a file that caused the error (decimal); optional, - * deprecated. Send as additional field instead. - * file string (UTF-8) the file (with path if you want) that caused the error - * (string); optional, deprecated. Send as additional field instead. - * _[additional field] string (UTF-8) or number every field you send and - * prefix with a _ (underscore) will be treated as an additional field. - * Allowed characters in field names are any word character (letter, - * number, underscore), dashes and dots. The verifying regular expression - * is: ^[\w\.\-]*$ - * Libraries SHOULD not allow to send id as additional field (_id). Graylog - * server nodes omit this field automatically. - */ - -/* - * Generate a unique message ID. The upper 48-bit is milliseconds - * since the Epoch, the lower 16-bit is a random nonce. - */ -static uint64_t message_id(void) -{ - uint64_t now; - uint16_t nonce; - struct flb_time tm; - - if (flb_time_get(&tm) != -1) { - now = (uint64_t) tm.tm.tv_sec * 1000 + tm.tm.tv_nsec / 1000000; - } - else { - now = (uint64_t) time(NULL) * 1000; - } - nonce = (uint16_t) rand(); - - return (now << 16) | nonce; -} - -/* - * A GELF header is 12 bytes in size. It has the following - * structure: - * - * +---+---+---+---+---+---+---+---+---+---+---+---+ - * | MAGIC | MESSAGE ID |SEQ|NUM| - * +---+---+---+---+---+---+---+---+---+---+---+---+ - * - * NUM is the total number of packets to send. SEQ is the - * unique sequence number for each packet (zero-indexed). - */ -#define GELF_MAGIC "\x1e\x0f" -#define GELF_HEADER_SIZE 12 - -static void init_chunk_header(uint8_t *buf, int count) -{ - uint64_t msgid = message_id(); - - memcpy(buf, GELF_MAGIC, 2); - memcpy(buf + 2, &msgid, 8); - buf[10] = 0; - buf[11] = count; -} - -/* - * Chunked GELF - * Prepend the following structure to your GELF message to make it chunked: - * Chunked GELF magic bytes 2 bytes 0x1e 0x0f - * Message ID 8 bytes Must be the same for every chunk of this message. - * Identifying the whole message and is used to reassemble the chunks later. - * Generate from millisecond timestamp + hostname for example. - * Sequence number 1 byte The sequence number of this chunk. Starting at 0 - * and always less than the sequence count. - * Sequence count 1 byte Total number of chunks this message has. - * All chunks MUST arrive within 5 seconds or the server will discard all - * already arrived and still arriving chunks. - * A message MUST NOT consist of more than 128 chunks. - */ -static int gelf_send_udp_chunked(struct flb_out_gelf_config *ctx, void *msg, - size_t msg_size) -{ - int ret; - uint8_t n; - size_t chunks; - size_t offset; - size_t len; - uint8_t *buf = (uint8_t *) ctx->pckt_buf; - - chunks = msg_size / ctx->pckt_size; - if (msg_size % ctx->pckt_size != 0) { - chunks++; - } - - if (chunks > 128) { - flb_plg_error(ctx->ins, "message too big: %zd bytes", msg_size); - return -1; - } - - init_chunk_header(buf, chunks); - - offset = 0; - for (n = 0; n < chunks; n++) { - buf[10] = n; - - len = msg_size - offset; - if (ctx->pckt_size < len) { - len = ctx->pckt_size; - } - memcpy(buf + GELF_HEADER_SIZE, (char *) msg + offset, len); - - ret = send(ctx->fd, buf, len + GELF_HEADER_SIZE, - MSG_DONTWAIT | MSG_NOSIGNAL); - if (ret == -1) { - flb_errno(); - } - offset += ctx->pckt_size; - } - return 0; -} - -static int gelf_send_udp_pckt (struct flb_out_gelf_config *ctx, char *msg, - size_t msg_size) -{ - int ret; - - if (msg_size > ctx->pckt_size) { - gelf_send_udp_chunked(ctx, msg, msg_size); - } - else { - ret = send(ctx->fd, msg, msg_size, MSG_DONTWAIT | MSG_NOSIGNAL); - if (ret == -1) { - flb_errno(); - return -1; - } - } - - return 0; -} - -static int gelf_send_udp(struct flb_out_gelf_config *ctx, char *msg, - size_t msg_size) -{ - int ret; - int status; - void *zdata; - size_t zdata_len; - - if (ctx->compress == FLB_TRUE || (msg_size > ctx->pckt_size)) { - ret = flb_gzip_compress(msg, msg_size, &zdata, &zdata_len); - if (ret != 0) { - return -1; - } - - status = gelf_send_udp_pckt (ctx, zdata, zdata_len); - flb_free(zdata); - if (status < 0) { - return status; - } - } - else { - status = send(ctx->fd, msg, msg_size, MSG_DONTWAIT | MSG_NOSIGNAL); - if (status < 0) { - return status; - } - } - - return 0; -} - -static void cb_gelf_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - int ret; - flb_sds_t s; - flb_sds_t tmp; - size_t off = 0; - size_t prev_off = 0; - size_t size = 0; - size_t bytes_sent; - msgpack_object map; - struct flb_connection *u_conn = NULL; - struct flb_out_gelf_config *ctx = out_context; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - if (ctx->mode != FLB_GELF_UDP) { - u_conn = flb_upstream_conn_get(ctx->u); - if (!u_conn) { - flb_plg_error(ctx->ins, "no upstream connections available"); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - } - - ret = flb_log_event_decoder_init(&log_decoder, - (char *) event_chunk->data, - event_chunk->size); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - if (ctx->mode != FLB_GELF_UDP) { - flb_upstream_conn_release(u_conn); - } - - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - off = log_decoder.offset; - size = off - prev_off; - prev_off = off; - - map = *log_event.body; - - size = (size * 1.4); - s = flb_sds_create_size(size); - if (s == NULL) { - flb_log_event_decoder_destroy(&log_decoder); - FLB_OUTPUT_RETURN(FLB_ERROR); - } - - tmp = flb_msgpack_to_gelf(&s, &map, &log_event.timestamp, - &(ctx->fields)); - if (tmp != NULL) { - s = tmp; - if (ctx->mode == FLB_GELF_UDP) { - ret = gelf_send_udp(ctx, s, flb_sds_len(s)); - if (ret == -1) { - if (ctx->mode != FLB_GELF_UDP) { - flb_upstream_conn_release(u_conn); - } - - flb_log_event_decoder_destroy(&log_decoder); - - flb_sds_destroy(s); - - FLB_OUTPUT_RETURN(FLB_RETRY); - } - } - else { - /* write gelf json plus \0 */ - ret = flb_io_net_write(u_conn, - s, flb_sds_len(s) + 1, &bytes_sent); - if (ret == -1) { - flb_errno(); - - if (ctx->mode != FLB_GELF_UDP) { - flb_upstream_conn_release(u_conn); - } - - flb_log_event_decoder_destroy(&log_decoder); - - flb_sds_destroy(s); - - FLB_OUTPUT_RETURN(FLB_RETRY); - } - } - } - else { - flb_plg_error(ctx->ins, "error encoding to GELF"); - } - - flb_sds_destroy(s); - } - - flb_log_event_decoder_destroy(&log_decoder); - - if (ctx->mode != FLB_GELF_UDP) { - flb_upstream_conn_release(u_conn); - } - - FLB_OUTPUT_RETURN(FLB_OK); -} - -static int cb_gelf_init(struct flb_output_instance *ins, struct flb_config *config, - void *data) -{ - int ret; - const char *tmp; - struct flb_out_gelf_config *ctx = NULL; - - /* Set default network configuration */ - flb_output_net_default("127.0.0.1", 12201, ins); - - /* Allocate plugin context */ - ctx = flb_calloc(1, sizeof(struct flb_out_gelf_config)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->ins = ins; - - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_plg_error(ins, "flb_output_config_map_set failed"); - flb_free(ctx); - return -1; - } - - /* Config Mode */ - tmp = flb_output_get_property("mode", ins); - if (tmp) { - if (!strcasecmp(tmp, "tcp")) { - ctx->mode = FLB_GELF_TCP; - } - else if (!strcasecmp(tmp, "tls")) { - ctx->mode = FLB_GELF_TLS; - } - else if (!strcasecmp(tmp, "udp")) { - ctx->mode = FLB_GELF_UDP; - } - else { - flb_plg_error(ctx->ins, "Unknown gelf mode %s", tmp); - flb_free(ctx); - return -1; - } - } - else { - ctx->mode = FLB_GELF_UDP; - } - - /* Config Gelf_Timestamp_Key */ - tmp = flb_output_get_property("gelf_timestamp_key", ins); - if (tmp) { - ctx->fields.timestamp_key = flb_sds_create(tmp); - } - - /* Config Gelf_Host_Key */ - tmp = flb_output_get_property("gelf_host_key", ins); - if (tmp) { - ctx->fields.host_key = flb_sds_create(tmp); - } - - /* Config Gelf_Short_Message_Key */ - tmp = flb_output_get_property("gelf_short_message_key", ins); - if (tmp) { - ctx->fields.short_message_key = flb_sds_create(tmp); - } - - /* Config Gelf_Full_Message_Key */ - tmp = flb_output_get_property("gelf_full_message_key", ins); - if (tmp) { - ctx->fields.full_message_key = flb_sds_create(tmp); - } - - /* Config Gelf_Level_Key */ - tmp = flb_output_get_property("gelf_level_key", ins); - if (tmp) { - ctx->fields.level_key = flb_sds_create(tmp); - } - - /* init random seed */ - if (flb_random_bytes((unsigned char *) &ctx->seed, sizeof(int))) { - ctx->seed = time(NULL); - } - srand(ctx->seed); - - ctx->fd = -1; - ctx->pckt_buf = NULL; - - if (ctx->mode == FLB_GELF_UDP) { - ctx->fd = flb_net_udp_connect(ins->host.name, ins->host.port, - ins->net_setup.source_address); - if (ctx->fd < 0) { - flb_free(ctx); - return -1; - } - ctx->pckt_buf = flb_malloc(GELF_HEADER_SIZE + ctx->pckt_size); - if (ctx->pckt_buf == NULL) { - flb_socket_close(ctx->fd); - flb_free(ctx); - return -1; - } - } - else { - int io_flags = FLB_IO_TCP; - - if (ctx->mode == FLB_GELF_TLS) { - io_flags = FLB_IO_TLS; - } - - if (ins->host.ipv6 == FLB_TRUE) { - io_flags |= FLB_IO_IPV6; - } - - ctx->u = flb_upstream_create(config, ins->host.name, ins->host.port, - io_flags, ins->tls); - if (!(ctx->u)) { - flb_free(ctx); - return -1; - } - flb_output_upstream_set(ctx->u, ins); - } - - /* Set the plugin context */ - flb_output_set_context(ins, ctx); - return 0; -} - -static int cb_gelf_exit(void *data, struct flb_config *config) -{ - struct flb_out_gelf_config *ctx = data; - - if (ctx == NULL) { - return 0; - } - - if (ctx->u) { - flb_upstream_destroy(ctx->u); - } - if (ctx->fd >= 0) { - close(ctx->fd); - } - - flb_sds_destroy(ctx->fields.timestamp_key); - flb_sds_destroy(ctx->fields.host_key); - flb_sds_destroy(ctx->fields.short_message_key); - flb_sds_destroy(ctx->fields.full_message_key); - flb_sds_destroy(ctx->fields.level_key); - - flb_free(ctx->pckt_buf); - flb_free(ctx); - - return 0; -} - - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "mode", "udp", - 0, FLB_FALSE, 0, - "The protocol to use. 'tls', 'tcp' or 'udp'" - }, - { - FLB_CONFIG_MAP_STR, "gelf_short_message_key", NULL, - 0, FLB_FALSE, 0, - "A short descriptive message (MUST be set in GELF)" - }, - { - FLB_CONFIG_MAP_STR, "gelf_timestamp_key", NULL, - 0, FLB_FALSE, 0, - "Timestamp key name (SHOULD be set in GELF)" - }, - { - FLB_CONFIG_MAP_STR, "gelf_host_key", NULL, - 0, FLB_FALSE, 0, - "Key which its value is used as the name of the host," - "source or application that sent this message. (MUST be set in GELF) " - }, - { - FLB_CONFIG_MAP_STR, "gelf_full_message_key", NULL, - 0, FLB_FALSE, 0, - "Key to use as the long message that can i.e. contain a backtrace. " - "(Optional in GELF)" - }, - { - FLB_CONFIG_MAP_STR, "gelf_level_key", NULL, - 0, FLB_FALSE, 0, - "Key to be used as the log level. " - "Its value must be in standard syslog levels (between 0 and 7). " - "(Optional in GELF)" - }, - { - FLB_CONFIG_MAP_INT, "packet_size", "1420", - 0, FLB_TRUE, offsetof(struct flb_out_gelf_config, pckt_size), - "If transport protocol is udp, you can set the size of packets to be sent." - }, - { - FLB_CONFIG_MAP_BOOL, "compress", "true", - 0, FLB_TRUE, offsetof(struct flb_out_gelf_config, compress), - "If transport protocol is udp, " - "you can set this if you want your UDP packets to be compressed." - }, - - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_output_plugin out_gelf_plugin = { - .name = "gelf", - .description = "GELF Output", - .cb_init = cb_gelf_init, - .cb_pre_run = NULL, - .cb_flush = cb_gelf_flush, - .cb_exit = cb_gelf_exit, - .flags = FLB_OUTPUT_NET | FLB_IO_OPT_TLS, - .config_map = config_map -}; diff --git a/fluent-bit/plugins/out_gelf/gelf.h b/fluent-bit/plugins/out_gelf/gelf.h deleted file mode 100644 index b834059cf..000000000 --- a/fluent-bit/plugins/out_gelf/gelf.h +++ /dev/null @@ -1,47 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_GELF_H -#define FLB_OUT_GELF_H - -#define FLB_GELF_UDP 0 -#define FLB_GELF_TCP 1 -#define FLB_GELF_TLS 2 - -#include - -struct flb_out_gelf_config { - - struct flb_gelf_fields fields; - - /* Upstream connection to the backend server */ - struct flb_upstream *u; - flb_sockfd_t fd; - - int pckt_size; - char *pckt_buf; - int compress; - unsigned int seed; - - int mode; - - struct flb_output_instance *ins; -}; - -#endif diff --git a/fluent-bit/plugins/out_http/CMakeLists.txt b/fluent-bit/plugins/out_http/CMakeLists.txt deleted file mode 100644 index 216561017..000000000 --- a/fluent-bit/plugins/out_http/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(src - http.c - http_conf.c - ) - -FLB_PLUGIN(out_http "${src}" "mk_core") diff --git a/fluent-bit/plugins/out_http/http.c b/fluent-bit/plugins/out_http/http.c deleted file mode 100644 index f88b6346d..000000000 --- a/fluent-bit/plugins/out_http/http.c +++ /dev/null @@ -1,774 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef FLB_HAVE_SIGNV4 -#ifdef FLB_HAVE_AWS -#include -#include -#endif -#endif - -#include -#include -#include -#include -#include - -#include "http.h" -#include "http_conf.h" - -#include - -static int cb_http_init(struct flb_output_instance *ins, - struct flb_config *config, void *data) -{ - struct flb_out_http *ctx = NULL; - (void) data; - - ctx = flb_http_conf_create(ins, config); - if (!ctx) { - return -1; - } - - /* Set the plugin context */ - flb_output_set_context(ins, ctx); - - /* - * This plugin instance uses the HTTP client interface, let's register - * it debugging callbacks. - */ - flb_output_set_http_debug_callbacks(ins); - - return 0; -} - -static void append_headers(struct flb_http_client *c, - char **headers) -{ - int i; - char *header_key; - char *header_value; - - i = 0; - header_key = NULL; - header_value = NULL; - while (*headers) { - if (i % 2 == 0) { - header_key = *headers; - } - else { - header_value = *headers; - } - if (header_key && header_value) { - flb_http_add_header(c, - header_key, - strlen(header_key), - header_value, - strlen(header_value)); - flb_free(header_key); - flb_free(header_value); - header_key = NULL; - header_value = NULL; - } - headers++; - i++; - } -} - -static int http_post(struct flb_out_http *ctx, - const void *body, size_t body_len, - const char *tag, int tag_len, - char **headers) -{ - int ret; - int out_ret = FLB_OK; - int compressed = FLB_FALSE; - size_t b_sent; - void *payload_buf = NULL; - size_t payload_size = 0; - struct flb_upstream *u; - struct flb_connection *u_conn; - struct flb_http_client *c; - struct mk_list *head; - struct flb_config_map_val *mv; - struct flb_slist_entry *key = NULL; - struct flb_slist_entry *val = NULL; - flb_sds_t signature = NULL; - - /* Get upstream context and connection */ - u = ctx->u; - u_conn = flb_upstream_conn_get(u); - if (!u_conn) { - flb_plg_error(ctx->ins, "no upstream connections available to %s:%i", - u->tcp_host, u->tcp_port); - return FLB_RETRY; - } - - /* Map payload */ - payload_buf = (void *) body; - payload_size = body_len; - - /* Should we compress the payload ? */ - if (ctx->compress_gzip == FLB_TRUE) { - ret = flb_gzip_compress((void *) body, body_len, - &payload_buf, &payload_size); - if (ret == -1) { - flb_plg_error(ctx->ins, - "cannot gzip payload, disabling compression"); - } - else { - compressed = FLB_TRUE; - } - } - - /* Create HTTP client context */ - c = flb_http_client(u_conn, FLB_HTTP_POST, ctx->uri, - payload_buf, payload_size, - ctx->host, ctx->port, - ctx->proxy, 0); - - - if (c->proxy.host) { - flb_plg_debug(ctx->ins, "[http_client] proxy host: %s port: %i", - c->proxy.host, c->proxy.port); - } - - /* Allow duplicated headers ? */ - flb_http_allow_duplicated_headers(c, ctx->allow_dup_headers); - - /* - * Direct assignment of the callback context to the HTTP client context. - * This needs to be improved through a more clean API. - */ - c->cb_ctx = ctx->ins->callback; - - /* Append headers */ - if (headers) { - append_headers(c, headers); - } - else if ((ctx->out_format == FLB_PACK_JSON_FORMAT_JSON) || - (ctx->out_format == FLB_PACK_JSON_FORMAT_STREAM) || - (ctx->out_format == FLB_PACK_JSON_FORMAT_LINES) || - (ctx->out_format == FLB_HTTP_OUT_GELF)) { - flb_http_add_header(c, - FLB_HTTP_CONTENT_TYPE, - sizeof(FLB_HTTP_CONTENT_TYPE) - 1, - FLB_HTTP_MIME_JSON, - sizeof(FLB_HTTP_MIME_JSON) - 1); - } - else { - flb_http_add_header(c, - FLB_HTTP_CONTENT_TYPE, - sizeof(FLB_HTTP_CONTENT_TYPE) - 1, - FLB_HTTP_MIME_MSGPACK, - sizeof(FLB_HTTP_MIME_MSGPACK) - 1); - } - - if (ctx->header_tag) { - flb_http_add_header(c, - ctx->header_tag, - flb_sds_len(ctx->header_tag), - tag, tag_len); - } - - /* Content Encoding: gzip */ - if (compressed == FLB_TRUE) { - flb_http_set_content_encoding_gzip(c); - } - - /* Basic Auth headers */ - if (ctx->http_user && ctx->http_passwd) { - flb_http_basic_auth(c, ctx->http_user, ctx->http_passwd); - } - - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - - flb_config_map_foreach(head, mv, ctx->headers) { - key = mk_list_entry_first(mv->val.list, struct flb_slist_entry, _head); - val = mk_list_entry_last(mv->val.list, struct flb_slist_entry, _head); - - flb_http_add_header(c, - key->str, flb_sds_len(key->str), - val->str, flb_sds_len(val->str)); - } - -#ifdef FLB_HAVE_SIGNV4 -#ifdef FLB_HAVE_AWS - /* AWS SigV4 headers */ - if (ctx->has_aws_auth == FLB_TRUE) { - flb_plg_debug(ctx->ins, "signing request with AWS Sigv4"); - signature = flb_signv4_do(c, - FLB_TRUE, /* normalize URI ? */ - FLB_TRUE, /* add x-amz-date header ? */ - time(NULL), - (char *) ctx->aws_region, - (char *) ctx->aws_service, - 0, NULL, - ctx->aws_provider); - - if (!signature) { - flb_plg_error(ctx->ins, "could not sign request with sigv4"); - out_ret = FLB_RETRY; - goto cleanup; - } - flb_sds_destroy(signature); - } -#endif -#endif - - ret = flb_http_do(c, &b_sent); - if (ret == 0) { - /* - * Only allow the following HTTP status: - * - * - 200: OK - * - 201: Created - * - 202: Accepted - * - 203: no authorative resp - * - 204: No Content - * - 205: Reset content - * - */ - if (c->resp.status < 200 || c->resp.status > 205) { - if (ctx->log_response_payload && - c->resp.payload && c->resp.payload_size > 0) { - flb_plg_error(ctx->ins, "%s:%i, HTTP status=%i\n%s", - ctx->host, ctx->port, - c->resp.status, c->resp.payload); - } - else { - flb_plg_error(ctx->ins, "%s:%i, HTTP status=%i", - ctx->host, ctx->port, c->resp.status); - } - out_ret = FLB_RETRY; - } - else { - if (ctx->log_response_payload && - c->resp.payload && c->resp.payload_size > 0) { - flb_plg_info(ctx->ins, "%s:%i, HTTP status=%i\n%s", - ctx->host, ctx->port, - c->resp.status, c->resp.payload); - } - else { - flb_plg_info(ctx->ins, "%s:%i, HTTP status=%i", - ctx->host, ctx->port, - c->resp.status); - } - } - } - else { - flb_plg_error(ctx->ins, "could not flush records to %s:%i (http_do=%i)", - ctx->host, ctx->port, ret); - out_ret = FLB_RETRY; - } - -cleanup: - /* - * If the payload buffer is different than incoming records in body, means - * we generated a different payload and must be freed. - */ - if (payload_buf != body) { - flb_free(payload_buf); - } - - /* Destroy HTTP client context */ - flb_http_client_destroy(c); - - /* Release the TCP connection */ - flb_upstream_conn_release(u_conn); - - return out_ret; -} - -static int compose_payload_gelf(struct flb_out_http *ctx, - const char *data, uint64_t bytes, - void **out_body, size_t *out_size) -{ - flb_sds_t s; - flb_sds_t tmp = NULL; - size_t size = 0; - msgpack_object map; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - int ret; - - size = bytes * 1.5; - - /* Allocate buffer for our new payload */ - s = flb_sds_create_size(size); - if (!s) { - flb_plg_error(ctx->ins, "flb_sds_create_size failed"); - return FLB_RETRY; - } - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - flb_sds_destroy(s); - - return FLB_RETRY; - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - map = *log_event.body; - - tmp = flb_msgpack_to_gelf(&s, &map, - &log_event.timestamp, - &(ctx->gelf_fields)); - if (!tmp) { - flb_plg_error(ctx->ins, "error encoding to GELF"); - - flb_sds_destroy(s); - flb_log_event_decoder_destroy(&log_decoder); - - return FLB_ERROR; - } - - /* Append new line */ - tmp = flb_sds_cat(s, "\n", 1); - if (!tmp) { - flb_plg_error(ctx->ins, "error concatenating records"); - - flb_sds_destroy(s); - flb_log_event_decoder_destroy(&log_decoder); - - return FLB_RETRY; - } - - s = tmp; - } - - *out_body = s; - *out_size = flb_sds_len(s); - - flb_log_event_decoder_destroy(&log_decoder); - - return FLB_OK; -} - -static int compose_payload(struct flb_out_http *ctx, - const void *in_body, size_t in_size, - void **out_body, size_t *out_size) -{ - flb_sds_t encoded; - - *out_body = NULL; - *out_size = 0; - - if ((ctx->out_format == FLB_PACK_JSON_FORMAT_JSON) || - (ctx->out_format == FLB_PACK_JSON_FORMAT_STREAM) || - (ctx->out_format == FLB_PACK_JSON_FORMAT_LINES)) { - - encoded = flb_pack_msgpack_to_json_format(in_body, - in_size, - ctx->out_format, - ctx->json_date_format, - ctx->date_key); - if (encoded == NULL) { - flb_plg_error(ctx->ins, "failed to convert json"); - return FLB_ERROR; - } - *out_body = (void*)encoded; - *out_size = flb_sds_len(encoded); - } - else if (ctx->out_format == FLB_HTTP_OUT_GELF) { - return compose_payload_gelf(ctx, in_body, in_size, out_body, out_size); - } - else { - /* Nothing to do, if the format is msgpack */ - *out_body = (void *)in_body; - *out_size = in_size; - } - - return FLB_OK; -} - -static char **extract_headers(msgpack_object *obj) { - size_t i; - char **headers = NULL; - size_t str_count; - msgpack_object_map map; - msgpack_object_str k; - msgpack_object_str v; - - if (obj->type != MSGPACK_OBJECT_MAP) { - goto err; - } - - map = obj->via.map; - str_count = map.size * 2 + 1; - headers = flb_calloc(str_count, sizeof *headers); - - if (!headers) { - goto err; - } - - for (i = 0; i < map.size; i++) { - if (map.ptr[i].key.type != MSGPACK_OBJECT_STR || - map.ptr[i].val.type != MSGPACK_OBJECT_STR) { - continue; - } - - k = map.ptr[i].key.via.str; - v = map.ptr[i].val.via.str; - - headers[i * 2] = strndup(k.ptr, k.size); - - if (!headers[i]) { - goto err; - } - - headers[i * 2 + 1] = strndup(v.ptr, v.size); - - if (!headers[i]) { - goto err; - } - } - - return headers; - -err: - if (headers) { - for (i = 0; i < str_count; i++) { - if (headers[i]) { - flb_free(headers[i]); - } - } - flb_free(headers); - } - return NULL; -} - -static int post_all_requests(struct flb_out_http *ctx, - const char *data, size_t size, - flb_sds_t body_key, - flb_sds_t headers_key, - struct flb_event_chunk *event_chunk) -{ - msgpack_object map; - msgpack_object *k; - msgpack_object *v; - msgpack_object *start_key; - const char *body; - size_t body_size; - bool body_found; - bool headers_found; - char **headers; - size_t record_count = 0; - int ret = 0; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, size); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return -1; - } - - while ((flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - headers = NULL; - body_found = false; - headers_found = false; - - map = *log_event.body; - - if (map.type != MSGPACK_OBJECT_MAP) { - ret = -1; - break; - } - - if (!flb_ra_get_kv_pair(ctx->body_ra, map, &start_key, &k, &v)) { - if (v->type == MSGPACK_OBJECT_STR || v->type == MSGPACK_OBJECT_BIN) { - body = v->via.str.ptr; - body_size = v->via.str.size; - body_found = true; - } - else { - flb_plg_warn(ctx->ins, - "failed to extract body using pattern \"%s\" " - "(must be a msgpack string or bin)", ctx->body_key); - } - } - - if (!flb_ra_get_kv_pair(ctx->headers_ra, map, &start_key, &k, &v)) { - headers = extract_headers(v); - if (headers) { - headers_found = true; - } - else { - flb_plg_warn(ctx->ins, - "error extracting headers using pattern \"%s\"", - ctx->headers_key); - } - } - - if (body_found && headers_found) { - flb_plg_trace(ctx->ins, "posting record %zu", record_count++); - ret = http_post(ctx, body, body_size, event_chunk->tag, - flb_sds_len(event_chunk->tag), headers); - } - else { - flb_plg_warn(ctx->ins, - "failed to extract body/headers using patterns " - "\"%s\" and \"%s\"", ctx->body_key, ctx->headers_key); - ret = -1; - continue; - } - - flb_free(headers); - } - - flb_log_event_decoder_destroy(&log_decoder); - - return ret; -} - -static void cb_http_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - int ret = FLB_ERROR; - struct flb_out_http *ctx = out_context; - void *out_body; - size_t out_size; - (void) i_ins; - - if (ctx->body_key) { - ret = post_all_requests(ctx, event_chunk->data, event_chunk->size, - ctx->body_key, ctx->headers_key, event_chunk); - if (ret < 0) { - flb_plg_error(ctx->ins, - "failed to post requests body key \"%s\"", ctx->body_key); - } - } - else { - ret = compose_payload(ctx, event_chunk->data, event_chunk->size, - &out_body, &out_size); - if (ret != FLB_OK) { - FLB_OUTPUT_RETURN(ret); - } - - if ((ctx->out_format == FLB_PACK_JSON_FORMAT_JSON) || - (ctx->out_format == FLB_PACK_JSON_FORMAT_STREAM) || - (ctx->out_format == FLB_PACK_JSON_FORMAT_LINES) || - (ctx->out_format == FLB_HTTP_OUT_GELF)) { - ret = http_post(ctx, out_body, out_size, - event_chunk->tag, flb_sds_len(event_chunk->tag), NULL); - flb_sds_destroy(out_body); - } - else { - /* msgpack */ - ret = http_post(ctx, - event_chunk->data, event_chunk->size, - event_chunk->tag, flb_sds_len(event_chunk->tag), NULL); - } - } - - FLB_OUTPUT_RETURN(ret); -} - -static int cb_http_exit(void *data, struct flb_config *config) -{ - struct flb_out_http *ctx = data; - - flb_http_conf_destroy(ctx); - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "proxy", NULL, - 0, FLB_FALSE, 0, - "Specify an HTTP Proxy. The expected format of this value is http://host:port. " - }, - { - FLB_CONFIG_MAP_BOOL, "allow_duplicated_headers", "true", - 0, FLB_TRUE, offsetof(struct flb_out_http, allow_dup_headers), - "Specify if duplicated headers are allowed or not" - }, - { - FLB_CONFIG_MAP_BOOL, "log_response_payload", "true", - 0, FLB_TRUE, offsetof(struct flb_out_http, log_response_payload), - "Specify if the response paylod should be logged or not" - }, - { - FLB_CONFIG_MAP_STR, "http_user", NULL, - 0, FLB_TRUE, offsetof(struct flb_out_http, http_user), - "Set HTTP auth user" - }, - { - FLB_CONFIG_MAP_STR, "http_passwd", "", - 0, FLB_TRUE, offsetof(struct flb_out_http, http_passwd), - "Set HTTP auth password" - }, -#ifdef FLB_HAVE_SIGNV4 -#ifdef FLB_HAVE_AWS - { - FLB_CONFIG_MAP_BOOL, "aws_auth", "false", - 0, FLB_TRUE, offsetof(struct flb_out_http, has_aws_auth), - "Enable AWS SigV4 authentication" - }, - { - FLB_CONFIG_MAP_STR, "aws_service", NULL, - 0, FLB_TRUE, offsetof(struct flb_out_http, aws_service), - "AWS destination service code, used by SigV4 authentication" - }, - FLB_AWS_CREDENTIAL_BASE_CONFIG_MAP(FLB_HTTP_AWS_CREDENTIAL_PREFIX), -#endif -#endif - { - FLB_CONFIG_MAP_STR, "header_tag", NULL, - 0, FLB_TRUE, offsetof(struct flb_out_http, header_tag), - "Set a HTTP header which value is the Tag" - }, - { - FLB_CONFIG_MAP_STR, "format", NULL, - 0, FLB_FALSE, 0, - "Set desired payload format: json, json_stream, json_lines, gelf or msgpack" - }, - { - FLB_CONFIG_MAP_STR, "json_date_format", NULL, - 0, FLB_FALSE, 0, - FBL_PACK_JSON_DATE_FORMAT_DESCRIPTION - }, - { - FLB_CONFIG_MAP_STR, "json_date_key", "date", - 0, FLB_TRUE, offsetof(struct flb_out_http, json_date_key), - "Specify the name of the date field in output" - }, - { - FLB_CONFIG_MAP_STR, "compress", NULL, - 0, FLB_FALSE, 0, - "Set payload compression mechanism. Option available is 'gzip'" - }, - { - FLB_CONFIG_MAP_SLIST_1, "header", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct flb_out_http, headers), - "Add a HTTP header key/value pair. Multiple headers can be set" - }, - { - FLB_CONFIG_MAP_STR, "uri", NULL, - 0, FLB_TRUE, offsetof(struct flb_out_http, uri), - "Specify an optional HTTP URI for the target web server, e.g: /something" - }, - - /* Gelf Properties */ - { - FLB_CONFIG_MAP_STR, "gelf_timestamp_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_out_http, gelf_fields.timestamp_key), - "Specify the key to use for 'timestamp' in gelf format" - }, - { - FLB_CONFIG_MAP_STR, "gelf_host_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_out_http, gelf_fields.host_key), - "Specify the key to use for the 'host' in gelf format" - }, - { - FLB_CONFIG_MAP_STR, "gelf_short_message_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_out_http, gelf_fields.short_message_key), - "Specify the key to use as the 'short' message in gelf format" - }, - { - FLB_CONFIG_MAP_STR, "gelf_full_message_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_out_http, gelf_fields.full_message_key), - "Specify the key to use for the 'full' message in gelf format" - }, - { - FLB_CONFIG_MAP_STR, "gelf_level_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_out_http, gelf_fields.level_key), - "Specify the key to use for the 'level' in gelf format" - }, - { - FLB_CONFIG_MAP_STR, "body_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_out_http, body_key), - "Specify the key which contains the body" - }, - { - FLB_CONFIG_MAP_STR, "headers_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_out_http, headers_key), - "Specify the key which contains the headers" - }, - - /* EOF */ - {0} -}; - -static int cb_http_format_test(struct flb_config *config, - struct flb_input_instance *ins, - void *plugin_context, - void *flush_ctx, - int event_type, - const char *tag, int tag_len, - const void *data, size_t bytes, - void **out_data, size_t *out_size) -{ - struct flb_out_http *ctx = plugin_context; - int ret; - - ret = compose_payload(ctx, data, bytes, out_data, out_size); - if (ret != FLB_OK) { - flb_error("ret=%d", ret); - return -1; - } - return 0; -} - -/* Plugin reference */ -struct flb_output_plugin out_http_plugin = { - .name = "http", - .description = "HTTP Output", - .cb_init = cb_http_init, - .cb_pre_run = NULL, - .cb_flush = cb_http_flush, - .cb_exit = cb_http_exit, - .config_map = config_map, - - /* for testing */ - .test_formatter.callback = cb_http_format_test, - - .flags = FLB_OUTPUT_NET | FLB_IO_OPT_TLS, - .workers = 2 -}; diff --git a/fluent-bit/plugins/out_http/http.h b/fluent-bit/plugins/out_http/http.h deleted file mode 100644 index 151658f31..000000000 --- a/fluent-bit/plugins/out_http/http.h +++ /dev/null @@ -1,103 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_HTTP_H -#define FLB_OUT_HTTP_H - -#define FLB_HTTP_OUT_MSGPACK FLB_PACK_JSON_FORMAT_NONE -#define FLB_HTTP_OUT_GELF 20 - -#define FLB_HTTP_CONTENT_TYPE "Content-Type" -#define FLB_HTTP_MIME_MSGPACK "application/msgpack" -#define FLB_HTTP_MIME_JSON "application/json" - -#ifdef FLB_HAVE_SIGNV4 -#ifdef FLB_HAVE_AWS -#define FLB_HTTP_AWS_CREDENTIAL_PREFIX "aws_" -#endif -#endif - -struct flb_out_http { - /* HTTP Auth */ - char *http_user; - char *http_passwd; - - /* AWS Auth */ -#ifdef FLB_HAVE_SIGNV4 -#ifdef FLB_HAVE_AWS - int has_aws_auth; - struct flb_aws_provider *aws_provider; - const char *aws_region; - const char *aws_service; -#endif -#endif - - /* Proxy */ - const char *proxy; - char *proxy_host; - int proxy_port; - - /* Output format */ - int out_format; - - int json_date_format; - flb_sds_t json_date_key; - flb_sds_t date_key; /* internal use */ - - /* HTTP URI */ - char *uri; - char *host; - int port; - - /* GELF fields */ - struct flb_gelf_fields gelf_fields; - - /* which record key to use as body */ - flb_sds_t body_key; - - struct flb_record_accessor *body_ra; - - /* override headers with contents of the map in the key specified here */ - flb_sds_t headers_key; - - struct flb_record_accessor *headers_ra; - - /* Include tag in header */ - flb_sds_t header_tag; - - /* Compression mode (gzip) */ - int compress_gzip; - - /* Allow duplicated headers */ - int allow_dup_headers; - - /* Log the response paylod */ - int log_response_payload; - - /* Upstream connection to the backend server */ - struct flb_upstream *u; - - /* Arbitrary HTTP headers */ - struct mk_list *headers; - - /* Plugin instance */ - struct flb_output_instance *ins; -}; - -#endif diff --git a/fluent-bit/plugins/out_http/http_conf.c b/fluent-bit/plugins/out_http/http_conf.c deleted file mode 100644 index cf3f2101a..000000000 --- a/fluent-bit/plugins/out_http/http_conf.c +++ /dev/null @@ -1,298 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#ifdef FLB_HAVE_SIGNV4 -#ifdef FLB_HAVE_AWS -#include -#endif -#endif -#include "http.h" -#include "http_conf.h" - -struct flb_out_http *flb_http_conf_create(struct flb_output_instance *ins, - struct flb_config *config) -{ - int ret; - int ulen; - int io_flags = 0; - char *protocol = NULL; - char *host = NULL; - char *port = NULL; - char *uri = NULL; - char *tmp_uri = NULL; - const char *tmp; - struct flb_upstream *upstream; - struct flb_out_http *ctx = NULL; - - /* Allocate plugin context */ - ctx = flb_calloc(1, sizeof(struct flb_out_http)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return NULL; - } - - if (ctx->headers_key && !ctx->body_key) { - flb_plg_error(ctx->ins, "when setting headers_key, body_key is also required"); - flb_free(ctx); - return NULL; - } - - if (ctx->body_key && !ctx->headers_key) { - flb_plg_error(ctx->ins, "when setting body_key, headers_key is also required"); - flb_free(ctx); - return NULL; - } - - if (ctx->body_key && ctx->headers_key) { - ctx->body_ra = flb_ra_create(ctx->body_key, FLB_FALSE); - if (!ctx->body_ra) { - flb_plg_error(ctx->ins, "failed to allocate body record accessor"); - flb_free(ctx); - return NULL; - } - - ctx->headers_ra = flb_ra_create(ctx->headers_key, FLB_FALSE); - if (!ctx->headers_ra) { - flb_plg_error(ctx->ins, "failed to allocate headers record accessor"); - flb_free(ctx); - return NULL; - } - } - - /* - * Check if a Proxy have been set, if so the Upstream manager will use - * the Proxy end-point and then we let the HTTP client know about it, so - * it can adjust the HTTP requests. - */ - tmp = flb_output_get_property("proxy", ins); - if (tmp) { - ret = flb_utils_url_split(tmp, &protocol, &host, &port, &uri); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not parse proxy parameter: '%s'", tmp); - flb_free(ctx); - return NULL; - } - - ctx->proxy_host = host; - ctx->proxy_port = atoi(port); - ctx->proxy = tmp; - flb_free(protocol); - flb_free(port); - flb_free(uri); - uri = NULL; - } - else { - flb_output_net_default("127.0.0.1", 80, ins); - } - - /* Check if AWS SigV4 authentication is enabled */ -#ifdef FLB_HAVE_SIGNV4 -#ifdef FLB_HAVE_AWS - if (ctx->has_aws_auth) { - ctx->aws_service = flb_output_get_property(FLB_HTTP_AWS_CREDENTIAL_PREFIX - "service", ctx->ins); - if (!ctx->aws_service) { - flb_plg_error(ins, "aws_auth option requires " FLB_HTTP_AWS_CREDENTIAL_PREFIX - "service to be set"); - flb_free(ctx); - return NULL; - } - - ctx->aws_provider = flb_managed_chain_provider_create( - ins, - config, - FLB_HTTP_AWS_CREDENTIAL_PREFIX, - NULL, - flb_aws_client_generator() - ); - if (!ctx->aws_provider) { - flb_plg_error(ins, "failed to create aws credential provider for sigv4 auth"); - flb_free(ctx); - return NULL; - } - - /* If managed provider creation succeeds, then region key is present */ - ctx->aws_region = flb_output_get_property(FLB_HTTP_AWS_CREDENTIAL_PREFIX - "region", ctx->ins); - } -#endif /* !FLB_HAVE_AWS */ -#endif /* !FLB_HAVE_SIGNV4 */ - - /* Check if SSL/TLS is enabled */ -#ifdef FLB_HAVE_TLS - if (ins->use_tls == FLB_TRUE) { - io_flags = FLB_IO_TLS; - } - else { - io_flags = FLB_IO_TCP; - } -#else - io_flags = FLB_IO_TCP; -#endif - - if (ins->host.ipv6 == FLB_TRUE) { - io_flags |= FLB_IO_IPV6; - } - - if (ctx->proxy) { - flb_plg_trace(ctx->ins, "Upstream Proxy=%s:%i", - ctx->proxy_host, ctx->proxy_port); - upstream = flb_upstream_create(config, - ctx->proxy_host, - ctx->proxy_port, - io_flags, ins->tls); - } - else { - upstream = flb_upstream_create(config, - ins->host.name, - ins->host.port, - io_flags, ins->tls); - } - - if (!upstream) { - flb_free(ctx); - return NULL; - } - - if (ins->host.uri) { - uri = flb_strdup(ins->host.uri->full); - } - else { - tmp = flb_output_get_property("uri", ins); - if (tmp) { - uri = flb_strdup(tmp); - } - } - - if (!uri) { - uri = flb_strdup("/"); - } - else if (uri[0] != '/') { - ulen = strlen(uri); - tmp_uri = flb_malloc(ulen + 2); - tmp_uri[0] = '/'; - memcpy(tmp_uri + 1, uri, ulen); - tmp_uri[ulen + 1] = '\0'; - flb_free(uri); - uri = tmp_uri; - } - - /* Output format */ - ctx->out_format = FLB_PACK_JSON_FORMAT_NONE; - tmp = flb_output_get_property("format", ins); - if (tmp) { - if (strcasecmp(tmp, "gelf") == 0) { - ctx->out_format = FLB_HTTP_OUT_GELF; - } - else { - ret = flb_pack_to_json_format_type(tmp); - if (ret == -1) { - flb_plg_error(ctx->ins, "unrecognized 'format' option. " - "Using 'msgpack'"); - } - else { - ctx->out_format = ret; - } - } - } - - /* Date key */ - ctx->date_key = ctx->json_date_key; - tmp = flb_output_get_property("json_date_key", ins); - if (tmp) { - /* Just check if we have to disable it */ - if (flb_utils_bool(tmp) == FLB_FALSE) { - ctx->date_key = NULL; - } - } - - /* Date format for JSON output */ - ctx->json_date_format = FLB_PACK_JSON_DATE_DOUBLE; - tmp = flb_output_get_property("json_date_format", ins); - if (tmp) { - ret = flb_pack_to_json_date_type(tmp); - if (ret == -1) { - flb_plg_error(ctx->ins, "unrecognized 'json_date_format' option. " - "Using 'double'."); - } - else { - ctx->json_date_format = ret; - } - } - - /* Compress (gzip) */ - tmp = flb_output_get_property("compress", ins); - ctx->compress_gzip = FLB_FALSE; - if (tmp) { - if (strcasecmp(tmp, "gzip") == 0) { - ctx->compress_gzip = FLB_TRUE; - } - } - - ctx->u = upstream; - ctx->uri = uri; - ctx->host = ins->host.name; - ctx->port = ins->host.port; - - /* Set instance flags into upstream */ - flb_output_upstream_set(ctx->u, ins); - - return ctx; -} - -void flb_http_conf_destroy(struct flb_out_http *ctx) -{ - if (!ctx) { - return; - } - - if (ctx->body_ra && ctx->headers_ra) { - flb_ra_destroy(ctx->body_ra); - flb_ra_destroy(ctx->headers_ra); - } - - if (ctx->u) { - flb_upstream_destroy(ctx->u); - } - -#ifdef FLB_HAVE_SIGNV4 -#ifdef FLB_HAVE_AWS - if (ctx->aws_provider) { - flb_aws_provider_destroy(ctx->aws_provider); - } -#endif -#endif - - flb_free(ctx->proxy_host); - flb_free(ctx->uri); - flb_free(ctx); -} diff --git a/fluent-bit/plugins/out_http/http_conf.h b/fluent-bit/plugins/out_http/http_conf.h deleted file mode 100644 index 9e87e1002..000000000 --- a/fluent-bit/plugins/out_http/http_conf.h +++ /dev/null @@ -1,32 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_HTTP_CONF_H -#define FLB_OUT_HTTP_CONF_H - -#include -#include - -#include "http.h" - -struct flb_out_http *flb_http_conf_create(struct flb_output_instance *ins, - struct flb_config *config); -void flb_http_conf_destroy(struct flb_out_http *ctx); - -#endif diff --git a/fluent-bit/plugins/out_influxdb/CMakeLists.txt b/fluent-bit/plugins/out_influxdb/CMakeLists.txt deleted file mode 100644 index 75d85a6b5..000000000 --- a/fluent-bit/plugins/out_influxdb/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -set(src - influxdb_bulk.c - influxdb.c) - -FLB_PLUGIN(out_influxdb "${src}" "") diff --git a/fluent-bit/plugins/out_influxdb/influxdb.c b/fluent-bit/plugins/out_influxdb/influxdb.c deleted file mode 100644 index 71e489fbc..000000000 --- a/fluent-bit/plugins/out_influxdb/influxdb.c +++ /dev/null @@ -1,682 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include - -#include - -#include "influxdb.h" -#include "influxdb_bulk.h" - -#include - -/* - * Returns FLB_TRUE when the specified key is in Tag_Keys list, - * otherwise FLB_FALSE - */ -static int is_tagged_key(struct flb_influxdb *ctx, - const char *key, int kl, int type); - -/* - * Increments the timestamp when it is duplicated - */ -static void influxdb_tsmod(struct flb_time *ts, struct flb_time *dupe, - struct flb_time *last) { - if (flb_time_equal(ts, last) || flb_time_equal(ts, dupe)) { - ++dupe->tm.tv_nsec; - flb_time_copy(last, ts); - flb_time_copy(ts, dupe); - } - else { - flb_time_copy(last, ts); - flb_time_copy(dupe, ts); - } -} - -/* - * Convert the internal Fluent Bit data representation to the required one - * by InfluxDB. - */ -static char *influxdb_format(const char *tag, int tag_len, - const void *data, size_t bytes, size_t *out_size, - struct flb_influxdb *ctx) -{ - int i; - int ret; - int n_size; - uint64_t seq = 0; - char *buf; - char *str = NULL; - size_t str_size; - char tmp[128]; - msgpack_object map; - struct flb_time tm; - struct influxdb_bulk *bulk = NULL; - struct influxdb_bulk *bulk_head = NULL; - struct influxdb_bulk *bulk_body = NULL; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return NULL; - } - - /* Create the bulk composer */ - bulk = influxdb_bulk_create(); - if (!bulk) { - goto error; - } - - bulk_head = influxdb_bulk_create(); - if (!bulk_head) { - goto error; - } - - bulk_body = influxdb_bulk_create(); - if (!bulk_body) { - goto error; - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - flb_time_copy(&tm, &log_event.timestamp); - - map = *log_event.body; - n_size = map.via.map.size + 1; - - seq = ctx->seq; - if (ctx->seq + 1 >= 100000) { - seq = 1; - } - else { - ctx->seq++; - } - - ret = influxdb_bulk_append_header(bulk_head, - tag, tag_len, - seq, - ctx->seq_name, ctx->seq_len); - if (ret == -1) { - goto error; - } - - for (i = 0; i < n_size - 1; i++) { - msgpack_object *k = &map.via.map.ptr[i].key; - msgpack_object *v = &map.via.map.ptr[i].val; - - if (k->type != MSGPACK_OBJECT_BIN && k->type != MSGPACK_OBJECT_STR) { - continue; - } - - int quote = FLB_FALSE; - - /* key */ - const char *key = NULL; - int key_len; - - /* val */ - const char *val = NULL; - int val_len; - - if (k->type == MSGPACK_OBJECT_STR) { - key = k->via.str.ptr; - key_len = k->via.str.size; - } - else { - key = k->via.bin.ptr; - key_len = k->via.bin.size; - } - - /* Store value */ - if (v->type == MSGPACK_OBJECT_NIL) { - /* Missing values are Null by default in InfluxDB */ - continue; - } - else if (v->type == MSGPACK_OBJECT_BOOLEAN) { - if (v->via.boolean) { - val = "TRUE"; - val_len = 4; - } - else { - val = "FALSE"; - val_len = 5; - } - } - else if (v->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - val = tmp; - val_len = snprintf(tmp, sizeof(tmp) - 1, "%" PRIu64, v->via.u64); - } - else if (v->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { - val = tmp; - val_len = snprintf(tmp, sizeof(tmp) - 1, "%" PRId64, v->via.i64); - } - else if (v->type == MSGPACK_OBJECT_FLOAT || v->type == MSGPACK_OBJECT_FLOAT32) { - val = tmp; - val_len = snprintf(tmp, sizeof(tmp) - 1, "%f", v->via.f64); - } - else if (v->type == MSGPACK_OBJECT_STR) { - /* String value */ - quote = FLB_TRUE; - val = v->via.str.ptr; - val_len = v->via.str.size; - } - else if (v->type == MSGPACK_OBJECT_BIN) { - /* Bin value */ - quote = FLB_TRUE; - val = v->via.bin.ptr; - val_len = v->via.bin.size; - } - - if (!val || !key) { - continue; - } - - /* is this a string ? */ - if (quote == FLB_TRUE) { - ret = flb_utils_write_str_buf(val, val_len, - &str, &str_size); - if (ret == -1) { - flb_errno(); - goto error; - } - - val = str; - val_len = str_size; - } - - if (is_tagged_key(ctx, key, key_len, v->type)) { - /* Append key/value data into the bulk_head */ - ret = influxdb_bulk_append_kv(bulk_head, - key, key_len, - val, val_len, - false); - } - else { - /* Append key/value data into the bulk_body */ - ret = influxdb_bulk_append_kv(bulk_body, - key, key_len, - val, val_len, - quote); - } - - if (quote == FLB_TRUE) { - flb_free(str); - str_size = 0; - } - - if (ret == -1) { - flb_plg_error(ctx->ins, "cannot append key/value"); - goto error; - } - } - - /* Check have data fields */ - if (bulk_body->len > 0) { - /* Modify timestamp in avoidance of duplication */ - influxdb_tsmod(&tm, &ctx->ts_dupe, &ctx->ts_last); - /* Append the timestamp */ - ret = influxdb_bulk_append_timestamp(bulk_body, &tm); - if (ret == -1) { - flb_plg_error(ctx->ins, "cannot append timestamp"); - goto error; - } - - /* Append collected data to final bulk */ - if (influxdb_bulk_append_bulk(bulk, bulk_head, '\n') != 0 || - influxdb_bulk_append_bulk(bulk, bulk_body, ' ') != 0) { - goto error; - } - } - else { - flb_plg_warn(ctx->ins, "skip send record, " - "since no record available " - "or all fields are tagged in record"); - /* Following records maybe ok, so continue processing */ - } - - /* Reset bulk_head and bulk_body */ - bulk_head->len = 0; - bulk_body->len = 0; - } - - flb_log_event_decoder_destroy(&log_decoder); - - *out_size = bulk->len; - buf = bulk->ptr; - - /* - * Note: we don't destroy the bulk as we need to keep the allocated - * buffer with the data. Instead we just release the bulk context and - * return the bulk->ptr buffer - */ - flb_free(bulk); - influxdb_bulk_destroy(bulk_head); - influxdb_bulk_destroy(bulk_body); - - return buf; - -error: - if (bulk != NULL) { - influxdb_bulk_destroy(bulk); - } - if (bulk_head != NULL) { - influxdb_bulk_destroy(bulk_head); - } - if (bulk_body != NULL) { - influxdb_bulk_destroy(bulk_body); - } - - flb_log_event_decoder_destroy(&log_decoder); - - return NULL; -} - -static int cb_influxdb_init(struct flb_output_instance *ins, struct flb_config *config, - void *data) -{ - int ret; - int io_flags = 0; - const char *tmp; - struct flb_upstream *upstream; - struct flb_influxdb *ctx; - - /* Set default network configuration */ - flb_output_net_default(FLB_INFLUXDB_HOST, FLB_INFLUXDB_PORT, ins); - - /* Allocate plugin context */ - ctx = flb_calloc(1, sizeof(struct flb_influxdb)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->ins = ins; - - /* Register context with plugin instance */ - flb_output_set_context(ins, ctx); - - /* - * This plugin instance uses the HTTP client interface, let's register - * it debugging callbacks. - */ - flb_output_set_http_debug_callbacks(ins); - - /* Load config map */ - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - return -1; - } - - if (ins->use_tls == FLB_TRUE) { - io_flags = FLB_IO_TLS; - } - else { - io_flags = FLB_IO_TCP; - } - - /* sequence tag */ - tmp = flb_output_get_property("sequence_tag", ins); - if (!tmp) { - ctx->seq_name = flb_strdup("_seq"); - } - else if (strcmp(tmp, "off") == 0) { - ctx->seq_name = flb_strdup(""); - } - else { - ctx->seq_name = flb_strdup(tmp); - } - ctx->seq_len = strlen(ctx->seq_name); - - if (ctx->custom_uri) { - /* custom URI endpoint (e.g: Grafana */ - if (ctx->custom_uri[0] != '/') { - flb_plg_error(ctx->ins, - "'custom_uri' value must start wih a forward slash '/'"); - return -1; - } - snprintf(ctx->uri, sizeof(ctx->uri) - 1, "%s", ctx->custom_uri); - } - else if (ctx->bucket) { - /* bucket: api v2 */ - snprintf(ctx->uri, sizeof(ctx->uri) - 1, - "/api/v2/write?org=%s&bucket=%s&precision=ns", - ctx->organization, ctx->bucket); - } - else { - snprintf(ctx->uri, sizeof(ctx->uri) - 1, - "/write?db=%s&precision=n", - ctx->database); - } - - if (ins->host.ipv6 == FLB_TRUE) { - io_flags |= FLB_IO_IPV6; - } - - - /* Tag_Keys */ - tmp = flb_output_get_property("tag_keys", ins); - if (tmp) { - ctx->tag_keys = flb_utils_split(tmp, ' ', 256); - } - else { - ctx->tag_keys = NULL; - } - - /* Prepare an upstream handler */ - upstream = flb_upstream_create(config, - ins->host.name, - ins->host.port, - io_flags, - ins->tls); - if (!upstream) { - flb_free(ctx); - return -1; - } - ctx->u = upstream; - ctx->seq = 0; - flb_output_upstream_set(ctx->u, ins); - - flb_time_zero(&ctx->ts_dupe); - flb_time_zero(&ctx->ts_last); - - flb_plg_debug(ctx->ins, "host=%s port=%i", ins->host.name, ins->host.port); - - return 0; -} - -static int format_metrics(struct flb_output_instance *ins, - const void *data, size_t bytes, - char **out_buf, size_t *out_size) -{ - int ret; - size_t off = 0; - cfl_sds_t text; - struct cmt *cmt = NULL; - - /* get cmetrics context */ - ret = cmt_decode_msgpack_create(&cmt, (char *) data, bytes, &off); - if (ret != 0) { - flb_plg_error(ins, "could not process metrics payload"); - return -1; - } - - /* convert to text representation */ - text = cmt_encode_influx_create(cmt); - if (!text) { - cmt_destroy(cmt); - return -1; - } - - /* destroy cmt context */ - cmt_destroy(cmt); - - *out_buf = text; - *out_size = flb_sds_len(text); - - return 0; -} - -static void cb_influxdb_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - int ret; - int out_ret = FLB_OK; - int is_metric = FLB_FALSE; - size_t b_sent; - size_t bytes_out; - char *pack; - char tmp[128]; - struct mk_list *head; - struct flb_connection *u_conn; - struct flb_http_client *c; - struct flb_config_map_val *mv; - struct flb_slist_entry *key = NULL; - struct flb_slist_entry *val = NULL; - struct flb_influxdb *ctx = out_context; - - /* Convert format: metrics / logs */ - if (event_chunk->type == FLB_EVENT_TYPE_METRICS) { - /* format metrics */ - ret = format_metrics(ctx->ins, - (char *) event_chunk->data, - event_chunk->size, - &pack, &bytes_out); - if (ret == -1) { - FLB_OUTPUT_RETURN(FLB_ERROR); - } - is_metric = FLB_TRUE; - } - else { - /* format logs */ - pack = influxdb_format(event_chunk->tag, flb_sds_len(event_chunk->tag), - event_chunk->data, event_chunk->size, - &bytes_out, ctx); - if (!pack) { - FLB_OUTPUT_RETURN(FLB_ERROR); - } - } - - /* Get upstream connection */ - u_conn = flb_upstream_conn_get(ctx->u); - if (!u_conn) { - if (is_metric) { - cmt_encode_influx_destroy(pack); - } - else { - flb_free(pack); - } - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Compose HTTP Client request */ - c = flb_http_client(u_conn, FLB_HTTP_POST, ctx->uri, - pack, bytes_out, NULL, 0, NULL, 0); - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - - if (ctx->http_token) { - ret = snprintf(tmp, sizeof(tmp) - 1, "Token %s", ctx->http_token); - flb_http_add_header(c, FLB_HTTP_HEADER_AUTH, sizeof FLB_HTTP_HEADER_AUTH - 1, tmp, ret); - } - else if (ctx->http_user && ctx->http_passwd) { - flb_http_basic_auth(c, ctx->http_user, ctx->http_passwd); - } - - /* Append custom headers if any */ - flb_config_map_foreach(head, mv, ctx->headers) { - key = mk_list_entry_first(mv->val.list, struct flb_slist_entry, _head); - val = mk_list_entry_last(mv->val.list, struct flb_slist_entry, _head); - - flb_http_add_header(c, - key->str, flb_sds_len(key->str), - val->str, flb_sds_len(val->str)); - } - - /* Map debug callbacks */ - flb_http_client_debug(c, ctx->ins->callback); - - ret = flb_http_do(c, &b_sent); - if (ret == 0) { - if (c->resp.status != 200 && c->resp.status != 204) { - if (c->resp.payload_size > 0) { - flb_plg_error(ctx->ins, "http_status=%i\n%s", - c->resp.status, c->resp.payload); - } - else { - flb_plg_debug(ctx->ins, "http_status=%i", - c->resp.status); - } - } - flb_plg_debug(ctx->ins, "http_do=%i OK", ret); - } - else { - flb_plg_error(ctx->ins, "http_do=%i", ret); - out_ret = FLB_RETRY; - } - - flb_http_client_destroy(c); - - if (is_metric) { - cmt_encode_influx_destroy(pack); - } - else { - flb_free(pack); - } - - /* Release the connection */ - flb_upstream_conn_release(u_conn); - - FLB_OUTPUT_RETURN(out_ret); -} - -static int cb_influxdb_exit(void *data, struct flb_config *config) -{ - struct flb_influxdb *ctx = data; - - if (!ctx) { - return 0; - } - - if (ctx->tag_keys) { - flb_utils_split_free(ctx->tag_keys); - } - - flb_upstream_destroy(ctx->u); - flb_free(ctx); - - return 0; -} - -int is_tagged_key(struct flb_influxdb *ctx, const char *key, int kl, int type) -{ - if (type == MSGPACK_OBJECT_STR) { - if (ctx->auto_tags) { - return FLB_TRUE; - } - } - - struct mk_list *head; - struct flb_split_entry *entry; - - if (ctx->tag_keys) { - mk_list_foreach(head, ctx->tag_keys) { - entry = mk_list_entry(head, struct flb_split_entry, _head); - if (kl == entry->len && strncmp(key, entry->value, kl) == 0) { - return FLB_TRUE; - } - } - } - - return FLB_FALSE; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "database", "fluentbit", - 0, FLB_TRUE, offsetof(struct flb_influxdb, database), - "Set the database name." - }, - { - FLB_CONFIG_MAP_STR, "bucket", NULL, - 0, FLB_TRUE, offsetof(struct flb_influxdb, bucket), - "Specify the bucket name, used on InfluxDB API v2." - }, - - { - FLB_CONFIG_MAP_STR, "org", "fluent", - 0, FLB_TRUE, offsetof(struct flb_influxdb, organization), - "Set the Organization name." - }, - - { - FLB_CONFIG_MAP_STR, "sequence_tag", NULL, - 0, FLB_FALSE, 0, - "Specify the sequence tag." - }, - - { - FLB_CONFIG_MAP_STR, "uri", NULL, - 0, FLB_TRUE, offsetof(struct flb_influxdb, custom_uri), - "Specify a custom URI endpoint (must start with '/')." - }, - - { - FLB_CONFIG_MAP_STR, "http_user", NULL, - 0, FLB_TRUE, offsetof(struct flb_influxdb, http_user), - "HTTP Basic Auth username." - }, - - { - FLB_CONFIG_MAP_STR, "http_passwd", "", - 0, FLB_TRUE, offsetof(struct flb_influxdb, http_passwd), - "HTTP Basic Auth password." - }, - - { - FLB_CONFIG_MAP_STR, "http_token", NULL, - 0, FLB_TRUE, offsetof(struct flb_influxdb, http_token), - "Set InfluxDB HTTP Token API v2." - }, - - { - FLB_CONFIG_MAP_SLIST_1, "http_header", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct flb_influxdb, headers), - "Add a HTTP header key/value pair. Multiple headers can be set" - }, - - { - FLB_CONFIG_MAP_BOOL, "auto_tags", "false", - 0, FLB_TRUE, offsetof(struct flb_influxdb, auto_tags), - "Automatically tag keys where value is string." - }, - - { - FLB_CONFIG_MAP_BOOL, "tag_keys", NULL, - 0, FLB_FALSE, 0, - "Space separated list of keys that needs to be tagged." - }, - - /* EOF */ - {0} -}; - -struct flb_output_plugin out_influxdb_plugin = { - .name = "influxdb", - .description = "InfluxDB Time Series", - .cb_init = cb_influxdb_init, - .cb_pre_run = NULL, - .cb_flush = cb_influxdb_flush, - .cb_exit = cb_influxdb_exit, - .config_map = config_map, - .flags = FLB_OUTPUT_NET | FLB_IO_OPT_TLS, - .event_type = FLB_OUTPUT_LOGS | FLB_OUTPUT_METRICS -}; diff --git a/fluent-bit/plugins/out_influxdb/influxdb.h b/fluent-bit/plugins/out_influxdb/influxdb.h deleted file mode 100644 index 13a72d4b8..000000000 --- a/fluent-bit/plugins/out_influxdb/influxdb.h +++ /dev/null @@ -1,78 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_INFLUXDB_H -#define FLB_OUT_INFLUXDB_H - -#include -#include - -#define FLB_INFLUXDB_HOST "127.0.0.1" -#define FLB_INFLUXDB_PORT 8086 - -struct flb_influxdb { - uint64_t seq; - - char uri[2048]; - - /* v1 */ - /* database */ - flb_sds_t database; - - /* HTTP Auth */ - flb_sds_t http_user; - flb_sds_t http_passwd; - - // v2 - /* bucket */ - flb_sds_t bucket; - - /* organization */ - flb_sds_t organization; - - /* custom HTTP URI */ - flb_sds_t custom_uri; - - /* HTTP Token */ - flb_sds_t http_token; - - /* sequence tag */ - char *seq_name; - int seq_len; - - /* auto_tags: on/off */ - int auto_tags; - - /* tag_keys: space separated list of key */ - struct mk_list *tag_keys; - - /* Arbitrary HTTP headers */ - struct mk_list *headers; - - /* Upstream connection to the backend server */ - struct flb_upstream *u; - - /* used for incrementing identical timestamps */ - struct flb_time ts_dupe; - struct flb_time ts_last; - - struct flb_output_instance *ins; -}; - -#endif diff --git a/fluent-bit/plugins/out_influxdb/influxdb_bulk.c b/fluent-bit/plugins/out_influxdb/influxdb_bulk.c deleted file mode 100644 index 2542ee3c4..000000000 --- a/fluent-bit/plugins/out_influxdb/influxdb_bulk.c +++ /dev/null @@ -1,233 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include -#include "influxdb.h" -#include "influxdb_bulk.h" - -static const uint64_t ONE_BILLION = 1000000000; - -static int influxdb_escape(char *out, const char *str, int size, bool quote) { - int out_size = 0; - int i; - for (i = 0; i < size; ++i) { - char ch = str[i]; - if (quote ? (ch == '"' || ch == '\\') : (isspace(ch) || ch == ',' || ch == '=')) { - out[out_size++] = '\\'; - } else if (ch == '\\') { - out[out_size++] = '\\'; - } - out[out_size++] = ch; - } - return out_size; -} - -static int influxdb_bulk_buffer(struct influxdb_bulk *bulk, int required) -{ - int new_size; - int available; - char *ptr; - - available = (bulk->size - bulk->len); - if (available < required) { - new_size = bulk->size + available + required + INFLUXDB_BULK_CHUNK; - ptr = flb_realloc(bulk->ptr, new_size); - if (!ptr) { - flb_errno(); - return -1; - } - bulk->ptr = ptr; - bulk->size = new_size; - } - - return 0; -} - -struct influxdb_bulk *influxdb_bulk_create() -{ - struct influxdb_bulk *b; - - b = flb_malloc(sizeof(struct influxdb_bulk)); - if (!b) { - perror("calloc"); - return NULL; - } - - b->ptr = flb_malloc(INFLUXDB_BULK_CHUNK); - if (!b->ptr) { - perror("malloc"); - flb_free(b); - return NULL; - } - - b->size = INFLUXDB_BULK_CHUNK; - b->len = 0; - - return b; -} - -void influxdb_bulk_destroy(struct influxdb_bulk *bulk) -{ - if (bulk->size > 0) { - flb_free(bulk->ptr); - } - flb_free(bulk); -} - -int influxdb_bulk_append_header(struct influxdb_bulk *bulk, - const char *tag, int tag_len, - uint64_t seq_n, const char *seq, int seq_len) -{ - int ret; - int required; - - required = tag_len + 1 + seq_len + 1 + 32; - - /* Make sure we have enough space */ - ret = influxdb_bulk_buffer(bulk, required); - if (ret != 0) { - return -1; - } - - /* Tag, sequence and final space */ - memcpy(bulk->ptr + bulk->len, tag, tag_len); - bulk->len += tag_len; - - if (seq_len != 0) { - bulk->ptr[bulk->len] = ','; - bulk->len++; - - /* Sequence number */ - memcpy(bulk->ptr + bulk->len, seq, seq_len); - bulk->len += seq_len; - - bulk->ptr[bulk->len] = '='; - bulk->len++; - - ret = snprintf(bulk->ptr + bulk->len, 32, "%" PRIu64, seq_n); - bulk->len += ret; - } - - /* Add a NULL byte for debugging purposes */ - bulk->ptr[bulk->len] = '\0'; - - return 0; -} - -int influxdb_bulk_append_kv(struct influxdb_bulk *bulk, - const char *key, int k_len, - const char *val, int v_len, - int quote) -{ - int ret; - int required; - - /* Reserve double space for keys and values in case of escaping */ - required = k_len * 2 + 1 + v_len * 2 + 1 + 1; - if (quote) { - required += 2; - } - - /* Make sure we have enough space */ - ret = influxdb_bulk_buffer(bulk, required); - if (ret != 0) { - return -1; - } - - if (bulk->len > 0) { - bulk->ptr[bulk->len] = ','; - bulk->len++; - } - - /* key */ - bulk->len += influxdb_escape(bulk->ptr + bulk->len, key, k_len, false); - - /* separator */ - bulk->ptr[bulk->len] = '='; - bulk->len++; - - /* value */ - if (quote) { - bulk->ptr[bulk->len] = '"'; - bulk->len++; - } - bulk->len += influxdb_escape(bulk->ptr + bulk->len, val, v_len, quote); - if (quote) { - bulk->ptr[bulk->len] = '"'; - bulk->len++; - } - - /* Add a NULL byte for debugging purposes */ - bulk->ptr[bulk->len] = '\0'; - - return 0; -}; - -int influxdb_bulk_append_timestamp(struct influxdb_bulk *bulk, - struct flb_time *t) -{ - int ret; - int len; - uint64_t timestamp; - - /* Make sure we have enough space */ - ret = influxdb_bulk_buffer(bulk, 128); - if (ret != 0) { - return -1; - } - - /* Timestamp is in Nanoseconds */ - timestamp = (t->tm.tv_sec * ONE_BILLION) + t->tm.tv_nsec; - len = snprintf(bulk->ptr + bulk->len, 127, " %" PRIu64, timestamp); - if (len == -1) { - return -1; - } - bulk->len += len; - bulk->ptr[bulk->len] = '\0'; - - return 0; -}; - -int influxdb_bulk_append_bulk(struct influxdb_bulk *bulk_to, - struct influxdb_bulk *bulk_from, - char separator) -{ - if (influxdb_bulk_buffer(bulk_to, bulk_from->len + 2) != 0) { - return -1; - } - - if (bulk_to->len > 0) { - bulk_to->ptr[bulk_to->len] = separator; - bulk_to->len += 1; - } - - memcpy(bulk_to->ptr + bulk_to->len, - bulk_from->ptr, bulk_from->len * sizeof(char)); - bulk_to->len += bulk_from->len; - - /* Add a NULL byte for always terminating with NULL */ - bulk_to->ptr[bulk_to->len] = '\0'; - - return 0; -}; diff --git a/fluent-bit/plugins/out_influxdb/influxdb_bulk.h b/fluent-bit/plugins/out_influxdb/influxdb_bulk.h deleted file mode 100644 index 66f914528..000000000 --- a/fluent-bit/plugins/out_influxdb/influxdb_bulk.h +++ /dev/null @@ -1,54 +0,0 @@ - /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_INFLUXDB_BULK_H -#define FLB_INFLUXDB_BULK_H - -#include -#include -#include - -#define INFLUXDB_BULK_CHUNK 4096 /* 4KB of buffer chunks */ - -struct influxdb_bulk { - char *ptr; - uint32_t len; - uint32_t size; -}; - -struct influxdb_bulk *influxdb_bulk_create(); - -int influxdb_bulk_append_header(struct influxdb_bulk *bulk, - const char *tag, int tag_len, - uint64_t seq_n, const char *seq, int seq_len); - -int influxdb_bulk_append_kv(struct influxdb_bulk *bulk, - const char *key, int k_len, - const char *val, int v_len, - int quote); - -int influxdb_bulk_append_bulk(struct influxdb_bulk *bulk_to, - struct influxdb_bulk *bulk_from, - char separator); - -void influxdb_bulk_destroy(struct influxdb_bulk *bulk); -int influxdb_bulk_append_timestamp(struct influxdb_bulk *bulk, - struct flb_time *t); - -#endif diff --git a/fluent-bit/plugins/out_kafka/CMakeLists.txt b/fluent-bit/plugins/out_kafka/CMakeLists.txt deleted file mode 100644 index 526910d49..000000000 --- a/fluent-bit/plugins/out_kafka/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Fluent Bit Kafka Output plugin -set(src - kafka_config.c - kafka_topic.c - kafka.c) - -FLB_PLUGIN(out_kafka "${src}" "rdkafka") -target_link_libraries(flb-plugin-out_kafka -lpthread) diff --git a/fluent-bit/plugins/out_kafka/kafka.c b/fluent-bit/plugins/out_kafka/kafka.c deleted file mode 100644 index ff700a687..000000000 --- a/fluent-bit/plugins/out_kafka/kafka.c +++ /dev/null @@ -1,658 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "kafka_config.h" -#include "kafka_topic.h" - -void cb_kafka_msg(rd_kafka_t *rk, const rd_kafka_message_t *rkmessage, - void *opaque) -{ - struct flb_out_kafka *ctx = (struct flb_out_kafka *) opaque; - - if (rkmessage->err) { - flb_plg_warn(ctx->ins, "message delivery failed: %s", - rd_kafka_err2str(rkmessage->err)); - } - else { - flb_plg_debug(ctx->ins, "message delivered (%zd bytes, " - "partition %"PRId32")", - rkmessage->len, rkmessage->partition); - } -} - -void cb_kafka_logger(const rd_kafka_t *rk, int level, - const char *fac, const char *buf) -{ - struct flb_out_kafka *ctx; - - ctx = (struct flb_out_kafka *) rd_kafka_opaque(rk); - - if (level <= FLB_KAFKA_LOG_ERR) { - flb_plg_error(ctx->ins, "%s: %s", - rk ? rd_kafka_name(rk) : NULL, buf); - } - else if (level == FLB_KAFKA_LOG_WARNING) { - flb_plg_warn(ctx->ins, "%s: %s", - rk ? rd_kafka_name(rk) : NULL, buf); - } - else if (level == FLB_KAFKA_LOG_NOTICE || level == FLB_KAFKA_LOG_INFO) { - flb_plg_info(ctx->ins, "%s: %s", - rk ? rd_kafka_name(rk) : NULL, buf); - } - else if (level == FLB_KAFKA_LOG_DEBUG) { - flb_plg_debug(ctx->ins, "%s: %s", - rk ? rd_kafka_name(rk) : NULL, buf); - } -} - -static int cb_kafka_init(struct flb_output_instance *ins, - struct flb_config *config, - void *data) -{ - struct flb_out_kafka *ctx; - - /* Configuration */ - ctx = flb_out_kafka_create(ins, config); - if (!ctx) { - flb_plg_error(ins, "failed to initialize"); - return -1; - } - - /* Set global context */ - flb_output_set_context(ins, ctx); - return 0; -} - -int produce_message(struct flb_time *tm, msgpack_object *map, - struct flb_out_kafka *ctx, struct flb_config *config) -{ - int i; - int ret; - int size; - int queue_full_retries = 0; - char *out_buf; - size_t out_size; - struct mk_list *head; - struct mk_list *topics; - struct flb_split_entry *entry; - char *dynamic_topic; - char *message_key = NULL; - size_t message_key_len = 0; - struct flb_kafka_topic *topic = NULL; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - msgpack_object key; - msgpack_object val; - flb_sds_t s; - -#ifdef FLB_HAVE_AVRO_ENCODER - // used to flag when a buffer needs to be freed for avro - bool avro_fast_buffer = true; - - // avro encoding uses a buffer - // the majority of lines are fairly small - // so using static buffer for these is much more efficient - // larger sizes will allocate -#ifndef AVRO_DEFAULT_BUFFER_SIZE -#define AVRO_DEFAULT_BUFFER_SIZE 2048 -#endif - static char avro_buff[AVRO_DEFAULT_BUFFER_SIZE]; - - // don't take lines that are too large - // these lines will log a warning - // this roughly a log line of 250000 chars -#ifndef AVRO_LINE_MAX_LEN -#define AVRO_LINE_MAX_LEN 1000000 - - // this is a convenience -#define AVRO_FREE(X, Y) if (!X) { flb_free(Y); } -#endif - - // this is just to keep the code cleaner - // the avro encoding includes - // an embedded schemaid which is used - // the embedding is a null byte - // followed by a 16 byte schemaid -#define AVRO_SCHEMA_OVERHEAD 16 + 1 -#endif - - flb_debug("in produce_message\n"); - if (flb_log_check(FLB_LOG_DEBUG)) - msgpack_object_print(stderr, *map); - - /* Init temporal buffers */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - if (ctx->format == FLB_KAFKA_FMT_JSON || ctx->format == FLB_KAFKA_FMT_MSGP) { - /* Make room for the timestamp */ - size = map->via.map.size + 1; - msgpack_pack_map(&mp_pck, size); - - /* Pack timestamp */ - msgpack_pack_str(&mp_pck, ctx->timestamp_key_len); - msgpack_pack_str_body(&mp_pck, - ctx->timestamp_key, ctx->timestamp_key_len); - switch (ctx->timestamp_format) { - case FLB_JSON_DATE_DOUBLE: - msgpack_pack_double(&mp_pck, flb_time_to_double(tm)); - break; - - case FLB_JSON_DATE_ISO8601: - case FLB_JSON_DATE_ISO8601_NS: - { - size_t date_len; - int len; - struct tm _tm; - char time_formatted[36]; - - /* Format the time; use microsecond precision (not nanoseconds). */ - gmtime_r(&tm->tm.tv_sec, &_tm); - date_len = strftime(time_formatted, sizeof(time_formatted) - 1, - FLB_JSON_DATE_ISO8601_FMT, &_tm); - - if (ctx->timestamp_format == FLB_JSON_DATE_ISO8601) { - len = snprintf(time_formatted + date_len, sizeof(time_formatted) - 1 - date_len, - ".%06" PRIu64 "Z", (uint64_t) tm->tm.tv_nsec / 1000); - } - else { - /* FLB_JSON_DATE_ISO8601_NS */ - len = snprintf(time_formatted + date_len, sizeof(time_formatted) - 1 - date_len, - ".%09" PRIu64 "Z", (uint64_t) tm->tm.tv_nsec); - } - date_len += len; - - msgpack_pack_str(&mp_pck, date_len); - msgpack_pack_str_body(&mp_pck, time_formatted, date_len); - } - break; - } - } - else { - size = map->via.map.size; - msgpack_pack_map(&mp_pck, size); - } - - for (i = 0; i < map->via.map.size; i++) { - key = map->via.map.ptr[i].key; - val = map->via.map.ptr[i].val; - - msgpack_pack_object(&mp_pck, key); - msgpack_pack_object(&mp_pck, val); - - /* Lookup message key */ - if (ctx->message_key_field && !message_key && val.type == MSGPACK_OBJECT_STR) { - if (key.via.str.size == ctx->message_key_field_len && - strncmp(key.via.str.ptr, ctx->message_key_field, ctx->message_key_field_len) == 0) { - message_key = (char *) val.via.str.ptr; - message_key_len = val.via.str.size; - } - } - - /* Lookup key/topic */ - if (ctx->topic_key && !topic && val.type == MSGPACK_OBJECT_STR) { - if (key.via.str.size == ctx->topic_key_len && - strncmp(key.via.str.ptr, ctx->topic_key, ctx->topic_key_len) == 0) { - topic = flb_kafka_topic_lookup((char *) val.via.str.ptr, - val.via.str.size, ctx); - /* Add extracted topic on the fly to topiclist */ - if (ctx->dynamic_topic) { - /* Only if default topic is set and this topicname is not set for this message */ - if (strncmp(topic->name, flb_kafka_topic_default(ctx)->name, val.via.str.size) == 0 && - (strncmp(topic->name, val.via.str.ptr, val.via.str.size) != 0) ) { - if (memchr(val.via.str.ptr, ',', val.via.str.size)) { - /* Don't allow commas in kafkatopic name */ - flb_warn("',' not allowed in dynamic_kafka topic names"); - continue; - } - if (val.via.str.size > 64) { - /* Don't allow length of dynamic kafka topics > 64 */ - flb_warn(" dynamic kafka topic length > 64 not allowed"); - continue; - } - dynamic_topic = flb_malloc(val.via.str.size + 1); - if (!dynamic_topic) { - /* Use default topic */ - flb_errno(); - continue; - } - strncpy(dynamic_topic, val.via.str.ptr, val.via.str.size); - dynamic_topic[val.via.str.size] = '\0'; - topics = flb_utils_split(dynamic_topic, ',', 0); - if (!topics) { - /* Use the default topic */ - flb_errno(); - flb_free(dynamic_topic); - continue; - } - mk_list_foreach(head, topics) { - /* Add the (one) found topicname to the topic configuration */ - entry = mk_list_entry(head, struct flb_split_entry, _head); - topic = flb_kafka_topic_create(entry->value, ctx); - if (!topic) { - /* Use default topic */ - flb_error("[out_kafka] cannot register topic '%s'", - entry->value); - topic = flb_kafka_topic_lookup((char *) val.via.str.ptr, - val.via.str.size, ctx); - } - else { - flb_info("[out_kafka] new topic added: %s", dynamic_topic); - } - } - flb_free(dynamic_topic); - } - } - } - } - } - - if (ctx->format == FLB_KAFKA_FMT_JSON) { - s = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); - if (!s) { - flb_plg_error(ctx->ins, "error encoding to JSON"); - msgpack_sbuffer_destroy(&mp_sbuf); - return FLB_ERROR; - } - out_buf = s; - out_size = flb_sds_len(out_buf); - } - else if (ctx->format == FLB_KAFKA_FMT_MSGP) { - out_buf = mp_sbuf.data; - out_size = mp_sbuf.size; - } - else if (ctx->format == FLB_KAFKA_FMT_GELF) { - s = flb_msgpack_raw_to_gelf(mp_sbuf.data, mp_sbuf.size, - tm, &(ctx->gelf_fields)); - if (s == NULL) { - flb_plg_error(ctx->ins, "error encoding to GELF"); - msgpack_sbuffer_destroy(&mp_sbuf); - return FLB_ERROR; - } - out_buf = s; - out_size = flb_sds_len(s); - } -#ifdef FLB_HAVE_AVRO_ENCODER - else if (ctx->format == FLB_KAFKA_FMT_AVRO) { - - flb_plg_debug(ctx->ins, "avro schema ID:%s:\n", ctx->avro_fields.schema_id); - flb_plg_debug(ctx->ins, "avro schema string:%s:\n", ctx->avro_fields.schema_str); - - // if there's no data then log it and return - if (mp_sbuf.size == 0) { - flb_plg_error(ctx->ins, "got zero bytes decoding to avro AVRO:schemaID:%s:\n", ctx->avro_fields.schema_id); - msgpack_sbuffer_destroy(&mp_sbuf); - return FLB_OK; - } - - // is the line is too long log it and return - if (mp_sbuf.size > AVRO_LINE_MAX_LEN) { - flb_plg_warn(ctx->ins, "skipping long line AVRO:len:%zu:limit:%zu:schemaID:%s:\n", (size_t)mp_sbuf.size, (size_t)AVRO_LINE_MAX_LEN, ctx->avro_fields.schema_id); - msgpack_sbuffer_destroy(&mp_sbuf); - return FLB_OK; - } - - flb_plg_debug(ctx->ins, "using default buffer AVRO:len:%zu:limit:%zu:schemaID:%s:\n", (size_t)mp_sbuf.size, (size_t)AVRO_DEFAULT_BUFFER_SIZE, ctx->avro_fields.schema_id); - out_buf = avro_buff; - out_size = AVRO_DEFAULT_BUFFER_SIZE; - - if (mp_sbuf.size + AVRO_SCHEMA_OVERHEAD >= AVRO_DEFAULT_BUFFER_SIZE) { - flb_plg_info(ctx->ins, "upsizing to dynamic buffer AVRO:len:%zu:schemaID:%s:\n", (size_t)mp_sbuf.size, ctx->avro_fields.schema_id); - avro_fast_buffer = false; - // avro will always be smaller than msgpack - // it contains no meta-info aside from the schemaid - // all the metadata is in the schema which is not part of the msg - // add schemaid + magic byte for safety buffer and allocate - // that's 16 byte schemaid and one byte magic byte - out_size = mp_sbuf.size + AVRO_SCHEMA_OVERHEAD; - out_buf = flb_malloc(out_size); - if (!out_buf) { - flb_plg_error(ctx->ins, "error allocating memory for decoding to AVRO:schema:%s:schemaID:%s:\n", ctx->avro_fields.schema_str, ctx->avro_fields.schema_id); - msgpack_sbuffer_destroy(&mp_sbuf); - return FLB_ERROR; - } - } - - if(!flb_msgpack_raw_to_avro_sds(mp_sbuf.data, mp_sbuf.size, &ctx->avro_fields, out_buf, &out_size)) { - flb_plg_error(ctx->ins, "error encoding to AVRO:schema:%s:schemaID:%s:\n", ctx->avro_fields.schema_str, ctx->avro_fields.schema_id); - msgpack_sbuffer_destroy(&mp_sbuf); - if (!avro_fast_buffer) { - flb_free(out_buf); - } - return FLB_ERROR; - } - - } -#endif - - if (!message_key) { - message_key = ctx->message_key; - message_key_len = ctx->message_key_len; - } - - if (!topic) { - topic = flb_kafka_topic_default(ctx); - } - if (!topic) { - flb_plg_error(ctx->ins, "no default topic found"); - msgpack_sbuffer_destroy(&mp_sbuf); -#ifdef FLB_HAVE_AVRO_ENCODER - if (ctx->format == FLB_KAFKA_FMT_AVRO) { - AVRO_FREE(avro_fast_buffer, out_buf) - } -#endif - return FLB_ERROR; - } - - retry: - /* - * If the local rdkafka queue is full, we retry up to 'queue_full_retries' - * times set by the configuration (default: 10). If the configuration - * property was set to '0' or 'false', we don't impose a limit. Use that - * value under your own risk. - */ - if (ctx->queue_full_retries > 0 && - queue_full_retries >= ctx->queue_full_retries) { - if (ctx->format != FLB_KAFKA_FMT_MSGP) { - flb_sds_destroy(s); - } - msgpack_sbuffer_destroy(&mp_sbuf); -#ifdef FLB_HAVE_AVRO_ENCODER - if (ctx->format == FLB_KAFKA_FMT_AVRO) { - AVRO_FREE(avro_fast_buffer, out_buf) - } -#endif - /* - * Unblock the flush requests so that the - * engine could try sending data again. - */ - ctx->blocked = FLB_FALSE; - return FLB_RETRY; - } - - ret = rd_kafka_produce(topic->tp, - RD_KAFKA_PARTITION_UA, - RD_KAFKA_MSG_F_COPY, - out_buf, out_size, - message_key, message_key_len, - ctx); - - if (ret == -1) { - flb_error( - "%% Failed to produce to topic %s: %s\n", - rd_kafka_topic_name(topic->tp), - rd_kafka_err2str(rd_kafka_last_error())); - - /* - * rdkafka queue is full, keep trying 'locally' for a few seconds, - * otherwise let the caller to issue a main retry againt the engine. - */ - if (rd_kafka_last_error() == RD_KAFKA_RESP_ERR__QUEUE_FULL) { - flb_plg_warn(ctx->ins, - "internal queue is full, retrying in one second"); - - /* - * If the queue is full, first make sure to discard any further - * flush request from the engine. This means 'the caller will - * issue a retry at a later time'. - */ - ctx->blocked = FLB_TRUE; - - /* - * Next step is to give it some time to the background rdkafka - * library to do it own work. By default rdkafka wait 1 second - * or up to 10000 messages to be enqueued before delivery. - * - * If the kafka broker is down we should try a couple of times - * to enqueue this message, if we exceed 10 times, we just - * issue a full retry of the data chunk. - */ - flb_time_sleep(1000); - rd_kafka_poll(ctx->kafka.rk, 0); - - /* Issue a re-try */ - queue_full_retries++; - goto retry; - } - } - else { - flb_plg_debug(ctx->ins, "enqueued message (%zd bytes) for topic '%s'", - out_size, rd_kafka_topic_name(topic->tp)); - } - ctx->blocked = FLB_FALSE; - - rd_kafka_poll(ctx->kafka.rk, 0); - if (ctx->format == FLB_KAFKA_FMT_JSON) { - flb_sds_destroy(s); - } - if (ctx->format == FLB_KAFKA_FMT_GELF) { - flb_sds_destroy(s); - } -#ifdef FLB_HAVE_AVRO_ENCODER - if (ctx->format == FLB_KAFKA_FMT_AVRO) { - AVRO_FREE(avro_fast_buffer, out_buf) - } -#endif - - msgpack_sbuffer_destroy(&mp_sbuf); - return FLB_OK; -} - -static void cb_kafka_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - - int ret; - struct flb_out_kafka *ctx = out_context; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - /* - * If the context is blocked, means rdkafka queue is full and no more - * messages can be appended. For our called (Fluent Bit engine) means - * that is not possible to work on this now and it need to 'retry'. - */ - if (ctx->blocked == FLB_TRUE) { - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - ret = flb_log_event_decoder_init(&log_decoder, - (char *) event_chunk->data, - event_chunk->size); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Iterate the original buffer and perform adjustments */ - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - ret = produce_message(&log_event.timestamp, - log_event.body, - ctx, config); - - if (ret != FLB_OK) { - flb_log_event_decoder_destroy(&log_decoder); - - FLB_OUTPUT_RETURN(ret); - } - } - - flb_log_event_decoder_destroy(&log_decoder); - - FLB_OUTPUT_RETURN(FLB_OK); -} - -static void kafka_flush_force(struct flb_out_kafka *ctx, - struct flb_config *config) -{ - int ret; - - if (!ctx) { - return; - } - - if (ctx->kafka.rk) { - ret = rd_kafka_flush(ctx->kafka.rk, config->grace * 1000); - if (ret != RD_KAFKA_RESP_ERR_NO_ERROR) { - flb_plg_warn(ctx->ins, "Failed to force flush: %s", - rd_kafka_err2str(ret)); - } - } -} - -static int cb_kafka_exit(void *data, struct flb_config *config) -{ - struct flb_out_kafka *ctx = data; - - kafka_flush_force(ctx, config); - flb_out_kafka_destroy(ctx); - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "topic_key", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_out_kafka, topic_key), - "Which record to use as the kafka topic." - }, - { - FLB_CONFIG_MAP_BOOL, "dynamic_topic", "false", - 0, FLB_TRUE, offsetof(struct flb_out_kafka, dynamic_topic), - "Activate dynamic topics." - }, - { - FLB_CONFIG_MAP_STR, "format", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_out_kafka, format_str), - "Set the record output format." - }, - { - FLB_CONFIG_MAP_STR, "message_key", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_out_kafka, message_key), - "Which record key to use as the message data." - }, - { - FLB_CONFIG_MAP_STR, "message_key_field", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_out_kafka, message_key_field), - "Which record key field to use as the message data." - }, - { - FLB_CONFIG_MAP_STR, "timestamp_key", FLB_KAFKA_TS_KEY, - 0, FLB_TRUE, offsetof(struct flb_out_kafka, timestamp_key), - "Set the key for the the timestamp." - }, - { - FLB_CONFIG_MAP_STR, "timestamp_format", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_out_kafka, timestamp_format_str), - "Set the format the timestamp is in." - }, - { - FLB_CONFIG_MAP_INT, "queue_full_retries", FLB_KAFKA_QUEUE_FULL_RETRIES, - 0, FLB_TRUE, offsetof(struct flb_out_kafka, queue_full_retries), - "Set the number of local retries to enqueue the data." - }, - { - FLB_CONFIG_MAP_STR, "gelf_timestamp_key", (char *)NULL, - 0, FLB_FALSE, 0, - "Set the timestamp key for gelf output." - }, - { - FLB_CONFIG_MAP_STR, "gelf_host_key", (char *)NULL, - 0, FLB_FALSE, 0, - "Set the host key for gelf output." - }, - { - FLB_CONFIG_MAP_STR, "gelf_short_message_key", (char *)NULL, - 0, FLB_FALSE, 0, - "Set the short message key for gelf output." - }, - { - FLB_CONFIG_MAP_STR, "gelf_full_message_key", (char *)NULL, - 0, FLB_FALSE, 0, - "Set the full message key for gelf output." - }, - { - FLB_CONFIG_MAP_STR, "gelf_level_key", (char *)NULL, - 0, FLB_FALSE, 0, - "Set the level key for gelf output." - }, -#ifdef FLB_HAVE_AVRO_ENCODER - { - FLB_CONFIG_MAP_STR, "schema_str", (char *)NULL, - 0, FLB_FALSE, 0, - "Set AVRO schema." - }, - { - FLB_CONFIG_MAP_STR, "schema_id", (char *)NULL, - 0, FLB_FALSE, 0, - "Set AVRO schema ID." - }, -#endif - { - FLB_CONFIG_MAP_STR, "topics", (char *)NULL, - 0, FLB_FALSE, 0, - "Set the kafka topics, delimited by commas." - }, - { - FLB_CONFIG_MAP_STR, "brokers", (char *)NULL, - 0, FLB_FALSE, 0, - "Set the kafka brokers, delimited by commas." - }, - { - FLB_CONFIG_MAP_STR, "client_id", (char *)NULL, - 0, FLB_FALSE, 0, - "Set the kafka client_id." - }, - { - FLB_CONFIG_MAP_STR, "group_id", (char *)NULL, - 0, FLB_FALSE, 0, - "Set the kafka group_id." - }, - { - FLB_CONFIG_MAP_STR_PREFIX, "rdkafka.", NULL, - //FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct flb_out_kafka, rdkafka_opts), - 0, FLB_FALSE, 0, - "Set the kafka group_id." - }, - /* EOF */ - {0} -}; - -struct flb_output_plugin out_kafka_plugin = { - .name = "kafka", - .description = "Kafka", - .cb_init = cb_kafka_init, - .cb_flush = cb_kafka_flush, - .cb_exit = cb_kafka_exit, - .config_map = config_map, - .flags = 0 -}; diff --git a/fluent-bit/plugins/out_kafka/kafka_callbacks.h b/fluent-bit/plugins/out_kafka/kafka_callbacks.h deleted file mode 100644 index f496cba8c..000000000 --- a/fluent-bit/plugins/out_kafka/kafka_callbacks.h +++ /dev/null @@ -1,31 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_KAFKA_CALLBACKS_H -#define FLB_OUT_KAFKA_CALLBACKS_H - -#include "rdkafka.h" - -void cb_kafka_msg(rd_kafka_t *rk, const rd_kafka_message_t *rkmessage, - void *opaque); - -void cb_kafka_logger(const rd_kafka_t *rk, int level, - const char *fac, const char *buf); - -#endif diff --git a/fluent-bit/plugins/out_kafka/kafka_config.c b/fluent-bit/plugins/out_kafka/kafka_config.c deleted file mode 100644 index 3c00f3682..000000000 --- a/fluent-bit/plugins/out_kafka/kafka_config.c +++ /dev/null @@ -1,253 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "kafka_config.h" -#include "kafka_topic.h" -#include "kafka_callbacks.h" - -struct flb_out_kafka *flb_out_kafka_create(struct flb_output_instance *ins, - struct flb_config *config) -{ - int ret; - const char *tmp; - char errstr[512]; - struct mk_list *head; - struct mk_list *topics; - struct flb_split_entry *entry; - struct flb_out_kafka *ctx; - - /* Configuration context */ - ctx = flb_calloc(1, sizeof(struct flb_out_kafka)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - ctx->blocked = FLB_FALSE; - - ret = flb_output_config_map_set(ins, (void*) ctx); - if (ret == -1) { - flb_plg_error(ins, "unable to load configuration."); - flb_free(ctx); - - return NULL; - } - - /* rdkafka config context */ - ctx->conf = flb_kafka_conf_create(&ctx->kafka, &ins->properties, 0); - if (!ctx->conf) { - flb_plg_error(ctx->ins, "error creating context"); - flb_free(ctx); - return NULL; - } - - /* Set our global opaque data (plugin context*/ - rd_kafka_conf_set_opaque(ctx->conf, ctx); - - /* Callback: message delivery */ - rd_kafka_conf_set_dr_msg_cb(ctx->conf, cb_kafka_msg); - - /* Callback: log */ - rd_kafka_conf_set_log_cb(ctx->conf, cb_kafka_logger); - - /* Config: Topic_Key */ - if (ctx->topic_key) { - ctx->topic_key_len = strlen(ctx->topic_key); - } - - /* Config: Format */ - if (ctx->format_str) { - if (strcasecmp(ctx->format_str, "json") == 0) { - ctx->format = FLB_KAFKA_FMT_JSON; - } - else if (strcasecmp(ctx->format_str, "msgpack") == 0) { - ctx->format = FLB_KAFKA_FMT_MSGP; - } - else if (strcasecmp(ctx->format_str, "gelf") == 0) { - ctx->format = FLB_KAFKA_FMT_GELF; - } -#ifdef FLB_HAVE_AVRO_ENCODER - else if (strcasecmp(ctx->format_str, "avro") == 0) { - ctx->format = FLB_KAFKA_FMT_AVRO; - } -#endif - } - else { - ctx->format = FLB_KAFKA_FMT_JSON; - } - - /* Config: Message_Key */ - if (ctx->message_key) { - ctx->message_key_len = strlen(ctx->message_key); - } - else { - ctx->message_key_len = 0; - } - - /* Config: Message_Key_Field */ - if (ctx->message_key_field) { - ctx->message_key_field_len = strlen(ctx->message_key_field); - } - else { - ctx->message_key_field_len = 0; - } - - /* Config: Timestamp_Key */ - if (ctx->timestamp_key) { - ctx->timestamp_key_len = strlen(ctx->timestamp_key); - } - - /* Config: Timestamp_Format */ - ctx->timestamp_format = FLB_JSON_DATE_DOUBLE; - if (ctx->timestamp_format_str) { - if (strcasecmp(ctx->timestamp_format_str, "iso8601") == 0) { - ctx->timestamp_format = FLB_JSON_DATE_ISO8601; - } - else if (strcasecmp(ctx->timestamp_format_str, "iso8601_ns") == 0) { - ctx->timestamp_format = FLB_JSON_DATE_ISO8601_NS; - } - } - - /* set number of retries: note that if the number is zero, means forever */ - if (ctx->queue_full_retries < 0) { - ctx->queue_full_retries = 0; - } - - /* Config Gelf_Short_Message_Key */ - tmp = flb_output_get_property("gelf_short_message_key", ins); - if (tmp) { - ctx->gelf_fields.short_message_key = flb_sds_create(tmp); - } - - /* Config Gelf_Full_Message_Key */ - tmp = flb_output_get_property("gelf_full_message_key", ins); - if (tmp) { - ctx->gelf_fields.full_message_key = flb_sds_create(tmp); - } - - /* Config Gelf_Level_Key */ - tmp = flb_output_get_property("gelf_level_key", ins); - if (tmp) { - ctx->gelf_fields.level_key = flb_sds_create(tmp); - } - - /* Kafka Producer */ - ctx->kafka.rk = rd_kafka_new(RD_KAFKA_PRODUCER, ctx->conf, - errstr, sizeof(errstr)); - if (!ctx->kafka.rk) { - flb_plg_error(ctx->ins, "failed to create producer: %s", - errstr); - flb_out_kafka_destroy(ctx); - return NULL; - } - -#ifdef FLB_HAVE_AVRO_ENCODER - /* Config AVRO */ - tmp = flb_output_get_property("schema_str", ins); - if (tmp) { - ctx->avro_fields.schema_str = flb_sds_create(tmp); - } - tmp = flb_output_get_property("schema_id", ins); - if (tmp) { - ctx->avro_fields.schema_id = flb_sds_create(tmp); - } -#endif - - /* Config: Topic */ - mk_list_init(&ctx->topics); - tmp = flb_output_get_property("topics", ins); - if (!tmp) { - flb_kafka_topic_create(FLB_KAFKA_TOPIC, ctx); - } - else { - topics = flb_utils_split(tmp, ',', -1); - if (!topics) { - flb_plg_warn(ctx->ins, "invalid topics defined, setting default"); - flb_kafka_topic_create(FLB_KAFKA_TOPIC, ctx); - } - else { - /* Register each topic */ - mk_list_foreach(head, topics) { - entry = mk_list_entry(head, struct flb_split_entry, _head); - if (!flb_kafka_topic_create(entry->value, ctx)) { - flb_plg_error(ctx->ins, "cannot register topic '%s'", - entry->value); - } - } - flb_utils_split_free(topics); - } - } - - flb_plg_info(ctx->ins, "brokers='%s' topics='%s'", ctx->kafka.brokers, tmp); -#ifdef FLB_HAVE_AVRO_ENCODER - flb_plg_info(ctx->ins, "schemaID='%s' schema='%s'", ctx->avro_fields.schema_id, ctx->avro_fields.schema_str); -#endif - - return ctx; -} - -int flb_out_kafka_destroy(struct flb_out_kafka *ctx) -{ - if (!ctx) { - return 0; - } - - if (ctx->kafka.brokers) { - flb_free(ctx->kafka.brokers); - } - - flb_kafka_topic_destroy_all(ctx); - - if (ctx->kafka.rk) { - rd_kafka_destroy(ctx->kafka.rk); - } - - if (ctx->topic_key) { - flb_free(ctx->topic_key); - } - - if (ctx->message_key) { - flb_free(ctx->message_key); - } - - if (ctx->message_key_field) { - flb_free(ctx->message_key_field); - } - - flb_sds_destroy(ctx->gelf_fields.timestamp_key); - flb_sds_destroy(ctx->gelf_fields.host_key); - flb_sds_destroy(ctx->gelf_fields.short_message_key); - flb_sds_destroy(ctx->gelf_fields.full_message_key); - flb_sds_destroy(ctx->gelf_fields.level_key); - -#ifdef FLB_HAVE_AVRO_ENCODER - // avro - flb_sds_destroy(ctx->avro_fields.schema_id); - flb_sds_destroy(ctx->avro_fields.schema_str); -#endif - - flb_free(ctx); - return 0; -} diff --git a/fluent-bit/plugins/out_kafka/kafka_config.h b/fluent-bit/plugins/out_kafka/kafka_config.h deleted file mode 100644 index 1ef2cce16..000000000 --- a/fluent-bit/plugins/out_kafka/kafka_config.h +++ /dev/null @@ -1,129 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_KAFKA_CONFIG_H -#define FLB_OUT_KAFKA_CONFIG_H - -#include -#include -#ifdef FLB_HAVE_AVRO_ENCODER -#include -#endif - -#include - -#define FLB_KAFKA_FMT_JSON 0 -#define FLB_KAFKA_FMT_MSGP 1 -#define FLB_KAFKA_FMT_GELF 2 -#ifdef FLB_HAVE_AVRO_ENCODER -#define FLB_KAFKA_FMT_AVRO 3 -#endif -#define FLB_KAFKA_TS_KEY "@timestamp" -#define FLB_KAFKA_QUEUE_FULL_RETRIES "10" - -/* rdkafka log levels based on syslog(3) */ -#define FLB_KAFKA_LOG_EMERG 0 -#define FLB_KAFKA_LOG_ALERT 1 -#define FLB_KAFKA_LOG_CRIT 2 -#define FLB_KAFKA_LOG_ERR 3 -#define FLB_KAFKA_LOG_WARNING 4 -#define FLB_KAFKA_LOG_NOTICE 5 -#define FLB_KAFKA_LOG_INFO 6 -#define FLB_KAFKA_LOG_DEBUG 7 - -#define FLB_JSON_DATE_DOUBLE 0 -#define FLB_JSON_DATE_ISO8601 1 -#define FLB_JSON_DATE_ISO8601_NS 2 -#define FLB_JSON_DATE_ISO8601_FMT "%Y-%m-%dT%H:%M:%S" - -struct flb_kafka_topic { - int name_len; - char *name; - rd_kafka_topic_t *tp; - struct mk_list _head; -}; - -struct flb_out_kafka { - struct flb_kafka kafka; - /* Config Parameters */ - int format; - flb_sds_t format_str; - - /* Optional topic key for routing */ - int topic_key_len; - char *topic_key; - - int timestamp_key_len; - char *timestamp_key; - int timestamp_format; - flb_sds_t timestamp_format_str; - - int message_key_len; - char *message_key; - - int message_key_field_len; - char *message_key_field; - - /* Gelf Keys */ - struct flb_gelf_fields gelf_fields; - - /* Head of defined topics by configuration */ - struct mk_list topics; - - /* - * Blocked Status: since rdkafka have it own buffering queue, there is a - * chance that the queue becomes full, when that happens our default - * behavior is the following: - * - * - out_kafka yields and try to continue every second until it succeed. In - * the meanwhile blocked flag gets FLB_TRUE value. - * - when flushing more records and blocked == FLB_TRUE, issue - * a retry. - */ - int blocked; - - int dynamic_topic; - - int queue_full_retries; - - /* Internal */ - rd_kafka_conf_t *conf; - - /* Plugin instance */ - struct flb_output_instance *ins; - -#ifdef FLB_HAVE_AVRO_ENCODER - // avro serialization requires a schema - // the schema is stored in json in avro_schema_str - // - // optionally the schema ID can be stashed in the avro data stream - // the schema ID is stored in avro_schema_id - // this is common at this time with large kafka installations and schema registries - // flb_sds_t avro_schema_str; - // flb_sds_t avro_schema_id; - struct flb_avro_fields avro_fields; -#endif - -}; - -struct flb_out_kafka *flb_out_kafka_create(struct flb_output_instance *ins, - struct flb_config *config); -int flb_out_kafka_destroy(struct flb_out_kafka *ctx); - -#endif diff --git a/fluent-bit/plugins/out_kafka/kafka_topic.c b/fluent-bit/plugins/out_kafka/kafka_topic.c deleted file mode 100644 index 2db8698b1..000000000 --- a/fluent-bit/plugins/out_kafka/kafka_topic.c +++ /dev/null @@ -1,120 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include "kafka_config.h" -#include "rdkafka.h" - -struct flb_kafka_topic *flb_kafka_topic_create(char *name, - struct flb_out_kafka *ctx) -{ - rd_kafka_topic_t *tp; - struct flb_kafka_topic *topic; - - tp = rd_kafka_topic_new(ctx->kafka.rk, name, NULL); - if (!tp) { - flb_plg_error(ctx->ins, "failed to create topic: %s", - rd_kafka_err2str(rd_kafka_last_error())); - return NULL; - } - - topic = flb_malloc(sizeof(struct flb_kafka_topic)); - if (!topic) { - flb_errno(); - return NULL; - } - - topic->name = flb_strdup(name); - topic->name_len = strlen(name); - topic->tp = tp; - mk_list_add(&topic->_head, &ctx->topics); - - return topic; -} - -int flb_kafka_topic_destroy(struct flb_kafka_topic *topic, - struct flb_out_kafka *ctx) -{ - mk_list_del(&topic->_head); - rd_kafka_topic_destroy(topic->tp); - flb_free(topic->name); - flb_free(topic); - - return 0; -} - -int flb_kafka_topic_destroy_all(struct flb_out_kafka *ctx) -{ - int c = 0; - struct mk_list *tmp; - struct mk_list *head; - struct flb_kafka_topic *topic; - - mk_list_foreach_safe(head, tmp, &ctx->topics) { - topic = mk_list_entry(head, struct flb_kafka_topic, _head); - flb_kafka_topic_destroy(topic, ctx); - c++; - } - - return c; -} - -/* Get first topic of the list (default topic) */ -struct flb_kafka_topic *flb_kafka_topic_default(struct flb_out_kafka *ctx) -{ - struct flb_kafka_topic *topic; - - if (mk_list_is_empty(&ctx->topics) == 0) { - return NULL; - } - - topic = mk_list_entry_first(&ctx->topics, struct flb_kafka_topic, - _head); - return topic; -} - -struct flb_kafka_topic *flb_kafka_topic_lookup(char *name, - int name_len, - struct flb_out_kafka *ctx) -{ - struct mk_list *head; - struct flb_kafka_topic *topic; - - if (!ctx->topic_key) { - return flb_kafka_topic_default(ctx); - } - - mk_list_foreach(head, &ctx->topics) { - topic = mk_list_entry(head, struct flb_kafka_topic, _head); - if (topic->name_len != name_len) { - continue; - } - - if (strncmp(name, topic->name, topic->name_len) == 0) { - return topic; - } - } - - /* No matches, return the default topic */ - return flb_kafka_topic_default(ctx); - -} diff --git a/fluent-bit/plugins/out_kafka/kafka_topic.h b/fluent-bit/plugins/out_kafka/kafka_topic.h deleted file mode 100644 index 9b1203b96..000000000 --- a/fluent-bit/plugins/out_kafka/kafka_topic.h +++ /dev/null @@ -1,34 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_KAFKA_TOPIC_H -#define FLB_KAFKA_TOPIC_H - -struct flb_kafka_topic *flb_kafka_topic_create(char *name, - struct flb_out_kafka *ctx); -int flb_kafka_topic_destroy(struct flb_kafka_topic *topic, - struct flb_out_kafka *ctx); -int flb_kafka_topic_destroy_all(struct flb_out_kafka *ctx); -struct flb_kafka_topic *flb_kafka_topic_default(struct flb_out_kafka *ctx); - -struct flb_kafka_topic *flb_kafka_topic_lookup(char *name, - int name_len, - struct flb_out_kafka *ctx); - -#endif diff --git a/fluent-bit/plugins/out_kafka_rest/CMakeLists.txt b/fluent-bit/plugins/out_kafka_rest/CMakeLists.txt deleted file mode 100644 index 39df92f77..000000000 --- a/fluent-bit/plugins/out_kafka_rest/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -set(src - kafka_conf.c - kafka.c) - -FLB_PLUGIN(out_kafka_rest "${src}" "") diff --git a/fluent-bit/plugins/out_kafka_rest/kafka.c b/fluent-bit/plugins/out_kafka_rest/kafka.c deleted file mode 100644 index f3b6153a6..000000000 --- a/fluent-bit/plugins/out_kafka_rest/kafka.c +++ /dev/null @@ -1,351 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "kafka.h" -#include "kafka_conf.h" - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "message_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_kafka_rest, message_key), - "Specify a message key. " - }, - - { - FLB_CONFIG_MAP_STR, "time_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_kafka_rest, time_key), - "Specify the name of the field that holds the record timestamp. " - }, - - { - FLB_CONFIG_MAP_STR, "topic", "fluent-bit", - 0, FLB_TRUE, offsetof(struct flb_kafka_rest, topic), - "Specify the kafka topic. " - }, - - { - FLB_CONFIG_MAP_STR, "url_path", NULL, - 0, FLB_TRUE, offsetof(struct flb_kafka_rest, url_path), - "Specify an optional HTTP URL path for the target web server, e.g: /something" - }, - - { - FLB_CONFIG_MAP_DOUBLE, "partition", "-1", - 0, FLB_TRUE, offsetof(struct flb_kafka_rest, partition), - "Specify kafka partition number. " - }, - - { - FLB_CONFIG_MAP_STR, "time_key_format", FLB_KAFKA_TIME_KEYF, - 0, FLB_TRUE, offsetof(struct flb_kafka_rest, time_key_format), - "Specify the format of the timestamp. " - }, - - { - FLB_CONFIG_MAP_BOOL, "include_tag_key", "false", - 0, FLB_TRUE, offsetof(struct flb_kafka_rest, include_tag_key), - "Specify whether to append tag name to final record. " - }, - - { - FLB_CONFIG_MAP_STR, "tag_key", "_flb-key", - 0, FLB_TRUE, offsetof(struct flb_kafka_rest, tag_key), - "Specify the key name of the record if include_tag_key is enabled. " - }, - { - FLB_CONFIG_MAP_BOOL, "avro_http_header", "false", - 0, FLB_TRUE, offsetof(struct flb_kafka_rest, avro_http_header), - "Specify if the format has avro header in http request" - }, - - /* EOF */ - {0} -}; -/* - * Convert the internal Fluent Bit data representation to the required - * one by Kafka REST Proxy. - */ -static flb_sds_t kafka_rest_format(const void *data, size_t bytes, - const char *tag, int tag_len, - size_t *out_size, - struct flb_kafka_rest *ctx) -{ - int i; - int len; - int arr_size = 0; - int map_size; - size_t s; - flb_sds_t out_buf; - char time_formatted[256]; - msgpack_object map; - msgpack_object key; - msgpack_object val; - struct tm tm; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - int ret; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return NULL; - } - - /* Init temporary buffers */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - /* Count number of entries */ - arr_size = flb_mp_count(data, bytes); - - /* Root map */ - msgpack_pack_map(&mp_pck, 1); - msgpack_pack_str(&mp_pck, 7); - msgpack_pack_str_body(&mp_pck, "records", 7); - - msgpack_pack_array(&mp_pck, arr_size); - - /* Iterate and compose array content */ - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - map = *log_event.body; - map_size = 1; - - if (ctx->partition >= 0) { - map_size++; - } - - if (ctx->message_key != NULL) { - map_size++; - } - - msgpack_pack_map(&mp_pck, map_size); - if (ctx->partition >= 0) { - msgpack_pack_str(&mp_pck, 9); - msgpack_pack_str_body(&mp_pck, "partition", 9); - msgpack_pack_int64(&mp_pck, ctx->partition); - } - - - if (ctx->message_key != NULL) { - msgpack_pack_str(&mp_pck, 3); - msgpack_pack_str_body(&mp_pck, "key", 3); - msgpack_pack_str(&mp_pck, ctx->message_key_len); - msgpack_pack_str_body(&mp_pck, ctx->message_key, ctx->message_key_len); - } - - /* Value Map Size */ - map_size = map.via.map.size; - map_size++; - if (ctx->include_tag_key == FLB_TRUE) { - map_size++; - } - - msgpack_pack_str(&mp_pck, 5); - msgpack_pack_str_body(&mp_pck, "value", 5); - - msgpack_pack_map(&mp_pck, map_size); - - /* Time key and time formatted */ - msgpack_pack_str(&mp_pck, ctx->time_key_len); - msgpack_pack_str_body(&mp_pck, ctx->time_key, ctx->time_key_len); - - /* Format the time */ - gmtime_r(&log_event.timestamp.tm.tv_sec, &tm); - s = strftime(time_formatted, sizeof(time_formatted) - 1, - ctx->time_key_format, &tm); - len = snprintf(time_formatted + s, sizeof(time_formatted) - 1 - s, - ".%09" PRIu64 "Z", (uint64_t) log_event.timestamp.tm.tv_nsec); - s += len; - msgpack_pack_str(&mp_pck, s); - msgpack_pack_str_body(&mp_pck, time_formatted, s); - - /* Tag Key */ - if (ctx->include_tag_key == FLB_TRUE) { - msgpack_pack_str(&mp_pck, ctx->tag_key_len); - msgpack_pack_str_body(&mp_pck, ctx->tag_key, ctx->tag_key_len); - msgpack_pack_str(&mp_pck, tag_len); - msgpack_pack_str_body(&mp_pck, tag, tag_len); - } - - for (i = 0; i < map.via.map.size; i++) { - key = map.via.map.ptr[i].key; - val = map.via.map.ptr[i].val; - - msgpack_pack_object(&mp_pck, key); - msgpack_pack_object(&mp_pck, val); - } - } - flb_log_event_decoder_destroy(&log_decoder); - - /* Convert to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); - msgpack_sbuffer_destroy(&mp_sbuf); - if (!out_buf) { - return NULL; - } - - *out_size = flb_sds_len(out_buf); - - return out_buf; -} - -static int cb_kafka_init(struct flb_output_instance *ins, - struct flb_config *config, - void *data) -{ - (void) ins; - (void) config; - (void) data; - struct flb_kafka_rest *ctx; - - ctx = flb_kr_conf_create(ins, config); - if (!ctx) { - flb_plg_error(ins, "cannot initialize plugin"); - return -1; - } - - flb_plg_debug(ctx->ins, "host=%s port=%i", - ins->host.name, ins->host.port); - flb_output_set_context(ins, ctx); - - return 0; -} - -static void cb_kafka_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - int ret; - flb_sds_t js; - size_t js_size; - size_t b_sent; - struct flb_http_client *c; - struct flb_connection *u_conn; - struct flb_kafka_rest *ctx = out_context; - (void) i_ins; - - /* Get upstream connection */ - u_conn = flb_upstream_conn_get(ctx->u); - if (!u_conn) { - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Convert format */ - js = kafka_rest_format(event_chunk->data, event_chunk->size, - event_chunk->tag, flb_sds_len(event_chunk->tag), - &js_size, ctx); - if (!js) { - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_ERROR); - } - - /* Compose HTTP Client request */ - c = flb_http_client(u_conn, FLB_HTTP_POST, ctx->uri, - js, js_size, NULL, 0, NULL, 0); - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - if (ctx->avro_http_header == FLB_TRUE) { - flb_http_add_header(c, - "Content-Type", 12, - "application/vnd.kafka.avro.v2+json", 34); - } - else { - flb_http_add_header(c, - "Content-Type", 12, - "application/vnd.kafka.json.v2+json", 34); - } - - if (ctx->http_user && ctx->http_passwd) { - flb_http_basic_auth(c, ctx->http_user, ctx->http_passwd); - } - - ret = flb_http_do(c, &b_sent); - if (ret != 0) { - flb_plg_warn(ctx->ins, "http_do=%i", ret); - goto retry; - } - else { - /* The request was issued successfully, validate the 'error' field */ - flb_plg_debug(ctx->ins, "HTTP Status=%i", c->resp.status); - if (c->resp.status != 200) { - if (c->resp.payload_size > 0) { - flb_plg_debug(ctx->ins, "Kafka REST response\n%s", - c->resp.payload); - } - goto retry; - } - - if (c->resp.payload_size > 0) { - flb_plg_debug(ctx->ins, "Kafka REST response\n%s", - c->resp.payload); - } - else { - goto retry; - } - } - - /* Cleanup */ - flb_http_client_destroy(c); - flb_sds_destroy(js); - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_OK); - - /* Issue a retry */ - retry: - flb_http_client_destroy(c); - flb_sds_destroy(js); - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_RETRY); -} - -static int cb_kafka_exit(void *data, struct flb_config *config) -{ - struct flb_kafka_rest *ctx = data; - - flb_kr_conf_destroy(ctx); - return 0; -} - -struct flb_output_plugin out_kafka_rest_plugin = { - .name = "kafka-rest", - .description = "Kafka REST Proxy", - .cb_init = cb_kafka_init, - .cb_flush = cb_kafka_flush, - .cb_exit = cb_kafka_exit, - .config_map = config_map, - .flags = FLB_OUTPUT_NET | FLB_IO_OPT_TLS, -}; diff --git a/fluent-bit/plugins/out_kafka_rest/kafka.h b/fluent-bit/plugins/out_kafka_rest/kafka.h deleted file mode 100644 index c2d220e7d..000000000 --- a/fluent-bit/plugins/out_kafka_rest/kafka.h +++ /dev/null @@ -1,66 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_KAFKA_REST_H -#define FLB_OUT_KAFKA_REST_H - -#define FLB_KAFKA_TIME_KEY "@timestamp" -#define FLB_KAFKA_TIME_KEYF "%Y-%m-%dT%H:%M:%S" -#define FLB_KAFKA_TAG_KEY "_flb-key" - -struct flb_kafka_rest { - /* Kafka specifics */ - long partition; - char *topic; - int message_key_len; - char *message_key; - - /* HTTP Auth */ - char *http_user; - char *http_passwd; - - /* time key */ - int time_key_len; - char *time_key; - - /* time key format */ - int time_key_format_len; - char *time_key_format; - - /* include_tag_key */ - int include_tag_key; - int tag_key_len; - char *tag_key; - - /* HTTP URI */ - char uri[256]; - char *url_path; - - /* Upstream connection to the backend server */ - struct flb_upstream *u; - - /* Plugin instance */ - struct flb_output_instance *ins; - - /* Avro http header*/ - int avro_http_header; -}; - - -#endif diff --git a/fluent-bit/plugins/out_kafka_rest/kafka_conf.c b/fluent-bit/plugins/out_kafka_rest/kafka_conf.c deleted file mode 100644 index 3df50eb8b..000000000 --- a/fluent-bit/plugins/out_kafka_rest/kafka_conf.c +++ /dev/null @@ -1,223 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include "kafka.h" -#include "kafka_conf.h" - -struct flb_kafka_rest *flb_kr_conf_create(struct flb_output_instance *ins, - struct flb_config *config) -{ - long part; - int io_flags = 0; - const char *tmp; - char *endptr; - struct flb_upstream *upstream; - struct flb_kafka_rest *ctx; - int ret; - - /* Allocate context */ - ctx = flb_calloc(1, sizeof(struct flb_kafka_rest)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return NULL; - } - - /* Get network configuration */ - flb_output_net_default("127.0.0.1", 8082, ins); - - /* use TLS ? */ - if (ins->use_tls == FLB_TRUE) { - io_flags = FLB_IO_TLS; - } - else { - io_flags = FLB_IO_TCP; - } - - if (ins->host.ipv6 == FLB_TRUE) { - io_flags |= FLB_IO_IPV6; - } - - /* Prepare an upstream handler */ - upstream = flb_upstream_create(config, - ins->host.name, - ins->host.port, - io_flags, - ins->tls); - if (!upstream) { - flb_plg_error(ctx->ins, "cannot create Upstream context"); - flb_kr_conf_destroy(ctx); - return NULL; - } - ctx->u = upstream; - flb_output_upstream_set(ctx->u, ins); - - flb_output_upstream_set(ctx->u, ins); - - /* HTTP Auth */ - tmp = flb_output_get_property("http_user", ins); - if (tmp) { - ctx->http_user = flb_strdup(tmp); - - tmp = flb_output_get_property("http_passwd", ins); - if (tmp) { - ctx->http_passwd = flb_strdup(tmp); - } - else { - ctx->http_passwd = flb_strdup(""); - } - } - - /* Time Key */ - tmp = flb_output_get_property("time_key", ins); - if (tmp) { - ctx->time_key = flb_strdup(tmp); - ctx->time_key_len = strlen(tmp); - } - else { - ctx->time_key = flb_strdup(FLB_KAFKA_TIME_KEY); - ctx->time_key_len = sizeof(FLB_KAFKA_TIME_KEY) - 1; - } - - /* Time Key Format */ - tmp = flb_output_get_property("time_key_format", ins); - if (tmp) { - ctx->time_key_format = flb_strdup(tmp); - ctx->time_key_format_len = strlen(tmp); - } - else { - ctx->time_key_format = flb_strdup(FLB_KAFKA_TIME_KEYF); - ctx->time_key_format_len = sizeof(FLB_KAFKA_TIME_KEYF) - 1; - } - - /* Include Tag key */ - tmp = flb_output_get_property("include_tag_key", ins); - if (tmp) { - ctx->include_tag_key = flb_utils_bool(tmp); - } - else { - ctx->include_tag_key = FLB_FALSE; - } - - /* Tag Key */ - if (ctx->include_tag_key == FLB_TRUE) { - tmp = flb_output_get_property("tag_key", ins); - if (tmp) { - ctx->tag_key = flb_strdup(tmp); - ctx->tag_key_len = strlen(tmp); - if (tmp[0] != '_') { - flb_plg_warn(ctx->ins, "consider use a tag_key " - "that starts with '_'"); - } - } - else { - ctx->tag_key = flb_strdup(FLB_KAFKA_TAG_KEY); - ctx->tag_key_len = sizeof(FLB_KAFKA_TAG_KEY) - 1; - } - } - - /* Kafka: partition */ - tmp = flb_output_get_property("partition", ins); - if (tmp) { - errno = 0; - part = strtol(tmp, &endptr, 10); - if ((errno == ERANGE && (part == LONG_MAX || part == LONG_MIN)) - || (errno != 0 && part == 0)) { - flb_plg_error(ctx->ins, "invalid partition number"); - } - - if (endptr == tmp) { - flb_plg_error(ctx->ins, "invalid partition number"); - } - ctx->partition = part; - } - else { - ctx->partition = -1; - } - - /* Kafka: topic */ - tmp = flb_output_get_property("topic", ins); - if (tmp) { - ctx->topic = flb_strdup(tmp); - } - else { - ctx->topic = flb_strdup("fluent-bit"); - } - - /* Set partition based on topic */ - tmp = flb_output_get_property("url_path", ins); - if (tmp) { - ctx->url_path = flb_strdup(tmp); - snprintf(ctx->uri, sizeof(ctx->uri) - 1, "%s/topics/%s", ctx->url_path, ctx->topic); - } else { - ctx->url_path = NULL; - snprintf(ctx->uri, sizeof(ctx->uri) - 1, "/topics/%s", ctx->topic); - } - - /* Kafka: message key */ - tmp = flb_output_get_property("message_key", ins); - if (tmp) { - ctx->message_key = flb_strdup(tmp); - ctx->message_key_len = strlen(tmp); - } - else { - ctx->message_key = NULL; - ctx->message_key_len = 0; - } - - return ctx; -} - -int flb_kr_conf_destroy(struct flb_kafka_rest *ctx) -{ - flb_free(ctx->topic); - flb_free(ctx->http_user); - flb_free(ctx->http_passwd); - - flb_free(ctx->time_key); - flb_free(ctx->time_key_format); - - if (ctx->url_path) { - flb_free(ctx->url_path); - } - - if (ctx->include_tag_key) { - flb_free(ctx->tag_key); - } - - if (ctx->message_key) { - flb_free(ctx->message_key); - } - - flb_upstream_destroy(ctx->u); - flb_free(ctx); - - return 0; -} diff --git a/fluent-bit/plugins/out_kafka_rest/kafka_conf.h b/fluent-bit/plugins/out_kafka_rest/kafka_conf.h deleted file mode 100644 index 1d80445b3..000000000 --- a/fluent-bit/plugins/out_kafka_rest/kafka_conf.h +++ /dev/null @@ -1,33 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_KAFKA_REST_CONF_H -#define FLB_OUT_KAFKA_REST_CONF_H - -#include -#include -#include - -#include "kafka.h" - -struct flb_kafka_rest *flb_kr_conf_create(struct flb_output_instance *ins, - struct flb_config *config); -int flb_kr_conf_destroy(struct flb_kafka_rest *ctx); - -#endif diff --git a/fluent-bit/plugins/out_kinesis_firehose/CMakeLists.txt b/fluent-bit/plugins/out_kinesis_firehose/CMakeLists.txt deleted file mode 100644 index 9cbf05d36..000000000 --- a/fluent-bit/plugins/out_kinesis_firehose/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -set(src - firehose.c - firehose_api.c) - -FLB_PLUGIN(out_kinesis_firehose "${src}" "") diff --git a/fluent-bit/plugins/out_kinesis_firehose/firehose.c b/fluent-bit/plugins/out_kinesis_firehose/firehose.c deleted file mode 100644 index a66d3f37c..000000000 --- a/fluent-bit/plugins/out_kinesis_firehose/firehose.c +++ /dev/null @@ -1,503 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include - -#include "firehose.h" -#include "firehose_api.h" - -static struct flb_aws_header content_type_header = { - .key = "Content-Type", - .key_len = 12, - .val = "application/x-amz-json-1.1", - .val_len = 26, -}; - -static int cb_firehose_init(struct flb_output_instance *ins, - struct flb_config *config, void *data) -{ - const char *tmp; - char *session_name = NULL; - struct flb_firehose *ctx = NULL; - int ret; - (void) config; - (void) data; - - ctx = flb_calloc(1, sizeof(struct flb_firehose)); - if (!ctx) { - flb_errno(); - return -1; - } - - ctx->ins = ins; - - /* Populate context with config map defaults and incoming properties */ - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_plg_error(ctx->ins, "configuration error"); - goto error; - } - - tmp = flb_output_get_property("delivery_stream", ins); - if (tmp) { - ctx->delivery_stream = tmp; - } else { - flb_plg_error(ctx->ins, "'delivery_stream' is a required field"); - goto error; - } - - tmp = flb_output_get_property("time_key", ins); - if (tmp) { - ctx->time_key = tmp; - } - - tmp = flb_output_get_property("time_key_format", ins); - if (tmp) { - ctx->time_key_format = tmp; - } else { - ctx->time_key_format = DEFAULT_TIME_KEY_FORMAT; - } - - tmp = flb_output_get_property("log_key", ins); - if (tmp) { - ctx->log_key = tmp; - } - - if (ctx->log_key && ctx->time_key) { - flb_plg_error(ctx->ins, "'time_key' and 'log_key' can not be used together"); - goto error; - } - - tmp = flb_output_get_property("endpoint", ins); - if (tmp) { - ctx->custom_endpoint = FLB_TRUE; - ctx->endpoint = removeProtocol((char *) tmp, "https://"); - } - else { - ctx->custom_endpoint = FLB_FALSE; - } - - tmp = flb_output_get_property("sts_endpoint", ins); - if (tmp) { - ctx->sts_endpoint = (char *) tmp; - } - - tmp = flb_output_get_property("compression", ins); - if (tmp) { - ret = flb_aws_compression_get_type(tmp); - if (ret == -1) { - flb_plg_error(ctx->ins, "unknown compression: %s", tmp); - goto error; - } - ctx->compression = ret; - } - - tmp = flb_output_get_property("log_key", ins); - if (tmp) { - ctx->log_key = tmp; - } - - tmp = flb_output_get_property("region", ins); - if (tmp) { - ctx->region = tmp; - } else { - flb_plg_error(ctx->ins, "'region' is a required field"); - goto error; - } - - tmp = flb_output_get_property("role_arn", ins); - if (tmp) { - ctx->role_arn = tmp; - } - - /* one tls instance for provider, one for cw client */ - ctx->cred_tls = flb_tls_create(FLB_TLS_CLIENT_MODE, - FLB_TRUE, - ins->tls_debug, - ins->tls_vhost, - ins->tls_ca_path, - ins->tls_ca_file, - ins->tls_crt_file, - ins->tls_key_file, - ins->tls_key_passwd); - - if (!ctx->cred_tls) { - flb_plg_error(ctx->ins, "Failed to create tls context"); - goto error; - } - - ctx->client_tls = flb_tls_create(FLB_TLS_CLIENT_MODE, - FLB_TRUE, - ins->tls_debug, - ins->tls_vhost, - ins->tls_ca_path, - ins->tls_ca_file, - ins->tls_crt_file, - ins->tls_key_file, - ins->tls_key_passwd); - if (!ctx->client_tls) { - flb_plg_error(ctx->ins, "Failed to create tls context"); - goto error; - } - - ctx->aws_provider = flb_standard_chain_provider_create(config, - ctx->cred_tls, - (char *) ctx->region, - ctx->sts_endpoint, - NULL, - flb_aws_client_generator(), - ctx->profile); - if (!ctx->aws_provider) { - flb_plg_error(ctx->ins, "Failed to create AWS Credential Provider"); - goto error; - } - - if(ctx->role_arn) { - /* set up sts assume role provider */ - session_name = flb_sts_session_name(); - if (!session_name) { - flb_plg_error(ctx->ins, - "Failed to generate random STS session name"); - goto error; - } - - /* STS provider needs yet another separate TLS instance */ - ctx->sts_tls = flb_tls_create(FLB_TLS_CLIENT_MODE, - FLB_TRUE, - ins->tls_debug, - ins->tls_vhost, - ins->tls_ca_path, - ins->tls_ca_file, - ins->tls_crt_file, - ins->tls_key_file, - ins->tls_key_passwd); - if (!ctx->sts_tls) { - flb_errno(); - goto error; - } - - ctx->base_aws_provider = ctx->aws_provider; - - ctx->aws_provider = flb_sts_provider_create(config, - ctx->sts_tls, - ctx->base_aws_provider, - (char *) ctx->external_id, - (char *) ctx->role_arn, - session_name, - (char *) ctx->region, - ctx->sts_endpoint, - NULL, - flb_aws_client_generator()); - if (!ctx->aws_provider) { - flb_plg_error(ctx->ins, - "Failed to create AWS STS Credential Provider"); - goto error; - } - /* session name can freed after provider is created */ - flb_free(session_name); - session_name = NULL; - } - - /* initialize credentials and set to sync mode */ - ctx->aws_provider->provider_vtable->sync(ctx->aws_provider); - ctx->aws_provider->provider_vtable->init(ctx->aws_provider); - ctx->aws_provider->provider_vtable->upstream_set(ctx->aws_provider, ctx->ins); - - if (ctx->endpoint == NULL) { - ctx->endpoint = flb_aws_endpoint("firehose", (char *) ctx->region); - if (!ctx->endpoint) { - goto error; - } - } - - struct flb_aws_client_generator *generator = flb_aws_client_generator(); - ctx->firehose_client = generator->create(); - if (!ctx->firehose_client) { - goto error; - } - ctx->firehose_client->name = "firehose_client"; - ctx->firehose_client->has_auth = FLB_TRUE; - ctx->firehose_client->provider = ctx->aws_provider; - ctx->firehose_client->region = (char *) ctx->region; - ctx->firehose_client->retry_requests = ctx->retry_requests; - ctx->firehose_client->service = "firehose"; - ctx->firehose_client->port = 443; - ctx->firehose_client->flags = 0; - ctx->firehose_client->proxy = NULL; - ctx->firehose_client->static_headers = &content_type_header; - ctx->firehose_client->static_headers_len = 1; - - struct flb_upstream *upstream = flb_upstream_create(config, ctx->endpoint, - 443, FLB_IO_TLS, - ctx->client_tls); - if (!upstream) { - flb_plg_error(ctx->ins, "Connection initialization error"); - goto error; - } - - ctx->firehose_client->upstream = upstream; - flb_output_upstream_set(upstream, ctx->ins); - - ctx->firehose_client->host = ctx->endpoint; - - /* Export context */ - flb_output_set_context(ins, ctx); - - return 0; - -error: - flb_free(session_name); - flb_plg_error(ctx->ins, "Initialization failed"); - flb_firehose_ctx_destroy(ctx); - return -1; -} - -struct flush *new_flush_buffer() -{ - struct flush *buf; - - - buf = flb_calloc(1, sizeof(struct flush)); - if (!buf) { - flb_errno(); - return NULL; - } - - buf->tmp_buf = flb_malloc(sizeof(char) * PUT_RECORD_BATCH_PAYLOAD_SIZE); - if (!buf->tmp_buf) { - flb_errno(); - flush_destroy(buf); - return NULL; - } - buf->tmp_buf_size = PUT_RECORD_BATCH_PAYLOAD_SIZE; - - buf->events = flb_malloc(sizeof(struct firehose_event) * MAX_EVENTS_PER_PUT); - if (!buf->events) { - flb_errno(); - flush_destroy(buf); - return NULL; - } - buf->events_capacity = MAX_EVENTS_PER_PUT; - - return buf; -} - -static void cb_firehose_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - struct flb_firehose *ctx = out_context; - int ret; - struct flush *buf; - (void) i_ins; - (void) config; - - buf = new_flush_buffer(); - if (!buf) { - flb_plg_error(ctx->ins, "Failed to construct flush buffer"); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - ret = process_and_send_records(ctx, buf, - event_chunk->data, event_chunk->size); - if (ret < 0) { - flb_plg_error(ctx->ins, "Failed to send records"); - flush_destroy(buf); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - flb_plg_debug(ctx->ins, "Processed %d records, sent %d to %s", - buf->records_processed, buf->records_sent, ctx->delivery_stream); - flush_destroy(buf); - - FLB_OUTPUT_RETURN(FLB_OK); -} - -void flb_firehose_ctx_destroy(struct flb_firehose *ctx) -{ - if (ctx != NULL) { - if (ctx->base_aws_provider) { - flb_aws_provider_destroy(ctx->base_aws_provider); - } - - if (ctx->aws_provider) { - flb_aws_provider_destroy(ctx->aws_provider); - } - - if (ctx->cred_tls) { - flb_tls_destroy(ctx->cred_tls); - } - - if (ctx->sts_tls) { - flb_tls_destroy(ctx->sts_tls); - } - - if (ctx->client_tls) { - flb_tls_destroy(ctx->client_tls); - } - - if (ctx->firehose_client) { - flb_aws_client_destroy(ctx->firehose_client); - } - - if (ctx->custom_endpoint == FLB_FALSE) { - flb_free(ctx->endpoint); - } - - flb_free(ctx); - } -} - -static int cb_firehose_exit(void *data, struct flb_config *config) -{ - struct flb_firehose *ctx = data; - - flb_firehose_ctx_destroy(ctx); - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "region", NULL, - 0, FLB_TRUE, offsetof(struct flb_firehose, region), - "The AWS region of your delivery stream" - }, - - { - FLB_CONFIG_MAP_STR, "delivery_stream", NULL, - 0, FLB_TRUE, offsetof(struct flb_firehose, delivery_stream), - "Firehose delivery stream name" - }, - - { - FLB_CONFIG_MAP_STR, "time_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_firehose, time_key), - "Add the timestamp to the record under this key. By default the timestamp " - "from Fluent Bit will not be added to records sent to Kinesis." - }, - - { - FLB_CONFIG_MAP_STR, "time_key_format", NULL, - 0, FLB_TRUE, offsetof(struct flb_firehose, time_key_format), - "strftime compliant format string for the timestamp; for example, " - "the default is '%Y-%m-%dT%H:%M:%S'. This option is used with time_key. " - }, - - { - FLB_CONFIG_MAP_STR, "role_arn", NULL, - 0, FLB_TRUE, offsetof(struct flb_firehose, role_arn), - "ARN of an IAM role to assume (ex. for cross account access)." - }, - - { - FLB_CONFIG_MAP_STR, "endpoint", NULL, - 0, FLB_FALSE, 0, - "Specify a custom endpoint for the Firehose API" - }, - - { - FLB_CONFIG_MAP_STR, "sts_endpoint", NULL, - 0, FLB_TRUE, offsetof(struct flb_firehose, sts_endpoint), - "Custom endpoint for the STS API." - }, - - { - FLB_CONFIG_MAP_STR, "external_id", NULL, - 0, FLB_TRUE, offsetof(struct flb_firehose, external_id), - "Specify an external ID for the STS API, can be used with the role_arn parameter if your role " - "requires an external ID." - }, - - { - FLB_CONFIG_MAP_STR, "compression", NULL, - 0, FLB_FALSE, 0, - "Compression type for Firehose records. Each log record is individually compressed " - "and sent to Firehose. 'gzip' and 'arrow' are the supported values. " - "'arrow' is only an available if Apache Arrow was enabled at compile time. " - "Defaults to no compression." - }, - - { - FLB_CONFIG_MAP_STR, "log_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_firehose, log_key), - "By default, the whole log record will be sent to Firehose. " - "If you specify a key name with this option, then only the value of " - "that key will be sent to Firehose. For example, if you are using " - "the Fluentd Docker log driver, you can specify `log_key log` and only " - "the log message will be sent to Firehose." - }, - - { - FLB_CONFIG_MAP_BOOL, "auto_retry_requests", "true", - 0, FLB_TRUE, offsetof(struct flb_firehose, retry_requests), - "Immediately retry failed requests to AWS services once. This option " - "does not affect the normal Fluent Bit retry mechanism with backoff. " - "Instead, it enables an immediate retry with no delay for networking " - "errors, which may help improve throughput when there are transient/random " - "networking issues." - }, - - { - FLB_CONFIG_MAP_STR, "profile", NULL, - 0, FLB_TRUE, offsetof(struct flb_firehose, profile), - "AWS Profile name. AWS Profiles can be configured with AWS CLI and are usually stored in " - "$HOME/.aws/ directory." - }, - /* EOF */ - {0} -}; - -/* Plugin registration */ -struct flb_output_plugin out_kinesis_firehose_plugin = { - .name = "kinesis_firehose", - .description = "Send logs to Amazon Kinesis Firehose", - .cb_init = cb_firehose_init, - .cb_flush = cb_firehose_flush, - .cb_exit = cb_firehose_exit, - .workers = 1, - .flags = 0, - - /* Configuration */ - .config_map = config_map, -}; diff --git a/fluent-bit/plugins/out_kinesis_firehose/firehose.h b/fluent-bit/plugins/out_kinesis_firehose/firehose.h deleted file mode 100644 index 98e2659d8..000000000 --- a/fluent-bit/plugins/out_kinesis_firehose/firehose.h +++ /dev/null @@ -1,104 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_FIREHOSE_H -#define FLB_OUT_FIREHOSE_H - -#include -#include -#include -#include -#include -#include - -#define DEFAULT_TIME_KEY_FORMAT "%Y-%m-%dT%H:%M:%S" - -/* buffers used for each flush */ -struct flush { - /* temporary buffer for storing the serialized event messages */ - char *tmp_buf; - size_t tmp_buf_size; - /* current index of tmp_buf */ - size_t tmp_buf_offset; - - /* projected final size of the payload for this flush */ - size_t data_size; - - /* log records- each of these has a pointer to their message in tmp_buf */ - struct firehose_event *events; - int events_capacity; - /* current event */ - int event_index; - - /* the payload of the API request */ - char *out_buf; - size_t out_buf_size; - - /* buffer used to temporarily hold an event during processing */ - char *event_buf; - size_t event_buf_size; - - int records_sent; - int records_processed; -}; - -struct firehose_event { - char *json; - size_t len; - struct timespec timestamp; -}; - -struct flb_firehose { - /* - * TLS instances can not be re-used. So we have one for: - * - Base cred provider (needed for EKS provider) - * - STS Assume role provider - * - The CloudWatch Logs client for this plugin - */ - struct flb_tls *cred_tls; - struct flb_tls *sts_tls; - struct flb_tls *client_tls; - struct flb_aws_provider *aws_provider; - struct flb_aws_provider *base_aws_provider; - struct flb_aws_client *firehose_client; - - /* configuration options */ - const char *delivery_stream; - const char *time_key; - const char *time_key_format; - const char *region; - const char *role_arn; - const char *log_key; - const char *external_id; - char *sts_endpoint; - char *profile; - int custom_endpoint; - int retry_requests; - int compression; - - /* must be freed on shutdown if custom_endpoint is not set */ - char *endpoint; - - /* Plugin output instance reference */ - struct flb_output_instance *ins; -}; - -void flb_firehose_ctx_destroy(struct flb_firehose *ctx); - -#endif diff --git a/fluent-bit/plugins/out_kinesis_firehose/firehose_api.c b/fluent-bit/plugins/out_kinesis_firehose/firehose_api.c deleted file mode 100644 index 5c2f0c2f9..000000000 --- a/fluent-bit/plugins/out_kinesis_firehose/firehose_api.c +++ /dev/null @@ -1,959 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include -#include - -#include -#include -#include -#include -#include - -#ifndef FLB_SYSTEM_WINDOWS -#include -#endif - -#include "firehose_api.h" - -#define ERR_CODE_SERVICE_UNAVAILABLE "ServiceUnavailableException" - -static struct flb_aws_header put_record_batch_header = { - .key = "X-Amz-Target", - .key_len = 12, - .val = "Firehose_20150804.PutRecordBatch", - .val_len = 32, -}; - -static inline int try_to_write(char *buf, int *off, size_t left, - const char *str, size_t str_len) -{ - if (str_len <= 0){ - str_len = strlen(str); - } - if (left <= *off+str_len) { - return FLB_FALSE; - } - memcpy(buf+*off, str, str_len); - *off += str_len; - return FLB_TRUE; -} - -/* - * Writes the "header" for a put_record_batch payload - */ -static int init_put_payload(struct flb_firehose *ctx, struct flush *buf, - int *offset) -{ - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - "{\"DeliveryStreamName\":\"", 23)) { - goto error; - } - - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - ctx->delivery_stream, 0)) { - goto error; - } - - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - "\",\"Records\":[", 13)) { - goto error; - } - return 0; - -error: - return -1; -} - -/* - * Writes a log event to the output buffer - */ -static int write_event(struct flb_firehose *ctx, struct flush *buf, - struct firehose_event *event, int *offset) -{ - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - "{\"Data\":\"", 9)) { - goto error; - } - - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - event->json, event->len)) { - goto error; - } - - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - "\"}", 2)) { - goto error; - } - - return 0; - -error: - return -1; -} - -/* Terminates a PutRecordBatch payload */ -static int end_put_payload(struct flb_firehose *ctx, struct flush *buf, - int *offset) -{ - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - "]}", 2)) { - return -1; - } - buf->out_buf[*offset] = '\0'; - - return 0; -} - - -/* - * Processes the msgpack object - * -1 = failure, record not added - * 0 = success, record added - * 1 = we ran out of space, send and retry - * 2 = record could not be processed, discard it - * Returns 0 on success, -1 on general errors, - * and 1 if we ran out of space to write the event - * which means a send must occur - */ -static int process_event(struct flb_firehose *ctx, struct flush *buf, - const msgpack_object *obj, struct flb_time *tms) -{ - size_t written = 0; - int ret; - size_t size; - size_t b64_len; - struct firehose_event *event; - char *tmp_buf_ptr; - char *time_key_ptr; - struct tm time_stamp; - struct tm *tmp; - size_t len; - size_t tmp_size; - void *compressed_tmp_buf; - char *out_buf; - - tmp_buf_ptr = buf->tmp_buf + buf->tmp_buf_offset; - ret = flb_msgpack_to_json(tmp_buf_ptr, - buf->tmp_buf_size - buf->tmp_buf_offset, - obj); - if (ret <= 0) { - /* - * negative value means failure to write to buffer, - * which means we ran out of space, and must send the logs - * - * TODO: This could also incorrectly be triggered if the record - * is larger than MAX_EVENT_SIZE - */ - return 1; - } - written = (size_t) ret; - - /* Discard empty messages (written == 2 means '""') */ - if (written <= 2) { - flb_plg_debug(ctx->ins, "Found empty log message, %s", ctx->delivery_stream); - return 2; - } - - if (ctx->log_key) { - /* - * flb_msgpack_to_json will encase the value in quotes - * We don't want that for log_key, so we ignore the first - * and last character - */ - written -= 2; - tmp_buf_ptr++; /* pass over the opening quote */ - buf->tmp_buf_offset++; - } - - /* is (written + 1) because we still have to append newline */ - if ((written + 1) >= MAX_EVENT_SIZE) { - flb_plg_warn(ctx->ins, "[size=%zu] Discarding record which is larger than " - "max size allowed by Firehose, %s", written + 1, - ctx->delivery_stream); - return 2; - } - - if (ctx->time_key) { - /* append time_key to end of json string */ - tmp = gmtime_r(&tms->tm.tv_sec, &time_stamp); - if (!tmp) { - flb_plg_error(ctx->ins, "Could not create time stamp for %lu unix " - "seconds, discarding record, %s", tms->tm.tv_sec, - ctx->delivery_stream); - return 2; - } - - /* format time output and return the length */ - len = flb_aws_strftime_precision(&out_buf, ctx->time_key_format, tms); - - /* how much space do we have left */ - tmp_size = (buf->tmp_buf_size - buf->tmp_buf_offset) - written; - if (len > tmp_size) { - /* not enough space - tell caller to retry */ - flb_free(out_buf); - return 1; - } - - if (len == 0) { - /* - * when the length of out_buf is not enough for time_key_format, - * time_key will not be added to record. - */ - flb_plg_error(ctx->ins, "Failed to add time_key %s to record, %s", - ctx->time_key, ctx->delivery_stream); - flb_free(out_buf); - } - else { - time_key_ptr = tmp_buf_ptr + written - 1; - memcpy(time_key_ptr, ",", 1); - time_key_ptr++; - memcpy(time_key_ptr, "\"", 1); - time_key_ptr++; - memcpy(time_key_ptr, ctx->time_key, strlen(ctx->time_key)); - time_key_ptr += strlen(ctx->time_key); - memcpy(time_key_ptr, "\":\"", 3); - time_key_ptr += 3; - tmp_size = buf->tmp_buf_size - buf->tmp_buf_offset; - tmp_size -= (time_key_ptr - tmp_buf_ptr); - - /* merge out_buf to time_key_ptr */ - memcpy(time_key_ptr, out_buf, len); - flb_free(out_buf); - time_key_ptr += len; - memcpy(time_key_ptr, "\"}", 2); - time_key_ptr += 2; - written = (time_key_ptr - tmp_buf_ptr); - } - } - - /* is (written + 1) because we still have to append newline */ - if ((written + 1) >= MAX_EVENT_SIZE) { - flb_plg_warn(ctx->ins, "[size=%zu] Discarding record which is larger than " - "max size allowed by Firehose, %s", written + 1, - ctx->delivery_stream); - return 2; - } - - /* append newline to record */ - - tmp_size = (buf->tmp_buf_size - buf->tmp_buf_offset) - written; - if (tmp_size <= 1) { - /* no space left- tell caller to retry */ - return 1; - } - - memcpy(tmp_buf_ptr + written, "\n", 1); - written++; - - if (ctx->compression == FLB_AWS_COMPRESS_NONE) { - /* - * check if event_buf is initialized and big enough - * Base64 encoding will increase size by ~4/3 - */ - size = (written * 1.5) + 4; - if (buf->event_buf == NULL || buf->event_buf_size < size) { - flb_free(buf->event_buf); - buf->event_buf = flb_malloc(size); - buf->event_buf_size = size; - if (buf->event_buf == NULL) { - flb_errno(); - return -1; - } - } - - tmp_buf_ptr = buf->tmp_buf + buf->tmp_buf_offset; - - ret = flb_base64_encode((unsigned char *) buf->event_buf, size, &b64_len, - (unsigned char *) tmp_buf_ptr, written); - if (ret != 0) { - flb_errno(); - return -1; - } - written = b64_len; - } - else { - /* - * compress event, truncating input if needed - * replace event buffer with compressed buffer - */ - ret = flb_aws_compression_b64_truncate_compress(ctx->compression, - MAX_B64_EVENT_SIZE, - tmp_buf_ptr, - written, &compressed_tmp_buf, - &size); /* evaluate size */ - if (ret == -1) { - flb_plg_error(ctx->ins, "Unable to compress record, discarding, " - "%s", ctx->delivery_stream); - return 2; - } - flb_free(buf->event_buf); - buf->event_buf = compressed_tmp_buf; - compressed_tmp_buf = NULL; - written = size; - } - - tmp_buf_ptr = buf->tmp_buf + buf->tmp_buf_offset; - if ((buf->tmp_buf_size - buf->tmp_buf_offset) < written) { - /* not enough space, send logs */ - return 1; - } - - /* copy serialized json to tmp_buf */ - memcpy(tmp_buf_ptr, buf->event_buf, written); - - buf->tmp_buf_offset += written; - event = &buf->events[buf->event_index]; - event->json = tmp_buf_ptr; - event->len = written; - event->timestamp.tv_sec = tms->tm.tv_sec; - event->timestamp.tv_nsec = tms->tm.tv_nsec; - - return 0; -} - -/* Resets or inits a flush struct */ -static void reset_flush_buf(struct flb_firehose *ctx, struct flush *buf) { - buf->event_index = 0; - buf->tmp_buf_offset = 0; - buf->data_size = PUT_RECORD_BATCH_HEADER_LEN + PUT_RECORD_BATCH_FOOTER_LEN; - buf->data_size += strlen(ctx->delivery_stream); -} - -/* constructs a put payload, and then sends */ -static int send_log_events(struct flb_firehose *ctx, struct flush *buf) { - int ret; - int offset; - int i; - struct firehose_event *event; - - if (buf->event_index <= 0) { - /* - * event_index should always be 1 more than the actual last event index - * when this function is called. - * Except in the case where send_log_events() is called at the end of - * process_and_send. If all records were already sent, event_index - * will be 0. Hence this check. - */ - return 0; - } - - /* alloc out_buf if needed */ - if (buf->out_buf == NULL || buf->out_buf_size < buf->data_size) { - if (buf->out_buf != NULL) { - flb_free(buf->out_buf); - } - buf->out_buf = flb_malloc(buf->data_size + 1); - if (!buf->out_buf) { - flb_errno(); - return -1; - } - buf->out_buf_size = buf->data_size; - } - - offset = 0; - ret = init_put_payload(ctx, buf, &offset); - if (ret < 0) { - flb_plg_error(ctx->ins, "Failed to initialize PutRecordBatch payload, %s", - ctx->delivery_stream); - return -1; - } - - for (i = 0; i < buf->event_index; i++) { - event = &buf->events[i]; - ret = write_event(ctx, buf, event, &offset); - if (ret < 0) { - flb_plg_error(ctx->ins, "Failed to write log record %d to " - "payload buffer, %s", i, ctx->delivery_stream); - return -1; - } - if (i != (buf->event_index -1)) { - if (!try_to_write(buf->out_buf, &offset, buf->out_buf_size, - ",", 1)) { - flb_plg_error(ctx->ins, "Could not terminate record with ','"); - return -1; - } - } - } - - ret = end_put_payload(ctx, buf, &offset); - if (ret < 0) { - flb_plg_error(ctx->ins, "Could not complete PutRecordBatch payload"); - return -1; - } - flb_plg_debug(ctx->ins, "firehose:PutRecordBatch: events=%d, payload=%d bytes", i, offset); - ret = put_record_batch(ctx, buf, (size_t) offset, i); - if (ret < 0) { - flb_plg_error(ctx->ins, "Failed to send log records"); - return -1; - } - buf->records_sent += i; - - return 0; -} - -/* - * Processes the msgpack object, sends the current batch if needed - */ -static int add_event(struct flb_firehose *ctx, struct flush *buf, - const msgpack_object *obj, struct flb_time *tms) -{ - int ret; - struct firehose_event *event; - int retry_add = FLB_FALSE; - size_t event_bytes = 0; - - if (buf->event_index == 0) { - /* init */ - reset_flush_buf(ctx, buf); - } - -retry_add_event: - retry_add = FLB_FALSE; - ret = process_event(ctx, buf, obj, tms); - if (ret < 0) { - return -1; - } - else if (ret == 1) { - if (buf->event_index <= 0) { - /* somehow the record was larger than our entire request buffer */ - flb_plg_warn(ctx->ins, "Discarding massive log record, %s", - ctx->delivery_stream); - return 0; /* discard this record and return to caller */ - } - /* send logs and then retry the add */ - retry_add = FLB_TRUE; - goto send; - } else if (ret == 2) { - /* discard this record and return to caller */ - flb_plg_warn(ctx->ins, "Discarding large or unprocessable record, %s", - ctx->delivery_stream); - return 0; - } - - event = &buf->events[buf->event_index]; - event_bytes = event->len + PUT_RECORD_BATCH_PER_RECORD_LEN; - - if ((buf->data_size + event_bytes) > PUT_RECORD_BATCH_PAYLOAD_SIZE) { - if (buf->event_index <= 0) { - /* somehow the record was larger than our entire request buffer */ - flb_plg_warn(ctx->ins, "[size=%zu] Discarding massive log record, %s", - event_bytes, ctx->delivery_stream); - return 0; /* discard this record and return to caller */ - } - /* do not send this event */ - retry_add = FLB_TRUE; - goto send; - } - - /* send is not needed yet, return to caller */ - buf->data_size += event_bytes; - buf->event_index++; - - if (buf->event_index == MAX_EVENTS_PER_PUT) { - goto send; - } - - return 0; - -send: - ret = send_log_events(ctx, buf); - reset_flush_buf(ctx, buf); - if (ret < 0) { - return -1; - } - - if (retry_add == FLB_TRUE) { - goto retry_add_event; - } - - return 0; -} - -/* - * Main routine- processes msgpack and sends in batches - * return value is the number of events processed (number sent is stored in buf) - */ -int process_and_send_records(struct flb_firehose *ctx, struct flush *buf, - const char *data, size_t bytes) -{ - // size_t off = 0; - int i = 0; - size_t map_size; - // msgpack_unpacked result; - // msgpack_object *obj; - msgpack_object map; - // msgpack_object root; - msgpack_object_kv *kv; - msgpack_object key; - msgpack_object val; - char *key_str = NULL; - size_t key_str_size = 0; - int j; - int ret; - int check = FLB_FALSE; - int found = FLB_FALSE; - // struct flb_time tms; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return -1; - } - - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - map = *log_event.body; - map_size = map.via.map.size; - - if (ctx->log_key) { - key_str = NULL; - key_str_size = 0; - check = FLB_FALSE; - found = FLB_FALSE; - - kv = map.via.map.ptr; - - for(j=0; j < map_size; j++) { - key = (kv+j)->key; - if (key.type == MSGPACK_OBJECT_BIN) { - key_str = (char *) key.via.bin.ptr; - key_str_size = key.via.bin.size; - check = FLB_TRUE; - } - if (key.type == MSGPACK_OBJECT_STR) { - key_str = (char *) key.via.str.ptr; - key_str_size = key.via.str.size; - check = FLB_TRUE; - } - - if (check == FLB_TRUE) { - if (strncmp(ctx->log_key, key_str, key_str_size) == 0) { - found = FLB_TRUE; - val = (kv+j)->val; - ret = add_event(ctx, buf, &val, &log_event.timestamp); - if (ret < 0 ) { - goto error; - } - } - } - - } - if (found == FLB_FALSE) { - flb_plg_error(ctx->ins, "Could not find log_key '%s' in record, %s", - ctx->log_key, ctx->delivery_stream); - } - else { - i++; - } - continue; - } - - ret = add_event(ctx, buf, &map, &log_event.timestamp); - if (ret < 0 ) { - goto error; - } - i++; - } - flb_log_event_decoder_destroy(&log_decoder); - - /* send any remaining events */ - ret = send_log_events(ctx, buf); - reset_flush_buf(ctx, buf); - - if (ret < 0) { - return -1; - } - - /* return number of events processed */ - buf->records_processed = i; - - return i; - -error: - flb_log_event_decoder_destroy(&log_decoder); - - return -1; -} - -/* - * Returns number of failed records on success, -1 on failure - */ -static int process_api_response(struct flb_firehose *ctx, - struct flb_http_client *c) -{ - int i; - int k; - int w; - int ret; - int failed_records = -1; - int root_type; - char *out_buf; - int throughput_exceeded = FLB_FALSE; - size_t off = 0; - size_t out_size; - msgpack_unpacked result; - msgpack_object root; - msgpack_object key; - msgpack_object val; - msgpack_object response; - msgpack_object response_key; - msgpack_object response_val; - - if (strstr(c->resp.payload, "\"FailedPutCount\":0")) { - return 0; - } - - /* Convert JSON payload to msgpack */ - ret = flb_pack_json(c->resp.payload, c->resp.payload_size, - &out_buf, &out_size, &root_type, NULL); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not pack/validate JSON API response\n%s", - c->resp.payload); - return -1; - } - - /* Lookup error field */ - msgpack_unpacked_init(&result); - ret = msgpack_unpack_next(&result, out_buf, out_size, &off); - if (ret != MSGPACK_UNPACK_SUCCESS) { - flb_plg_error(ctx->ins, "Cannot unpack response to find error\n%s", - c->resp.payload); - failed_records = -1; - goto done; - } - - root = result.data; - if (root.type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "unexpected payload type=%i", - root.type); - failed_records = -1; - goto done; - } - - for (i = 0; i < root.via.map.size; i++) { - key = root.via.map.ptr[i].key; - if (key.type != MSGPACK_OBJECT_STR) { - flb_plg_error(ctx->ins, "unexpected key type=%i", - key.type); - failed_records = -1; - goto done; - } - - if (key.via.str.size >= 14 && - strncmp(key.via.str.ptr, "FailedPutCount", 14) == 0) { - val = root.via.map.ptr[i].val; - if (val.type != MSGPACK_OBJECT_POSITIVE_INTEGER) { - flb_plg_error(ctx->ins, "unexpected 'FailedPutCount' value type=%i", - val.type); - failed_records = -1; - goto done; - } - - failed_records = val.via.u64; - if (failed_records == 0) { - /* no need to check RequestResponses field */ - goto done; - } - } - - if (key.via.str.size >= 14 && - strncmp(key.via.str.ptr, "RequestResponses", 16) == 0) { - val = root.via.map.ptr[i].val; - if (val.type != MSGPACK_OBJECT_ARRAY) { - flb_plg_error(ctx->ins, "unexpected 'RequestResponses' value type=%i", - val.type); - failed_records = -1; - goto done; - } - - if (val.via.array.size == 0) { - flb_plg_error(ctx->ins, "'RequestResponses' field in response is empty"); - failed_records = -1; - goto done; - } - - for (k = 0; k < val.via.array.size; k++) { - /* iterate through the responses */ - response = val.via.array.ptr[k]; - if (response.type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "unexpected 'RequestResponses[%d]' value type=%i", - k, response.type); - failed_records = -1; - goto done; - } - for (w = 0; w < response.via.map.size; w++) { - /* iterate through the response's keys */ - response_key = response.via.map.ptr[w].key; - if (response_key.type != MSGPACK_OBJECT_STR) { - flb_plg_error(ctx->ins, "unexpected key type=%i", - response_key.type); - failed_records = -1; - goto done; - } - if (response_key.via.str.size >= 9 && - strncmp(response_key.via.str.ptr, "ErrorCode", 9) == 0) { - response_val = response.via.map.ptr[w].val; - if (!throughput_exceeded && - response_val.via.str.size >= 27 && - (strncmp(response_val.via.str.ptr, - ERR_CODE_SERVICE_UNAVAILABLE, 27) == 0)) { - throughput_exceeded = FLB_TRUE; - flb_plg_error(ctx->ins, "Throughput limits may have been exceeded, %s", - ctx->delivery_stream); - } - flb_plg_debug(ctx->ins, "Record %i failed with err_code=%.*s", - k, response_val.via.str.size, - response_val.via.str.ptr); - } - if (response_key.via.str.size >= 12 && - strncmp(response_key.via.str.ptr, "ErrorMessage", 12) == 0) { - response_val = response.via.map.ptr[w].val; - flb_plg_debug(ctx->ins, "Record %i failed with err_msg=%.*s", - k, response_val.via.str.size, - response_val.via.str.ptr); - } - } - } - } - } - - done: - flb_free(out_buf); - msgpack_unpacked_destroy(&result); - return failed_records; -} - -static int plugin_under_test() -{ - if (getenv("FLB_FIREHOSE_PLUGIN_UNDER_TEST") != NULL) { - return FLB_TRUE; - } - - return FLB_FALSE; -} - -static char *mock_error_response(char *error_env_var) -{ - char *err_val = NULL; - char *error = NULL; - int len = 0; - - err_val = getenv(error_env_var); - if (err_val != NULL && strlen(err_val) > 0) { - error = flb_malloc(strlen(err_val) + sizeof(char)); - if (error == NULL) { - flb_errno(); - return NULL; - } - - len = strlen(err_val); - memcpy(error, err_val, len); - error[len] = '\0'; - return error; - } - - return NULL; -} - -int partial_success() -{ - char *err_val = NULL; - - err_val = getenv("PARTIAL_SUCCESS_CASE"); - if (err_val != NULL && strlen(err_val) > 0) { - return FLB_TRUE; - } - - return FLB_FALSE; -} - -static struct flb_http_client *mock_http_call(char *error_env_var) -{ - /* create an http client so that we can set the response */ - struct flb_http_client *c = NULL; - char *error = mock_error_response(error_env_var); - - c = flb_calloc(1, sizeof(struct flb_http_client)); - if (!c) { - flb_errno(); - flb_free(error); - return NULL; - } - mk_list_init(&c->headers); - - if (error != NULL) { - c->resp.status = 400; - /* resp.data is freed on destroy, payload is supposed to reference it */ - c->resp.data = error; - c->resp.payload = c->resp.data; - c->resp.payload_size = strlen(error); - } - else { - c->resp.status = 200; - c->resp.payload = ""; - c->resp.payload_size = 0; - if (partial_success() == FLB_TRUE) { - /* mocked partial failure response */ - c->resp.payload = "{\"Encrypted\": false,\"FailedPutCount\": 1,\"RequestResponses\":[{\"RecordId\": \"Me0CqhxK3BK3MiBWgy/AydQrVUg7vbc40Z4zNds3jiiJDscqGtWFz9bJugbrAoN70YCaxpXgmyR9R+LFxS2rleDepqFljYArBtXnRmVzSMOAzTJZlwsO84+757kBvA5RUycF3wC3XZjFtUFP0Q4QTdhuD8HMJBvKGiBY9Yy5jBUmZuKhXxCLQ/YTwKQaQKn4fnc5iISxaErPXsWMI7OApHZ1eFGvcHVZ\"},{\"RecordId\": \"NRAZVkblYgWWDSvTAF/9jBR4MlciEUFV+QIjb1D8uar7YbC3wqeLQuSZ0GEopGlE/8JAK9h9aAyTub5lH5V+bZuR3SeKKABWoJ788/tI455Kup9oRzmXTKWiXeklxmAe9MtsSz0y4t3oIrSLq8e3QVH9DJKWdhDkIXd8lXK1wuJi8tKmnNgxFob/Cz398kQFXPc4JwKj3Dv3Ou0qibZiusko6f7yBUve\",\"ErrorCode\":\"ServiceUnavailableException\",\"ErrorMessage\": \"Catsssss\"},{\"RecordId\": \"InFGTFvML/MGCLtnC3moI/zCISrKSScu/D8oCGmeIIeVaYUfywHpr2NmsQiZsxUL9+4ThOm2ypxqFGudZvgXQ45gUWMG+R4Y5xzS03N+vQ71+UaL392jY6HUs2SxYkZQe6vpdK+xHaJJ1b8uE++Laxg9rmsXtNt193WjmH3FhU1veu9pnSiGZgqC7czpyVgvZBNeWc+hTjEVicj3VAHBg/9yRN0sC30C\",\"ErrorCode\":\"ServiceUnavailableException\",\"ErrorMessage\": \"Catsssss 2\"},{\"RecordId\":\"KufmrRJ2z8zAgYAYGz6rm4BQC8SA7g87lQJQl2DQ+Be5EiEpr5bG33ilnQVvo1Q05BJuQBnjbw2cm919Ya72awapxfOBdZcPPKJN7KDZV/n1DFCDDrJ2vgyNK4qhKdo3Mr7nyrBpkLIs93PdxOdrTh11Y9HHEaFtim0cHJYpKCSZBjNObfWjfjHx5TuB7L3PHQqMKMu0MT5L9gPgVXHElGalqKZGTcfB\"}]}"; - c->resp.payload_size = strlen(c->resp.payload); - } - else { - /* mocked success response */ - c->resp.payload = "{\"Encrypted\": false,\"FailedPutCount\": 0,\"RequestResponses\":[{\"RecordId\": \"Me0CqhxK3BK3MiBWgy/AydQrVUg7vbc40Z4zNds3jiiJDscqGtWFz9bJugbrAoN70YCaxpXgmyR9R+LFxS2rleDepqFljYArBtXnRmVzSMOAzTJZlwsO84+757kBvA5RUycF3wC3XZjFtUFP0Q4QTdhuD8HMJBvKGiBY9Yy5jBUmZuKhXxCLQ/YTwKQaQKn4fnc5iISxaErPXsWMI7OApHZ1eFGvcHVZ\"},{\"RecordId\": \"NRAZVkblYgWWDSvTAF/9jBR4MlciEUFV+QIjb1D8uar7YbC3wqeLQuSZ0GEopGlE/8JAK9h9aAyTub5lH5V+bZuR3SeKKABWoJ788/tI455Kup9oRzmXTKWiXeklxmAe9MtsSz0y4t3oIrSLq8e3QVH9DJKWdhDkIXd8lXK1wuJi8tKmnNgxFob/Cz398kQFXPc4JwKj3Dv3Ou0qibZiusko6f7yBUve\"},{\"RecordId\": \"InFGTFvML/MGCLtnC3moI/zCISrKSScu/D8oCGmeIIeVaYUfywHpr2NmsQiZsxUL9+4ThOm2ypxqFGudZvgXQ45gUWMG+R4Y5xzS03N+vQ71+UaL392jY6HUs2SxYkZQe6vpdK+xHaJJ1b8uE++Laxg9rmsXtNt193WjmH3FhU1veu9pnSiGZgqC7czpyVgvZBNeWc+hTjEVicj3VAHBg/9yRN0sC30C\"},{\"RecordId\": \"KufmrRJ2z8zAgYAYGz6rm4BQC8SA7g87lQJQl2DQ+Be5EiEpr5bG33ilnQVvo1Q05BJuQBnjbw2cm919Ya72awapxfOBdZcPPKJN7KDZV/n1DFCDDrJ2vgyNK4qhKdo3Mr7nyrBpkLIs93PdxOdrTh11Y9HHEaFtim0cHJYpKCSZBjNObfWjfjHx5TuB7L3PHQqMKMu0MT5L9gPgVXHElGalqKZGTcfB\"}]}"; - c->resp.payload_size = strlen(c->resp.payload); - } - } - - return c; -} - - -/* - * Returns -1 on failure, 0 on success - */ -int put_record_batch(struct flb_firehose *ctx, struct flush *buf, - size_t payload_size, int num_records) -{ - - struct flb_http_client *c = NULL; - struct flb_aws_client *firehose_client; - flb_sds_t error; - int failed_records = 0; - - flb_plg_debug(ctx->ins, "Sending log records to delivery stream %s", - ctx->delivery_stream); - - if (plugin_under_test() == FLB_TRUE) { - c = mock_http_call("TEST_PUT_RECORD_BATCH_ERROR"); - } - else { - firehose_client = ctx->firehose_client; - c = firehose_client->client_vtable->request(firehose_client, FLB_HTTP_POST, - "/", buf->out_buf, payload_size, - &put_record_batch_header, 1); - } - - if (c) { - flb_plg_debug(ctx->ins, "PutRecordBatch http status=%d", c->resp.status); - - if (c->resp.status == 200) { - /* Firehose API can return partial success- check response */ - if (c->resp.payload_size > 0) { - failed_records = process_api_response(ctx, c); - if (failed_records < 0) { - flb_plg_error(ctx->ins, "PutRecordBatch response " - "could not be parsed, %s", - c->resp.payload); - flb_http_client_destroy(c); - return -1; - } - if (failed_records == num_records) { - flb_plg_error(ctx->ins, "PutRecordBatch request returned " - "with no records successfully recieved, %s", - ctx->delivery_stream); - flb_http_client_destroy(c); - return -1; - } - if (failed_records > 0) { - flb_plg_error(ctx->ins, "%d out of %d records failed to be " - "delivered, will retry this batch, %s", - failed_records, num_records, - ctx->delivery_stream); - flb_http_client_destroy(c); - return -1; - } - } - flb_plg_debug(ctx->ins, "Sent events to %s", ctx->delivery_stream); - flb_http_client_destroy(c); - return 0; - } - - /* Check error */ - if (c->resp.payload_size > 0) { - error = flb_aws_error(c->resp.payload, c->resp.payload_size); - if (error != NULL) { - if (strcmp(error, ERR_CODE_SERVICE_UNAVAILABLE) == 0) { - flb_plg_error(ctx->ins, "Throughput limits for %s " - "may have been exceeded.", - ctx->delivery_stream); - } - if (strncmp(error, "SerializationException", 22) == 0) { - /* - * If this happens, we habe a bug in the code - * User should send us the output to debug - */ - flb_plg_error(ctx->ins, "<<------Bug in Code------>>"); - printf("Malformed request: %s", buf->out_buf); - } - flb_aws_print_error(c->resp.payload, c->resp.payload_size, - "PutRecordBatch", ctx->ins); - flb_sds_destroy(error); - } - else { - /* error could not be parsed, print raw response to debug */ - flb_plg_debug(ctx->ins, "Raw response: %s", c->resp.payload); - } - } - } - - flb_plg_error(ctx->ins, "Failed to send log records to %s", ctx->delivery_stream); - if (c) { - flb_http_client_destroy(c); - } - return -1; -} - - -void flush_destroy(struct flush *buf) -{ - if (buf) { - flb_free(buf->tmp_buf); - flb_free(buf->out_buf); - flb_free(buf->events); - flb_free(buf->event_buf); - flb_free(buf); - } -} diff --git a/fluent-bit/plugins/out_kinesis_firehose/firehose_api.h b/fluent-bit/plugins/out_kinesis_firehose/firehose_api.h deleted file mode 100644 index 55e107a55..000000000 --- a/fluent-bit/plugins/out_kinesis_firehose/firehose_api.h +++ /dev/null @@ -1,45 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_CLOUDWATCH_API -#define FLB_OUT_CLOUDWATCH_API - -#define PUT_RECORD_BATCH_PAYLOAD_SIZE 4194304 -#define MAX_EVENTS_PER_PUT 500 -#define MAX_EVENT_SIZE 1024000 -#define MAX_B64_EVENT_SIZE 1365336 /* ceil(1024000 / 3) * 4 */ - -/* number of characters needed to 'start' a PutRecordBatch payload */ -#define PUT_RECORD_BATCH_HEADER_LEN 42 -/* number of characters needed per record in a PutRecordBatch payload */ -#define PUT_RECORD_BATCH_PER_RECORD_LEN 12 -/* number of characters needed to 'end' a PutRecordBatch payload */ -#define PUT_RECORD_BATCH_FOOTER_LEN 4 - -#include "firehose.h" - -void flush_destroy(struct flush *buf); - -int process_and_send_records(struct flb_firehose *ctx, struct flush *buf, - const char *data, size_t bytes); - -int put_record_batch(struct flb_firehose *ctx, struct flush *buf, - size_t payload_size, int num_records); - -#endif diff --git a/fluent-bit/plugins/out_kinesis_streams/CMakeLists.txt b/fluent-bit/plugins/out_kinesis_streams/CMakeLists.txt deleted file mode 100644 index d95110ee2..000000000 --- a/fluent-bit/plugins/out_kinesis_streams/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -set(src - kinesis.c - kinesis_api.c) - -FLB_PLUGIN(out_kinesis_streams "${src}" "") diff --git a/fluent-bit/plugins/out_kinesis_streams/kinesis.c b/fluent-bit/plugins/out_kinesis_streams/kinesis.c deleted file mode 100644 index 85dd37948..000000000 --- a/fluent-bit/plugins/out_kinesis_streams/kinesis.c +++ /dev/null @@ -1,499 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "kinesis.h" -#include "kinesis_api.h" - -static struct flb_aws_header content_type_header = { - .key = "Content-Type", - .key_len = 12, - .val = "application/x-amz-json-1.1", - .val_len = 26, -}; - -static int cb_kinesis_init(struct flb_output_instance *ins, - struct flb_config *config, void *data) -{ - const char *tmp; - char *session_name = NULL; - struct flb_kinesis *ctx = NULL; - int ret; - (void) config; - (void) data; - - ctx = flb_calloc(1, sizeof(struct flb_kinesis)); - if (!ctx) { - flb_errno(); - return -1; - } - - ctx->ins = ins; - - /* Populate context with config map defaults and incoming properties */ - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_plg_error(ctx->ins, "configuration error"); - goto error; - } - - tmp = flb_output_get_property("stream", ins); - if (tmp) { - ctx->stream_name = tmp; - } else { - flb_plg_error(ctx->ins, "'stream' is a required field"); - goto error; - } - - tmp = flb_output_get_property("time_key", ins); - if (tmp) { - ctx->time_key = tmp; - } - - tmp = flb_output_get_property("time_key_format", ins); - if (tmp) { - ctx->time_key_format = tmp; - } else { - ctx->time_key_format = DEFAULT_TIME_KEY_FORMAT; - } - - tmp = flb_output_get_property("log_key", ins); - if (tmp) { - ctx->log_key = tmp; - } - - if (ctx->log_key && ctx->time_key) { - flb_plg_error(ctx->ins, "'time_key' and 'log_key' can not be used together"); - goto error; - } - - tmp = flb_output_get_property("endpoint", ins); - if (tmp) { - ctx->custom_endpoint = FLB_TRUE; - ctx->endpoint = removeProtocol((char *) tmp, "https://"); - } - else { - ctx->custom_endpoint = FLB_FALSE; - } - - tmp = flb_output_get_property("sts_endpoint", ins); - if (tmp) { - ctx->sts_endpoint = (char *) tmp; - } - - - tmp = flb_output_get_property("log_key", ins); - if (tmp) { - ctx->log_key = tmp; - } - - tmp = flb_output_get_property("region", ins); - if (tmp) { - ctx->region = tmp; - } else { - flb_plg_error(ctx->ins, "'region' is a required field"); - goto error; - } - - tmp = flb_output_get_property("role_arn", ins); - if (tmp) { - ctx->role_arn = tmp; - } - - /* one tls instance for provider, one for cw client */ - ctx->cred_tls = flb_tls_create(FLB_TLS_CLIENT_MODE, - FLB_TRUE, - ins->tls_debug, - ins->tls_vhost, - ins->tls_ca_path, - ins->tls_ca_file, - ins->tls_crt_file, - ins->tls_key_file, - ins->tls_key_passwd); - - if (!ctx->cred_tls) { - flb_plg_error(ctx->ins, "Failed to create tls context"); - goto error; - } - - ctx->client_tls = flb_tls_create(FLB_TLS_CLIENT_MODE, - FLB_TRUE, - ins->tls_debug, - ins->tls_vhost, - ins->tls_ca_path, - ins->tls_ca_file, - ins->tls_crt_file, - ins->tls_key_file, - ins->tls_key_passwd); - if (!ctx->client_tls) { - flb_plg_error(ctx->ins, "Failed to create tls context"); - goto error; - } - - ctx->aws_provider = flb_standard_chain_provider_create(config, - ctx->cred_tls, - (char *) ctx->region, - ctx->sts_endpoint, - NULL, - flb_aws_client_generator(), - ctx->profile); - if (!ctx->aws_provider) { - flb_plg_error(ctx->ins, "Failed to create AWS Credential Provider"); - goto error; - } - - ctx->uuid = flb_sts_session_name(); - if (!ctx->uuid) { - flb_plg_error(ctx->ins, - "Failed to generate plugin instance UUID"); - goto error; - } - - if(ctx->role_arn) { - /* set up sts assume role provider */ - session_name = flb_sts_session_name(); - if (!session_name) { - flb_plg_error(ctx->ins, - "Failed to generate random STS session name"); - goto error; - } - - /* STS provider needs yet another separate TLS instance */ - ctx->sts_tls = flb_tls_create(FLB_TLS_CLIENT_MODE, - FLB_TRUE, - ins->tls_debug, - ins->tls_vhost, - ins->tls_ca_path, - ins->tls_ca_file, - ins->tls_crt_file, - ins->tls_key_file, - ins->tls_key_passwd); - if (!ctx->sts_tls) { - flb_errno(); - goto error; - } - - ctx->base_aws_provider = ctx->aws_provider; - - ctx->aws_provider = flb_sts_provider_create(config, - ctx->sts_tls, - ctx->base_aws_provider, - (char *) ctx->external_id, - (char *) ctx->role_arn, - session_name, - (char *) ctx->region, - ctx->sts_endpoint, - NULL, - flb_aws_client_generator()); - if (!ctx->aws_provider) { - flb_plg_error(ctx->ins, - "Failed to create AWS STS Credential Provider"); - goto error; - } - /* session name can freed after provider is created */ - flb_free(session_name); - session_name = NULL; - } - - /* initialize credentials and set to sync mode */ - ctx->aws_provider->provider_vtable->sync(ctx->aws_provider); - ctx->aws_provider->provider_vtable->init(ctx->aws_provider); - ctx->aws_provider->provider_vtable->upstream_set(ctx->aws_provider, ctx->ins); - - if (ctx->endpoint == NULL) { - ctx->endpoint = flb_aws_endpoint("kinesis", (char *) ctx->region); - if (!ctx->endpoint) { - goto error; - } - } - - struct flb_aws_client_generator *generator = flb_aws_client_generator(); - ctx->kinesis_client = generator->create(); - if (!ctx->kinesis_client) { - goto error; - } - ctx->kinesis_client->name = "kinesis_client"; - ctx->kinesis_client->has_auth = FLB_TRUE; - ctx->kinesis_client->provider = ctx->aws_provider; - ctx->kinesis_client->region = (char *) ctx->region; - ctx->kinesis_client->retry_requests = ctx->retry_requests; - ctx->kinesis_client->service = "kinesis"; - ctx->kinesis_client->port = 443; - ctx->kinesis_client->flags = 0; - ctx->kinesis_client->proxy = NULL; - ctx->kinesis_client->static_headers = &content_type_header; - ctx->kinesis_client->static_headers_len = 1; - - struct flb_upstream *upstream = flb_upstream_create(config, ctx->endpoint, - 443, FLB_IO_TLS, - ctx->client_tls); - if (!upstream) { - flb_plg_error(ctx->ins, "Connection initialization error"); - goto error; - } - - ctx->kinesis_client->upstream = upstream; - flb_output_upstream_set(upstream, ctx->ins); - - ctx->kinesis_client->host = ctx->endpoint; - - /* Export context */ - flb_output_set_context(ins, ctx); - - return 0; - -error: - flb_free(session_name); - flb_plg_error(ctx->ins, "Initialization failed"); - flb_kinesis_ctx_destroy(ctx); - return -1; -} - -static struct flush *new_flush_buffer(const char *tag, int tag_len) -{ - struct flush *buf; - - - buf = flb_calloc(1, sizeof(struct flush)); - if (!buf) { - flb_errno(); - return NULL; - } - - buf->tmp_buf = flb_malloc(sizeof(char) * PUT_RECORDS_PAYLOAD_SIZE); - if (!buf->tmp_buf) { - flb_errno(); - kinesis_flush_destroy(buf); - return NULL; - } - buf->tmp_buf_size = PUT_RECORDS_PAYLOAD_SIZE; - - buf->events = flb_malloc(sizeof(struct kinesis_event) * MAX_EVENTS_PER_PUT); - if (!buf->events) { - flb_errno(); - kinesis_flush_destroy(buf); - return NULL; - } - buf->events_capacity = MAX_EVENTS_PER_PUT; - - buf->tag = tag; - buf->tag_len = tag_len; - - return buf; -} - -static void cb_kinesis_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - struct flb_kinesis *ctx = out_context; - int ret; - struct flush *buf; - (void) i_ins; - (void) config; - - buf = new_flush_buffer(event_chunk->tag, flb_sds_len(event_chunk->tag)); - if (!buf) { - flb_plg_error(ctx->ins, "Failed to construct flush buffer"); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - ret = process_and_send_to_kinesis(ctx, buf, - event_chunk->data, - event_chunk->size); - if (ret < 0) { - flb_plg_error(ctx->ins, "Failed to send records to kinesis"); - kinesis_flush_destroy(buf); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - flb_plg_debug(ctx->ins, "Processed %d records, sent %d to %s", - buf->records_processed, buf->records_sent, ctx->stream_name); - kinesis_flush_destroy(buf); - - FLB_OUTPUT_RETURN(FLB_OK); -} - -void flb_kinesis_ctx_destroy(struct flb_kinesis *ctx) -{ - if (ctx != NULL) { - if (ctx->base_aws_provider) { - flb_aws_provider_destroy(ctx->base_aws_provider); - } - - if (ctx->aws_provider) { - flb_aws_provider_destroy(ctx->aws_provider); - } - - if (ctx->cred_tls) { - flb_tls_destroy(ctx->cred_tls); - } - - if (ctx->sts_tls) { - flb_tls_destroy(ctx->sts_tls); - } - - if (ctx->client_tls) { - flb_tls_destroy(ctx->client_tls); - } - - if (ctx->kinesis_client) { - flb_aws_client_destroy(ctx->kinesis_client); - } - - if (ctx->custom_endpoint == FLB_FALSE) { - flb_free(ctx->endpoint); - } - - if (ctx->uuid) { - flb_free(ctx->uuid); - } - - flb_free(ctx); - } -} - -static int cb_kinesis_exit(void *data, struct flb_config *config) -{ - struct flb_kinesis *ctx = data; - - flb_kinesis_ctx_destroy(ctx); - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "region", NULL, - 0, FLB_TRUE, offsetof(struct flb_kinesis, region), - "The AWS region of your kinesis stream" - }, - - { - FLB_CONFIG_MAP_STR, "stream", NULL, - 0, FLB_TRUE, offsetof(struct flb_kinesis, stream_name), - "Kinesis stream name" - }, - - { - FLB_CONFIG_MAP_STR, "time_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_kinesis, time_key), - "Add the timestamp to the record under this key. By default the timestamp " - "from Fluent Bit will not be added to records sent to Kinesis." - }, - - { - FLB_CONFIG_MAP_STR, "time_key_format", NULL, - 0, FLB_TRUE, offsetof(struct flb_kinesis, time_key_format), - "strftime compliant format string for the timestamp; for example, " - "the default is '%Y-%m-%dT%H:%M:%S'. This option is used with time_key. " - }, - - { - FLB_CONFIG_MAP_STR, "role_arn", NULL, - 0, FLB_TRUE, offsetof(struct flb_kinesis, role_arn), - "ARN of an IAM role to assume (ex. for cross account access)." - }, - - { - FLB_CONFIG_MAP_STR, "endpoint", NULL, - 0, FLB_FALSE, 0, - "Specify a custom endpoint for the Kinesis API" - }, - - { - FLB_CONFIG_MAP_STR, "sts_endpoint", NULL, - 0, FLB_TRUE, offsetof(struct flb_kinesis, sts_endpoint), - "Custom endpoint for the STS API." - }, - - { - FLB_CONFIG_MAP_STR, "external_id", NULL, - 0, FLB_TRUE, offsetof(struct flb_kinesis, external_id), - "Specify an external ID for the STS API, can be used with the role_arn parameter if your role " - "requires an external ID." - }, - - { - FLB_CONFIG_MAP_STR, "log_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_kinesis, log_key), - "By default, the whole log record will be sent to Kinesis. " - "If you specify a key name with this option, then only the value of " - "that key will be sent to Kinesis. For example, if you are using " - "the Fluentd Docker log driver, you can specify `log_key log` and only " - "the log message will be sent to Kinesis." - }, - - { - FLB_CONFIG_MAP_BOOL, "auto_retry_requests", "true", - 0, FLB_TRUE, offsetof(struct flb_kinesis, retry_requests), - "Immediately retry failed requests to AWS services once. This option " - "does not affect the normal Fluent Bit retry mechanism with backoff. " - "Instead, it enables an immediate retry with no delay for networking " - "errors, which may help improve throughput when there are transient/random " - "networking issues." - }, - - { - FLB_CONFIG_MAP_STR, "profile", NULL, - 0, FLB_TRUE, offsetof(struct flb_kinesis, profile), - "AWS Profile name. AWS Profiles can be configured with AWS CLI and are usually stored in " - "$HOME/.aws/ directory." - }, - - /* EOF */ - {0} -}; - -/* Plugin registration */ -struct flb_output_plugin out_kinesis_streams_plugin = { - .name = "kinesis_streams", - .description = "Send logs to Amazon Kinesis Streams", - .cb_init = cb_kinesis_init, - .cb_flush = cb_kinesis_flush, - .cb_exit = cb_kinesis_exit, - .workers = 1, - .flags = 0, - - /* Configuration */ - .config_map = config_map, -}; diff --git a/fluent-bit/plugins/out_kinesis_streams/kinesis.h b/fluent-bit/plugins/out_kinesis_streams/kinesis.h deleted file mode 100644 index 75d41e107..000000000 --- a/fluent-bit/plugins/out_kinesis_streams/kinesis.h +++ /dev/null @@ -1,109 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_KINESIS_H -#define FLB_OUT_KINESIS_H - -#include -#include -#include -#include -#include -#include - -#define DEFAULT_TIME_KEY_FORMAT "%Y-%m-%dT%H:%M:%S" - -/* buffers used for each flush */ -struct flush { - /* temporary buffer for storing the serialized event messages */ - char *tmp_buf; - size_t tmp_buf_size; - /* current index of tmp_buf */ - size_t tmp_buf_offset; - - /* projected final size of the payload for this flush */ - size_t data_size; - - /* log records- each of these has a pointer to their message in tmp_buf */ - struct kinesis_event *events; - int events_capacity; - /* current event */ - int event_index; - - /* the payload of the API request */ - char *out_buf; - size_t out_buf_size; - - /* buffer used to temporarily hold an event during processing */ - char *event_buf; - size_t event_buf_size; - - int records_sent; - int records_processed; - - const char *tag; - int tag_len; -}; - -struct kinesis_event { - char *json; - size_t len; - struct timespec timestamp; -}; - -struct flb_kinesis { - /* - * TLS instances can not be re-used. So we have one for: - * - Base cred provider (needed for EKS provider) - * - STS Assume role provider - * - The CloudWatch Logs client for this plugin - */ - struct flb_tls *cred_tls; - struct flb_tls *sts_tls; - struct flb_tls *client_tls; - struct flb_aws_provider *aws_provider; - struct flb_aws_provider *base_aws_provider; - struct flb_aws_client *kinesis_client; - - /* configuration options */ - const char *stream_name; - const char *time_key; - const char *time_key_format; - const char *region; - const char *role_arn; - const char *log_key; - const char *external_id; - int retry_requests; - char *sts_endpoint; - int custom_endpoint; - char *profile; - - /* in this plugin the 'random' partition key is a uuid + fluent tag + timestamp */ - char *uuid; - - /* must be freed on shutdown if custom_endpoint is not set */ - char *endpoint; - - /* Plugin output instance reference */ - struct flb_output_instance *ins; -}; - -void flb_kinesis_ctx_destroy(struct flb_kinesis *ctx); - -#endif diff --git a/fluent-bit/plugins/out_kinesis_streams/kinesis_api.c b/fluent-bit/plugins/out_kinesis_streams/kinesis_api.c deleted file mode 100644 index 9124657bc..000000000 --- a/fluent-bit/plugins/out_kinesis_streams/kinesis_api.c +++ /dev/null @@ -1,987 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#ifndef FLB_SYSTEM_WINDOWS -#include -#endif - -#include "kinesis_api.h" - -#define ERR_CODE_EXCEEDED_THROUGHPUT "ProvisionedThroughputExceededException" - -static struct flb_aws_header put_records_target_header = { - .key = "X-Amz-Target", - .key_len = 12, - .val = "Kinesis_20131202.PutRecords", - .val_len = 27, -}; - -static inline int try_to_write(char *buf, int *off, size_t left, - const char *str, size_t str_len) -{ - if (str_len <= 0){ - str_len = strlen(str); - } - if (left <= *off+str_len) { - return FLB_FALSE; - } - memcpy(buf+*off, str, str_len); - *off += str_len; - return FLB_TRUE; -} - -/* - * Writes the "header" for a put_records payload - */ -static int init_put_payload(struct flb_kinesis *ctx, struct flush *buf, - int *offset) -{ - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - "{\"StreamName\":\"", 15)) { - goto error; - } - - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - ctx->stream_name, 0)) { - goto error; - } - - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - "\",\"Records\":[", 13)) { - goto error; - } - return 0; - -error: - return -1; -} - -/* - * Simple and fast hashing algorithm to create random partition keys - */ -static flb_sds_t random_partition_key(const char *tag) -{ - int c; - unsigned long hash = 5381; - unsigned long hash2 = 5381; - flb_sds_t hash_str; - flb_sds_t tmp; - struct flb_time tm; - - /* get current time */ - flb_time_get(&tm); - - /* compose hash */ - while ((c = *tag++)) { - hash = ((hash << 5) + hash) + c; /* hash * 33 + c */ - } - hash2 = (unsigned long) hash2 * tm.tm.tv_sec * tm.tm.tv_nsec; - - /* flb_sds_printf allocs if the incoming sds is not at least 64 bytes */ - hash_str = flb_sds_create_size(64); - if (!hash_str) { - flb_errno(); - return NULL; - } - tmp = flb_sds_printf(&hash_str, "%lu%lu", hash % 7919, hash2 % 7919); - if (!tmp) { - flb_errno(); - flb_sds_destroy(hash_str); - return NULL; - } - hash_str = tmp; - - return hash_str; -} - -/* - * Writes a log event to the output buffer - */ -static int write_event(struct flb_kinesis *ctx, struct flush *buf, - struct kinesis_event *event, int *offset) -{ - flb_sds_t tag_timestamp = NULL; - - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - "{\"Data\":\"", 9)) { - goto error; - } - - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - event->json, event->len)) { - goto error; - } - - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - "\",\"PartitionKey\":\"", 18)) { - goto error; - } - - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - ctx->uuid, 10)) { - goto error; - } - - tag_timestamp = random_partition_key(buf->tag); - if (!tag_timestamp) { - flb_plg_error(ctx->ins, "failed to generate partition key for %s", buf->tag); - goto error; - } - - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - tag_timestamp, 0)) { - flb_sds_destroy(tag_timestamp); - goto error; - } - flb_sds_destroy(tag_timestamp); - - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - "\"}", 2)) { - goto error; - } - - return 0; - -error: - return -1; -} - -/* Terminates a PutRecords payload */ -static int end_put_payload(struct flb_kinesis *ctx, struct flush *buf, - int *offset) -{ - if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, - "]}", 2)) { - return -1; - } - buf->out_buf[*offset] = '\0'; - - return 0; -} - - -/* - * Processes the msgpack object - * -1 = failure, record not added - * 0 = success, record added - * 1 = we ran out of space, send and retry - * 2 = record could not be processed, discard it - * Returns 0 on success, -1 on general errors, - * and 1 if we ran out of space to write the event - * which means a send must occur - */ -static int process_event(struct flb_kinesis *ctx, struct flush *buf, - const msgpack_object *obj, struct flb_time *tms) -{ - size_t written = 0; - int ret; - size_t size; - size_t b64_len; - struct kinesis_event *event; - char *tmp_buf_ptr; - char *time_key_ptr; - struct tm time_stamp; - struct tm *tmp; - size_t len; - size_t tmp_size; - char *out_buf; - - tmp_buf_ptr = buf->tmp_buf + buf->tmp_buf_offset; - ret = flb_msgpack_to_json(tmp_buf_ptr, - buf->tmp_buf_size - buf->tmp_buf_offset, - obj); - if (ret <= 0) { - /* - * negative value means failure to write to buffer, - * which means we ran out of space, and must send the logs - * - * TODO: This could also incorrectly be triggered if the record - * is larger than MAX_EVENT_SIZE - */ - return 1; - } - written = (size_t) ret; - - /* Discard empty messages (written == 2 means '""') */ - if (written <= 2) { - flb_plg_debug(ctx->ins, "Found empty log message, %s", ctx->stream_name); - return 2; - } - - if (ctx->log_key) { - /* - * flb_msgpack_to_json will encase the value in quotes - * We don't want that for log_key, so we ignore the first - * and last character - */ - written -= 2; - tmp_buf_ptr++; /* pass over the opening quote */ - buf->tmp_buf_offset++; - } - - /* is (written + 1) because we still have to append newline */ - if ((written + 1) >= MAX_EVENT_SIZE) { - flb_plg_warn(ctx->ins, "[size=%zu] Discarding record which is larger than " - "max size allowed by Kinesis, %s", written + 1, - ctx->stream_name); - return 2; - } - - if (ctx->time_key) { - /* append time_key to end of json string */ - tmp = gmtime_r(&tms->tm.tv_sec, &time_stamp); - if (!tmp) { - flb_plg_error(ctx->ins, "Could not create time stamp for %lu unix " - "seconds, discarding record, %s", tms->tm.tv_sec, - ctx->stream_name); - return 2; - } - - /* format time output and return the length */ - len = flb_aws_strftime_precision(&out_buf, ctx->time_key_format, tms); - - /* how much space do we have left */ - tmp_size = (buf->tmp_buf_size - buf->tmp_buf_offset) - written; - if (len > tmp_size) { - /* not enough space - tell caller to retry */ - flb_free(out_buf); - return 1; - } - - if (len == 0) { - /* - * when the length of out_buf is not enough for time_key_format, - * time_key will not be added to record. - */ - flb_plg_error(ctx->ins, "Failed to add time_key %s to record, %s", - ctx->time_key, ctx->stream_name); - flb_free(out_buf); - } - else { - time_key_ptr = tmp_buf_ptr + written - 1; - memcpy(time_key_ptr, ",", 1); - time_key_ptr++; - memcpy(time_key_ptr, "\"", 1); - time_key_ptr++; - memcpy(time_key_ptr, ctx->time_key, strlen(ctx->time_key)); - time_key_ptr += strlen(ctx->time_key); - memcpy(time_key_ptr, "\":\"", 3); - time_key_ptr += 3; - - /* merge out_buf to time_key_ptr */ - memcpy(time_key_ptr, out_buf, len); - flb_free(out_buf); - time_key_ptr += len; - memcpy(time_key_ptr, "\"}", 2); - time_key_ptr += 2; - written = (time_key_ptr - tmp_buf_ptr); - } - } - - /* is (written + 1) because we still have to append newline */ - if ((written + 1) >= MAX_EVENT_SIZE) { - flb_plg_warn(ctx->ins, "[size=%zu] Discarding record which is larger than " - "max size allowed by Kinesis, %s", written + 1, - ctx->stream_name); - return 2; - } - - /* append newline to record */ - - tmp_size = (buf->tmp_buf_size - buf->tmp_buf_offset) - written; - if (tmp_size <= 1) { - /* no space left- tell caller to retry */ - return 1; - } - - memcpy(tmp_buf_ptr + written, "\n", 1); - written++; - - /* - * check if event_buf is initialized and big enough - * Base64 encoding will increase size by ~4/3 - */ - size = (written * 1.5) + 4; - if (buf->event_buf == NULL || buf->event_buf_size < size) { - flb_free(buf->event_buf); - buf->event_buf = flb_malloc(size); - buf->event_buf_size = size; - if (buf->event_buf == NULL) { - flb_errno(); - return -1; - } - } - - tmp_buf_ptr = buf->tmp_buf + buf->tmp_buf_offset; - ret = flb_base64_encode((unsigned char *) buf->event_buf, size, &b64_len, - (unsigned char *) tmp_buf_ptr, written); - if (ret != 0) { - flb_errno(); - return -1; - } - written = b64_len; - - tmp_buf_ptr = buf->tmp_buf + buf->tmp_buf_offset; - if ((buf->tmp_buf_size - buf->tmp_buf_offset) < written) { - /* not enough space, send logs */ - return 1; - } - - /* copy serialized json to tmp_buf */ - memcpy(tmp_buf_ptr, buf->event_buf, written); - - buf->tmp_buf_offset += written; - event = &buf->events[buf->event_index]; - event->json = tmp_buf_ptr; - event->len = written; - event->timestamp.tv_sec = tms->tm.tv_sec; - event->timestamp.tv_nsec = tms->tm.tv_nsec; - - return 0; -} - -/* Resets or inits a flush struct */ -static void reset_flush_buf(struct flb_kinesis *ctx, struct flush *buf) { - buf->event_index = 0; - buf->tmp_buf_offset = 0; - buf->data_size = PUT_RECORDS_HEADER_LEN + PUT_RECORDS_FOOTER_LEN; - buf->data_size += strlen(ctx->stream_name); -} - -/* constructs a put payload, and then sends */ -static int send_log_events(struct flb_kinesis *ctx, struct flush *buf) { - int ret; - int offset; - int i; - struct kinesis_event *event; - - if (buf->event_index <= 0) { - /* - * event_index should always be 1 more than the actual last event index - * when this function is called. - * Except in the case where send_log_events() is called at the end of - * process_and_send_to_kinesis. If all records were already sent, event_index - * will be 0. Hence this check. - */ - return 0; - } - - /* alloc out_buf if needed */ - if (buf->out_buf == NULL || buf->out_buf_size < buf->data_size) { - if (buf->out_buf != NULL) { - flb_free(buf->out_buf); - } - buf->out_buf = flb_malloc(buf->data_size + 1); - if (!buf->out_buf) { - flb_errno(); - return -1; - } - buf->out_buf_size = buf->data_size; - } - - offset = 0; - ret = init_put_payload(ctx, buf, &offset); - if (ret < 0) { - flb_plg_error(ctx->ins, "Failed to initialize PutRecords payload, %s", - ctx->stream_name); - return -1; - } - - for (i = 0; i < buf->event_index; i++) { - event = &buf->events[i]; - ret = write_event(ctx, buf, event, &offset); - if (ret < 0) { - flb_plg_error(ctx->ins, "Failed to write log record %d to " - "payload buffer, %s", i, ctx->stream_name); - return -1; - } - if (i != (buf->event_index -1)) { - if (!try_to_write(buf->out_buf, &offset, buf->out_buf_size, - ",", 1)) { - flb_plg_error(ctx->ins, "Could not terminate record with ','"); - return -1; - } - } - } - - ret = end_put_payload(ctx, buf, &offset); - if (ret < 0) { - flb_plg_error(ctx->ins, "Could not complete PutRecords payload"); - return -1; - } - flb_plg_debug(ctx->ins, "kinesis:PutRecords: events=%d, payload=%d bytes", i, offset); - ret = put_records(ctx, buf, (size_t) offset, i); - if (ret < 0) { - flb_plg_error(ctx->ins, "Failed to send log records"); - return -1; - } - buf->records_sent += i; - - return 0; -} - -/* - * Processes the msgpack object, sends the current batch if needed - */ -static int add_event(struct flb_kinesis *ctx, struct flush *buf, - const msgpack_object *obj, struct flb_time *tms) -{ - int ret; - struct kinesis_event *event; - int retry_add = FLB_FALSE; - size_t event_bytes = 0; - - if (buf->event_index == 0) { - /* init */ - reset_flush_buf(ctx, buf); - } - -retry_add_event: - retry_add = FLB_FALSE; - ret = process_event(ctx, buf, obj, tms); - if (ret < 0) { - return -1; - } - else if (ret == 1) { - if (buf->event_index <= 0) { - /* somehow the record was larger than our entire request buffer */ - flb_plg_warn(ctx->ins, "Discarding massive log record, %s", - ctx->stream_name); - return 0; /* discard this record and return to caller */ - } - /* send logs and then retry the add */ - retry_add = FLB_TRUE; - goto send; - } else if (ret == 2) { - /* discard this record and return to caller */ - flb_plg_warn(ctx->ins, "Discarding large or unprocessable record, %s", - ctx->stream_name); - return 0; - } - - event = &buf->events[buf->event_index]; - event_bytes = event->len + PUT_RECORDS_PER_RECORD_LEN; - - if ((buf->data_size + event_bytes) > PUT_RECORDS_PAYLOAD_SIZE) { - if (buf->event_index <= 0) { - /* somehow the record was larger than our entire request buffer */ - flb_plg_warn(ctx->ins, "[size=%zu] Discarding massive log record, %s", - event_bytes, ctx->stream_name); - return 0; /* discard this record and return to caller */ - } - /* do not send this event */ - retry_add = FLB_TRUE; - goto send; - } - - /* send is not needed yet, return to caller */ - buf->data_size += event_bytes; - buf->event_index++; - - if (buf->event_index == MAX_EVENTS_PER_PUT) { - goto send; - } - - return 0; - -send: - ret = send_log_events(ctx, buf); - reset_flush_buf(ctx, buf); - if (ret < 0) { - return -1; - } - - if (retry_add == FLB_TRUE) { - goto retry_add_event; - } - - return 0; -} - -/* - * Main routine- processes msgpack and sends in batches - * return value is the number of events processed (number sent is stored in buf) - */ -int process_and_send_to_kinesis(struct flb_kinesis *ctx, struct flush *buf, - const char *data, size_t bytes) -{ - int i = 0; - size_t map_size; - msgpack_object map; - msgpack_object_kv *kv; - msgpack_object key; - msgpack_object val; - char *key_str = NULL; - size_t key_str_size = 0; - int j; - int ret; - int check = FLB_FALSE; - int found = FLB_FALSE; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return -1; - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - map = *log_event.body; - map_size = map.via.map.size; - - if (ctx->log_key) { - key_str = NULL; - key_str_size = 0; - check = FLB_FALSE; - found = FLB_FALSE; - - kv = map.via.map.ptr; - - for(j=0; j < map_size; j++) { - key = (kv+j)->key; - if (key.type == MSGPACK_OBJECT_BIN) { - key_str = (char *) key.via.bin.ptr; - key_str_size = key.via.bin.size; - check = FLB_TRUE; - } - if (key.type == MSGPACK_OBJECT_STR) { - key_str = (char *) key.via.str.ptr; - key_str_size = key.via.str.size; - check = FLB_TRUE; - } - - if (check == FLB_TRUE) { - if (strncmp(ctx->log_key, key_str, key_str_size) == 0) { - found = FLB_TRUE; - val = (kv+j)->val; - ret = add_event(ctx, buf, &val, &log_event.timestamp); - if (ret < 0 ) { - goto error; - } - } - } - - } - if (found == FLB_FALSE) { - flb_plg_error(ctx->ins, "Could not find log_key '%s' in record, %s", - ctx->log_key, ctx->stream_name); - } - else { - i++; - } - continue; - } - - ret = add_event(ctx, buf, &map, &log_event.timestamp); - if (ret < 0 ) { - goto error; - } - i++; - } - - flb_log_event_decoder_destroy(&log_decoder); - - /* send any remaining events */ - ret = send_log_events(ctx, buf); - reset_flush_buf(ctx, buf); - if (ret < 0) { - return -1; - } - - /* return number of events processed */ - buf->records_processed = i; - return i; - -error: - flb_log_event_decoder_destroy(&log_decoder); - - return -1; -} - -/* - * Returns number of failed records on success, -1 on failure - */ -static int process_api_response(struct flb_kinesis *ctx, - struct flb_http_client *c) -{ - int i; - int k; - int w; - int ret; - int failed_records = -1; - int root_type; - char *out_buf; - int throughput_exceeded = FLB_FALSE; - size_t off = 0; - size_t out_size; - msgpack_unpacked result; - msgpack_object root; - msgpack_object key; - msgpack_object val; - msgpack_object response; - msgpack_object response_key; - msgpack_object response_val; - - if (strstr(c->resp.payload, "\"FailedRecordCount\":0")) { - return 0; - } - - /* Convert JSON payload to msgpack */ - ret = flb_pack_json(c->resp.payload, c->resp.payload_size, - &out_buf, &out_size, &root_type, NULL); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not pack/validate JSON API response\n%s", - c->resp.payload); - return -1; - } - - /* Lookup error field */ - msgpack_unpacked_init(&result); - ret = msgpack_unpack_next(&result, out_buf, out_size, &off); - if (ret != MSGPACK_UNPACK_SUCCESS) { - flb_plg_error(ctx->ins, "Cannot unpack response to find error\n%s", - c->resp.payload); - failed_records = -1; - goto done; - } - - root = result.data; - if (root.type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "unexpected payload type=%i", - root.type); - failed_records = -1; - goto done; - } - - for (i = 0; i < root.via.map.size; i++) { - key = root.via.map.ptr[i].key; - if (key.type != MSGPACK_OBJECT_STR) { - flb_plg_error(ctx->ins, "unexpected key type=%i", - key.type); - failed_records = -1; - goto done; - } - - if (key.via.str.size >= 14 && - strncmp(key.via.str.ptr, "FailedRecordCount", 14) == 0) { - val = root.via.map.ptr[i].val; - if (val.type != MSGPACK_OBJECT_POSITIVE_INTEGER) { - flb_plg_error(ctx->ins, "unexpected 'FailedRecordCount' value type=%i", - val.type); - failed_records = -1; - goto done; - } - - failed_records = val.via.u64; - if (failed_records == 0) { - /* no need to check RequestResponses field */ - goto done; - } - } - - if (key.via.str.size >= 14 && - strncmp(key.via.str.ptr, "Records", 7) == 0) { - val = root.via.map.ptr[i].val; - if (val.type != MSGPACK_OBJECT_ARRAY) { - flb_plg_error(ctx->ins, "unexpected 'Records' value type=%i", - val.type); - failed_records = -1; - goto done; - } - - if (val.via.array.size == 0) { - flb_plg_error(ctx->ins, "'Records' field in response is empty"); - failed_records = -1; - goto done; - } - - for (k = 0; k < val.via.array.size; k++) { - /* iterate through the responses */ - response = val.via.array.ptr[k]; - if (response.type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "unexpected 'Records[%d]' value type=%i", - k, response.type); - failed_records = -1; - goto done; - } - for (w = 0; w < response.via.map.size; w++) { - /* iterate through the response's keys */ - response_key = response.via.map.ptr[w].key; - if (response_key.type != MSGPACK_OBJECT_STR) { - flb_plg_error(ctx->ins, "unexpected key type=%i", - response_key.type); - failed_records = -1; - goto done; - } - if (response_key.via.str.size >= 9 && - strncmp(response_key.via.str.ptr, "ErrorCode", 9) == 0) { - response_val = response.via.map.ptr[w].val; - if (!throughput_exceeded && - response_val.via.str.size >= 38 && - (strncmp(response_val.via.str.ptr, - ERR_CODE_EXCEEDED_THROUGHPUT, 38) == 0)) { - throughput_exceeded = FLB_TRUE; - flb_plg_error(ctx->ins, "Throughput limits may have been exceeded, %s", - ctx->stream_name); - } - flb_plg_debug(ctx->ins, "Record %i failed with err_code=%.*s", - k, response_val.via.str.size, - response_val.via.str.ptr); - } - if (response_key.via.str.size >= 12 && - strncmp(response_key.via.str.ptr, "ErrorMessage", 12) == 0) { - response_val = response.via.map.ptr[w].val; - flb_plg_debug(ctx->ins, "Record %i failed with err_msg=%.*s", - k, response_val.via.str.size, - response_val.via.str.ptr); - } - } - } - } - } - - done: - flb_free(out_buf); - msgpack_unpacked_destroy(&result); - return failed_records; -} - -static int plugin_under_test() -{ - if (getenv("FLB_KINESIS_PLUGIN_UNDER_TEST") != NULL) { - return FLB_TRUE; - } - - return FLB_FALSE; -} - -static char *mock_error_response(char *error_env_var) -{ - char *err_val = NULL; - char *error = NULL; - int len = 0; - - err_val = getenv(error_env_var); - if (err_val != NULL && strlen(err_val) > 0) { - error = flb_malloc(strlen(err_val) + sizeof(char)); - if (error == NULL) { - flb_errno(); - return NULL; - } - - len = strlen(err_val); - memcpy(error, err_val, len); - error[len] = '\0'; - return error; - } - - return NULL; -} - -static int partial_success() -{ - char *err_val = NULL; - - err_val = getenv("PARTIAL_SUCCESS_CASE"); - if (err_val != NULL && strlen(err_val) > 0) { - return FLB_TRUE; - } - - return FLB_FALSE; -} - -static struct flb_http_client *mock_http_call(char *error_env_var) -{ - /* create an http client so that we can set the response */ - struct flb_http_client *c = NULL; - char *error = mock_error_response(error_env_var); - - c = flb_calloc(1, sizeof(struct flb_http_client)); - if (!c) { - flb_errno(); - flb_free(error); - return NULL; - } - mk_list_init(&c->headers); - - if (error != NULL) { - c->resp.status = 400; - /* resp.data is freed on destroy, payload is supposed to reference it */ - c->resp.data = error; - c->resp.payload = c->resp.data; - c->resp.payload_size = strlen(error); - } - else { - c->resp.status = 200; - c->resp.payload = ""; - c->resp.payload_size = 0; - if (partial_success() == FLB_TRUE) { - /* mocked partial failure response */ - c->resp.payload = "{\"FailedRecordCount\":2,\"Records\":[{\"SequenceNumber\":\"49543463076548007577105092703039560359975228518395012686\",\"ShardId\":\"shardId-000000000000\"},{\"ErrorCode\":\"ProvisionedThroughputExceededException\",\"ErrorMessage\":\"Rate exceeded for shard shardId-000000000001 in stream exampleStreamName under account 111111111111.\"},{\"ErrorCode\":\"InternalFailure\",\"ErrorMessage\":\"Internal service failure.\"}]}"; - c->resp.payload_size = strlen(c->resp.payload); - } - else { - /* mocked success response */ - c->resp.payload = "{\"FailedRecordCount\":0,\"Records\":[{\"SequenceNumber\":\"49543463076548007577105092703039560359975228518395019266\",\"ShardId\":\"shardId-000000000000\"},{\"SequenceNumber\":\"49543463076570308322303623326179887152428262250726293522\",\"ShardId\":\"shardId-000000000001\"},{\"SequenceNumber\":\"49543463076570308322303623326179887152428262250726293588\",\"ShardId\":\"shardId-000000000003\"}]}"; - c->resp.payload_size = strlen(c->resp.payload); - } - } - - return c; -} - - -/* - * Returns -1 on failure, 0 on success - */ -int put_records(struct flb_kinesis *ctx, struct flush *buf, - size_t payload_size, int num_records) -{ - - struct flb_http_client *c = NULL; - struct flb_aws_client *kinesis_client; - flb_sds_t error; - int failed_records = 0; - - flb_plg_debug(ctx->ins, "Sending log records to stream %s", - ctx->stream_name); - - if (plugin_under_test() == FLB_TRUE) { - c = mock_http_call("TEST_PUT_RECORDS_ERROR"); - } - else { - kinesis_client = ctx->kinesis_client; - c = kinesis_client->client_vtable->request(kinesis_client, FLB_HTTP_POST, - "/", buf->out_buf, payload_size, - &put_records_target_header, 1); - } - - if (c) { - flb_plg_debug(ctx->ins, "PutRecords http status=%d", c->resp.status); - - if (c->resp.status == 200) { - /* Kinesis API can return partial success- check response */ - if (c->resp.payload_size > 0) { - failed_records = process_api_response(ctx, c); - if (failed_records < 0) { - flb_plg_error(ctx->ins, "PutRecords response " - "could not be parsed, %s", - c->resp.payload); - flb_http_client_destroy(c); - return -1; - } - if (failed_records == num_records) { - flb_plg_error(ctx->ins, "PutRecords request returned " - "with no records successfully recieved, %s", - ctx->stream_name); - flb_http_client_destroy(c); - return -1; - } - if (failed_records > 0) { - flb_plg_error(ctx->ins, "%d out of %d records failed to be " - "delivered, will retry this batch, %s", - failed_records, num_records, - ctx->stream_name); - flb_http_client_destroy(c); - return -1; - } - } - flb_plg_debug(ctx->ins, "Sent events to %s", ctx->stream_name); - flb_http_client_destroy(c); - return 0; - } - - /* Check error */ - if (c->resp.payload_size > 0) { - error = flb_aws_error(c->resp.payload, c->resp.payload_size); - if (error != NULL) { - if (strcmp(error, ERR_CODE_EXCEEDED_THROUGHPUT) == 0) { - flb_plg_error(ctx->ins, "Throughput limits for %s " - "may have been exceeded.", - ctx->stream_name); - } - if (strncmp(error, "SerializationException", 22) == 0) { - /* - * If this happens, we habe a bug in the code - * User should send us the output to debug - */ - flb_plg_error(ctx->ins, "<<------Bug in Code------>>"); - printf("Malformed request: %s", buf->out_buf); - } - flb_aws_print_error(c->resp.payload, c->resp.payload_size, - "PutRecords", ctx->ins); - flb_sds_destroy(error); - } - else { - /* error could not be parsed, print raw response to debug */ - flb_plg_debug(ctx->ins, "Raw response: %s", c->resp.payload); - } - } - } - - flb_plg_error(ctx->ins, "Failed to send log records to %s", ctx->stream_name); - if (c) { - flb_http_client_destroy(c); - } - return -1; -} - - -void kinesis_flush_destroy(struct flush *buf) -{ - if (buf) { - flb_free(buf->tmp_buf); - flb_free(buf->out_buf); - flb_free(buf->events); - flb_free(buf->event_buf); - flb_free(buf); - } -} diff --git a/fluent-bit/plugins/out_kinesis_streams/kinesis_api.h b/fluent-bit/plugins/out_kinesis_streams/kinesis_api.h deleted file mode 100644 index e44de6d4d..000000000 --- a/fluent-bit/plugins/out_kinesis_streams/kinesis_api.h +++ /dev/null @@ -1,44 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_KINESIS_API -#define FLB_OUT_KINESIS_API - -#define PUT_RECORDS_PAYLOAD_SIZE 5242880 -#define MAX_EVENTS_PER_PUT 500 -#define MAX_EVENT_SIZE 1048556 /* 1048576 - 20 bytes for partition key */ - -/* number of characters needed to 'start' a PutRecords payload */ -#define PUT_RECORDS_HEADER_LEN 30 -/* number of characters needed per record in a PutRecords payload */ -#define PUT_RECORDS_PER_RECORD_LEN 48 -/* number of characters needed to 'end' a PutRecords payload */ -#define PUT_RECORDS_FOOTER_LEN 4 - -#include "kinesis.h" - -void kinesis_flush_destroy(struct flush *buf); - -int process_and_send_to_kinesis(struct flb_kinesis *ctx, struct flush *buf, - const char *data, size_t bytes); - -int put_records(struct flb_kinesis *ctx, struct flush *buf, - size_t payload_size, int num_records); - -#endif diff --git a/fluent-bit/plugins/out_lib/CMakeLists.txt b/fluent-bit/plugins/out_lib/CMakeLists.txt deleted file mode 100644 index b50ecc2ed..000000000 --- a/fluent-bit/plugins/out_lib/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -set(src - out_lib.c) - -FLB_PLUGIN(out_lib "${src}" "") -#target_link_libraries(flb-plugin-out_lib msgpack) diff --git a/fluent-bit/plugins/out_lib/out_lib.c b/fluent-bit/plugins/out_lib/out_lib.c deleted file mode 100644 index da0972243..000000000 --- a/fluent-bit/plugins/out_lib/out_lib.c +++ /dev/null @@ -1,222 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include -#include -#include -#include -#include - -#include "out_lib.h" - -#define PLUGIN_NAME "out_lib" - -static int configure(struct flb_out_lib_config *ctx, - struct flb_output_instance *ins) -{ - const char *tmp; - - tmp = flb_output_get_property("format", ins); - if (!tmp) { - ctx->format = FLB_OUT_LIB_FMT_MSGPACK; - } - else { - if (strcasecmp(tmp, FLB_FMT_STR_MSGPACK) == 0) { - ctx->format = FLB_OUT_LIB_FMT_MSGPACK; - } - else if (strcasecmp(tmp, FLB_FMT_STR_JSON) == 0) { - ctx->format = FLB_OUT_LIB_FMT_JSON; - } - } - - tmp = flb_output_get_property("max_records", ins); - if (tmp) { - ctx->max_records = atoi(tmp); - } - else { - ctx->max_records = 0; - } - - return 0; -} - - -/** - * User callback is passed from flb_output(ctx, output, callback) - * - * The prototype of callback should be - * int (*callback)(void* data, size_t size ); - * @param data The data which comes from input plugin. - * @param size The size of data. - * @return success ? 0 : negative value - * - */ -static int out_lib_init(struct flb_output_instance *ins, - struct flb_config *config, - void *data) -{ - struct flb_out_lib_config *ctx = NULL; - struct flb_lib_out_cb *cb_data = data; - (void) config; - - ctx = flb_calloc(1, sizeof(struct flb_out_lib_config)); - if (ctx == NULL) { - flb_errno(); - return -1; - } - ctx->ins = ins; - - if (cb_data) { - /* Set user callback and data */ - ctx->cb_func = cb_data->cb; - ctx->cb_data = cb_data->data; - } - else { - flb_plg_error(ctx->ins, "Callback is not set"); - flb_free(ctx); - return -1; - } - - configure(ctx, ins); - flb_output_set_context(ins, ctx); - - return 0; -} - -static void out_lib_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - int len; - int count = 0; - size_t off = 0; - size_t last_off = 0; - size_t data_size = 0; - size_t alloc_size = 0; - size_t out_size = 0; - char *buf = NULL; - char *out_buf = NULL; - char *data_for_user = NULL; - msgpack_object *obj; - msgpack_unpacked result; - struct flb_time tm; - struct flb_out_lib_config *ctx = out_context; - (void) i_ins; - (void) config; - - msgpack_unpacked_init(&result); - while (msgpack_unpack_next(&result, - event_chunk->data, - event_chunk->size, &off) == MSGPACK_UNPACK_SUCCESS) { - if (ctx->max_records > 0 && count >= ctx->max_records) { - break; - } - switch(ctx->format) { - case FLB_OUT_LIB_FMT_MSGPACK: - alloc_size = (off - last_off); - - /* copy raw bytes */ - data_for_user = flb_malloc(alloc_size); - if (!data_for_user) { - flb_errno(); - msgpack_unpacked_destroy(&result); - FLB_OUTPUT_RETURN(FLB_ERROR); - } - - memcpy(data_for_user, - (char *) event_chunk->data + last_off, alloc_size); - data_size = alloc_size; - break; - case FLB_OUT_LIB_FMT_JSON: -#ifdef FLB_HAVE_METRICS - if (event_chunk->type == FLB_EVENT_TYPE_METRICS) { - alloc_size = (off - last_off) + 4096; - buf = flb_msgpack_to_json_str(alloc_size, &result.data); - if (buf == NULL) { - msgpack_unpacked_destroy(&result); - FLB_OUTPUT_RETURN(FLB_ERROR); - } - data_size = strlen(buf); - data_for_user = buf; - } - else { -#endif - /* JSON is larger than msgpack */ - alloc_size = (off - last_off) + 128; - - flb_time_pop_from_msgpack(&tm, &result, &obj); - buf = flb_msgpack_to_json_str(alloc_size, obj); - if (!buf) { - msgpack_unpacked_destroy(&result); - FLB_OUTPUT_RETURN(FLB_ERROR); - } - - len = strlen(buf); - out_size = len + 32; - out_buf = flb_malloc(out_size); - if (!out_buf) { - flb_errno(); - msgpack_unpacked_destroy(&result); - FLB_OUTPUT_RETURN(FLB_ERROR); - } - - len = snprintf(out_buf, out_size, "[%f,%s]", - flb_time_to_double(&tm), - buf); - flb_free(buf); - data_for_user = out_buf; - data_size = len; -#ifdef FLB_HAVE_METRICS - } -#endif - break; - } - - /* Invoke user callback */ - ctx->cb_func(data_for_user, data_size, ctx->cb_data); - last_off = off; - count++; - } - - msgpack_unpacked_destroy(&result); - FLB_OUTPUT_RETURN(FLB_OK); -} - -static int out_lib_exit(void *data, struct flb_config *config) -{ - struct flb_out_lib_config *ctx = data; - - flb_free(ctx); - return 0; -} - -struct flb_output_plugin out_lib_plugin = { - .name = "lib", - .description = "Library mode Output", - .cb_init = out_lib_init, - .cb_flush = out_lib_flush, - .cb_exit = out_lib_exit, - .event_type = FLB_OUTPUT_LOGS | FLB_OUTPUT_METRICS, - .flags = 0, -}; diff --git a/fluent-bit/plugins/out_lib/out_lib.h b/fluent-bit/plugins/out_lib/out_lib.h deleted file mode 100644 index 5e0925ede..000000000 --- a/fluent-bit/plugins/out_lib/out_lib.h +++ /dev/null @@ -1,42 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_LIB -#define FLB_OUT_LIB - -#include - -enum { - FLB_OUT_LIB_FMT_MSGPACK = 0, - FLB_OUT_LIB_FMT_JSON, - FLB_OUT_LIB_FMT_ERROR, -}; - -#define FLB_FMT_STR_MSGPACK "msgpack" -#define FLB_FMT_STR_JSON "json" - -struct flb_out_lib_config { - int format; - int max_records; - int (*cb_func)(void *record, size_t size, void *data); - void *cb_data; - struct flb_output_instance *ins; -}; - -#endif diff --git a/fluent-bit/plugins/out_logdna/CMakeLists.txt b/fluent-bit/plugins/out_logdna/CMakeLists.txt deleted file mode 100644 index 2bfa1f9d8..000000000 --- a/fluent-bit/plugins/out_logdna/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -set(src - logdna.c - ) - -FLB_PLUGIN(out_logdna "${src}" "") diff --git a/fluent-bit/plugins/out_logdna/logdna.c b/fluent-bit/plugins/out_logdna/logdna.c deleted file mode 100644 index e3ab9f56f..000000000 --- a/fluent-bit/plugins/out_logdna/logdna.c +++ /dev/null @@ -1,591 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include - -#include "logdna.h" - -static inline int primary_key_check(msgpack_object k, char *name, int len) -{ - if (k.type != MSGPACK_OBJECT_STR) { - return FLB_FALSE; - } - - if (k.via.str.size != len) { - return FLB_FALSE; - } - - if (memcmp(k.via.str.ptr, name, len) == 0) { - return FLB_TRUE; - } - - return FLB_FALSE; -} - -/* - * This function looks for the following keys and add them to the buffer - * - * - level or severity - * - file - * - app - * - meta - */ -static int record_append_primary_keys(struct flb_logdna *ctx, - msgpack_object *map, - msgpack_packer *mp_sbuf) -{ - int i; - int c = 0; - msgpack_object *level = NULL; - msgpack_object *file = NULL; - msgpack_object *app = NULL; - msgpack_object *meta = NULL; - msgpack_object k; - msgpack_object v; - - for (i = 0; i < map->via.array.size; i++) { - k = map->via.map.ptr[i].key; - v = map->via.map.ptr[i].val; - - /* Level - optional */ - if (!level && - (primary_key_check(k, "level", 5) == FLB_TRUE || - primary_key_check(k, "severity", 8) == FLB_TRUE)) { - level = &k; - msgpack_pack_str(mp_sbuf, 5); - msgpack_pack_str_body(mp_sbuf, "level", 5); - msgpack_pack_object(mp_sbuf, v); - c++; - } - - /* Meta - optional */ - if (!meta && primary_key_check(k, "meta", 4) == FLB_TRUE) { - meta = &k; - msgpack_pack_str(mp_sbuf, 4); - msgpack_pack_str_body(mp_sbuf, "meta", 4); - msgpack_pack_object(mp_sbuf, v); - c++; - } - - /* File */ - if (!file && primary_key_check(k, "file", 4) == FLB_TRUE) { - file = &k; - msgpack_pack_str(mp_sbuf, 4); - msgpack_pack_str_body(mp_sbuf, "file", 4); - msgpack_pack_object(mp_sbuf, v); - c++; - } - - /* App */ - if (primary_key_check(k, "app", 3) == FLB_TRUE) { - app = &k; - msgpack_pack_str(mp_sbuf, 3); - msgpack_pack_str_body(mp_sbuf, "app", 3); - msgpack_pack_object(mp_sbuf, v); - c++; - } - } - - /* Set the global file name if the record did not provided one */ - if (!file && ctx->file) { - msgpack_pack_str(mp_sbuf, 4); - msgpack_pack_str_body(mp_sbuf, "file", 4); - msgpack_pack_str(mp_sbuf, flb_sds_len(ctx->file)); - msgpack_pack_str_body(mp_sbuf, ctx->file, flb_sds_len(ctx->file)); - c++; - } - - - /* If no application name is set, set the default */ - if (!app) { - msgpack_pack_str(mp_sbuf, 3); - msgpack_pack_str_body(mp_sbuf, "app", 3); - msgpack_pack_str(mp_sbuf, flb_sds_len(ctx->app)); - msgpack_pack_str_body(mp_sbuf, ctx->app, flb_sds_len(ctx->app)); - c++; - } - - return c; -} - -static flb_sds_t logdna_compose_payload(struct flb_logdna *ctx, - const void *data, size_t bytes, - const char *tag, int tag_len) -{ - int ret; - int len; - int total_lines; - int array_size = 0; - off_t map_off; - char *line_json; - flb_sds_t json; - msgpack_packer mp_pck; - msgpack_sbuffer mp_sbuf; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return NULL; - } - - /* Count number of records */ - total_lines = flb_mp_count(data, bytes); - - /* Initialize msgpack buffers */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - msgpack_pack_map(&mp_pck, 1); - - msgpack_pack_str(&mp_pck, 5); - msgpack_pack_str_body(&mp_pck, "lines", 5); - - msgpack_pack_array(&mp_pck, total_lines); - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - map_off = mp_sbuf.size; - - array_size = 2; - msgpack_pack_map(&mp_pck, array_size); - - /* - * Append primary keys found, the return values is the number of appended - * keys to the record, we use that to adjust the map header size. - */ - ret = record_append_primary_keys(ctx, log_event.body, &mp_pck); - array_size += ret; - - /* Timestamp */ - msgpack_pack_str(&mp_pck, 9); - msgpack_pack_str_body(&mp_pck, "timestamp", 9); - msgpack_pack_int(&mp_pck, (int) flb_time_to_double(&log_event.timestamp)); - - /* Line */ - msgpack_pack_str(&mp_pck, 4); - msgpack_pack_str_body(&mp_pck, "line", 4); - - line_json = flb_msgpack_to_json_str(1024, log_event.body); - len = strlen(line_json); - msgpack_pack_str(&mp_pck, len); - msgpack_pack_str_body(&mp_pck, line_json, len); - flb_free(line_json); - - /* Adjust map header size */ - flb_mp_set_map_header_size(mp_sbuf.data + map_off, array_size); - } - - flb_log_event_decoder_destroy(&log_decoder); - - json = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); - msgpack_sbuffer_destroy(&mp_sbuf); - - return json; -} - -static void logdna_config_destroy(struct flb_logdna *ctx) -{ - if (ctx->u) { - flb_upstream_destroy(ctx->u); - } - - if (ctx->tags_formatted) { - flb_sds_destroy(ctx->tags_formatted); - } - - flb_free(ctx); -} - -static struct flb_logdna *logdna_config_create(struct flb_output_instance *ins, - struct flb_config *config) -{ - int ret; - int len = 0; - char *hostname; - flb_sds_t tmp; - flb_sds_t encoded; - struct mk_list *head; - struct flb_slist_entry *tag_entry; - struct flb_logdna *ctx; - struct flb_upstream *upstream; - - /* Create context */ - ctx = flb_calloc(1, sizeof(struct flb_logdna)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - - /* Load config map */ - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - logdna_config_destroy(ctx); - return NULL; - } - - /* validate API key */ - if (!ctx->api_key) { - flb_plg_error(ins, "no `api_key` was set, this is a mandatory property"); - logdna_config_destroy(ctx); - return NULL; - } - - /* - * Tags: this value is a linked list of values created by the config map - * reader. - */ - if (ctx->tags) { - /* For every tag, make sure no empty spaces exists */ - mk_list_foreach(head, ctx->tags) { - tag_entry = mk_list_entry(head, struct flb_slist_entry, _head); - len += flb_sds_len(tag_entry->str) + 1; - } - - /* Compose a full tag for URI request */ - ctx->tags_formatted = flb_sds_create_size(len); - if (!ctx->tags_formatted) { - logdna_config_destroy(ctx); - return NULL; - } - - mk_list_foreach(head, ctx->tags) { - tag_entry = mk_list_entry(head, struct flb_slist_entry, _head); - - encoded = flb_uri_encode(tag_entry->str, - flb_sds_len(tag_entry->str)); - tmp = flb_sds_cat(ctx->tags_formatted, - encoded, flb_sds_len(encoded)); - ctx->tags_formatted = tmp; - flb_sds_destroy(encoded); - - if (tag_entry->_head.next != ctx->tags) { - tmp = flb_sds_cat(ctx->tags_formatted, ",", 1); - ctx->tags_formatted = tmp; - } - } - } - - /* - * Hostname: if the hostname was not set manually, try to get it from the - * environment variable. - * - * Note that hostname is populated by a config map, and config maps are - * immutable so we use an internal variable to do a final composition - * if required. - */ - if (!ctx->hostname) { - tmp = NULL; - hostname = (char *) flb_env_get(config->env, "HOSTNAME"); - if (hostname) { - ctx->_hostname = flb_sds_create(hostname); - } - else { - ctx->_hostname = flb_sds_create("unknown"); - } - } - else { - ctx->_hostname = flb_sds_create(ctx->hostname); - } - - /* Bail if unsuccessful hostname creation */ - if (!ctx->_hostname) { - flb_free(ctx); - return NULL; - } - - /* Create Upstream connection context */ - upstream = flb_upstream_create(config, - ctx->logdna_host, - ctx->logdna_port, - FLB_IO_TLS, ins->tls); - if (!upstream) { - flb_free(ctx); - return NULL; - } - ctx->u = upstream; - flb_output_upstream_set(ctx->u, ins); - - /* Set networking defaults */ - flb_output_net_default(FLB_LOGDNA_HOST, atoi(FLB_LOGDNA_PORT), ins); - return ctx; -} - -static int cb_logdna_init(struct flb_output_instance *ins, - struct flb_config *config, void *data) -{ - struct flb_logdna *ctx; - - ctx = logdna_config_create(ins, config); - if (!ctx) { - flb_plg_error(ins, "cannot initialize configuration"); - return -1; - } - - flb_output_set_context(ins, ctx); - - /* - * This plugin instance uses the HTTP client interface, let's register - * it debugging callbacks. - */ - flb_output_set_http_debug_callbacks(ins); - - flb_plg_info(ins, "configured, hostname=%s", ctx->hostname); - return 0; -} - -static void cb_logdna_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - int ret; - int out_ret = FLB_OK; - size_t b_sent; - flb_sds_t uri; - flb_sds_t tmp; - flb_sds_t payload; - struct flb_logdna *ctx = out_context; - struct flb_connection *u_conn; - struct flb_http_client *c; - - /* Format the data to the expected LogDNA Payload */ - payload = logdna_compose_payload(ctx, - event_chunk->data, - event_chunk->size, - event_chunk->tag, - flb_sds_len(event_chunk->tag)); - if (!payload) { - flb_plg_error(ctx->ins, "cannot compose request payload"); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Lookup an available connection context */ - u_conn = flb_upstream_conn_get(ctx->u); - if (!u_conn) { - flb_plg_error(ctx->ins, "no upstream connections available"); - flb_sds_destroy(payload); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Compose the HTTP URI */ - uri = flb_sds_create_size(256); - if (!uri) { - flb_plg_error(ctx->ins, "cannot allocate buffer for URI"); - flb_sds_destroy(payload); - flb_free(ctx); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - tmp = flb_sds_printf(&uri, - "/logs/ingest?hostname=%s&mac=%s&ip=%s&now=%lu&tags=%s", - ctx->_hostname, - ctx->mac_addr, - ctx->ip_addr, - time(NULL), - ctx->tags_formatted); - if (!tmp) { - flb_plg_error(ctx->ins, "error formatting URI"); - flb_sds_destroy(payload); - flb_free(ctx); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Create HTTP client context */ - c = flb_http_client(u_conn, FLB_HTTP_POST, uri, - payload, flb_sds_len(payload), - ctx->logdna_host, ctx->logdna_port, - NULL, 0); - if (!c) { - flb_plg_error(ctx->ins, "cannot create HTTP client context"); - flb_sds_destroy(uri); - flb_sds_destroy(payload); - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Set callback context to the HTTP client context */ - flb_http_set_callback_context(c, ctx->ins->callback); - - /* User Agent */ - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - - /* Add Content-Type header */ - flb_http_add_header(c, - FLB_LOGDNA_CT, sizeof(FLB_LOGDNA_CT) - 1, - FLB_LOGDNA_CT_JSON, sizeof(FLB_LOGDNA_CT_JSON) - 1); - - /* Add auth */ - flb_http_basic_auth(c, ctx->api_key, ""); - - flb_http_strip_port_from_host(c); - - /* Send HTTP request */ - ret = flb_http_do(c, &b_sent); - - /* Destroy buffers */ - flb_sds_destroy(uri); - flb_sds_destroy(payload); - - /* Validate HTTP client return status */ - if (ret == 0) { - /* - * Only allow the following HTTP status: - * - * - 200: OK - * - 201: Created - * - 202: Accepted - * - 203: no authorative resp - * - 204: No Content - * - 205: Reset content - * - */ - if (c->resp.status < 200 || c->resp.status > 205) { - if (c->resp.payload) { - flb_plg_error(ctx->ins, "%s:%i, HTTP status=%i\n%s", - ctx->logdna_host, ctx->logdna_port, c->resp.status, - c->resp.payload); - } - else { - flb_plg_error(ctx->ins, "%s:%i, HTTP status=%i", - ctx->logdna_host, ctx->logdna_port, c->resp.status); - } - out_ret = FLB_RETRY; - } - else { - if (c->resp.payload) { - flb_plg_info(ctx->ins, "%s:%i, HTTP status=%i\n%s", - ctx->logdna_host, ctx->logdna_port, - c->resp.status, c->resp.payload); - } - else { - flb_plg_info(ctx->ins, "%s:%i, HTTP status=%i", - ctx->logdna_host, ctx->logdna_port, - c->resp.status); - } - } - } - else { - flb_plg_error(ctx->ins, "could not flush records to %s:%s (http_do=%i)", - FLB_LOGDNA_HOST, FLB_LOGDNA_PORT, ret); - out_ret = FLB_RETRY; - } - - flb_http_client_destroy(c); - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(out_ret); -} - -static int cb_logdna_exit(void *data, struct flb_config *config) -{ - struct flb_logdna *ctx = data; - - if (!ctx) { - return 0; - } - - if (ctx->_hostname) { - flb_sds_destroy(ctx->_hostname); - } - logdna_config_destroy(ctx); - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "logdna_host", FLB_LOGDNA_HOST, - 0, FLB_TRUE, offsetof(struct flb_logdna, logdna_host), - "LogDNA Host address" - }, - - { - FLB_CONFIG_MAP_INT, "logdna_port", FLB_LOGDNA_PORT, - 0, FLB_TRUE, offsetof(struct flb_logdna, logdna_port), - "LogDNA TCP port" - }, - - { - FLB_CONFIG_MAP_STR, "api_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_logdna, api_key), - "Logdna API key" - }, - - { - FLB_CONFIG_MAP_STR, "hostname", NULL, - 0, FLB_TRUE, offsetof(struct flb_logdna, hostname), - "Local Server or device host name" - }, - - { - FLB_CONFIG_MAP_STR, "mac", "", - 0, FLB_TRUE, offsetof(struct flb_logdna, mac_addr), - "MAC address (optional)" - }, - - { - FLB_CONFIG_MAP_STR, "ip", "", - 0, FLB_TRUE, offsetof(struct flb_logdna, ip_addr), - "IP address (optional)" - }, - - { - FLB_CONFIG_MAP_CLIST, "tags", "", - 0, FLB_TRUE, offsetof(struct flb_logdna, tags), - "Tags (optional)" - }, - - { - FLB_CONFIG_MAP_STR, "file", NULL, - 0, FLB_TRUE, offsetof(struct flb_logdna, file), - "Name of the monitored file (optional)" - }, - - { - FLB_CONFIG_MAP_STR, "app", "Fluent Bit", - 0, FLB_TRUE, offsetof(struct flb_logdna, app), - "Name of the application generating the data (optional)" - }, - - /* EOF */ - {0} - -}; - -/* Plugin reference */ -struct flb_output_plugin out_logdna_plugin = { - .name = "logdna", - .description = "LogDNA", - .cb_init = cb_logdna_init, - .cb_flush = cb_logdna_flush, - .cb_exit = cb_logdna_exit, - .config_map = config_map, - .flags = FLB_OUTPUT_NET | FLB_IO_TLS, -}; diff --git a/fluent-bit/plugins/out_logdna/logdna.h b/fluent-bit/plugins/out_logdna/logdna.h deleted file mode 100644 index f7ca19a43..000000000 --- a/fluent-bit/plugins/out_logdna/logdna.h +++ /dev/null @@ -1,51 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_LOGDNA_H -#define FLB_OUT_LOGDNA_H - -#include -#include - -#define FLB_LOGDNA_HOST "logs.logdna.com" -#define FLB_LOGDNA_PORT "443" -#define FLB_LOGDNA_CT "Content-Type" -#define FLB_LOGDNA_CT_JSON "application/json; charset=UTF-8" - -struct flb_logdna { - /* Incoming Configuration Properties */ - flb_sds_t logdna_host; - int logdna_port; - flb_sds_t api_key; - flb_sds_t hostname; - flb_sds_t mac_addr; - flb_sds_t ip_addr; - flb_sds_t file; - flb_sds_t app; - struct mk_list *tags; - - /* Internal */ - flb_sds_t _hostname; - flb_sds_t tags_formatted; - struct flb_upstream *u; - struct flb_output_instance *ins; -}; - - -#endif diff --git a/fluent-bit/plugins/out_loki/CMakeLists.txt b/fluent-bit/plugins/out_loki/CMakeLists.txt deleted file mode 100644 index d91f0aa73..000000000 --- a/fluent-bit/plugins/out_loki/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -set(src - loki.c - ) - -FLB_PLUGIN(out_loki "${src}" "") diff --git a/fluent-bit/plugins/out_loki/loki.c b/fluent-bit/plugins/out_loki/loki.c deleted file mode 100644 index d93a3f9aa..000000000 --- a/fluent-bit/plugins/out_loki/loki.c +++ /dev/null @@ -1,1868 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "loki.h" - -struct flb_loki_dynamic_tenant_id_entry { - flb_sds_t value; - struct cfl_list _head; -}; - -pthread_once_t initialization_guard = PTHREAD_ONCE_INIT; - -FLB_TLS_DEFINE(struct flb_loki_dynamic_tenant_id_entry, - thread_local_tenant_id); - -void initialize_thread_local_storage() -{ - FLB_TLS_INIT(thread_local_tenant_id); -} - -static struct flb_loki_dynamic_tenant_id_entry *dynamic_tenant_id_create() { - struct flb_loki_dynamic_tenant_id_entry *entry; - - entry = (struct flb_loki_dynamic_tenant_id_entry *) \ - flb_calloc(1, sizeof(struct flb_loki_dynamic_tenant_id_entry)); - - if (entry != NULL) { - entry->value = NULL; - - cfl_list_entry_init(&entry->_head); - } - - return entry; -} - -static void dynamic_tenant_id_destroy(struct flb_loki_dynamic_tenant_id_entry *entry) { - if (entry != NULL) { - if (entry->value != NULL) { - flb_sds_destroy(entry->value); - - entry->value = NULL; - } - - if (!cfl_list_entry_is_orphan(&entry->_head)) { - cfl_list_del(&entry->_head); - } - - flb_free(entry); - } -} - -static void flb_loki_kv_init(struct mk_list *list) -{ - mk_list_init(list); -} - -static inline void safe_sds_cat(flb_sds_t *buf, const char *str, int len) -{ - flb_sds_t tmp; - - tmp = flb_sds_cat(*buf, str, len); - if (tmp) { - *buf = tmp; - } -} - -static inline void normalize_cat(struct flb_ra_parser *rp, flb_sds_t *name) -{ - int sub; - int len; - char tmp[64]; - struct mk_list *s_head; - struct flb_ra_key *key; - struct flb_ra_subentry *entry; - - /* Iterate record accessor keys */ - key = rp->key; - if (rp->type == FLB_RA_PARSER_STRING) { - safe_sds_cat(name, key->name, flb_sds_len(key->name)); - } - else if (rp->type == FLB_RA_PARSER_KEYMAP) { - safe_sds_cat(name, key->name, flb_sds_len(key->name)); - if (mk_list_size(key->subkeys) > 0) { - safe_sds_cat(name, "_", 1); - } - - sub = 0; - mk_list_foreach(s_head, key->subkeys) { - entry = mk_list_entry(s_head, struct flb_ra_subentry, _head); - - if (sub > 0) { - safe_sds_cat(name, "_", 1); - } - if (entry->type == FLB_RA_PARSER_STRING) { - safe_sds_cat(name, entry->str, flb_sds_len(entry->str)); - } - else if (entry->type == FLB_RA_PARSER_ARRAY_ID) { - len = snprintf(tmp, sizeof(tmp) -1, "%d", - entry->array_id); - safe_sds_cat(name, tmp, len); - } - sub++; - } - } -} - -static flb_sds_t normalize_ra_key_name(struct flb_loki *ctx, - struct flb_record_accessor *ra) -{ - int c = 0; - flb_sds_t name; - struct mk_list *head; - struct flb_ra_parser *rp; - - name = flb_sds_create_size(128); - if (!name) { - return NULL; - } - - mk_list_foreach(head, &ra->list) { - rp = mk_list_entry(head, struct flb_ra_parser, _head); - if (c > 0) { - flb_sds_cat(name, "_", 1); - } - normalize_cat(rp, &name); - c++; - } - - return name; -} - -void flb_loki_kv_destroy(struct flb_loki_kv *kv) -{ - /* destroy key and value */ - flb_sds_destroy(kv->key); - if (kv->val_type == FLB_LOKI_KV_STR) { - flb_sds_destroy(kv->str_val); - } - else if (kv->val_type == FLB_LOKI_KV_RA) { - flb_ra_destroy(kv->ra_val); - } - - if (kv->ra_key) { - flb_ra_destroy(kv->ra_key); - } - - if (kv->key_normalized) { - flb_sds_destroy(kv->key_normalized); - } - - flb_free(kv); -} - -int flb_loki_kv_append(struct flb_loki *ctx, char *key, char *val) -{ - int ra_count = 0; - int k_len; - int ret; - struct flb_loki_kv *kv; - - if (!key) { - return -1; - } - - if (!val && key[0] != '$') { - return -1; - } - - kv = flb_calloc(1, sizeof(struct flb_loki_kv)); - if (!kv) { - flb_errno(); - return -1; - } - - k_len = strlen(key); - if (key[0] == '$' && k_len >= 2 && isdigit(key[1])) { - flb_plg_error(ctx->ins, - "key name for record accessor cannot start with a number: %s", - key); - flb_free(kv); - return -1; - } - - kv->key = flb_sds_create(key); - if (!kv->key) { - flb_free(kv); - return -1; - } - - /* - * If the key starts with a '$', it means its a record accessor pattern and - * the key value pair will be formed using the key name and it proper value. - */ - if (key[0] == '$' && val == NULL) { - kv->ra_key = flb_ra_create(key, FLB_TRUE); - if (!kv->ra_key) { - flb_plg_error(ctx->ins, - "invalid key record accessor pattern for key '%s'", - key); - flb_loki_kv_destroy(kv); - return -1; - } - - /* Normalize 'key name' using record accessor pattern */ - kv->key_normalized = normalize_ra_key_name(ctx, kv->ra_key); - if (!kv->key_normalized) { - flb_plg_error(ctx->ins, - "could not normalize key pattern name '%s'\n", - kv->ra_key->pattern); - flb_loki_kv_destroy(kv); - return -1; - } - /* remove record keys placed as stream labels via 'labels' and 'label_keys' */ - ret = flb_slist_add(&ctx->remove_keys_derived, key); - if (ret < 0) { - flb_loki_kv_destroy(kv); - return -1; - } - ra_count++; - } - else if (val[0] == '$') { - /* create a record accessor context */ - kv->val_type = FLB_LOKI_KV_RA; - kv->ra_val = flb_ra_create(val, FLB_TRUE); - if (!kv->ra_val) { - flb_plg_error(ctx->ins, - "invalid record accessor pattern for key '%s': %s", - key, val); - flb_loki_kv_destroy(kv); - return -1; - } - ret = flb_slist_add(&ctx->remove_keys_derived, val); - if (ret < 0) { - flb_loki_kv_destroy(kv); - return -1; - } - ra_count++; - } - else { - kv->val_type = FLB_LOKI_KV_STR; - kv->str_val = flb_sds_create(val); - if (!kv->str_val) { - flb_loki_kv_destroy(kv); - return -1; - } - } - mk_list_add(&kv->_head, &ctx->labels_list); - - /* return the number of record accessor values */ - return ra_count; -} - -static void flb_loki_kv_exit(struct flb_loki *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_loki_kv *kv; - - mk_list_foreach_safe(head, tmp, &ctx->labels_list) { - kv = mk_list_entry(head, struct flb_loki_kv, _head); - - /* unlink and destroy */ - mk_list_del(&kv->_head); - flb_loki_kv_destroy(kv); - } -} - -/* Pack a label key, it also perform sanitization of the characters */ -static int pack_label_key(msgpack_packer *mp_pck, char *key, int key_len) -{ - int i; - int k_len = key_len; - int is_digit = FLB_FALSE; - char *p; - size_t prev_size; - - /* Normalize key name using the packed value */ - if (isdigit(*key)) { - is_digit = FLB_TRUE; - k_len++; - } - - /* key: pack the length */ - msgpack_pack_str(mp_pck, k_len); - if (is_digit) { - msgpack_pack_str_body(mp_pck, "_", 1); - } - - /* save the current offset */ - prev_size = ((msgpack_sbuffer *) mp_pck->data)->size; - - /* Pack the key name */ - msgpack_pack_str_body(mp_pck, key, key_len); - - /* 'p' will point to where the key was written */ - p = (char *) (((msgpack_sbuffer*) mp_pck->data)->data + prev_size); - - /* and sanitize the key characters */ - for (i = 0; i < key_len; i++) { - if (!isalnum(p[i]) && p[i] != '_') { - p[i] = '_'; - } - } - - return 0; -} - -static flb_sds_t pack_labels(struct flb_loki *ctx, - msgpack_packer *mp_pck, - char *tag, int tag_len, - msgpack_object *map) -{ - int i; - flb_sds_t ra_val; - struct mk_list *head; - struct flb_ra_value *rval = NULL; - struct flb_loki_kv *kv; - msgpack_object k; - msgpack_object v; - struct flb_mp_map_header mh; - - - /* Initialize dynamic map header */ - flb_mp_map_header_init(&mh, mp_pck); - - mk_list_foreach(head, &ctx->labels_list) { - kv = mk_list_entry(head, struct flb_loki_kv, _head); - - /* record accessor key/value pair */ - if (kv->ra_key != NULL && kv->ra_val == NULL) { - ra_val = flb_ra_translate(kv->ra_key, tag, tag_len, *(map), NULL); - if (!ra_val || flb_sds_len(ra_val) == 0) { - /* if no value is retruned or if it's empty, just skip it */ - flb_plg_warn(ctx->ins, - "empty record accessor key translation for pattern: %s", - kv->ra_key->pattern); - } - else { - /* Pack the key and value */ - flb_mp_map_header_append(&mh); - - /* We skip the first '$' character since it won't be valid in Loki */ - pack_label_key(mp_pck, kv->key_normalized, - flb_sds_len(kv->key_normalized)); - - msgpack_pack_str(mp_pck, flb_sds_len(ra_val)); - msgpack_pack_str_body(mp_pck, ra_val, flb_sds_len(ra_val)); - } - - if (ra_val) { - flb_sds_destroy(ra_val); - } - continue; - } - - /* - * The code is a bit duplicated to be able to manage the exception of an - * invalid or empty value, on that case the k/v is skipped. - */ - if (kv->val_type == FLB_LOKI_KV_STR) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, flb_sds_len(kv->key)); - msgpack_pack_str_body(mp_pck, kv->key, flb_sds_len(kv->key)); - msgpack_pack_str(mp_pck, flb_sds_len(kv->str_val)); - msgpack_pack_str_body(mp_pck, kv->str_val, flb_sds_len(kv->str_val)); - } - else if (kv->val_type == FLB_LOKI_KV_RA) { - /* record accessor type */ - ra_val = flb_ra_translate(kv->ra_val, tag, tag_len, *(map), NULL); - if (!ra_val || flb_sds_len(ra_val) == 0) { - flb_plg_debug(ctx->ins, "could not translate record accessor"); - } - else { - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, flb_sds_len(kv->key)); - msgpack_pack_str_body(mp_pck, kv->key, flb_sds_len(kv->key)); - msgpack_pack_str(mp_pck, flb_sds_len(ra_val)); - msgpack_pack_str_body(mp_pck, ra_val, flb_sds_len(ra_val)); - } - - if (ra_val) { - flb_sds_destroy(ra_val); - } - } - } - - if (ctx->auto_kubernetes_labels == FLB_TRUE) { - rval = flb_ra_get_value_object(ctx->ra_k8s, *map); - if (rval && rval->o.type == MSGPACK_OBJECT_MAP) { - for (i = 0; i < rval->o.via.map.size; i++) { - k = rval->o.via.map.ptr[i].key; - v = rval->o.via.map.ptr[i].val; - - if (k.type != MSGPACK_OBJECT_STR || v.type != MSGPACK_OBJECT_STR) { - continue; - } - - /* append the key/value pair */ - flb_mp_map_header_append(&mh); - - /* Pack key */ - pack_label_key(mp_pck, (char *) k.via.str.ptr, k.via.str.size); - - /* Pack the value */ - msgpack_pack_str(mp_pck, v.via.str.size); - msgpack_pack_str_body(mp_pck, v.via.str.ptr, v.via.str.size); - } - } - - if (rval) { - flb_ra_key_value_destroy(rval); - } - } - - /* Check if we added any label, if no one has been set, set the defaul 'job' */ - if (mh.entries == 0) { - /* pack the default entry */ - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, 3); - msgpack_pack_str_body(mp_pck, "job", 3); - msgpack_pack_str(mp_pck, 10); - msgpack_pack_str_body(mp_pck, "fluent-bit", 10); - } - flb_mp_map_header_end(&mh); - return 0; -} - -static int create_label_map_entry(struct flb_loki *ctx, - struct flb_sds_list *list, msgpack_object *val, int *ra_used) -{ - msgpack_object key; - flb_sds_t label_key; - flb_sds_t val_str; - int i; - int len; - int ret; - - if (ctx == NULL || list == NULL || val == NULL || ra_used == NULL) { - return -1; - } - - switch (val->type) { - case MSGPACK_OBJECT_STR: - label_key = flb_sds_create_len(val->via.str.ptr, val->via.str.size); - if (label_key == NULL) { - flb_errno(); - return -1; - } - - val_str = flb_ra_create_str_from_list(list); - if (val_str == NULL) { - flb_plg_error(ctx->ins, "[%s] flb_ra_create_from_list failed", __FUNCTION__); - flb_sds_destroy(label_key); - return -1; - } - - /* for debugging - printf("label_key=%s val_str=%s\n", label_key, val_str); - */ - - ret = flb_loki_kv_append(ctx, label_key, val_str); - flb_sds_destroy(label_key); - flb_sds_destroy(val_str); - if (ret == -1) { - return -1; - } - *ra_used = *ra_used + 1; - - break; - case MSGPACK_OBJECT_MAP: - len = val->via.map.size; - for (i=0; ivia.map.ptr[i].key; - if (key.type != MSGPACK_OBJECT_STR) { - flb_plg_error(ctx->ins, "[%s] key is not string", __FUNCTION__); - return -1; - } - ret = flb_sds_list_add(list, (char*)key.via.str.ptr, key.via.str.size); - if (ret < 0) { - flb_plg_error(ctx->ins, "[%s] flb_sds_list_add failed", __FUNCTION__); - return -1; - } - - ret = create_label_map_entry(ctx, list, &val->via.map.ptr[i].val, ra_used); - if (ret < 0) { - return -1; - } - - ret = flb_sds_list_del_last_entry(list); - if (ret < 0) { - flb_plg_error(ctx->ins, "[%s] flb_sds_list_del_last_entry failed", __FUNCTION__); - return -1; - } - } - - break; - default: - flb_plg_error(ctx->ins, "[%s] value type is not str or map. type=%d", __FUNCTION__, val->type); - return -1; - } - return 0; -} - -static int create_label_map_entries(struct flb_loki *ctx, - char *msgpack_buf, size_t msgpack_size, int *ra_used) -{ - struct flb_sds_list *list = NULL; - msgpack_unpacked result; - size_t off = 0; - int i; - int len; - int ret; - msgpack_object key; - - if (ctx == NULL || msgpack_buf == NULL || ra_used == NULL) { - return -1; - } - - msgpack_unpacked_init(&result); - while(msgpack_unpack_next(&result, msgpack_buf, msgpack_size, &off) == MSGPACK_UNPACK_SUCCESS) { - if (result.data.type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "[%s] data type is not map", __FUNCTION__); - msgpack_unpacked_destroy(&result); - return -1; - } - - len = result.data.via.map.size; - for (i=0; iins, "[%s] flb_sds_list_create failed", __FUNCTION__); - msgpack_unpacked_destroy(&result); - return -1; - } - key = result.data.via.map.ptr[i].key; - if (key.type != MSGPACK_OBJECT_STR) { - flb_plg_error(ctx->ins, "[%s] key is not string", __FUNCTION__); - flb_sds_list_destroy(list); - msgpack_unpacked_destroy(&result); - return -1; - } - - ret = flb_sds_list_add(list, (char*)key.via.str.ptr, key.via.str.size); - if (ret < 0) { - flb_plg_error(ctx->ins, "[%s] flb_sds_list_add failed", __FUNCTION__); - flb_sds_list_destroy(list); - msgpack_unpacked_destroy(&result); - return -1; - } - - ret = create_label_map_entry(ctx, list, &result.data.via.map.ptr[i].val, ra_used); - if (ret < 0) { - flb_plg_error(ctx->ins, "[%s] create_label_map_entry failed", __FUNCTION__); - flb_sds_list_destroy(list); - msgpack_unpacked_destroy(&result); - return -1; - } - - flb_sds_list_destroy(list); - list = NULL; - } - } - - msgpack_unpacked_destroy(&result); - - return 0; -} - -static int read_label_map_path_file(struct flb_output_instance *ins, flb_sds_t path, - char **out_buf, size_t *out_size) -{ - int ret; - int root_type; - char *buf = NULL; - char *msgp_buf = NULL; - FILE *fp = NULL; - struct stat st; - size_t file_size; - size_t ret_size; - - ret = access(path, R_OK); - if (ret < 0) { - flb_errno(); - flb_plg_error(ins, "can't access %s", path); - return -1; - } - - ret = stat(path, &st); - if (ret < 0) { - flb_errno(); - flb_plg_error(ins, "stat failed %s", path); - return -1; - } - file_size = st.st_size; - - fp = fopen(path, "r"); - if (fp == NULL) { - flb_plg_error(ins, "can't open %s", path); - return -1; - } - - buf = flb_malloc(file_size); - if (buf == NULL) { - flb_plg_error(ins, "malloc failed"); - fclose(fp); - return -1; - } - - ret_size = fread(buf, 1, file_size, fp); - if (ret_size < file_size && feof(fp) != 0) { - flb_plg_error(ins, "fread failed"); - fclose(fp); - flb_free(buf); - return -1; - } - - ret = flb_pack_json(buf, file_size, &msgp_buf, &ret_size, &root_type, NULL); - if (ret < 0) { - flb_plg_error(ins, "flb_pack_json failed"); - fclose(fp); - flb_free(buf); - return -1; - } - - *out_buf = msgp_buf; - *out_size = ret_size; - - fclose(fp); - flb_free(buf); - return 0; -} - -static int load_label_map_path(struct flb_loki *ctx, flb_sds_t path, int *ra_used) -{ - int ret; - char *msgpack_buf = NULL; - size_t msgpack_size; - - ret = read_label_map_path_file(ctx->ins, path, &msgpack_buf, &msgpack_size); - if (ret < 0) { - return -1; - } - - ret = create_label_map_entries(ctx, msgpack_buf, msgpack_size, ra_used); - if (ret < 0) { - flb_free(msgpack_buf); - return -1; - } - - if (msgpack_buf != NULL) { - flb_free(msgpack_buf); - } - - return 0; -} - -static int parse_labels(struct flb_loki *ctx) -{ - int ret; - int ra_used = 0; - char *p; - flb_sds_t key; - flb_sds_t val; - struct mk_list *head; - struct flb_slist_entry *entry; - - flb_loki_kv_init(&ctx->labels_list); - - if (ctx->labels) { - mk_list_foreach(head, ctx->labels) { - entry = mk_list_entry(head, struct flb_slist_entry, _head); - - /* record accessor label key ? */ - if (entry->str[0] == '$') { - ret = flb_loki_kv_append(ctx, entry->str, NULL); - if (ret == -1) { - return -1; - } - else if (ret > 0) { - ra_used++; - } - continue; - } - - p = strchr(entry->str, '='); - if (!p) { - flb_plg_error(ctx->ins, "invalid key value pair on '%s'", - entry->str); - return -1; - } - - key = flb_sds_create_size((p - entry->str) + 1); - flb_sds_cat(key, entry->str, p - entry->str); - val = flb_sds_create(p + 1); - if (!key) { - flb_plg_error(ctx->ins, - "invalid key value pair on '%s'", - entry->str); - return -1; - } - if (!val || flb_sds_len(val) == 0) { - flb_plg_error(ctx->ins, - "invalid key value pair on '%s'", - entry->str); - flb_sds_destroy(key); - return -1; - } - - ret = flb_loki_kv_append(ctx, key, val); - flb_sds_destroy(key); - flb_sds_destroy(val); - - if (ret == -1) { - return -1; - } - else if (ret > 0) { - ra_used++; - } - } - } - - /* Append label keys set in the configuration */ - if (ctx->label_keys) { - mk_list_foreach(head, ctx->label_keys) { - entry = mk_list_entry(head, struct flb_slist_entry, _head); - if (entry->str[0] != '$') { - flb_plg_error(ctx->ins, - "invalid label key, the name must start with '$'"); - return -1; - } - - ret = flb_loki_kv_append(ctx, entry->str, NULL); - if (ret == -1) { - return -1; - } - else if (ret > 0) { - ra_used++; - } - } - } - - /* label_map_path */ - if (ctx->label_map_path) { - ret = load_label_map_path(ctx, ctx->label_map_path, &ra_used); - if (ret != 0) { - flb_plg_error(ctx->ins, "failed to load label_map_path"); - } - } - - if (ctx->auto_kubernetes_labels == FLB_TRUE) { - ctx->ra_k8s = flb_ra_create("$kubernetes['labels']", FLB_TRUE); - if (!ctx->ra_k8s) { - flb_plg_error(ctx->ins, - "could not create record accessor for Kubernetes labels"); - return -1; - } - } - - /* - * If the variable 'ra_used' is greater than zero, means that record accessor is - * being used to compose the stream labels. - */ - ctx->ra_used = ra_used; - return 0; -} - -static int key_is_duplicated(struct mk_list *list, char *str, int len) -{ - struct mk_list *head; - struct flb_slist_entry *entry; - - mk_list_foreach(head, list) { - entry = mk_list_entry(head, struct flb_slist_entry, _head); - if (flb_sds_len(entry->str) == len && - strncmp(entry->str, str, len) == 0) { - return FLB_TRUE; - } - } - - return FLB_FALSE; -} - -static int prepare_remove_keys(struct flb_loki *ctx) -{ - int ret; - int len; - int size; - char *tmp; - struct mk_list *head; - struct flb_slist_entry *entry; - struct mk_list *patterns; - - patterns = &ctx->remove_keys_derived; - - /* Add remove keys set in the configuration */ - if (ctx->remove_keys) { - mk_list_foreach(head, ctx->remove_keys) { - entry = mk_list_entry(head, struct flb_slist_entry, _head); - - if (entry->str[0] != '$') { - tmp = flb_malloc(flb_sds_len(entry->str) + 2); - if (!tmp) { - flb_errno(); - continue; - } - else { - tmp[0] = '$'; - len = flb_sds_len(entry->str); - memcpy(tmp + 1, entry->str, len); - tmp[len + 1] = '\0'; - len++; - } - } - else { - tmp = entry->str; - len = flb_sds_len(entry->str); - } - - ret = key_is_duplicated(patterns, tmp, len); - if (ret == FLB_TRUE) { - if (entry->str != tmp) { - flb_free(tmp); - } - continue; - } - - ret = flb_slist_add_n(patterns, tmp, len); - if (entry->str != tmp) { - flb_free(tmp); - } - if (ret < 0) { - return -1; - } - } - size = mk_list_size(patterns); - flb_plg_debug(ctx->ins, "remove_mpa size: %d", size); - if (size > 0) { - ctx->remove_mpa = flb_mp_accessor_create(patterns); - if (ctx->remove_mpa == NULL) { - return -1; - } - } - } - - return 0; -} - -static void loki_config_destroy(struct flb_loki *ctx) -{ - if (ctx->u) { - flb_upstream_destroy(ctx->u); - } - - if (ctx->ra_k8s) { - flb_ra_destroy(ctx->ra_k8s); - } - if (ctx->ra_tenant_id_key) { - flb_ra_destroy(ctx->ra_tenant_id_key); - } - - if (ctx->remove_mpa) { - flb_mp_accessor_destroy(ctx->remove_mpa); - } - flb_slist_destroy(&ctx->remove_keys_derived); - - flb_loki_kv_exit(ctx); - flb_free(ctx); -} - -static struct flb_loki *loki_config_create(struct flb_output_instance *ins, - struct flb_config *config) -{ - int ret; - int io_flags = 0; - struct flb_loki *ctx; - struct flb_upstream *upstream; - char *compress; - - /* Create context */ - ctx = flb_calloc(1, sizeof(struct flb_loki)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - flb_loki_kv_init(&ctx->labels_list); - - /* Register context with plugin instance */ - flb_output_set_context(ins, ctx); - - /* Set networking defaults */ - flb_output_net_default(FLB_LOKI_HOST, FLB_LOKI_PORT, ins); - - /* Load config map */ - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - return NULL; - } - - /* Initialize final remove_keys list */ - flb_slist_create(&ctx->remove_keys_derived); - - /* Parse labels */ - ret = parse_labels(ctx); - if (ret == -1) { - return NULL; - } - - /* Load remove keys */ - ret = prepare_remove_keys(ctx); - if (ret == -1) { - return NULL; - } - - /* tenant_id_key */ - if (ctx->tenant_id_key_config) { - ctx->ra_tenant_id_key = flb_ra_create(ctx->tenant_id_key_config, FLB_FALSE); - if (!ctx->ra_tenant_id_key) { - flb_plg_error(ctx->ins, - "could not create record accessor for Tenant ID"); - } - } - - /* Compress (gzip) */ - compress = (char *) flb_output_get_property("compress", ins); - ctx->compress_gzip = FLB_FALSE; - if (compress) { - if (strcasecmp(compress, "gzip") == 0) { - ctx->compress_gzip = FLB_TRUE; - } - } - - /* Line Format */ - if (strcasecmp(ctx->line_format, "json") == 0) { - ctx->out_line_format = FLB_LOKI_FMT_JSON; - } - else if (strcasecmp(ctx->line_format, "key_value") == 0) { - ctx->out_line_format = FLB_LOKI_FMT_KV; - } - else { - flb_plg_error(ctx->ins, "invalid 'line_format' value: %s", - ctx->line_format); - return NULL; - } - - /* use TLS ? */ - if (ins->use_tls == FLB_TRUE) { - io_flags = FLB_IO_TLS; - } - else { - io_flags = FLB_IO_TCP; - } - - if (ins->host.ipv6 == FLB_TRUE) { - io_flags |= FLB_IO_IPV6; - } - - /* Create Upstream connection context */ - upstream = flb_upstream_create(config, - ins->host.name, - ins->host.port, - io_flags, - ins->tls); - if (!upstream) { - return NULL; - } - ctx->u = upstream; - flb_output_upstream_set(ctx->u, ins); - ctx->tcp_port = ins->host.port; - ctx->tcp_host = ins->host.name; - - return ctx; -} - -/* - * Convert struct flb_tm timestamp value to nanoseconds and then it pack it as - * a string. - */ -static void pack_timestamp(msgpack_packer *mp_pck, struct flb_time *tms) -{ - int len; - char buf[64]; - uint64_t nanosecs; - - /* convert to nanoseconds */ - nanosecs = flb_time_to_nanosec(tms); - - /* format as a string */ - len = snprintf(buf, sizeof(buf) - 1, "%" PRIu64, nanosecs); - - /* pack the value */ - msgpack_pack_str(mp_pck, len); - msgpack_pack_str_body(mp_pck, buf, len); -} - - -static void pack_format_line_value(flb_sds_t *buf, msgpack_object *val) -{ - int i; - int len; - char temp[512]; - msgpack_object k; - msgpack_object v; - - if (val->type == MSGPACK_OBJECT_STR) { - safe_sds_cat(buf, "\"", 1); - safe_sds_cat(buf, val->via.str.ptr, val->via.str.size); - safe_sds_cat(buf, "\"", 1); - } - else if (val->type == MSGPACK_OBJECT_NIL) { - safe_sds_cat(buf, "null", 4); - } - else if (val->type == MSGPACK_OBJECT_BOOLEAN) { - if (val->via.boolean) { - safe_sds_cat(buf, "true", 4); - } - else { - safe_sds_cat(buf, "false", 5); - } - } - else if (val->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - len = snprintf(temp, sizeof(temp)-1, "%"PRIu64, val->via.u64); - safe_sds_cat(buf, temp, len); - } - else if (val->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { - len = snprintf(temp, sizeof(temp)-1, "%"PRId64, val->via.i64); - safe_sds_cat(buf, temp, len); - } - else if (val->type == MSGPACK_OBJECT_FLOAT32 || - val->type == MSGPACK_OBJECT_FLOAT64) { - if (val->via.f64 == (double)(long long int) val->via.f64) { - len = snprintf(temp, sizeof(temp)-1, "%.1f", val->via.f64); - } - else { - len = snprintf(temp, sizeof(temp)-1, "%.16g", val->via.f64); - } - safe_sds_cat(buf, temp, len); - } - else if (val->type == MSGPACK_OBJECT_ARRAY) { - safe_sds_cat(buf, "\"[", 2); - for (i = 0; i < val->via.array.size; i++) { - v = val->via.array.ptr[i]; - if (i > 0) { - safe_sds_cat(buf, " ", 1); - } - pack_format_line_value(buf, &v); - } - safe_sds_cat(buf, "]\"", 2); - } - else if (val->type == MSGPACK_OBJECT_MAP) { - safe_sds_cat(buf, "\"map[", 5); - - for (i = 0; i < val->via.map.size; i++) { - k = val->via.map.ptr[i].key; - v = val->via.map.ptr[i].val; - - if (k.type != MSGPACK_OBJECT_STR) { - continue; - } - - if (i > 0) { - safe_sds_cat(buf, " ", 1); - } - - safe_sds_cat(buf, k.via.str.ptr, k.via.str.size); - safe_sds_cat(buf, ":", 1); - pack_format_line_value(buf, &v); - } - safe_sds_cat(buf, "]\"", 2); - } - else { - - return; - } -} - -// seek tenant id from map and set it to dynamic_tenant_id -static int get_tenant_id_from_record(struct flb_loki *ctx, msgpack_object *map, - flb_sds_t *dynamic_tenant_id) -{ - struct flb_ra_value *rval = NULL; - flb_sds_t tmp_str; - int cmp_len; - - rval = flb_ra_get_value_object(ctx->ra_tenant_id_key, *map); - - if (rval == NULL) { - flb_plg_warn(ctx->ins, "the value of %s is missing", - ctx->tenant_id_key_config); - return -1; - } - else if (rval->o.type != MSGPACK_OBJECT_STR) { - flb_plg_warn(ctx->ins, "the value of %s is not string", - ctx->tenant_id_key_config); - flb_ra_key_value_destroy(rval); - return -1; - } - - tmp_str = flb_sds_create_len(rval->o.via.str.ptr, - rval->o.via.str.size); - if (tmp_str == NULL) { - flb_plg_warn(ctx->ins, "cannot create tenant ID string from record"); - flb_ra_key_value_destroy(rval); - return -1; - } - - // check if already dynamic_tenant_id is set. - if (*dynamic_tenant_id != NULL) { - cmp_len = flb_sds_len(*dynamic_tenant_id); - - if ((rval->o.via.str.size == cmp_len) && - flb_sds_cmp(tmp_str, *dynamic_tenant_id, cmp_len) == 0) { - // tenant_id is same. nothing to do. - flb_ra_key_value_destroy(rval); - flb_sds_destroy(tmp_str); - - return 0; - } - - flb_plg_warn(ctx->ins, "Tenant ID is overwritten %s -> %s", - *dynamic_tenant_id, tmp_str); - - flb_sds_destroy(*dynamic_tenant_id); - } - - // this sds will be released after setting http header. - *dynamic_tenant_id = tmp_str; - flb_plg_debug(ctx->ins, "Tenant ID is %s", *dynamic_tenant_id); - - flb_ra_key_value_destroy(rval); - return 0; -} - -static int pack_record(struct flb_loki *ctx, - msgpack_packer *mp_pck, msgpack_object *rec, - flb_sds_t *dynamic_tenant_id) -{ - int i; - int skip = 0; - int len; - int ret; - int size_hint = 1024; - char *line; - flb_sds_t buf; - msgpack_object key; - msgpack_object val; - char *tmp_sbuf_data = NULL; - size_t tmp_sbuf_size; - msgpack_unpacked mp_buffer; - size_t off = 0; - - /* - * Get tenant id from record before removing keys. - * https://github.com/fluent/fluent-bit/issues/6207 - */ - if (ctx->ra_tenant_id_key && rec->type == MSGPACK_OBJECT_MAP) { - get_tenant_id_from_record(ctx, rec, dynamic_tenant_id); - } - - /* Remove keys in remove_keys */ - msgpack_unpacked_init(&mp_buffer); - if (ctx->remove_mpa) { - ret = flb_mp_accessor_keys_remove(ctx->remove_mpa, rec, - (void *) &tmp_sbuf_data, &tmp_sbuf_size); - if (ret == FLB_TRUE) { - ret = msgpack_unpack_next(&mp_buffer, tmp_sbuf_data, tmp_sbuf_size, &off); - if (ret != MSGPACK_UNPACK_SUCCESS) { - flb_free(tmp_sbuf_data); - msgpack_unpacked_destroy(&mp_buffer); - return -1; - } - rec = &mp_buffer.data; - } - } - - /* Drop single key */ - if (ctx->drop_single_key == FLB_TRUE && rec->type == MSGPACK_OBJECT_MAP && rec->via.map.size == 1) { - if (ctx->out_line_format == FLB_LOKI_FMT_JSON) { - rec = &rec->via.map.ptr[0].val; - } else if (ctx->out_line_format == FLB_LOKI_FMT_KV) { - val = rec->via.map.ptr[0].val; - - if (val.type == MSGPACK_OBJECT_STR) { - msgpack_pack_str(mp_pck, val.via.str.size); - msgpack_pack_str_body(mp_pck, val.via.str.ptr, val.via.str.size); - } else { - buf = flb_sds_create_size(size_hint); - if (!buf) { - msgpack_unpacked_destroy(&mp_buffer); - if (tmp_sbuf_data) { - flb_free(tmp_sbuf_data); - } - return -1; - } - pack_format_line_value(&buf, &val); - msgpack_pack_str(mp_pck, flb_sds_len(buf)); - msgpack_pack_str_body(mp_pck, buf, flb_sds_len(buf)); - flb_sds_destroy(buf); - } - - msgpack_unpacked_destroy(&mp_buffer); - if (tmp_sbuf_data) { - flb_free(tmp_sbuf_data); - } - - return 0; - } - } - - if (ctx->out_line_format == FLB_LOKI_FMT_JSON) { - line = flb_msgpack_to_json_str(size_hint, rec); - if (!line) { - if (tmp_sbuf_data) { - flb_free(tmp_sbuf_data); - } - msgpack_unpacked_destroy(&mp_buffer); - return -1; - } - len = strlen(line); - msgpack_pack_str(mp_pck, len); - msgpack_pack_str_body(mp_pck, line, len); - flb_free(line); - } - else if (ctx->out_line_format == FLB_LOKI_FMT_KV) { - if (rec->type != MSGPACK_OBJECT_MAP) { - msgpack_unpacked_destroy(&mp_buffer); - if (tmp_sbuf_data) { - flb_free(tmp_sbuf_data); - } - return -1; - } - - buf = flb_sds_create_size(size_hint); - if (!buf) { - msgpack_unpacked_destroy(&mp_buffer); - if (tmp_sbuf_data) { - flb_free(tmp_sbuf_data); - } - return -1; - } - - for (i = 0; i < rec->via.map.size; i++) { - key = rec->via.map.ptr[i].key; - val = rec->via.map.ptr[i].val; - - if (key.type != MSGPACK_OBJECT_STR) { - skip++; - continue; - } - - if (i > skip) { - safe_sds_cat(&buf, " ", 1); - } - - safe_sds_cat(&buf, key.via.str.ptr, key.via.str.size); - safe_sds_cat(&buf, "=", 1); - pack_format_line_value(&buf, &val); - } - - msgpack_pack_str(mp_pck, flb_sds_len(buf)); - msgpack_pack_str_body(mp_pck, buf, flb_sds_len(buf)); - flb_sds_destroy(buf); - } - - msgpack_unpacked_destroy(&mp_buffer); - if (tmp_sbuf_data) { - flb_free(tmp_sbuf_data); - } - - return 0; -} - -/* Initialization callback */ -static int cb_loki_init(struct flb_output_instance *ins, - struct flb_config *config, void *data) -{ - int result; - struct flb_loki *ctx; - - /* Create plugin context */ - ctx = loki_config_create(ins, config); - if (!ctx) { - flb_plg_error(ins, "cannot initialize configuration"); - return -1; - } - - result = pthread_mutex_init(&ctx->dynamic_tenant_list_lock, NULL); - - if (result != 0) { - flb_errno(); - - flb_plg_error(ins, "cannot initialize dynamic tenant id list lock"); - - loki_config_destroy(ctx); - - return -1; - } - - result = pthread_once(&initialization_guard, - initialize_thread_local_storage); - - if (result != 0) { - flb_errno(); - - flb_plg_error(ins, "cannot initialize thread local storage"); - - loki_config_destroy(ctx); - - return -1; - } - - cfl_list_init(&ctx->dynamic_tenant_list); - - /* - * This plugin instance uses the HTTP client interface, let's register - * it debugging callbacks. - */ - flb_output_set_http_debug_callbacks(ins); - - flb_plg_info(ins, - "configured, hostname=%s:%i", - ctx->tcp_host, ctx->tcp_port); - return 0; -} - -static flb_sds_t loki_compose_payload(struct flb_loki *ctx, - int total_records, - char *tag, int tag_len, - const void *data, size_t bytes, - flb_sds_t *dynamic_tenant_id) -{ - // int mp_ok = MSGPACK_UNPACK_SUCCESS; - // size_t off = 0; - flb_sds_t json; - // struct flb_time tms; - // msgpack_unpacked result; - msgpack_packer mp_pck; - msgpack_sbuffer mp_sbuf; - // msgpack_object *obj; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - int ret; - - /* - * Fluent Bit uses Loki API v1 to push records in JSON format, this - * is the expected structure: - * - * { - * "streams": [ - * { - * "stream": { - * "label": "value" - * }, - * "values": [ - * [ "", "" ], - * [ "", "" ] - * ] - * } - * ] - * } - */ - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return NULL; - } - - /* Initialize msgpack buffers */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - /* Main map */ - msgpack_pack_map(&mp_pck, 1); - - /* streams */ - msgpack_pack_str(&mp_pck, 7); - msgpack_pack_str_body(&mp_pck, "streams", 7); - - if (ctx->ra_used == 0 && ctx->auto_kubernetes_labels == FLB_FALSE) { - /* - * If labels are cached, there is no record accessor or custom - * keys, so it's safe to put one main stream and attach all the - * values. - */ - msgpack_pack_array(&mp_pck, 1); - - /* map content: streams['stream'] & streams['values'] */ - msgpack_pack_map(&mp_pck, 2); - - /* streams['stream'] */ - msgpack_pack_str(&mp_pck, 6); - msgpack_pack_str_body(&mp_pck, "stream", 6); - - /* Pack stream labels */ - pack_labels(ctx, &mp_pck, tag, tag_len, NULL); - - /* streams['values'] */ - msgpack_pack_str(&mp_pck, 6); - msgpack_pack_str_body(&mp_pck, "values", 6); - msgpack_pack_array(&mp_pck, total_records); - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - msgpack_pack_array(&mp_pck, 2); - - /* Append the timestamp */ - pack_timestamp(&mp_pck, &log_event.timestamp); - pack_record(ctx, &mp_pck, log_event.body, dynamic_tenant_id); - } - } - else { - /* - * Here there are no cached labels and the labels are composed by - * each record content. To simplify the operation just create - * one stream per record. - */ - msgpack_pack_array(&mp_pck, total_records); - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - /* map content: streams['stream'] & streams['values'] */ - msgpack_pack_map(&mp_pck, 2); - - /* streams['stream'] */ - msgpack_pack_str(&mp_pck, 6); - msgpack_pack_str_body(&mp_pck, "stream", 6); - - /* Pack stream labels */ - pack_labels(ctx, &mp_pck, tag, tag_len, log_event.body); - - /* streams['values'] */ - msgpack_pack_str(&mp_pck, 6); - msgpack_pack_str_body(&mp_pck, "values", 6); - msgpack_pack_array(&mp_pck, 1); - - msgpack_pack_array(&mp_pck, 2); - - /* Append the timestamp */ - pack_timestamp(&mp_pck, &log_event.timestamp); - pack_record(ctx, &mp_pck, log_event.body, dynamic_tenant_id); - } - } - - flb_log_event_decoder_destroy(&log_decoder); - - json = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); - - msgpack_sbuffer_destroy(&mp_sbuf); - - return json; -} - -static void payload_release(void *payload, int compressed) -{ - if (compressed) { - flb_free(payload); - } - else { - flb_sds_destroy(payload); - } -} - -static void cb_loki_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - int ret; - int out_ret = FLB_OK; - size_t b_sent; - flb_sds_t payload = NULL; - flb_sds_t out_buf = NULL; - size_t out_size; - int compressed = FLB_FALSE; - struct flb_loki *ctx = out_context; - struct flb_connection *u_conn; - struct flb_http_client *c; - struct flb_loki_dynamic_tenant_id_entry *dynamic_tenant_id; - - dynamic_tenant_id = FLB_TLS_GET(thread_local_tenant_id); - - if (dynamic_tenant_id == NULL) { - dynamic_tenant_id = dynamic_tenant_id_create(); - - if (dynamic_tenant_id == NULL) { - flb_errno(); - flb_plg_error(ctx->ins, "cannot allocate dynamic tenant id"); - - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - FLB_TLS_SET(thread_local_tenant_id, dynamic_tenant_id); - - pthread_mutex_lock(&ctx->dynamic_tenant_list_lock); - - cfl_list_add(&dynamic_tenant_id->_head, &ctx->dynamic_tenant_list); - - pthread_mutex_unlock(&ctx->dynamic_tenant_list_lock); - } - - /* Format the data to the expected Newrelic Payload */ - payload = loki_compose_payload(ctx, - event_chunk->total_events, - (char *) event_chunk->tag, - flb_sds_len(event_chunk->tag), - event_chunk->data, event_chunk->size, - &dynamic_tenant_id->value); - - if (!payload) { - flb_plg_error(ctx->ins, "cannot compose request payload"); - - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Map buffer */ - out_buf = payload; - out_size = flb_sds_len(payload); - - if (ctx->compress_gzip == FLB_TRUE) { - ret = flb_gzip_compress((void *) payload, flb_sds_len(payload), (void **) &out_buf, &out_size); - if (ret == -1) { - flb_plg_error(ctx->ins, - "cannot gzip payload, disabling compression"); - } else { - compressed = FLB_TRUE; - /* payload is not longer needed */ - flb_sds_destroy(payload); - } - } - - /* Lookup an available connection context */ - u_conn = flb_upstream_conn_get(ctx->u); - if (!u_conn) { - flb_plg_error(ctx->ins, "no upstream connections available"); - - payload_release(out_buf, compressed); - - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Create HTTP client context */ - c = flb_http_client(u_conn, FLB_HTTP_POST, FLB_LOKI_URI, - out_buf, out_size, - ctx->tcp_host, ctx->tcp_port, - NULL, 0); - if (!c) { - flb_plg_error(ctx->ins, "cannot create HTTP client context"); - - payload_release(out_buf, compressed); - flb_upstream_conn_release(u_conn); - - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Set callback context to the HTTP client context */ - flb_http_set_callback_context(c, ctx->ins->callback); - - /* User Agent */ - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - - /* Auth headers */ - if (ctx->http_user && ctx->http_passwd) { /* Basic */ - flb_http_basic_auth(c, ctx->http_user, ctx->http_passwd); - } else if (ctx->bearer_token) { /* Bearer token */ - flb_http_bearer_auth(c, ctx->bearer_token); - } - - /* Add Content-Type header */ - flb_http_add_header(c, - FLB_LOKI_CT, sizeof(FLB_LOKI_CT) - 1, - FLB_LOKI_CT_JSON, sizeof(FLB_LOKI_CT_JSON) - 1); - - if (compressed == FLB_TRUE) { - flb_http_set_content_encoding_gzip(c); - } - - /* Add X-Scope-OrgID header */ - if (dynamic_tenant_id->value != NULL) { - flb_http_add_header(c, - FLB_LOKI_HEADER_SCOPE, sizeof(FLB_LOKI_HEADER_SCOPE) - 1, - dynamic_tenant_id->value, - flb_sds_len(dynamic_tenant_id->value)); - } - else if (ctx->tenant_id) { - flb_http_add_header(c, - FLB_LOKI_HEADER_SCOPE, sizeof(FLB_LOKI_HEADER_SCOPE) - 1, - ctx->tenant_id, flb_sds_len(ctx->tenant_id)); - } - - /* Send HTTP request */ - ret = flb_http_do(c, &b_sent); - payload_release(out_buf, compressed); - - /* Validate HTTP client return status */ - if (ret == 0) { - /* - * Only allow the following HTTP status: - * - * - 200: OK - * - 201: Created - * - 202: Accepted - * - 203: no authorative resp - * - 204: No Content - * - 205: Reset content - * - */ - if (c->resp.status == 400) { - /* - * Loki will return 400 if incoming data is out of order. - * We should not retry such data. - */ - flb_plg_error(ctx->ins, "%s:%i, HTTP status=%i Not retrying.\n%s", - ctx->tcp_host, ctx->tcp_port, c->resp.status, - c->resp.payload); - out_ret = FLB_ERROR; - } - else if (c->resp.status < 200 || c->resp.status > 205) { - if (c->resp.payload) { - flb_plg_error(ctx->ins, "%s:%i, HTTP status=%i\n%s", - ctx->tcp_host, ctx->tcp_port, c->resp.status, - c->resp.payload); - } - else { - flb_plg_error(ctx->ins, "%s:%i, HTTP status=%i", - ctx->tcp_host, ctx->tcp_port, c->resp.status); - } - out_ret = FLB_RETRY; - } - else { - if (c->resp.payload) { - flb_plg_debug(ctx->ins, "%s:%i, HTTP status=%i\n%s", - ctx->tcp_host, ctx->tcp_port, - c->resp.status, c->resp.payload); - } - else { - flb_plg_debug(ctx->ins, "%s:%i, HTTP status=%i", - ctx->tcp_host, ctx->tcp_port, - c->resp.status); - } - } - } - else { - flb_plg_error(ctx->ins, "could not flush records to %s:%i (http_do=%i)", - ctx->tcp_host, ctx->tcp_port, ret); - out_ret = FLB_RETRY; - } - - flb_http_client_destroy(c); - flb_upstream_conn_release(u_conn); - - FLB_OUTPUT_RETURN(out_ret); -} - -static void release_dynamic_tenant_ids(struct cfl_list *dynamic_tenant_list) -{ - struct cfl_list *iterator; - struct cfl_list *backup; - struct flb_loki_dynamic_tenant_id_entry *entry; - - cfl_list_foreach_safe(iterator, backup, dynamic_tenant_list) { - entry = cfl_list_entry(iterator, - struct flb_loki_dynamic_tenant_id_entry, - _head); - - dynamic_tenant_id_destroy(entry); - } -} - -static int cb_loki_exit(void *data, struct flb_config *config) -{ - struct flb_loki *ctx = data; - - if (!ctx) { - return 0; - } - - pthread_mutex_lock(&ctx->dynamic_tenant_list_lock); - - release_dynamic_tenant_ids(&ctx->dynamic_tenant_list); - - pthread_mutex_unlock(&ctx->dynamic_tenant_list_lock); - - loki_config_destroy(ctx); - - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "tenant_id", NULL, - 0, FLB_TRUE, offsetof(struct flb_loki, tenant_id), - "Tenant ID used by default to push logs to Loki. If omitted or empty " - "it assumes Loki is running in single-tenant mode and no X-Scope-OrgID " - "header is sent." - }, - { - FLB_CONFIG_MAP_STR, "tenant_id_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_loki, tenant_id_key_config), - "If set, X-Scope-OrgID will be the value of the key from incoming record. " - "It is useful to set X-Scode-OrgID dynamically." - }, - - { - FLB_CONFIG_MAP_CLIST, "labels", NULL, - 0, FLB_TRUE, offsetof(struct flb_loki, labels), - "labels for API requests. If no value is set, the default label is 'job=fluent-bit'" - }, - - { - FLB_CONFIG_MAP_BOOL, "auto_kubernetes_labels", "false", - 0, FLB_TRUE, offsetof(struct flb_loki, auto_kubernetes_labels), - "If set to true, it will add all Kubernetes labels to Loki labels.", - }, - - { - FLB_CONFIG_MAP_BOOL, "drop_single_key", "false", - 0, FLB_TRUE, offsetof(struct flb_loki, drop_single_key), - "If set to true and only a single key remains, the log line sent to Loki " - "will be the value of that key.", - }, - - { - FLB_CONFIG_MAP_CLIST, "label_keys", NULL, - 0, FLB_TRUE, offsetof(struct flb_loki, label_keys), - "Comma separated list of keys to use as stream labels." - }, - - { - FLB_CONFIG_MAP_CLIST, "remove_keys", NULL, - 0, FLB_TRUE, offsetof(struct flb_loki, remove_keys), - "Comma separated list of keys to remove." - }, - - { - FLB_CONFIG_MAP_STR, "line_format", "json", - 0, FLB_TRUE, offsetof(struct flb_loki, line_format), - "Format to use when flattening the record to a log line. Valid values are " - "'json' or 'key_value'. If set to 'json' the log line sent to Loki will be " - "the Fluent Bit record dumped as json. If set to 'key_value', the log line " - "will be each item in the record concatenated together (separated by a " - "single space) in the format '='." - }, - - { - FLB_CONFIG_MAP_STR, "label_map_path", NULL, - 0, FLB_TRUE, offsetof(struct flb_loki, label_map_path), - "A label map file path" - }, - - { - FLB_CONFIG_MAP_STR, "http_user", NULL, - 0, FLB_TRUE, offsetof(struct flb_loki, http_user), - "Set HTTP auth user" - }, - - { - FLB_CONFIG_MAP_STR, "http_passwd", "", - 0, FLB_TRUE, offsetof(struct flb_loki, http_passwd), - "Set HTTP auth password" - }, - - { - FLB_CONFIG_MAP_STR, "bearer_token", NULL, - 0, FLB_TRUE, offsetof(struct flb_loki, bearer_token), - "Set bearer token auth" - }, - - { - FLB_CONFIG_MAP_STR, "compress", NULL, - 0, FLB_FALSE, 0, - "Set payload compression in network transfer. Option available is 'gzip'" - }, - - /* EOF */ - {0} -}; - -/* for testing */ -static int cb_loki_format_test(struct flb_config *config, - struct flb_input_instance *ins, - void *plugin_context, - void *flush_ctx, - int event_type, - const char *tag, int tag_len, - const void *data, size_t bytes, - void **out_data, size_t *out_size) -{ - int total_records; - flb_sds_t payload = NULL; - flb_sds_t dynamic_tenant_id; - struct flb_loki *ctx = plugin_context; - - dynamic_tenant_id = NULL; - - /* Count number of records */ - total_records = flb_mp_count(data, bytes); - - payload = loki_compose_payload(ctx, total_records, - (char *) tag, tag_len, data, bytes, - &dynamic_tenant_id); - if (payload == NULL) { - if (dynamic_tenant_id != NULL) { - flb_sds_destroy(dynamic_tenant_id); - } - - return -1; - } - - *out_data = payload; - *out_size = flb_sds_len(payload); - - return 0; -} - -/* Plugin reference */ -struct flb_output_plugin out_loki_plugin = { - .name = "loki", - .description = "Loki", - .cb_init = cb_loki_init, - .cb_flush = cb_loki_flush, - .cb_exit = cb_loki_exit, - .config_map = config_map, - - /* for testing */ - .test_formatter.callback = cb_loki_format_test, - - .flags = FLB_OUTPUT_NET | FLB_IO_OPT_TLS, -}; diff --git a/fluent-bit/plugins/out_loki/loki.h b/fluent-bit/plugins/out_loki/loki.h deleted file mode 100644 index 2011cee3d..000000000 --- a/fluent-bit/plugins/out_loki/loki.h +++ /dev/null @@ -1,98 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_LOKI_H -#define FLB_OUT_LOKI_H - -#include -#include -#include -#include -#include - -#define FLB_LOKI_CT "Content-Type" -#define FLB_LOKI_CT_JSON "application/json" -#define FLB_LOKI_URI "/loki/api/v1/push" -#define FLB_LOKI_HOST "127.0.0.1" -#define FLB_LOKI_PORT 3100 -#define FLB_LOKI_HEADER_SCOPE "X-Scope-OrgID" - -#define FLB_LOKI_KV_STR 0 /* sds string */ -#define FLB_LOKI_KV_RA 1 /* record accessor */ -#define FLB_LOKI_KV_K8S 2 /* kubernetes label */ - -/* Output line format */ -#define FLB_LOKI_FMT_JSON 0 -#define FLB_LOKI_FMT_KV 1 - -struct flb_loki_kv { - int val_type; /* FLB_LOKI_KV_STR or FLB_LOKI_KV_RA */ - flb_sds_t key; /* string key */ - flb_sds_t str_val; /* string value */ - flb_sds_t key_normalized; /* normalized key name when using ra */ - struct flb_record_accessor *ra_key; /* record accessor key context */ - struct flb_record_accessor *ra_val; /* record accessor value context */ - struct mk_list _head; /* link to flb_loki->labels_list */ -}; - -struct flb_loki { - /* Public configuration properties */ - int auto_kubernetes_labels; - int drop_single_key; - flb_sds_t line_format; - flb_sds_t tenant_id; - flb_sds_t tenant_id_key_config; - int compress_gzip; - - /* HTTP Auth */ - flb_sds_t http_user; - flb_sds_t http_passwd; - - /* Bearer Token Auth */ - flb_sds_t bearer_token; - - /* Labels */ - struct mk_list *labels; - struct mk_list *label_keys; - struct mk_list *remove_keys; - - flb_sds_t label_map_path; - - /* Private */ - int tcp_port; - char *tcp_host; - int out_line_format; - int ra_used; /* number of record accessor label keys */ - struct flb_record_accessor *ra_k8s; /* kubernetes record accessor */ - struct mk_list labels_list; /* list of flb_loki_kv nodes */ - struct mk_list remove_keys_derived; /* remove_keys with label RAs */ - struct flb_mp_accessor *remove_mpa; /* remove_keys multi-pattern accessor */ - struct flb_record_accessor *ra_tenant_id_key; /* dynamic tenant id key */ - - struct cfl_list dynamic_tenant_list; - pthread_mutex_t dynamic_tenant_list_lock; - - /* Upstream Context */ - struct flb_upstream *u; - - /* Plugin instance */ - struct flb_output_instance *ins; -}; - -#endif diff --git a/fluent-bit/plugins/out_nats/CMakeLists.txt b/fluent-bit/plugins/out_nats/CMakeLists.txt deleted file mode 100644 index 9dabb2f2c..000000000 --- a/fluent-bit/plugins/out_nats/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - nats.c) - -FLB_PLUGIN(out_nats "${src}" "") diff --git a/fluent-bit/plugins/out_nats/nats.c b/fluent-bit/plugins/out_nats/nats.c deleted file mode 100644 index aac615c13..000000000 --- a/fluent-bit/plugins/out_nats/nats.c +++ /dev/null @@ -1,252 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "nats.h" - -static int cb_nats_init(struct flb_output_instance *ins, struct flb_config *config, - void *data) -{ - int io_flags; - int ret; - struct flb_upstream *upstream; - struct flb_out_nats_config *ctx; - - /* Set default network configuration */ - flb_output_net_default("127.0.0.1", 4222, ins); - - /* Allocate plugin context */ - ctx = flb_malloc(sizeof(struct flb_out_nats_config)); - if (!ctx) { - flb_errno(); - return -1; - } - - /* Set default values */ - ret = flb_output_config_map_set(ins, ctx); - if (ret == -1) { - flb_plg_error(ins, "flb_output_config_map_set failed"); - flb_free(ctx); - return -1; - } - - io_flags = FLB_IO_TCP; - if (ins->host.ipv6 == FLB_TRUE) { - io_flags |= FLB_IO_IPV6; - } - - /* Prepare an upstream handler */ - upstream = flb_upstream_create(config, - ins->host.name, - ins->host.port, - io_flags, - NULL); - if (!upstream) { - flb_free(ctx); - return -1; - } - ctx->u = upstream; - ctx->ins = ins; - flb_output_upstream_set(ctx->u, ins); - flb_output_set_context(ins, ctx); - - return 0; -} - -static int msgpack_to_json(struct flb_out_nats_config *ctx, - const void *data, size_t bytes, - const char *tag, int tag_len, - char **out_json, size_t *out_size) -{ - int i; - int map_size; - size_t array_size = 0; - flb_sds_t out_buf; - msgpack_object map; - msgpack_object m_key; - msgpack_object m_val; - msgpack_packer mp_pck; - msgpack_sbuffer mp_sbuf; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - int ret; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return -1; - } - - array_size = flb_mp_count(data, bytes); - - /* Convert MsgPack to JSON */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - msgpack_pack_array(&mp_pck, array_size); - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - map = *log_event.body; - map_size = map.via.map.size; - - msgpack_pack_array(&mp_pck, 2); - msgpack_pack_double(&mp_pck, flb_time_to_double(&log_event.timestamp)); - - msgpack_pack_map(&mp_pck, map_size + 1); - msgpack_pack_str(&mp_pck, 3); - msgpack_pack_str_body(&mp_pck, "tag", 3); - msgpack_pack_str(&mp_pck, tag_len); - msgpack_pack_str_body(&mp_pck, tag, tag_len); - - for (i = 0; i < map_size; i++) { - m_key = map.via.map.ptr[i].key; - m_val = map.via.map.ptr[i].val; - - msgpack_pack_object(&mp_pck, m_key); - msgpack_pack_object(&mp_pck, m_val); - } - } - - flb_log_event_decoder_destroy(&log_decoder); - - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); - msgpack_sbuffer_destroy(&mp_sbuf); - - if (!out_buf) { - return -1; - } - - *out_json = out_buf; - *out_size = flb_sds_len(out_buf); - - return 0; -} - -static void cb_nats_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - int ret; - size_t bytes_sent; - size_t json_len; - flb_sds_t json_msg; - char *request; - int req_len; - struct flb_out_nats_config *ctx = out_context; - struct flb_connection *u_conn; - - u_conn = flb_upstream_conn_get(ctx->u); - if (!u_conn) { - flb_plg_error(ctx->ins, "no upstream connections available"); - FLB_OUTPUT_RETURN(FLB_ERROR); - } - - /* Before to flush the content check if we need to start the handshake */ - ret = flb_io_net_write(u_conn, - NATS_CONNECT, - sizeof(NATS_CONNECT) - 1, - &bytes_sent); - if (ret == -1) { - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Convert original Fluent Bit MsgPack format to JSON */ - ret = msgpack_to_json(ctx, - event_chunk->data, event_chunk->size, - event_chunk->tag, flb_sds_len(event_chunk->tag), - &json_msg, &json_len); - if (ret == -1) { - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_ERROR); - } - - /* Compose the NATS Publish request */ - request = flb_malloc(json_len + flb_sds_len(event_chunk->tag) + 32); - if (!request) { - flb_errno(); - flb_sds_destroy(json_msg); - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - req_len = snprintf(request, flb_sds_len(event_chunk->tag)+ 32, - "PUB %s %zu\r\n", - event_chunk->tag, json_len); - - /* Append JSON message and ending CRLF */ - memcpy(request + req_len, json_msg, json_len); - req_len += json_len; - request[req_len++] = '\r'; - request[req_len++] = '\n'; - flb_sds_destroy(json_msg); - - ret = flb_io_net_write(u_conn, request, req_len, &bytes_sent); - if (ret == -1) { - flb_errno(); - flb_free(request); - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - flb_free(request); - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_OK); -} - -int cb_nats_exit(void *data, struct flb_config *config) -{ - (void) config; - struct flb_out_nats_config *ctx = data; - - flb_upstream_destroy(ctx->u); - flb_free(ctx); - - return 0; -} - -static struct flb_config_map config_map[] = { - /* EOF */ - {0} -}; - -struct flb_output_plugin out_nats_plugin = { - .name = "nats", - .description = "NATS Server", - .cb_init = cb_nats_init, - .cb_flush = cb_nats_flush, - .cb_exit = cb_nats_exit, - .flags = FLB_OUTPUT_NET, - .config_map = config_map -}; diff --git a/fluent-bit/plugins/out_nats/nats.h b/fluent-bit/plugins/out_nats/nats.h deleted file mode 100644 index ae0586d82..000000000 --- a/fluent-bit/plugins/out_nats/nats.h +++ /dev/null @@ -1,33 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_NATS_H -#define FLB_OUT_NATS_H - -#include -#include - -#define NATS_CONNECT "CONNECT {\"verbose\":false,\"pedantic\":false,\"ssl_required\":false,\"name\":\"fluent-bit\",\"lang\":\"c\",\"version\":\"" FLB_VERSION_STR "\"}\r\n" - -struct flb_out_nats_config { - struct flb_upstream *u; - struct flb_output_instance *ins; -}; - -#endif diff --git a/fluent-bit/plugins/out_nrlogs/CMakeLists.txt b/fluent-bit/plugins/out_nrlogs/CMakeLists.txt deleted file mode 100644 index c4ae7a224..000000000 --- a/fluent-bit/plugins/out_nrlogs/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -set(src - newrelic.c - ) - -FLB_PLUGIN(out_nrlogs "${src}" "") diff --git a/fluent-bit/plugins/out_nrlogs/newrelic.c b/fluent-bit/plugins/out_nrlogs/newrelic.c deleted file mode 100644 index 54163a66a..000000000 --- a/fluent-bit/plugins/out_nrlogs/newrelic.c +++ /dev/null @@ -1,566 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "newrelic.h" - -static inline uint64_t time_to_milliseconds(struct flb_time *tms) -{ - return ((tms->tm.tv_sec * 1000) + (tms->tm.tv_nsec / 1000000)); -} - -static inline int key_matches(msgpack_object k, char *name, int len) -{ - if (k.type != MSGPACK_OBJECT_STR) { - return FLB_FALSE; - } - - if (k.via.str.size != len) { - return FLB_FALSE; - } - - if (memcmp(k.via.str.ptr, name, len) == 0) { - return FLB_TRUE; - } - - return FLB_FALSE; -} - -static int package_record(struct flb_time *ts, msgpack_object *map, - msgpack_packer *mp_pck) -{ - int i; - int map_size = 0; - uint64_t timestamp_ms; - int log = -1; - int message = -1; - msgpack_object k; - msgpack_object v; - - /* Check if 'message' or 'log' key exists in the record */ - for (i = 0; i < map->via.map.size; i++) { - k = map->via.map.ptr[i].key; - - if (message == -1 && key_matches(k, "message", 7) == FLB_TRUE) { - message = i; - continue; - } - - /* If we find 'log', just stop iterating */ - if (log == -1 && key_matches(k, "log", 3) == FLB_TRUE) { - log = i; - break; - } - } - - /* The log map contains at least 2 entries: 'timestamp' and 'attributes' */ - map_size = 2; - - /* If 'log' or 'message' are set, we add the 'message' key */ - if (log >= 0 || message >= 0) { - map_size++; - } - - /* Package the final record */ - msgpack_pack_map(mp_pck, map_size); - - /* Convert timestamp to milliseconds */ - timestamp_ms = time_to_milliseconds(ts); - - /* Pack timestamp */ - msgpack_pack_str(mp_pck, 9); - msgpack_pack_str_body(mp_pck, "timestamp", 9); - msgpack_pack_uint64(mp_pck, timestamp_ms); - - /* Keep 'log' over 'message' */ - if (log >= 0) { - message = -1; - msgpack_pack_str(mp_pck, 7); - msgpack_pack_str_body(mp_pck, "message", 7); - v = map->via.map.ptr[log].val; - msgpack_pack_object(mp_pck, v); - } - else if (message >= 0) { - msgpack_pack_str(mp_pck, 7); - msgpack_pack_str_body(mp_pck, "message", 7); - v = map->via.map.ptr[message].val; - msgpack_pack_object(mp_pck, v); - } - - /* Adjust attributes map size */ - map_size = map->via.map.size; - if (log >= 0 || message >= 0) { - map_size--; - } - - msgpack_pack_str(mp_pck, 10); - msgpack_pack_str_body(mp_pck, "attributes", 10); - msgpack_pack_map(mp_pck, map_size); - - /* Pack remaining attributes */ - for (i = 0; i < map->via.map.size; i++) { - k = map->via.map.ptr[i].key; - v = map->via.map.ptr[i].val; - - if (log >= 0 && key_matches(k, "log", 3) == FLB_TRUE) { - continue; - } - - if (message >= 0 && key_matches(k, "message", 7) == FLB_TRUE) { - continue; - } - - msgpack_pack_object(mp_pck, k); - msgpack_pack_object(mp_pck, v); - } - - return 0; -} - -static flb_sds_t newrelic_compose_payload(struct flb_newrelic *ctx, - const void *data, size_t bytes) -{ - int total_records; - flb_sds_t json; - msgpack_packer mp_pck; - msgpack_sbuffer mp_sbuf; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - int ret; - - /* - * Following the New Relic Fluentd implementation, this is the - * suggested structure for our payload: - * - * payload = {[ - * 'common' => { - * 'attributes' => { - * 'plugin' => { - * 'type' => 'fluentd', - * 'version' => NewrelicFluentdOutput::VERSION, - * } - * } - * }, - * 'logs' => [] - * ]} - */ - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return NULL; - } - - /* Count number of records */ - total_records = flb_mp_count(data, bytes); - - /* Initialize msgpack buffers */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - /* The New Relic MELT API format is wrapped in an array */ - msgpack_pack_array(&mp_pck, 1); - - /* Map for 'common' and 'logs' */ - msgpack_pack_map(&mp_pck, 2); - - /* 'common' map */ - msgpack_pack_str(&mp_pck, 6); - msgpack_pack_str_body(&mp_pck, "common", 6); - msgpack_pack_map(&mp_pck, 1); - - /* common['attributes'] */ - msgpack_pack_str(&mp_pck, 10); - msgpack_pack_str_body(&mp_pck, "attributes", 10); - msgpack_pack_map(&mp_pck, 1); - - /* common['attributes']['plugin'] */ - msgpack_pack_str(&mp_pck, 6); - msgpack_pack_str_body(&mp_pck, "plugin", 6); - msgpack_pack_map(&mp_pck, 2); - - /* common['attributes']['plugin']['type'] = 'Fluent Bit' */ - msgpack_pack_str(&mp_pck, 4); - msgpack_pack_str_body(&mp_pck, "type", 4); - msgpack_pack_str(&mp_pck, 10); - msgpack_pack_str_body(&mp_pck, "Fluent Bit", 10); - - /* common['attributes']['plugin']['version'] = 'FLB_VERSION_STR' */ - msgpack_pack_str(&mp_pck, 7); - msgpack_pack_str_body(&mp_pck, "version", 7); - msgpack_pack_str(&mp_pck, sizeof(FLB_VERSION_STR) - 1); - msgpack_pack_str_body(&mp_pck, FLB_VERSION_STR, sizeof(FLB_VERSION_STR) - 1); - - /* 'logs' array */ - msgpack_pack_str(&mp_pck, 4); - msgpack_pack_str_body(&mp_pck, "logs", 4); - msgpack_pack_array(&mp_pck, total_records); - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - /* Package the record */ - package_record(&log_event.timestamp, log_event.body, &mp_pck); - } - - flb_log_event_decoder_destroy(&log_decoder); - - json = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); - - msgpack_sbuffer_destroy(&mp_sbuf); - - return json; -} - -static void newrelic_config_destroy(struct flb_newrelic *ctx) -{ - flb_free(ctx->nr_protocol); - flb_free(ctx->nr_host); - flb_free(ctx->nr_uri); - - if (ctx->u) { - flb_upstream_destroy(ctx->u); - } - flb_free(ctx); -} - -static struct flb_newrelic *newrelic_config_create(struct flb_output_instance *ins, - struct flb_config *config) -{ - int ret; - char *port = NULL; - struct flb_newrelic *ctx; - struct flb_upstream *upstream; - - /* Create context */ - ctx = flb_calloc(1, sizeof(struct flb_newrelic)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - - /* Load config map */ - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - newrelic_config_destroy(ctx); - return NULL; - } - - /* At least we need one of api_key or license_key */ - if (!ctx->api_key && !ctx->license_key) { - flb_plg_error(ctx->ins, "no 'api_key' or 'license_key' was configured"); - newrelic_config_destroy(ctx); - return NULL; - } - - /* Parse Base URL */ - ret = flb_utils_url_split(ctx->base_uri, - &ctx->nr_protocol, - &ctx->nr_host, - &port, - &ctx->nr_uri); - if (ret == -1) { - flb_plg_error(ctx->ins, "error parsing base_uri '%s'", ctx->base_uri); - newrelic_config_destroy(ctx); - return NULL; - } - ctx->nr_port = atoi(port); - flb_free(port); - - if (strcasecmp(ctx->compress, "gzip") == 0) { - ctx->compress_gzip = FLB_TRUE; - } - else if (flb_utils_bool(ctx->compress) == FLB_FALSE) { - ctx->compress_gzip = FLB_FALSE; - } - else { - flb_plg_warn(ctx->ins, - "unknown compress encoding value '%s', " - "payload compression has been disabled", - ctx->compress); - ctx->compress_gzip = FLB_FALSE; - } - - /* Create Upstream connection context */ - upstream = flb_upstream_create(config, - ctx->nr_host, - ctx->nr_port, - FLB_IO_TLS, ins->tls); - if (!upstream) { - flb_free(ctx); - return NULL; - } - ctx->u = upstream; - flb_output_upstream_set(ctx->u, ins); - - return ctx; -} - -static int cb_newrelic_init(struct flb_output_instance *ins, - struct flb_config *config, void *data) -{ - struct flb_newrelic *ctx; - - /* Create plugin context */ - ctx = newrelic_config_create(ins, config); - if (!ctx) { - flb_plg_error(ins, "cannot initialize configuration"); - return -1; - } - - /* Register context with plugin instance */ - flb_output_set_context(ins, ctx); - - /* - * This plugin instance uses the HTTP client interface, let's register - * it debugging callbacks. - */ - flb_output_set_http_debug_callbacks(ins); - - flb_plg_info(ins, "configured, hostname=%s:%i", ctx->nr_host, ctx->nr_port); - return 0; -} - -static void cb_newrelic_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - int ret; - int out_ret = FLB_OK; - int compressed = FLB_FALSE; - size_t b_sent; - flb_sds_t payload; - void *payload_buf = NULL; - size_t payload_size = 0; - struct flb_newrelic *ctx = out_context; - struct flb_connection *u_conn; - struct flb_http_client *c; - - /* Format the data to the expected Newrelic Payload */ - payload = newrelic_compose_payload(ctx, - event_chunk->data, event_chunk->size); - if (!payload) { - flb_plg_error(ctx->ins, "cannot compose request payload"); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Map payload */ - payload_buf = (void *) payload; - payload_size = flb_sds_len(payload); - - /* Should we compress the payload ? */ - if (ctx->compress_gzip == FLB_TRUE) { - ret = flb_gzip_compress(payload, flb_sds_len(payload), - &payload_buf, &payload_size); - if (ret == -1) { - flb_plg_error(ctx->ins, - "cannot gzip payload, disabling compression"); - } - else { - compressed = FLB_TRUE; - flb_sds_destroy(payload); - } - } - - /* Lookup an available connection context */ - u_conn = flb_upstream_conn_get(ctx->u); - if (!u_conn) { - flb_plg_error(ctx->ins, "no upstream connections available"); - if (compressed == FLB_TRUE) { - flb_free(payload_buf); - } - else { - flb_sds_destroy(payload); - } - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Create HTTP client context */ - c = flb_http_client(u_conn, FLB_HTTP_POST, ctx->nr_uri, - payload_buf, payload_size, - ctx->nr_host, ctx->nr_port, - NULL, 0); - if (!c) { - flb_plg_error(ctx->ins, "cannot create HTTP client context"); - if (compressed == FLB_TRUE) { - flb_free(payload_buf); - } - else { - flb_sds_destroy(payload); - } - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Set callback context to the HTTP client context */ - flb_http_set_callback_context(c, ctx->ins->callback); - - /* User Agent */ - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - - /* API / License Key */ - if (ctx->license_key) { - flb_http_add_header(c, - "X-License-Key", 13, - ctx->license_key, flb_sds_len(ctx->license_key)); - } - else if (ctx->api_key) { - flb_http_add_header(c, - "X-Insert-Key", 12, - ctx->api_key, flb_sds_len(ctx->api_key)); - } - - /* Add Content-Type header */ - flb_http_add_header(c, - FLB_NEWRELIC_CT, sizeof(FLB_NEWRELIC_CT) - 1, - FLB_NEWRELIC_CT_JSON, sizeof(FLB_NEWRELIC_CT_JSON) - 1); - - /* Encoding */ - if (compressed == FLB_TRUE) { - flb_http_set_content_encoding_gzip(c); - } - - /* Send HTTP request */ - ret = flb_http_do(c, &b_sent); - - /* Destroy buffers */ - if (compressed == FLB_FALSE) { - flb_sds_destroy(payload); - } - else { - flb_free(payload_buf); - } - - /* Validate HTTP client return status */ - if (ret == 0) { - /* - * Only allow the following HTTP status: - * - * - 200: OK - * - 201: Created - * - 202: Accepted - * - 203: no authorative resp - * - 204: No Content - * - 205: Reset content - * - */ - if (c->resp.status < 200 || c->resp.status > 205) { - if (c->resp.payload) { - flb_plg_error(ctx->ins, "%s:%i, HTTP status=%i\n%s", - ctx->nr_host, ctx->nr_port, c->resp.status, - c->resp.payload); - } - else { - flb_plg_error(ctx->ins, "%s:%i, HTTP status=%i", - ctx->nr_host, ctx->nr_port, c->resp.status); - } - out_ret = FLB_RETRY; - } - else { - if (c->resp.payload) { - flb_plg_info(ctx->ins, "%s:%i, HTTP status=%i\n%s", - ctx->nr_host, ctx->nr_port, - c->resp.status, c->resp.payload); - } - else { - flb_plg_info(ctx->ins, "%s:%i, HTTP status=%i", - ctx->nr_host, ctx->nr_port, - c->resp.status); - } - } - } - else { - flb_plg_error(ctx->ins, "could not flush records to %s:%i (http_do=%i)", - ctx->nr_host, ctx->nr_port, ret); - out_ret = FLB_RETRY; - } - - flb_http_client_destroy(c); - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(out_ret); -} - -static int cb_newrelic_exit(void *data, struct flb_config *config) -{ - struct flb_newrelic *ctx = data; - - if (!ctx) { - return 0; - } - - newrelic_config_destroy(ctx); - - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "base_uri", FLB_NEWRELIC_BASE_URI, - 0, FLB_TRUE, offsetof(struct flb_newrelic, base_uri), - "New Relic Host address" - }, - - { - FLB_CONFIG_MAP_STR, "api_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_newrelic, api_key), - "New Relic API Key" - }, - - { - FLB_CONFIG_MAP_STR, "license_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_newrelic, license_key), - "New Relic License Key" - }, - - { - FLB_CONFIG_MAP_STR, "compress", "gzip", - 0, FLB_TRUE, offsetof(struct flb_newrelic, compress), - "Set payload compression mechanism", - }, - - /* EOF */ - {0} - -}; - -/* Plugin reference */ -struct flb_output_plugin out_nrlogs_plugin = { - .name = "nrlogs", - .description = "New Relic", - .cb_init = cb_newrelic_init, - .cb_flush = cb_newrelic_flush, - .cb_exit = cb_newrelic_exit, - .config_map = config_map, - .flags = FLB_OUTPUT_NET | FLB_IO_TLS, -}; diff --git a/fluent-bit/plugins/out_nrlogs/newrelic.h b/fluent-bit/plugins/out_nrlogs/newrelic.h deleted file mode 100644 index 60a8b92e6..000000000 --- a/fluent-bit/plugins/out_nrlogs/newrelic.h +++ /dev/null @@ -1,52 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_NEWRELIC_H -#define FLB_OUT_NEWRELIC_H - -#include -#include - -#define FLB_NEWRELIC_BASE_URI "https://log-api.newrelic.com/log/v1" - -#define FLB_NEWRELIC_CT "Content-Type" -#define FLB_NEWRELIC_CT_JSON "application/json" - -struct flb_newrelic { - /* Incoming Configuration Properties */ - flb_sds_t base_uri; - flb_sds_t api_key; - flb_sds_t license_key; - flb_sds_t compress; - - /* Internal parsed URL */ - char *nr_protocol; - char *nr_host; - int nr_port; - char *nr_uri; - int compress_gzip; - - /* Upstream Context */ - struct flb_upstream *u; - - /* Plugin instance */ - struct flb_output_instance *ins; -}; - -#endif diff --git a/fluent-bit/plugins/out_null/CMakeLists.txt b/fluent-bit/plugins/out_null/CMakeLists.txt deleted file mode 100644 index 77a80fe90..000000000 --- a/fluent-bit/plugins/out_null/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - null.c) - -FLB_PLUGIN(out_null "${src}" "") diff --git a/fluent-bit/plugins/out_null/null.c b/fluent-bit/plugins/out_null/null.c deleted file mode 100644 index a1e1689a0..000000000 --- a/fluent-bit/plugins/out_null/null.c +++ /dev/null @@ -1,178 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -struct flb_null { - struct flb_output_instance *ins; - - /* config map properties */ - int out_format; - int json_date_format; - flb_sds_t json_date_key; - flb_sds_t date_key; -}; - -int cb_null_init(struct flb_output_instance *ins, struct flb_config *config, - void *data) -{ - int ret; - (void) config; - (void) data; - const char *tmp; - struct flb_null *ctx; - - ctx = flb_malloc(sizeof(struct flb_null)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->ins = ins; - - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return -1; - } - - ctx->out_format = FLB_PACK_JSON_FORMAT_NONE; - tmp = flb_output_get_property("format", ins); - if (tmp) { - ret = flb_pack_to_json_format_type(tmp); - if (ret == -1) { - flb_plg_error(ctx->ins, "unrecognized 'format' option. " - "Using 'msgpack'"); - } - else { - ctx->out_format = ret; - } - } - - /* Date key */ - ctx->date_key = ctx->json_date_key; - tmp = flb_output_get_property("json_date_key", ins); - if (tmp) { - /* Just check if we have to disable it */ - if (flb_utils_bool(tmp) == FLB_FALSE) { - ctx->date_key = NULL; - } - } - - /* Date format for JSON output */ - ctx->json_date_format = FLB_PACK_JSON_DATE_DOUBLE; - tmp = flb_output_get_property("json_date_format", ins); - if (tmp) { - ret = flb_pack_to_json_date_type(tmp); - if (ret == -1) { - flb_plg_error(ctx->ins, "invalid json_date_format '%s'. " - "Using 'double' type", tmp); - } - else { - ctx->json_date_format = ret; - } - } - - flb_output_set_context(ins, ctx); - - return 0; -} - -static void cb_null_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - (void) out_context; - (void) config; - flb_sds_t json; - struct flb_null *ctx = out_context; - -#ifdef FLB_HAVE_METRICS - /* Check if the event type is metrics, just return */ - if (event_chunk->type == FLB_EVENT_TYPE_METRICS) { - FLB_OUTPUT_RETURN(FLB_OK); - } -#endif - - /* - * There are cases where the user might want to test the performance - * of msgpack payload conversion to JSON. Nothing will be printed, - * just encodeed and destroyed. - */ - if (ctx->out_format != FLB_PACK_JSON_FORMAT_NONE) { - json = flb_pack_msgpack_to_json_format(event_chunk->data, - event_chunk->size, - ctx->out_format, - ctx->json_date_format, - ctx->date_key); - flb_sds_destroy(json); - } - - flb_plg_debug(ctx->ins, "discarding %lu bytes", event_chunk->size); - FLB_OUTPUT_RETURN(FLB_OK); -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "format", NULL, - 0, FLB_FALSE, 0, - "Specifies the data format to be printed. Supported formats are msgpack json, json_lines and json_stream." - }, - { - FLB_CONFIG_MAP_STR, "json_date_format", NULL, - 0, FLB_FALSE, 0, - "Specifies the name of the date field in output." - }, - { - FLB_CONFIG_MAP_STR, "json_date_key", "date", - 0, FLB_TRUE, offsetof(struct flb_null, json_date_key), - FBL_PACK_JSON_DATE_FORMAT_DESCRIPTION - }, - - /* EOF */ - {0} -}; - -static int cb_null_exit(void *data, struct flb_config *config) -{ - struct flb_null *ctx = data; - - if (!ctx) { - return 0; - } - - flb_free(ctx); - return 0; -} - -struct flb_output_plugin out_null_plugin = { - .name = "null", - .description = "Throws away events", - .cb_init = cb_null_init, - .cb_flush = cb_null_flush, - .cb_exit = cb_null_exit, - .event_type = FLB_OUTPUT_LOGS | FLB_OUTPUT_METRICS, - .config_map = config_map, - .flags = 0, - .workers = 1, -}; diff --git a/fluent-bit/plugins/out_opensearch/CMakeLists.txt b/fluent-bit/plugins/out_opensearch/CMakeLists.txt deleted file mode 100644 index 0e2bf59fe..000000000 --- a/fluent-bit/plugins/out_opensearch/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(src - os_conf.c - opensearch.c - ) - -FLB_PLUGIN(out_opensearch "${src}" "") diff --git a/fluent-bit/plugins/out_opensearch/opensearch.c b/fluent-bit/plugins/out_opensearch/opensearch.c deleted file mode 100644 index dbd0fa8d0..000000000 --- a/fluent-bit/plugins/out_opensearch/opensearch.c +++ /dev/null @@ -1,1291 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "opensearch.h" -#include "os_conf.h" - -static int os_pack_array_content(msgpack_packer *tmp_pck, - msgpack_object array, - struct flb_opensearch *ctx); - -#ifdef FLB_HAVE_AWS -static flb_sds_t add_aws_auth(struct flb_http_client *c, - struct flb_opensearch *ctx) -{ - flb_sds_t signature = NULL; - int ret; - - flb_plg_debug(ctx->ins, "Signing request with AWS Sigv4"); - - /* Amazon OpenSearch Sigv4 does not allow the host header to include the port */ - ret = flb_http_strip_port_from_host(c); - if (ret < 0) { - flb_plg_error(ctx->ins, "could not strip port from host for sigv4"); - return NULL; - } - - /* AWS Fluent Bit user agent */ - flb_http_add_header(c, "User-Agent", 10, "aws-fluent-bit-plugin", 21); - - signature = flb_signv4_do(c, FLB_TRUE, FLB_TRUE, time(NULL), - ctx->aws_region, ctx->aws_service_name, - S3_MODE_SIGNED_PAYLOAD, ctx->aws_unsigned_headers, - ctx->aws_provider); - if (!signature) { - flb_plg_error(ctx->ins, "could not sign request with sigv4"); - return NULL; - } - return signature; -} -#endif /* FLB_HAVE_AWS */ - -static int os_pack_map_content(msgpack_packer *tmp_pck, - msgpack_object map, - struct flb_opensearch *ctx) -{ - int i; - char *ptr_key = NULL; - char buf_key[256]; - msgpack_object *k; - msgpack_object *v; - - for (i = 0; i < map.via.map.size; i++) { - k = &map.via.map.ptr[i].key; - v = &map.via.map.ptr[i].val; - ptr_key = NULL; - - /* Store key */ - const char *key_ptr = NULL; - size_t key_size = 0; - - if (k->type == MSGPACK_OBJECT_BIN) { - key_ptr = k->via.bin.ptr; - key_size = k->via.bin.size; - } - else if (k->type == MSGPACK_OBJECT_STR) { - key_ptr = k->via.str.ptr; - key_size = k->via.str.size; - } - - if (key_size < (sizeof(buf_key) - 1)) { - memcpy(buf_key, key_ptr, key_size); - buf_key[key_size] = '\0'; - ptr_key = buf_key; - } - else { - /* Long map keys have a performance penalty */ - ptr_key = flb_malloc(key_size + 1); - if (!ptr_key) { - flb_errno(); - return -1; - } - - memcpy(ptr_key, key_ptr, key_size); - ptr_key[key_size] = '\0'; - } - - /* - * Sanitize key name, it don't allow dots in field names: - * - * https://goo.gl/R5NMTr - */ - if (ctx->replace_dots == FLB_TRUE) { - char *p = ptr_key; - char *end = ptr_key + key_size; - while (p != end) { - if (*p == '.') *p = '_'; - p++; - } - } - - /* Append the key */ - msgpack_pack_str(tmp_pck, key_size); - msgpack_pack_str_body(tmp_pck, ptr_key, key_size); - - /* Release temporary key if was allocated */ - if (ptr_key && ptr_key != buf_key) { - flb_free(ptr_key); - } - ptr_key = NULL; - - /* - * The value can be any data type, if it's a map we need to - * sanitize to avoid dots. - */ - if (v->type == MSGPACK_OBJECT_MAP) { - msgpack_pack_map(tmp_pck, v->via.map.size); - os_pack_map_content(tmp_pck, *v, ctx); - } - /* - * The value can be any data type, if it's an array we need to - * pass it to os_pack_array_content. - */ - else if (v->type == MSGPACK_OBJECT_ARRAY) { - msgpack_pack_array(tmp_pck, v->via.array.size); - os_pack_array_content(tmp_pck, *v, ctx); - } - else { - msgpack_pack_object(tmp_pck, *v); - } - } - return 0; -} - -/* - * Iterate through the array and sanitize elements. - * Mutual recursion with os_pack_map_content. - */ -static int os_pack_array_content(msgpack_packer *tmp_pck, - msgpack_object array, - struct flb_opensearch *ctx) -{ - int i; - msgpack_object *e; - - for (i = 0; i < array.via.array.size; i++) { - e = &array.via.array.ptr[i]; - if (e->type == MSGPACK_OBJECT_MAP) { - msgpack_pack_map(tmp_pck, e->via.map.size); - os_pack_map_content(tmp_pck, *e, ctx); - } - else if (e->type == MSGPACK_OBJECT_ARRAY) { - msgpack_pack_array(tmp_pck, e->via.array.size); - os_pack_array_content(tmp_pck, *e, ctx); - } - else { - msgpack_pack_object(tmp_pck, *e); - } - } - return 0; -} - -/* - * Get _id value from incoming record. - * If it successed, return the value as flb_sds_t. - * If it failed, return NULL. -*/ -static flb_sds_t os_get_id_value(struct flb_opensearch *ctx, - msgpack_object *map) -{ - struct flb_ra_value *rval = NULL; - flb_sds_t tmp_str; - rval = flb_ra_get_value_object(ctx->ra_id_key, *map); - if (rval == NULL) { - flb_plg_warn(ctx->ins, "the value of %s is missing", - ctx->id_key); - return NULL; - } - else if(rval->o.type != MSGPACK_OBJECT_STR) { - flb_plg_warn(ctx->ins, "the value of %s is not string", - ctx->id_key); - flb_ra_key_value_destroy(rval); - return NULL; - } - - tmp_str = flb_sds_create_len(rval->o.via.str.ptr, - rval->o.via.str.size); - if (tmp_str == NULL) { - flb_plg_warn(ctx->ins, "cannot create ID string from record"); - flb_ra_key_value_destroy(rval); - return NULL; - } - flb_ra_key_value_destroy(rval); - return tmp_str; -} - -static int compose_index_header(struct flb_opensearch *ctx, - int index_custom_len, - char *logstash_index, size_t logstash_index_size, - char *separator_str, - struct tm *tm) -{ - int ret; - int len; - char *p; - size_t s; - - /* Compose Index header */ - if (index_custom_len > 0) { - p = logstash_index + index_custom_len; - } else { - p = logstash_index + flb_sds_len(ctx->logstash_prefix); - } - len = p - logstash_index; - ret = snprintf(p, logstash_index_size - len, "%s", - separator_str); - if (ret > logstash_index_size - len) { - /* exceed limit */ - return -1; - } - p += strlen(separator_str); - len += strlen(separator_str); - - s = strftime(p, logstash_index_size - len, - ctx->logstash_dateformat, tm); - if (s==0) { - /* exceed limit */ - return -1; - } - p += s; - *p++ = '\0'; - - return 0; -} - -/* - * Convert the internal Fluent Bit data representation to the required - * one by OpenSearch. - */ -static int opensearch_format(struct flb_config *config, - struct flb_input_instance *ins, - void *plugin_context, - void *flush_ctx, - int event_type, - const char *tag, int tag_len, - const void *data, size_t bytes, - void **out_data, size_t *out_size) -{ - int ret; - int len; - int map_size; - int index_len = 0; - int write_op_update = FLB_FALSE; - int write_op_upsert = FLB_FALSE; - flb_sds_t ra_index = NULL; - size_t s = 0; - char *index = NULL; - char logstash_index[256]; - char time_formatted[256]; - char index_formatted[256]; - char uuid[37]; - flb_sds_t out_buf; - flb_sds_t id_key_str = NULL; - msgpack_object map; - flb_sds_t bulk; - struct tm tm; - struct flb_time tms; - msgpack_sbuffer tmp_sbuf; - msgpack_packer tmp_pck; - cfl_hash_128bits_t hash; - unsigned char h[sizeof(cfl_hash_128bits_t)]; - int index_custom_len; - struct flb_opensearch *ctx = plugin_context; - flb_sds_t j_index; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return -1; - } - - j_index = flb_sds_create_size(FLB_OS_HEADER_SIZE); - if (j_index == NULL) { - flb_log_event_decoder_destroy(&log_decoder); - - return -1; - } - - bulk = flb_sds_create_size(bytes * 2); - if (!bulk) { - flb_log_event_decoder_destroy(&log_decoder); - flb_sds_destroy(j_index); - - return -1; - } - - /* Copy logstash prefix if logstash format is enabled */ - if (ctx->logstash_format == FLB_TRUE) { - strncpy(logstash_index, ctx->logstash_prefix, sizeof(logstash_index)); - logstash_index[sizeof(logstash_index) - 1] = '\0'; - } - - /* - * If logstash format and id generation are disabled, pre-generate - * the index line for all records. - * - * The header stored in 'j_index' will be used for the all records on - * this payload. - */ - if (ctx->logstash_format == FLB_FALSE && ctx->generate_id == FLB_FALSE && ctx->ra_index == NULL) { - flb_time_get(&tms); - gmtime_r(&tms.tm.tv_sec, &tm); - strftime(index_formatted, sizeof(index_formatted) - 1, - ctx->index, &tm); - index = index_formatted; - if (ctx->suppress_type_name) { - index_len = flb_sds_snprintf(&j_index, - flb_sds_alloc(j_index), - OS_BULK_INDEX_FMT_NO_TYPE, - ctx->action, - index); - } - else { - index_len = flb_sds_snprintf(&j_index, - flb_sds_alloc(j_index), - OS_BULK_INDEX_FMT, - ctx->action, - index, ctx->type); - } - - if (index_len == -1) { - flb_log_event_decoder_destroy(&log_decoder); - flb_sds_destroy(bulk); - flb_sds_destroy(j_index); - return -1; - } - } - - /* - * Some broken clients may have time drift up to year 1970 - * this will generate corresponding index in OpenSearch - * in order to prevent generating millions of indexes - * we can set to always use current time for index generation - */ - if (ctx->current_time_index == FLB_TRUE) { - flb_time_get(&tms); - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - /* Only pop time from record if current_time_index is disabled */ - if (!ctx->current_time_index) { - flb_time_copy(&tms, &log_event.timestamp); - } - - map = *log_event.body; - map_size = map.via.map.size; - - index_custom_len = 0; - if (ctx->logstash_prefix_key) { - flb_sds_t v = flb_ra_translate(ctx->ra_prefix_key, - (char *) tag, tag_len, - map, NULL); - if (v) { - len = flb_sds_len(v); - if (len > 128) { - len = 128; - memcpy(logstash_index, v, 128); - } - else { - memcpy(logstash_index, v, len); - } - - index_custom_len = len; - flb_sds_destroy(v); - } - } - - /* Create temporary msgpack buffer */ - msgpack_sbuffer_init(&tmp_sbuf); - msgpack_packer_init(&tmp_pck, &tmp_sbuf, msgpack_sbuffer_write); - - if (ctx->include_tag_key) { - map_size++; - } - - /* Set the new map size */ - msgpack_pack_map(&tmp_pck, map_size + 1); - - /* Append the time key */ - msgpack_pack_str(&tmp_pck, flb_sds_len(ctx->time_key)); - msgpack_pack_str_body(&tmp_pck, ctx->time_key, flb_sds_len(ctx->time_key)); - - /* Format the time */ - gmtime_r(&tms.tm.tv_sec, &tm); - s = strftime(time_formatted, sizeof(time_formatted) - 1, - ctx->time_key_format, &tm); - if (ctx->time_key_nanos) { - len = snprintf(time_formatted + s, sizeof(time_formatted) - 1 - s, - ".%09" PRIu64 "Z", (uint64_t) tms.tm.tv_nsec); - } else { - len = snprintf(time_formatted + s, sizeof(time_formatted) - 1 - s, - ".%03" PRIu64 "Z", - (uint64_t) tms.tm.tv_nsec / 1000000); - } - - s += len; - msgpack_pack_str(&tmp_pck, s); - msgpack_pack_str_body(&tmp_pck, time_formatted, s); - - index = ctx->index; - if (ctx->logstash_format == FLB_TRUE) { - ret = compose_index_header(ctx, index_custom_len, - &logstash_index[0], sizeof(logstash_index), - ctx->logstash_prefix_separator, &tm); - if (ret < 0) { - /* retry with default separator */ - compose_index_header(ctx, index_custom_len, - &logstash_index[0], sizeof(logstash_index), - "-", &tm); - } - index = logstash_index; - if (ctx->generate_id == FLB_FALSE) { - if (ctx->suppress_type_name) { - index_len = flb_sds_snprintf(&j_index, - flb_sds_alloc(j_index), - OS_BULK_INDEX_FMT_NO_TYPE, - ctx->action, - index); - } - else { - index_len = flb_sds_snprintf(&j_index, - flb_sds_alloc(j_index), - OS_BULK_INDEX_FMT, - ctx->action, - index, ctx->type); - } - } - } - else if (ctx->current_time_index == FLB_TRUE) { - /* Make sure we handle index time format for index */ - strftime(index_formatted, sizeof(index_formatted) - 1, - ctx->index, &tm); - index = index_formatted; - } - else if (ctx->ra_index) { - // free any previous ra_index to avoid memory leaks. - if (ra_index != NULL) { - flb_sds_destroy(ra_index); - } - /* a record accessor pattern exists for the index */ - ra_index = flb_ra_translate(ctx->ra_index, - (char *) tag, tag_len, - map, NULL); - if (!ra_index) { - flb_plg_warn(ctx->ins, "invalid index translation from record accessor pattern, default to static index"); - } - else { - index = ra_index; - } - - if (ctx->suppress_type_name) { - index_len = flb_sds_snprintf(&j_index, - flb_sds_alloc(j_index), - OS_BULK_INDEX_FMT_NO_TYPE, - ctx->action, - index); - } - else { - index_len = flb_sds_snprintf(&j_index, - flb_sds_alloc(j_index), - OS_BULK_INDEX_FMT, - ctx->action, - index, ctx->type); - } - } - - /* Tag Key */ - if (ctx->include_tag_key == FLB_TRUE) { - msgpack_pack_str(&tmp_pck, flb_sds_len(ctx->tag_key)); - msgpack_pack_str_body(&tmp_pck, ctx->tag_key, flb_sds_len(ctx->tag_key)); - msgpack_pack_str(&tmp_pck, tag_len); - msgpack_pack_str_body(&tmp_pck, tag, tag_len); - } - - /* - * The map_content routine iterate over each Key/Value pair found in - * the map and do some sanitization for the key names. - * - * There is a restriction that key names cannot contain a dot; if some - * dot is found, it's replaced with an underscore. - */ - ret = os_pack_map_content(&tmp_pck, map, ctx); - if (ret == -1) { - flb_log_event_decoder_destroy(&log_decoder); - msgpack_sbuffer_destroy(&tmp_sbuf); - flb_sds_destroy(bulk); - flb_sds_destroy(j_index); - if (ra_index != NULL) { - flb_sds_destroy(ra_index); - } - return -1; - } - - if (ctx->generate_id == FLB_TRUE) { - /* use a 128 bit hash and copy it to a buffer */ - hash = cfl_hash_128bits(tmp_sbuf.data, tmp_sbuf.size); - memcpy(h, &hash, sizeof(hash)); - snprintf(uuid, sizeof(uuid), - "%02X%02X%02X%02X-%02X%02X-%02X%02X-" - "%02X%02X-%02X%02X%02X%02X%02X%02X", - h[0], h[1], h[2], h[3], h[4], h[5], h[6], h[7], - h[8], h[9], h[10], h[11], h[12], h[13], h[14], h[15]); - - if (ctx->suppress_type_name) { - index_len = flb_sds_snprintf(&j_index, - flb_sds_alloc(j_index), - OS_BULK_INDEX_FMT_ID_NO_TYPE, - ctx->action, - index, uuid); - } - else { - index_len = flb_sds_snprintf(&j_index, - flb_sds_alloc(j_index), - OS_BULK_INDEX_FMT_ID, - ctx->action, - index, ctx->type, uuid); - } - } - if (ctx->ra_id_key) { - id_key_str = os_get_id_value(ctx ,&map); - if (id_key_str) { - if (ctx->suppress_type_name) { - index_len = flb_sds_snprintf(&j_index, - flb_sds_alloc(j_index), - OS_BULK_INDEX_FMT_ID_NO_TYPE, - ctx->action, - index, id_key_str); - } - else { - index_len = flb_sds_snprintf(&j_index, - flb_sds_alloc(j_index), - OS_BULK_INDEX_FMT_ID, - ctx->action, - index, ctx->type, id_key_str); - } - flb_sds_destroy(id_key_str); - id_key_str = NULL; - } - } - - /* Convert msgpack to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(tmp_sbuf.data, tmp_sbuf.size); - msgpack_sbuffer_destroy(&tmp_sbuf); - if (!out_buf) { - flb_log_event_decoder_destroy(&log_decoder); - flb_sds_destroy(bulk); - flb_sds_destroy(j_index); - if (ra_index != NULL) { - flb_sds_destroy(ra_index); - } - return -1; - } - - ret = flb_sds_cat_safe(&bulk, j_index, flb_sds_len(j_index)); - if (ret == -1) { - flb_log_event_decoder_destroy(&log_decoder); - *out_size = 0; - flb_sds_destroy(bulk); - flb_sds_destroy(j_index); - flb_sds_destroy(out_buf); - if (ra_index != NULL) { - flb_sds_destroy(ra_index); - } - return -1; - } - - if (strcasecmp(ctx->write_operation, FLB_OS_WRITE_OP_UPDATE) == 0) { - write_op_update = FLB_TRUE; - } - else if (strcasecmp(ctx->write_operation, FLB_OS_WRITE_OP_UPSERT) == 0) { - write_op_upsert = FLB_TRUE; - } - - /* UPDATE | UPSERT */ - if (write_op_update) { - flb_sds_cat_safe(&bulk, - OS_BULK_UPDATE_OP_BODY, - sizeof(OS_BULK_UPDATE_OP_BODY) - 1); - } - else if (write_op_upsert) { - flb_sds_cat_safe(&bulk, - OS_BULK_UPSERT_OP_BODY, - sizeof(OS_BULK_UPSERT_OP_BODY) - 1); - } - - ret = flb_sds_cat_safe(&bulk, out_buf, flb_sds_len(out_buf)); - if (ret == -1) { - flb_log_event_decoder_destroy(&log_decoder); - *out_size = 0; - flb_sds_destroy(bulk); - flb_sds_destroy(j_index); - flb_sds_destroy(out_buf); - if (ra_index != NULL) { - flb_sds_destroy(ra_index); - } - return -1; - } - - /* finish UPDATE | UPSERT */ - if (write_op_update || write_op_upsert) { - flb_sds_cat_safe(&bulk, "}", 1); - } - - flb_sds_cat_safe(&bulk, "\n", 1); - flb_sds_destroy(out_buf); - } - - flb_log_event_decoder_destroy(&log_decoder); - - /* Set outgoing data */ - *out_data = bulk; - *out_size = flb_sds_len(bulk); - - if (ra_index != NULL) { - flb_sds_destroy(ra_index); - } - /* - * Note: we don't destroy the bulk as we need to keep the allocated - * buffer with the data. Instead we just release the bulk context and - * return the bulk->ptr buffer - */ - if (ctx->trace_output) { - fwrite(*out_data, 1, *out_size, stdout); - fflush(stdout); - } - flb_sds_destroy(j_index); - return 0; -} - -static int cb_opensearch_init(struct flb_output_instance *ins, - struct flb_config *config, - void *data) -{ - struct flb_opensearch *ctx; - - ctx = flb_os_conf_create(ins, config); - if (!ctx) { - flb_plg_error(ins, "cannot initialize plugin"); - return -1; - } - - flb_plg_debug(ctx->ins, "host=%s port=%i uri=%s index=%s type=%s", - ins->host.name, ins->host.port, ctx->uri, - ctx->index, ctx->type); - - flb_output_set_context(ins, ctx); - - /* - * This plugin instance uses the HTTP client interface, let's register - * it debugging callbacks. - */ - flb_output_set_http_debug_callbacks(ins); - - return 0; -} - -static int opensearch_error_check(struct flb_opensearch *ctx, - struct flb_http_client *c) -{ - int i, j, k; - int ret; - int check = FLB_FALSE; - int root_type; - char *out_buf; - size_t off = 0; - size_t out_size; - msgpack_unpacked result; - msgpack_object root; - msgpack_object key; - msgpack_object val; - msgpack_object item; - msgpack_object item_key; - msgpack_object item_val; - - /* - * Check if our payload is complete: there is such situations where - * the OpenSearch HTTP response body is bigger than the HTTP client - * buffer so payload can be incomplete. - */ - /* Convert JSON payload to msgpack */ - ret = flb_pack_json(c->resp.payload, c->resp.payload_size, - &out_buf, &out_size, &root_type, NULL); - if (ret == -1) { - /* Is this an incomplete HTTP Request ? */ - if (c->resp.payload_size <= 0) { - return FLB_TRUE; - } - - /* Lookup error field */ - if (strstr(c->resp.payload, "\"errors\":false,\"items\":[")) { - return FLB_FALSE; - } - - flb_plg_error(ctx->ins, "could not pack/validate JSON response\n%s", - c->resp.payload); - return FLB_TRUE; - } - - /* Lookup error field */ - msgpack_unpacked_init(&result); - ret = msgpack_unpack_next(&result, out_buf, out_size, &off); - if (ret != MSGPACK_UNPACK_SUCCESS) { - flb_plg_error(ctx->ins, "Cannot unpack response to find error\n%s", - c->resp.payload); - return FLB_TRUE; - } - - root = result.data; - if (root.type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "unexpected payload type=%i", - root.type); - check = FLB_TRUE; - goto done; - } - - for (i = 0; i < root.via.map.size; i++) { - key = root.via.map.ptr[i].key; - if (key.type != MSGPACK_OBJECT_STR) { - flb_plg_error(ctx->ins, "unexpected key type=%i", - key.type); - check = FLB_TRUE; - goto done; - } - - if (key.via.str.size == 6 && strncmp(key.via.str.ptr, "errors", 6) == 0) { - val = root.via.map.ptr[i].val; - if (val.type != MSGPACK_OBJECT_BOOLEAN) { - flb_plg_error(ctx->ins, "unexpected 'error' value type=%i", - val.type); - check = FLB_TRUE; - goto done; - } - - /* If error == false, we are OK (no errors = FLB_FALSE) */ - if (!val.via.boolean) { - /* no errors */ - check = FLB_FALSE; - goto done; - } - } - else if (key.via.str.size == 5 && strncmp(key.via.str.ptr, "items", 5) == 0) { - val = root.via.map.ptr[i].val; - if (val.type != MSGPACK_OBJECT_ARRAY) { - flb_plg_error(ctx->ins, "unexpected 'items' value type=%i", - val.type); - check = FLB_TRUE; - goto done; - } - - for (j = 0; j < val.via.array.size; j++) { - item = val.via.array.ptr[j]; - if (item.type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "unexpected 'item' outer value type=%i", - item.type); - check = FLB_TRUE; - goto done; - } - - if (item.via.map.size != 1) { - flb_plg_error(ctx->ins, "unexpected 'item' size=%i", - item.via.map.size); - check = FLB_TRUE; - goto done; - } - - item = item.via.map.ptr[0].val; - if (item.type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "unexpected 'item' inner value type=%i", - item.type); - check = FLB_TRUE; - goto done; - } - - for (k = 0; k < item.via.map.size; k++) { - item_key = item.via.map.ptr[k].key; - if (item_key.type != MSGPACK_OBJECT_STR) { - flb_plg_error(ctx->ins, "unexpected key type=%i", - item_key.type); - check = FLB_TRUE; - goto done; - } - - if (item_key.via.str.size == 6 && strncmp(item_key.via.str.ptr, "status", 6) == 0) { - item_val = item.via.map.ptr[k].val; - - if (item_val.type != MSGPACK_OBJECT_POSITIVE_INTEGER) { - flb_plg_error(ctx->ins, "unexpected 'status' value type=%i", - item_val.type); - check = FLB_TRUE; - goto done; - } - /* Check for errors other than version conflict (document already exists) */ - if (item_val.via.i64 != 409) { - check = FLB_TRUE; - goto done; - } - } - } - } - } - } - - done: - flb_free(out_buf); - msgpack_unpacked_destroy(&result); - return check; -} - -static void cb_opensearch_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *ins, void *out_context, - struct flb_config *config) -{ - int ret = -1; - size_t pack_size; - flb_sds_t pack; - void *out_buf; - size_t out_size; - size_t b_sent; - struct flb_opensearch *ctx = out_context; - struct flb_connection *u_conn; - struct flb_http_client *c; - flb_sds_t signature = NULL; - int compressed = FLB_FALSE; - void *final_payload_buf = NULL; - size_t final_payload_size = 0; - - /* Get upstream connection */ - u_conn = flb_upstream_conn_get(ctx->u); - if (!u_conn) { - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Convert format */ - if (event_chunk->type == FLB_EVENT_TYPE_TRACES) { - pack = flb_msgpack_raw_to_json_sds(event_chunk->data, event_chunk->size); - if (pack) { - ret = 0; - } - else { - ret = -1; - } - } - else if (event_chunk->type == FLB_EVENT_TYPE_LOGS) { - ret = opensearch_format(config, ins, - ctx, NULL, - event_chunk->type, - event_chunk->tag, flb_sds_len(event_chunk->tag), - event_chunk->data, event_chunk->size, - &out_buf, &out_size); - } - - if (ret != 0) { - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_ERROR); - } - - pack = (char *) out_buf; - pack_size = out_size; - - final_payload_buf = pack; - final_payload_size = pack_size; - /* Should we compress the payload ? */ - if (ctx->compression == FLB_OS_COMPRESSION_GZIP) { - ret = flb_gzip_compress((void *) pack, pack_size, - &out_buf, &out_size); - if (ret == -1) { - flb_plg_error(ctx->ins, - "cannot gzip payload, disabling compression"); - } - else { - compressed = FLB_TRUE; - final_payload_buf = out_buf; - final_payload_size = out_size; - } - } - - /* Compose HTTP Client request */ - c = flb_http_client(u_conn, FLB_HTTP_POST, ctx->uri, - final_payload_buf, final_payload_size, NULL, 0, NULL, 0); - - flb_http_buffer_size(c, ctx->buffer_size); - -#ifndef FLB_HAVE_AWS - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); -#endif - - flb_http_add_header(c, "Content-Type", 12, "application/x-ndjson", 20); - - if (ctx->http_user && ctx->http_passwd) { - flb_http_basic_auth(c, ctx->http_user, ctx->http_passwd); - } - -#ifdef FLB_HAVE_AWS - if (ctx->has_aws_auth == FLB_TRUE) { - signature = add_aws_auth(c, ctx); - if (!signature) { - goto retry; - } - } - else { - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - } -#endif - - /* Set Content-Encoding of compressed payload */ - if (compressed == FLB_TRUE) { - if (ctx->compression == FLB_OS_COMPRESSION_GZIP) { - flb_http_set_content_encoding_gzip(c); - } - } - - /* Map debug callbacks */ - flb_http_client_debug(c, ctx->ins->callback); - - ret = flb_http_do(c, &b_sent); - if (ret != 0) { - flb_plg_warn(ctx->ins, "http_do=%i URI=%s", ret, ctx->uri); - goto retry; - } - else { - /* The request was issued successfully, validate the 'error' field */ - flb_plg_debug(ctx->ins, "HTTP Status=%i URI=%s", c->resp.status, ctx->uri); - if (c->resp.status != 200 && c->resp.status != 201) { - if (c->resp.payload_size > 0) { - flb_plg_error(ctx->ins, "HTTP status=%i URI=%s, response:\n%s\n", - c->resp.status, ctx->uri, c->resp.payload); - } - else { - flb_plg_error(ctx->ins, "HTTP status=%i URI=%s", - c->resp.status, ctx->uri); - } - goto retry; - } - - if (c->resp.payload_size > 0) { - /* - * OpenSearch payload should be JSON, we convert it to msgpack - * and lookup the 'error' field. - */ - ret = opensearch_error_check(ctx, c); - if (ret == FLB_TRUE) { - /* we got an error */ - if (ctx->trace_error) { - /* - * If trace_error is set, trace the actual - * response from Elasticsearch explaining the problem. - * Trace_Output can be used to see the request. - */ - if (pack_size < 4000) { - flb_plg_debug(ctx->ins, "error caused by: Input\n%.*s\n", - (int) pack_size, pack); - } - if (c->resp.payload_size < 4000) { - flb_plg_error(ctx->ins, "error: Output\n%s", - c->resp.payload); - } else { - /* - * We must use fwrite since the flb_log functions - * will truncate data at 4KB - */ - fwrite(c->resp.payload, 1, c->resp.payload_size, stderr); - fflush(stderr); - } - } - goto retry; - } - else { - flb_plg_debug(ctx->ins, "OpenSearch response\n%s", - c->resp.payload); - } - } - else { - goto retry; - } - } - - /* Cleanup */ - flb_http_client_destroy(c); - flb_sds_destroy(pack); - - if (final_payload_buf != pack) { - flb_free(final_payload_buf); - } - - flb_upstream_conn_release(u_conn); - if (signature) { - flb_sds_destroy(signature); - } - FLB_OUTPUT_RETURN(FLB_OK); - - /* Issue a retry */ - retry: - flb_http_client_destroy(c); - flb_sds_destroy(pack); - - if (final_payload_buf != pack) { - flb_free(final_payload_buf); - } - - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_RETRY); -} - -static int cb_opensearch_exit(void *data, struct flb_config *config) -{ - struct flb_opensearch *ctx = data; - - flb_os_conf_destroy(ctx); - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "index", FLB_OS_DEFAULT_INDEX, - 0, FLB_TRUE, offsetof(struct flb_opensearch, index), - "Set an index name" - }, - { - FLB_CONFIG_MAP_STR, "type", FLB_OS_DEFAULT_TYPE, - 0, FLB_TRUE, offsetof(struct flb_opensearch, type), - "Set the document type property" - }, - { - FLB_CONFIG_MAP_BOOL, "suppress_type_name", "false", - 0, FLB_TRUE, offsetof(struct flb_opensearch, suppress_type_name), - "If true, mapping types is removed. (for v7.0.0 or later)" - }, - - /* HTTP Authentication */ - { - FLB_CONFIG_MAP_STR, "http_user", NULL, - 0, FLB_TRUE, offsetof(struct flb_opensearch, http_user), - "Optional username credential for access" - }, - { - FLB_CONFIG_MAP_STR, "http_passwd", "", - 0, FLB_TRUE, offsetof(struct flb_opensearch, http_passwd), - "Password for user defined in 'http_user'" - }, - - /* AWS Authentication */ -#ifdef FLB_HAVE_AWS - { - FLB_CONFIG_MAP_BOOL, "aws_auth", "false", - 0, FLB_TRUE, offsetof(struct flb_opensearch, has_aws_auth), - "Enable AWS Sigv4 Authentication" - }, - { - FLB_CONFIG_MAP_STR, "aws_region", NULL, - 0, FLB_TRUE, offsetof(struct flb_opensearch, aws_region), - "AWS Region of your Amazon OpenSearch Service cluster" - }, - { - FLB_CONFIG_MAP_STR, "aws_profile", "default", - 0, FLB_TRUE, offsetof(struct flb_opensearch, aws_profile), - "AWS Profile name. AWS Profiles can be configured with AWS CLI and are usually stored in " - "$HOME/.aws/ directory." - }, - { - FLB_CONFIG_MAP_STR, "aws_sts_endpoint", NULL, - 0, FLB_TRUE, offsetof(struct flb_opensearch, aws_sts_endpoint), - "Custom endpoint for the AWS STS API, used with the AWS_Role_ARN option" - }, - { - FLB_CONFIG_MAP_STR, "aws_role_arn", NULL, - 0, FLB_FALSE, 0, - "AWS IAM Role to assume to put records to your Amazon OpenSearch cluster" - }, - { - FLB_CONFIG_MAP_STR, "aws_external_id", NULL, - 0, FLB_FALSE, 0, - "External ID for the AWS IAM Role specified with `aws_role_arn`" - }, - { - FLB_CONFIG_MAP_STR, "aws_service_name", "es", - 0, FLB_TRUE, offsetof(struct flb_opensearch, aws_service_name), - "AWS Service Name" - }, -#endif - - /* Logstash compatibility */ - { - FLB_CONFIG_MAP_BOOL, "logstash_format", "false", - 0, FLB_TRUE, offsetof(struct flb_opensearch, logstash_format), - "Enable Logstash format compatibility" - }, - { - FLB_CONFIG_MAP_STR, "logstash_prefix", FLB_OS_DEFAULT_PREFIX, - 0, FLB_TRUE, offsetof(struct flb_opensearch, logstash_prefix), - "When Logstash_Format is enabled, the Index name is composed using a prefix " - "and the date, e.g: If Logstash_Prefix is equals to 'mydata' your index will " - "become 'mydata-YYYY.MM.DD'. The last string appended belongs to the date " - "when the data is being generated" - }, - { - FLB_CONFIG_MAP_STR, "logstash_prefix_separator", "-", - 0, FLB_TRUE, offsetof(struct flb_opensearch, logstash_prefix_separator), - "Set a separator between logstash_prefix and date." - }, - { - FLB_CONFIG_MAP_STR, "logstash_prefix_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_opensearch, logstash_prefix_key), - "When included: the value in the record that belongs to the key will be looked " - "up and over-write the Logstash_Prefix for index generation. If the key/value " - "is not found in the record then the Logstash_Prefix option will act as a " - "fallback. Nested keys are supported through record accessor pattern" - }, - { - FLB_CONFIG_MAP_STR, "logstash_dateformat", FLB_OS_DEFAULT_TIME_FMT, - 0, FLB_TRUE, offsetof(struct flb_opensearch, logstash_dateformat), - "Time format (based on strftime) to generate the second part of the Index name" - }, - - /* Custom Time and Tag keys */ - { - FLB_CONFIG_MAP_STR, "time_key", FLB_OS_DEFAULT_TIME_KEY, - 0, FLB_TRUE, offsetof(struct flb_opensearch, time_key), - "When Logstash_Format is enabled, each record will get a new timestamp field. " - "The Time_Key property defines the name of that field" - }, - { - FLB_CONFIG_MAP_STR, "time_key_format", FLB_OS_DEFAULT_TIME_KEYF, - 0, FLB_TRUE, offsetof(struct flb_opensearch, time_key_format), - "When Logstash_Format is enabled, this property defines the format of the " - "timestamp" - }, - { - FLB_CONFIG_MAP_BOOL, "time_key_nanos", "false", - 0, FLB_TRUE, offsetof(struct flb_opensearch, time_key_nanos), - "When Logstash_Format is enabled, enabling this property sends nanosecond " - "precision timestamps" - }, - { - FLB_CONFIG_MAP_BOOL, "include_tag_key", "false", - 0, FLB_TRUE, offsetof(struct flb_opensearch, include_tag_key), - "When enabled, it append the Tag name to the record" - }, - { - FLB_CONFIG_MAP_STR, "tag_key", FLB_OS_DEFAULT_TAG_KEY, - 0, FLB_TRUE, offsetof(struct flb_opensearch, tag_key), - "When Include_Tag_Key is enabled, this property defines the key name for the tag" - }, - { - FLB_CONFIG_MAP_SIZE, "buffer_size", FLB_OS_DEFAULT_HTTP_MAX, - 0, FLB_TRUE, offsetof(struct flb_opensearch, buffer_size), - "Specify the buffer size used to read the response from the OpenSearch HTTP " - "service. This option is useful for debugging purposes where is required to read " - "full responses, note that response size grows depending of the number of records " - "inserted. To set an unlimited amount of memory set this value to 'false', " - "otherwise the value must be according to the Unit Size specification" - }, - - /* OpenSearch specifics */ - { - FLB_CONFIG_MAP_STR, "path", NULL, - 0, FLB_FALSE, 0, - "OpenSearch accepts new data on HTTP query path '/_bulk'. But it is also " - "possible to serve OpenSearch behind a reverse proxy on a subpath. This " - "option defines such path on the fluent-bit side. It simply adds a path " - "prefix in the indexing HTTP POST URI" - }, - { - FLB_CONFIG_MAP_STR, "pipeline", NULL, - 0, FLB_FALSE, 0, - "OpenSearch allows to setup filters called pipelines. " - "This option allows to define which pipeline the database should use. For " - "performance reasons is strongly suggested to do parsing and filtering on " - "Fluent Bit side, avoid pipelines" - }, - { - FLB_CONFIG_MAP_BOOL, "generate_id", "false", - 0, FLB_TRUE, offsetof(struct flb_opensearch, generate_id), - "When enabled, generate _id for outgoing records. This prevents duplicate " - "records when retrying" - }, - { - FLB_CONFIG_MAP_STR, "write_operation", "create", - 0, FLB_TRUE, offsetof(struct flb_opensearch, write_operation), - "Operation to use to write in bulk requests" - }, - { - FLB_CONFIG_MAP_STR, "id_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_opensearch, id_key), - "If set, _id will be the value of the key from incoming record." - }, - { - FLB_CONFIG_MAP_BOOL, "replace_dots", "false", - 0, FLB_TRUE, offsetof(struct flb_opensearch, replace_dots), - "When enabled, replace field name dots with underscore." - }, - - { - FLB_CONFIG_MAP_BOOL, "current_time_index", "false", - 0, FLB_TRUE, offsetof(struct flb_opensearch, current_time_index), - "Use current time for index generation instead of message record" - }, - - /* Trace */ - { - FLB_CONFIG_MAP_BOOL, "trace_output", "false", - 0, FLB_TRUE, offsetof(struct flb_opensearch, trace_output), - "When enabled print the OpenSearch API calls to stdout (for diag only)" - }, - { - FLB_CONFIG_MAP_BOOL, "trace_error", "false", - 0, FLB_TRUE, offsetof(struct flb_opensearch, trace_error), - "When enabled print the OpenSearch exception to stderr (for diag only)" - }, - - /* HTTP Compression */ - { - FLB_CONFIG_MAP_STR, "compress", NULL, - 0, FLB_TRUE, offsetof(struct flb_opensearch, compression_str), - "Set payload compression mechanism. Option available is 'gzip'" - }, - - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_output_plugin out_opensearch_plugin = { - .name = "opensearch", - .description = "OpenSearch", - .cb_init = cb_opensearch_init, - .cb_pre_run = NULL, - .cb_flush = cb_opensearch_flush, - .cb_exit = cb_opensearch_exit, - - /* Configuration */ - .config_map = config_map, - - /* Events supported */ - .event_type = FLB_OUTPUT_LOGS | FLB_OUTPUT_TRACES, - - /* Test */ - .test_formatter.callback = opensearch_format, - - /* Plugin flags */ - .flags = FLB_OUTPUT_NET | FLB_IO_OPT_TLS, -}; diff --git a/fluent-bit/plugins/out_opensearch/opensearch.h b/fluent-bit/plugins/out_opensearch/opensearch.h deleted file mode 100644 index a1087c1da..000000000 --- a/fluent-bit/plugins/out_opensearch/opensearch.h +++ /dev/null @@ -1,155 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_OPENSEARCH_H -#define FLB_OUT_OPENSEARCH_H - -/* config defaults */ -#define FLB_OS_DEFAULT_HOST "127.0.0.1" -#define FLB_OS_DEFAULT_PORT 92000 -#define FLB_OS_DEFAULT_INDEX "fluent-bit" -#define FLB_OS_DEFAULT_TYPE "_doc" -#define FLB_OS_DEFAULT_PREFIX "logstash" -#define FLB_OS_DEFAULT_TIME_FMT "%Y.%m.%d" -#define FLB_OS_DEFAULT_TIME_KEY "@timestamp" -#define FLB_OS_DEFAULT_TIME_KEYF "%Y-%m-%dT%H:%M:%S" -#define FLB_OS_DEFAULT_TAG_KEY "flb-key" -#define FLB_OS_DEFAULT_HTTP_MAX "512k" -#define FLB_OS_WRITE_OP_INDEX "index" -#define FLB_OS_WRITE_OP_CREATE "create" -#define FLB_OS_WRITE_OP_UPDATE "update" -#define FLB_OS_WRITE_OP_UPSERT "upsert" - -/* macros */ -#define FLB_OS_HEADER_SIZE 1024 -#define OS_BULK_CHUNK 4096 /* Size of buffer chunks */ -#define OS_BULK_HEADER 165 /* Bulk API prefix line */ - -/* Bulk formats */ -#define OS_BULK_INDEX_FMT "{\"%s\":{\"_index\":\"%s\",\"_type\":\"%s\"}}\n" -#define OS_BULK_INDEX_FMT_ID "{\"%s\":{\"_index\":\"%s\",\"_type\":\"%s\",\"_id\":\"%s\"}}\n" -#define OS_BULK_INDEX_FMT_NO_TYPE "{\"%s\":{\"_index\":\"%s\"}}\n" -#define OS_BULK_INDEX_FMT_ID_NO_TYPE "{\"%s\":{\"_index\":\"%s\",\"_id\":\"%s\"}}\n" - -/* Bulk write-type operations */ -#define OS_BULK_UPDATE_OP_BODY "{\"doc\":" -#define OS_BULK_UPSERT_OP_BODY "{\"doc_as_upsert\":true,\"doc\":" - -/* Supported compression algorithms */ -#define FLB_OS_COMPRESSION_NONE 0 -#define FLB_OS_COMPRESSION_GZIP 1 - -struct flb_opensearch { - /* OpenSearch index (database) and type (table) */ - flb_sds_t index; - struct flb_record_accessor *ra_index; - - char *type; - char suppress_type_name; - - /* HTTP Auth */ - char *http_user; - char *http_passwd; - - /* AWS Auth */ -#ifdef FLB_HAVE_AWS - int has_aws_auth; - char *aws_region; - char *aws_sts_endpoint; - char *aws_profile; - struct flb_aws_provider *aws_provider; - struct flb_aws_provider *base_aws_provider; - /* tls instances can't be re-used; aws provider requires a separate one */ - struct flb_tls *aws_tls; - /* one for the standard chain provider, one for sts assume role */ - struct flb_tls *aws_sts_tls; - char *aws_session_name; - char *aws_service_name; - struct mk_list *aws_unsigned_headers; -#endif - - /* HTTP Client Setup */ - size_t buffer_size; - - /* If enabled, replace field name dots with underscore */ - int replace_dots; - - int trace_output; - int trace_error; - - /* - * Logstash compatibility options - * ============================== - */ - - /* enabled/disabled */ - int logstash_format; - int generate_id; - int current_time_index; - - /* prefix */ - flb_sds_t logstash_prefix; - flb_sds_t logstash_prefix_separator; - - /* prefix key */ - flb_sds_t logstash_prefix_key; - - /* date format */ - flb_sds_t logstash_dateformat; - - /* time key */ - flb_sds_t time_key; - - /* time key format */ - flb_sds_t time_key_format; - - /* time key nanoseconds */ - int time_key_nanos; - - /* write operation config value */ - flb_sds_t write_operation; - - /* write operation / action */ - char *action; - - /* id_key */ - flb_sds_t id_key; - struct flb_record_accessor *ra_id_key; - - /* include_tag_key */ - int include_tag_key; - flb_sds_t tag_key; - - /* HTTP API */ - char uri[1024]; - - struct flb_record_accessor *ra_prefix_key; - - /* Upstream connection to the backend server */ - struct flb_upstream *u; - - /* Plugin output instance reference */ - struct flb_output_instance *ins; - - /* Compression algorithm */ - int compression; - flb_sds_t compression_str; -}; - -#endif diff --git a/fluent-bit/plugins/out_opensearch/os_conf.c b/fluent-bit/plugins/out_opensearch/os_conf.c deleted file mode 100644 index b814bd35f..000000000 --- a/fluent-bit/plugins/out_opensearch/os_conf.c +++ /dev/null @@ -1,411 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "opensearch.h" -#include "os_conf.h" - -struct flb_opensearch *flb_os_conf_create(struct flb_output_instance *ins, - struct flb_config *config) -{ - int len; - int io_flags = 0; - ssize_t ret; - char *buf; - const char *tmp; - const char *path; -#ifdef FLB_HAVE_AWS - char *aws_role_arn = NULL; - char *aws_external_id = NULL; - char *aws_session_name = NULL; -#endif - struct flb_uri *uri = ins->host.uri; - struct flb_uri_field *f_index = NULL; - struct flb_uri_field *f_type = NULL; - struct flb_upstream *upstream; - struct flb_opensearch *ctx; - - /* Allocate context */ - ctx = flb_calloc(1, sizeof(struct flb_opensearch)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - - /* only used if the config has been set from the command line */ - if (uri) { - if (uri->count >= 2) { - f_index = flb_uri_get(uri, 0); - f_type = flb_uri_get(uri, 1); - } - } - - /* Set default network configuration */ - flb_output_net_default("127.0.0.1", 9200, ins); - - /* Populate context with config map defaults and incoming properties */ - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_plg_error(ctx->ins, "configuration error"); - flb_os_conf_destroy(ctx); - return NULL; - } - - /* use TLS ? */ - if (ins->use_tls == FLB_TRUE) { - io_flags = FLB_IO_TLS; - } - else { - io_flags = FLB_IO_TCP; - } - - if (ins->host.ipv6 == FLB_TRUE) { - io_flags |= FLB_IO_IPV6; - } - - /* Prepare an upstream handler */ - upstream = flb_upstream_create(config, - ins->host.name, - ins->host.port, - io_flags, - ins->tls); - if (!upstream) { - flb_plg_error(ctx->ins, "cannot create Upstream context"); - flb_os_conf_destroy(ctx); - return NULL; - } - ctx->u = upstream; - - /* Set instance flags into upstream */ - flb_output_upstream_set(ctx->u, ins); - - /* Set manual Index and Type */ - if (f_index) { - ctx->index = flb_strdup(f_index->value); - } - else { - /* Check if the index has been set in the configuration */ - if (ctx->index) { - /* do we have a record accessor pattern ? */ - if (strchr(ctx->index, '$')) { - ctx->ra_index = flb_ra_create(ctx->index, FLB_TRUE); - if (!ctx->ra_index) { - flb_plg_error(ctx->ins, "invalid record accessor pattern set for 'index' property"); - flb_os_conf_destroy(ctx); - return NULL; - } - } - } - } - - if (f_type) { - ctx->type = flb_strdup(f_type->value); /* FIXME */ - } - - /* HTTP Payload (response) maximum buffer size (0 == unlimited) */ - if (ctx->buffer_size == -1) { - ctx->buffer_size = 0; - } - - /* Path */ - path = flb_output_get_property("path", ins); - if (!path) { - path = ""; - } - - /* Pipeline */ - tmp = flb_output_get_property("pipeline", ins); - if (tmp) { - snprintf(ctx->uri, sizeof(ctx->uri) - 1, "%s/_bulk/?pipeline=%s", path, tmp); - } - else { - snprintf(ctx->uri, sizeof(ctx->uri) - 1, "%s/_bulk", path); - } - - - if (ctx->id_key) { - ctx->ra_id_key = flb_ra_create(ctx->id_key, FLB_FALSE); - if (ctx->ra_id_key == NULL) { - flb_plg_error(ins, "could not create record accessor for Id Key"); - } - if (ctx->generate_id == FLB_TRUE) { - flb_plg_warn(ins, "Generate_ID is ignored when ID_key is set"); - ctx->generate_id = FLB_FALSE; - } - } - - if (ctx->write_operation) { - if (strcasecmp(ctx->write_operation, FLB_OS_WRITE_OP_INDEX) == 0) { - ctx->action = FLB_OS_WRITE_OP_INDEX; - } - else if (strcasecmp(ctx->write_operation, FLB_OS_WRITE_OP_CREATE) == 0) { - ctx->action = FLB_OS_WRITE_OP_CREATE; - } - else if (strcasecmp(ctx->write_operation, FLB_OS_WRITE_OP_UPDATE) == 0 - || strcasecmp(ctx->write_operation, FLB_OS_WRITE_OP_UPSERT) == 0) { - ctx->action = FLB_OS_WRITE_OP_UPDATE; - } - else { - flb_plg_error(ins, - "wrong Write_Operation (should be one of index, " - "create, update, upsert)"); - flb_os_conf_destroy(ctx); - return NULL; - } - - if (strcasecmp(ctx->action, FLB_OS_WRITE_OP_UPDATE) == 0 - && !ctx->ra_id_key && ctx->generate_id == FLB_FALSE) { - flb_plg_error(ins, - "id_key or generate_id must be set when Write_Operation " - "update or upsert"); - flb_os_conf_destroy(ctx); - return NULL; - } - } - - if (ctx->logstash_prefix_key) { - if (ctx->logstash_prefix_key[0] != '$') { - len = flb_sds_len(ctx->logstash_prefix_key); - buf = flb_malloc(len + 2); - if (!buf) { - flb_errno(); - flb_os_conf_destroy(ctx); - return NULL; - } - buf[0] = '$'; - memcpy(buf + 1, ctx->logstash_prefix_key, len); - buf[len + 1] = '\0'; - - ctx->ra_prefix_key = flb_ra_create(buf, FLB_TRUE); - flb_free(buf); - } - else { - ctx->ra_prefix_key = flb_ra_create(ctx->logstash_prefix_key, FLB_TRUE); - } - - if (!ctx->ra_prefix_key) { - flb_plg_error(ins, "invalid logstash_prefix_key pattern '%s'", tmp); - flb_os_conf_destroy(ctx); - return NULL; - } - } - - if (ctx->compression_str) { - if (strcasecmp(ctx->compression_str, "gzip") == 0) { - ctx->compression = FLB_OS_COMPRESSION_GZIP; - } - else { - ctx->compression = FLB_OS_COMPRESSION_NONE; - } - } - else { - ctx->compression = FLB_OS_COMPRESSION_NONE; - } - -#ifdef FLB_HAVE_AWS - /* AWS Auth Unsigned Headers */ - ctx->aws_unsigned_headers = flb_malloc(sizeof(struct mk_list)); - if (!ctx->aws_unsigned_headers) { - flb_os_conf_destroy(ctx); - return NULL; - } - flb_slist_create(ctx->aws_unsigned_headers); - ret = flb_slist_add(ctx->aws_unsigned_headers, "Content-Length"); - if (ret != 0) { - flb_os_conf_destroy(ctx); - return NULL; - } - - /* AWS Auth */ - ctx->has_aws_auth = FLB_FALSE; - tmp = flb_output_get_property("aws_auth", ins); - if (tmp) { - if (strncasecmp(tmp, "On", 2) == 0) { - ctx->has_aws_auth = FLB_TRUE; - flb_debug("[out_es] Enabled AWS Auth"); - - /* AWS provider needs a separate TLS instance */ - ctx->aws_tls = flb_tls_create(FLB_TLS_CLIENT_MODE, - FLB_TRUE, - ins->tls_debug, - ins->tls_vhost, - ins->tls_ca_path, - ins->tls_ca_file, - ins->tls_crt_file, - ins->tls_key_file, - ins->tls_key_passwd); - if (!ctx->aws_tls) { - flb_errno(); - flb_os_conf_destroy(ctx); - return NULL; - } - - tmp = flb_output_get_property("aws_region", ins); - if (!tmp) { - flb_error("[out_es] aws_auth enabled but aws_region not set"); - flb_os_conf_destroy(ctx); - return NULL; - } - ctx->aws_region = (char *) tmp; - - tmp = flb_output_get_property("aws_sts_endpoint", ins); - if (tmp) { - ctx->aws_sts_endpoint = (char *) tmp; - } - - ctx->aws_provider = flb_standard_chain_provider_create(config, - ctx->aws_tls, - ctx->aws_region, - ctx->aws_sts_endpoint, - NULL, - flb_aws_client_generator(), - ctx->aws_profile); - if (!ctx->aws_provider) { - flb_error("[out_es] Failed to create AWS Credential Provider"); - flb_os_conf_destroy(ctx); - return NULL; - } - - tmp = flb_output_get_property("aws_role_arn", ins); - if (tmp) { - /* Use the STS Provider */ - ctx->base_aws_provider = ctx->aws_provider; - aws_role_arn = (char *) tmp; - aws_external_id = NULL; - tmp = flb_output_get_property("aws_external_id", ins); - if (tmp) { - aws_external_id = (char *) tmp; - } - - aws_session_name = flb_sts_session_name(); - if (!aws_session_name) { - flb_error("[out_es] Failed to create aws iam role " - "session name"); - flb_os_conf_destroy(ctx); - return NULL; - } - - /* STS provider needs yet another separate TLS instance */ - ctx->aws_sts_tls = flb_tls_create(FLB_TLS_CLIENT_MODE, - FLB_TRUE, - ins->tls_debug, - ins->tls_vhost, - ins->tls_ca_path, - ins->tls_ca_file, - ins->tls_crt_file, - ins->tls_key_file, - ins->tls_key_passwd); - if (!ctx->aws_sts_tls) { - flb_errno(); - flb_os_conf_destroy(ctx); - return NULL; - } - - ctx->aws_provider = flb_sts_provider_create(config, - ctx->aws_sts_tls, - ctx-> - base_aws_provider, - aws_external_id, - aws_role_arn, - aws_session_name, - ctx->aws_region, - ctx->aws_sts_endpoint, - NULL, - flb_aws_client_generator()); - /* Session name can be freed once provider is created */ - flb_free(aws_session_name); - if (!ctx->aws_provider) { - flb_error("[out_es] Failed to create AWS STS Credential " - "Provider"); - flb_os_conf_destroy(ctx); - return NULL; - } - - } - - /* initialize credentials in sync mode */ - ctx->aws_provider->provider_vtable->sync(ctx->aws_provider); - ctx->aws_provider->provider_vtable->init(ctx->aws_provider); - /* set back to async */ - ctx->aws_provider->provider_vtable->async(ctx->aws_provider); - ctx->aws_provider->provider_vtable->upstream_set(ctx->aws_provider, ctx->ins); - } - } -#endif - - return ctx; -} - -int flb_os_conf_destroy(struct flb_opensearch *ctx) -{ - if (!ctx) { - return 0; - } - - if (ctx->u) { - flb_upstream_destroy(ctx->u); - } - if (ctx->ra_id_key) { - flb_ra_destroy(ctx->ra_id_key); - ctx->ra_id_key = NULL; - } - -#ifdef FLB_HAVE_AWS - if (ctx->base_aws_provider) { - flb_aws_provider_destroy(ctx->base_aws_provider); - } - - if (ctx->aws_provider) { - flb_aws_provider_destroy(ctx->aws_provider); - } - - if (ctx->aws_tls) { - flb_tls_destroy(ctx->aws_tls); - } - - if (ctx->aws_sts_tls) { - flb_tls_destroy(ctx->aws_sts_tls); - } - - if (ctx->aws_unsigned_headers) { - flb_slist_destroy(ctx->aws_unsigned_headers); - flb_free(ctx->aws_unsigned_headers); - } -#endif - - if (ctx->ra_prefix_key) { - flb_ra_destroy(ctx->ra_prefix_key); - } - - if (ctx->ra_index) { - flb_ra_destroy(ctx->ra_index); - } - - flb_free(ctx); - - return 0; -} diff --git a/fluent-bit/plugins/out_opensearch/os_conf.h b/fluent-bit/plugins/out_opensearch/os_conf.h deleted file mode 100644 index a48376a07..000000000 --- a/fluent-bit/plugins/out_opensearch/os_conf.h +++ /dev/null @@ -1,33 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_OPENSEARCH_CONF_H -#define FLB_OUT_OPENSEARCH_CONF_H - -#include -#include -#include - -#include "opensearch.h" - -struct flb_opensearch *flb_os_conf_create(struct flb_output_instance *ins, - struct flb_config *config); -int flb_os_conf_destroy(struct flb_opensearch *ctx); - -#endif diff --git a/fluent-bit/plugins/out_opentelemetry/CMakeLists.txt b/fluent-bit/plugins/out_opentelemetry/CMakeLists.txt deleted file mode 100644 index 03c697ab2..000000000 --- a/fluent-bit/plugins/out_opentelemetry/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(src - opentelemetry.c - opentelemetry_conf.c - ) - -FLB_PLUGIN(out_opentelemetry "${src}" "") diff --git a/fluent-bit/plugins/out_opentelemetry/opentelemetry.c b/fluent-bit/plugins/out_opentelemetry/opentelemetry.c deleted file mode 100644 index c981cc27c..000000000 --- a/fluent-bit/plugins/out_opentelemetry/opentelemetry.c +++ /dev/null @@ -1,1207 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include -#include -#include - -#include -#include - -extern cfl_sds_t cmt_encode_opentelemetry_create(struct cmt *cmt); -extern void cmt_encode_opentelemetry_destroy(cfl_sds_t text); - -#include "opentelemetry.h" -#include "opentelemetry_conf.h" - -static inline Opentelemetry__Proto__Common__V1__AnyValue *msgpack_object_to_otlp_any_value(struct msgpack_object *o); - -static inline void otlp_any_value_destroy(Opentelemetry__Proto__Common__V1__AnyValue *value); -static inline void otlp_kvarray_destroy(Opentelemetry__Proto__Common__V1__KeyValue **kvarray, size_t entry_count); -static inline void otlp_kvpair_destroy(Opentelemetry__Proto__Common__V1__KeyValue *kvpair); -static inline void otlp_kvlist_destroy(Opentelemetry__Proto__Common__V1__KeyValueList *kvlist); -static inline void otlp_array_destroy(Opentelemetry__Proto__Common__V1__ArrayValue *array); - -static inline void otlp_kvarray_destroy(Opentelemetry__Proto__Common__V1__KeyValue **kvarray, size_t entry_count) -{ - size_t index; - - if (kvarray != NULL) { - for (index = 0 ; index < entry_count ; index++) { - if (kvarray[index] != NULL) { - otlp_kvpair_destroy(kvarray[index]); - kvarray[index] = NULL; - } - } - - flb_free(kvarray); - } -} - -static inline void otlp_kvpair_destroy(Opentelemetry__Proto__Common__V1__KeyValue *kvpair) -{ - if (kvpair != NULL) { - if (kvpair->key != NULL) { - flb_free(kvpair->key); - } - - if (kvpair->value != NULL) { - otlp_any_value_destroy(kvpair->value); - } - - flb_free(kvpair); - } -} - -static inline void otlp_kvlist_destroy(Opentelemetry__Proto__Common__V1__KeyValueList *kvlist) -{ - size_t index; - - if (kvlist != NULL) { - if (kvlist->values != NULL) { - for (index = 0 ; index < kvlist->n_values ; index++) { - otlp_kvpair_destroy(kvlist->values[index]); - } - - flb_free(kvlist->values); - } - - flb_free(kvlist); - } -} - -static inline void otlp_array_destroy(Opentelemetry__Proto__Common__V1__ArrayValue *array) -{ - size_t index; - - if (array != NULL) { - if (array->values != NULL) { - for (index = 0 ; index < array->n_values ; index++) { - otlp_any_value_destroy(array->values[index]); - } - - flb_free(array->values); - } - - flb_free(array); - } -} - -static inline void otlp_any_value_destroy(Opentelemetry__Proto__Common__V1__AnyValue *value) -{ - if (value != NULL) { - if (value->value_case == OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE) { - if (value->string_value != NULL) { - flb_free(value->string_value); - } - } - else if (value->value_case == OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_ARRAY_VALUE) { - if (value->array_value != NULL) { - otlp_array_destroy(value->array_value); - } - } - else if (value->value_case == OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_KVLIST_VALUE) { - if (value->kvlist_value != NULL) { - otlp_kvlist_destroy(value->kvlist_value); - } - } - else if (value->value_case == OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BYTES_VALUE) { - if (value->bytes_value.data != NULL) { - flb_free(value->bytes_value.data); - } - } - - value->string_value = NULL; - - flb_free(value); - } -} - -static int http_post(struct opentelemetry_context *ctx, - const void *body, size_t body_len, - const char *tag, int tag_len, - const char *uri) -{ - size_t final_body_len; - void *final_body; - int compressed; - int out_ret; - size_t b_sent; - struct flb_connection *u_conn; - struct mk_list *head; - int ret; - struct flb_slist_entry *key; - struct flb_slist_entry *val; - struct flb_config_map_val *mv; - struct flb_http_client *c; - - compressed = FLB_FALSE; - - u_conn = flb_upstream_conn_get(ctx->u); - - if (u_conn == NULL) { - flb_plg_error(ctx->ins, - "no upstream connections available to %s:%i", - ctx->u->tcp_host, - ctx->u->tcp_port); - - return FLB_RETRY; - } - - if (ctx->compress_gzip) { - ret = flb_gzip_compress((void *) body, body_len, - &final_body, &final_body_len); - - if (ret == 0) { - compressed = FLB_TRUE; - } else { - flb_plg_error(ctx->ins, "cannot gzip payload, disabling compression"); - } - } else { - final_body = (void *) body; - final_body_len = body_len; - } - - /* Create HTTP client context */ - c = flb_http_client(u_conn, FLB_HTTP_POST, uri, - final_body, final_body_len, - ctx->host, ctx->port, - ctx->proxy, 0); - - if (c == NULL) { - flb_plg_error(ctx->ins, "error initializing http client"); - - if (compressed) { - flb_free(final_body); - } - - flb_upstream_conn_release(u_conn); - - return FLB_RETRY; - } - - if (c->proxy.host != NULL) { - flb_plg_debug(ctx->ins, "[http_client] proxy host: %s port: %i", - c->proxy.host, c->proxy.port); - } - - /* Allow duplicated headers ? */ - flb_http_allow_duplicated_headers(c, FLB_FALSE); - - /* - * Direct assignment of the callback context to the HTTP client context. - * This needs to be improved through a more clean API. - */ - c->cb_ctx = ctx->ins->callback; - - flb_http_add_header(c, - FLB_OPENTELEMETRY_CONTENT_TYPE_HEADER_NAME, - sizeof(FLB_OPENTELEMETRY_CONTENT_TYPE_HEADER_NAME) - 1, - FLB_OPENTELEMETRY_MIME_PROTOBUF_LITERAL, - sizeof(FLB_OPENTELEMETRY_MIME_PROTOBUF_LITERAL) - 1); - - /* Basic Auth headers */ - if (ctx->http_user != NULL && - ctx->http_passwd != NULL) { - flb_http_basic_auth(c, ctx->http_user, ctx->http_passwd); - } - - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - - flb_config_map_foreach(head, mv, ctx->headers) { - key = mk_list_entry_first(mv->val.list, struct flb_slist_entry, _head); - val = mk_list_entry_last(mv->val.list, struct flb_slist_entry, _head); - - flb_http_add_header(c, - key->str, flb_sds_len(key->str), - val->str, flb_sds_len(val->str)); - } - - if (compressed) { - flb_http_set_content_encoding_gzip(c); - } - - ret = flb_http_do(c, &b_sent); - - if (ret == 0) { - /* - * Only allow the following HTTP status: - * - * - 200: OK - * - 201: Created - * - 202: Accepted - * - 203: no authorative resp - * - 204: No Content - * - 205: Reset content - * - */ - if (c->resp.status < 200 || c->resp.status > 205) { - if (ctx->log_response_payload && - c->resp.payload != NULL && - c->resp.payload_size > 0) { - flb_plg_error(ctx->ins, "%s:%i, HTTP status=%i\n%.*s", - ctx->host, ctx->port, - c->resp.status, - (int) c->resp.payload_size, - c->resp.payload); - } - else { - flb_plg_error(ctx->ins, "%s:%i, HTTP status=%i", - ctx->host, ctx->port, c->resp.status); - } - - out_ret = FLB_RETRY; - } - else { - if (ctx->log_response_payload && - c->resp.payload != NULL && - c->resp.payload_size > 0) { - flb_plg_info(ctx->ins, "%s:%i, HTTP status=%i\n%.*s", - ctx->host, ctx->port, - c->resp.status, - (int) c->resp.payload_size, - c->resp.payload); - } - else { - flb_plg_info(ctx->ins, "%s:%i, HTTP status=%i", - ctx->host, ctx->port, - c->resp.status); - } - - out_ret = FLB_OK; - } - } - else { - flb_plg_error(ctx->ins, "could not flush records to %s:%i (http_do=%i)", - ctx->host, ctx->port, ret); - - out_ret = FLB_RETRY; - } - - if (compressed) { - flb_free(final_body); - } - - /* Destroy HTTP client context */ - flb_http_client_destroy(c); - - /* Release the TCP connection */ - flb_upstream_conn_release(u_conn); - - return out_ret; -} - -static void append_labels(struct opentelemetry_context *ctx, - struct cmt *cmt) -{ - struct flb_kv *kv; - struct mk_list *head; - - mk_list_foreach(head, &ctx->kv_labels) { - kv = mk_list_entry(head, struct flb_kv, _head); - cmt_label_add(cmt, kv->key, kv->val); - } -} - -static void clear_array(Opentelemetry__Proto__Logs__V1__LogRecord **logs, - size_t log_count) -{ - size_t index; - - if (logs == NULL){ - return; - } - - for (index = 0 ; index < log_count ; index++) { - if (logs[index]->body != NULL) { - otlp_any_value_destroy(logs[index]->body); - - logs[index]->body = NULL; - } - - if (logs[index]->attributes != NULL) { - otlp_kvarray_destroy(logs[index]->attributes, - logs[index]->n_attributes); - - logs[index]->attributes = NULL; - } - } -} - -static Opentelemetry__Proto__Common__V1__ArrayValue *otlp_array_value_initialize(size_t entry_count) -{ - Opentelemetry__Proto__Common__V1__ArrayValue *value; - - value = flb_calloc(1, sizeof(Opentelemetry__Proto__Common__V1__ArrayValue)); - - if (value != NULL) { - opentelemetry__proto__common__v1__array_value__init(value); - - if (entry_count > 0) { - value->values = \ - flb_calloc(entry_count, - sizeof(Opentelemetry__Proto__Common__V1__AnyValue *)); - - if (value->values == NULL) { - flb_free(value); - - value = NULL; - } - else { - value->n_values = entry_count; - } - } - } - - return value; -} - -static Opentelemetry__Proto__Common__V1__KeyValue *otlp_kvpair_value_initialize() -{ - Opentelemetry__Proto__Common__V1__KeyValue *value; - - value = flb_calloc(1, sizeof(Opentelemetry__Proto__Common__V1__KeyValue)); - - if (value != NULL) { - opentelemetry__proto__common__v1__key_value__init(value); - } - - return value; -} - -static Opentelemetry__Proto__Common__V1__KeyValueList *otlp_kvlist_value_initialize(size_t entry_count) -{ - Opentelemetry__Proto__Common__V1__KeyValueList *value; - - value = flb_calloc(1, sizeof(Opentelemetry__Proto__Common__V1__KeyValueList)); - - if (value != NULL) { - opentelemetry__proto__common__v1__key_value_list__init(value); - - if (entry_count > 0) { - value->values = \ - flb_calloc(entry_count, - sizeof(Opentelemetry__Proto__Common__V1__KeyValue *)); - - if (value->values == NULL) { - flb_free(value); - - value = NULL; - } - else { - value->n_values = entry_count; - } - } - } - - return value; -} - -static Opentelemetry__Proto__Common__V1__AnyValue *otlp_any_value_initialize(int data_type, size_t entry_count) -{ - Opentelemetry__Proto__Common__V1__AnyValue *value; - - value = flb_calloc(1, sizeof(Opentelemetry__Proto__Common__V1__AnyValue)); - - if (value == NULL) { - return NULL; - } - - opentelemetry__proto__common__v1__any_value__init(value); - - if (data_type == MSGPACK_OBJECT_STR) { - value->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE; - } - else if (data_type == MSGPACK_OBJECT_NIL) { - value->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE__NOT_SET; - } - else if (data_type == MSGPACK_OBJECT_BOOLEAN) { - value->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BOOL_VALUE; - } - else if (data_type == MSGPACK_OBJECT_POSITIVE_INTEGER || data_type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { - value->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_INT_VALUE; - } - else if (data_type == MSGPACK_OBJECT_FLOAT32 || data_type == MSGPACK_OBJECT_FLOAT64) { - value->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_DOUBLE_VALUE; - } - else if (data_type == MSGPACK_OBJECT_ARRAY) { - value->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_ARRAY_VALUE; - value->array_value = otlp_array_value_initialize(entry_count); - - if (value->array_value == NULL) { - flb_free(value); - - value = NULL; - } - } - else if (data_type == MSGPACK_OBJECT_MAP) { - value->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_KVLIST_VALUE; - - value->kvlist_value = otlp_kvlist_value_initialize(entry_count); - - if (value->kvlist_value == NULL) { - flb_free(value); - - value = NULL; - } - } - else if (data_type == MSGPACK_OBJECT_BIN) { - value->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BYTES_VALUE; - } - else { - flb_free(value); - - value = NULL; - } - - return value; -} - -static inline Opentelemetry__Proto__Common__V1__AnyValue *msgpack_boolean_to_otlp_any_value(struct msgpack_object *o) -{ - Opentelemetry__Proto__Common__V1__AnyValue *result; - - result = otlp_any_value_initialize(MSGPACK_OBJECT_BOOLEAN, 0); - - if (result != NULL) { - result->bool_value = o->via.boolean; - } - - return result; -} - -static inline Opentelemetry__Proto__Common__V1__AnyValue *msgpack_integer_to_otlp_any_value(struct msgpack_object *o) -{ - Opentelemetry__Proto__Common__V1__AnyValue *result; - - result = otlp_any_value_initialize(o->type, 0); - - if (result != NULL) { - if (o->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - result->int_value = (int64_t) o->via.u64; - } - else { - result->int_value = o->via.i64; - } - } - - return result; -} - -static inline Opentelemetry__Proto__Common__V1__AnyValue *msgpack_float_to_otlp_any_value(struct msgpack_object *o) -{ - Opentelemetry__Proto__Common__V1__AnyValue *result; - - result = otlp_any_value_initialize(o->type, 0); - - if (result != NULL) { - result->double_value = o->via.f64; - } - - return result; -} - -static inline Opentelemetry__Proto__Common__V1__AnyValue *msgpack_string_to_otlp_any_value(struct msgpack_object *o) -{ - Opentelemetry__Proto__Common__V1__AnyValue *result; - - result = otlp_any_value_initialize(MSGPACK_OBJECT_STR, 0); - - if (result != NULL) { - result->string_value = flb_strndup(o->via.str.ptr, o->via.str.size); - - if (result->string_value == NULL) { - otlp_any_value_destroy(result); - - result = NULL; - } - } - - return result; -} - -static inline Opentelemetry__Proto__Common__V1__AnyValue *msgpack_nil_to_otlp_any_value(struct msgpack_object *o) -{ - Opentelemetry__Proto__Common__V1__AnyValue *result; - - result = otlp_any_value_initialize(MSGPACK_OBJECT_NIL, 0); - - if (result != NULL) { - result->string_value = NULL; - } - - return result; -} - -static inline Opentelemetry__Proto__Common__V1__AnyValue *msgpack_bin_to_otlp_any_value(struct msgpack_object *o) -{ - Opentelemetry__Proto__Common__V1__AnyValue *result; - - result = otlp_any_value_initialize(MSGPACK_OBJECT_BIN, 0); - - if (result != NULL) { - result->bytes_value.len = o->via.bin.size; - result->bytes_value.data = flb_malloc(o->via.bin.size); - - if (result->bytes_value.data == NULL) { - otlp_any_value_destroy(result); - - result = NULL; - } - - memcpy(result->bytes_value.data, o->via.bin.ptr, o->via.bin.size); - } - - return result; -} - -static inline Opentelemetry__Proto__Common__V1__AnyValue *msgpack_array_to_otlp_any_value(struct msgpack_object *o) -{ - size_t entry_count; - Opentelemetry__Proto__Common__V1__AnyValue *entry_value; - Opentelemetry__Proto__Common__V1__AnyValue *result; - size_t index; - msgpack_object *p; - - entry_count = o->via.array.size; - result = otlp_any_value_initialize(MSGPACK_OBJECT_ARRAY, entry_count); - - p = o->via.array.ptr; - - if (result != NULL) { - index = 0; - - for (index = 0 ; index < entry_count ; index++) { - entry_value = msgpack_object_to_otlp_any_value(&p[index]); - - if (entry_value == NULL) { - otlp_any_value_destroy(result); - - result = NULL; - - break; - } - - result->array_value->values[index] = entry_value; - } - } - - return result; -} - -static inline Opentelemetry__Proto__Common__V1__KeyValue *msgpack_kv_to_otlp_any_value(struct msgpack_object_kv *input_pair) -{ - Opentelemetry__Proto__Common__V1__KeyValue *kv; - - kv = otlp_kvpair_value_initialize(); - if (kv == NULL) { - flb_errno(); - - return NULL; - } - - kv->key = flb_strndup(input_pair->key.via.str.ptr, input_pair->key.via.str.size); - if (kv->key == NULL) { - flb_errno(); - flb_free(kv); - - return NULL; - } - - kv->value = msgpack_object_to_otlp_any_value(&input_pair->val); - if (kv->value == NULL) { - flb_free(kv->key); - flb_free(kv); - - return NULL; - } - - return kv; -} - -static inline Opentelemetry__Proto__Common__V1__KeyValue **msgpack_map_to_otlp_kvarray(struct msgpack_object *o, size_t *entry_count) -{ - Opentelemetry__Proto__Common__V1__KeyValue **result; - size_t index; - msgpack_object_kv *kv; - - *entry_count = o->via.map.size; - result = flb_calloc(*entry_count, sizeof(Opentelemetry__Proto__Common__V1__KeyValue *)); - - if (result != NULL) { - for (index = 0; index < *entry_count; index++) { - kv = &o->via.map.ptr[index]; - result[index] = msgpack_kv_to_otlp_any_value(kv); - } - } - else { - *entry_count = 0; - } - - return result; -} - -static inline Opentelemetry__Proto__Common__V1__AnyValue *msgpack_map_to_otlp_any_value(struct msgpack_object *o) -{ - size_t entry_count; - Opentelemetry__Proto__Common__V1__AnyValue *result; - Opentelemetry__Proto__Common__V1__KeyValue *keyvalue; - size_t index; - msgpack_object_kv *kv; - - entry_count = o->via.map.size; - result = otlp_any_value_initialize(MSGPACK_OBJECT_MAP, entry_count); - - if (result != NULL) { - - for (index = 0; index < entry_count; index++) { - kv = &o->via.map.ptr[index]; - keyvalue = msgpack_kv_to_otlp_any_value(kv); - result->kvlist_value->values[index] = keyvalue; - } - } - - return result; -} - -static inline Opentelemetry__Proto__Common__V1__AnyValue *msgpack_object_to_otlp_any_value(struct msgpack_object *o) -{ - Opentelemetry__Proto__Common__V1__AnyValue *result; - - result = NULL; - - switch (o->type) { - case MSGPACK_OBJECT_NIL: - result = msgpack_nil_to_otlp_any_value(o); - break; - - case MSGPACK_OBJECT_BOOLEAN: - result = msgpack_boolean_to_otlp_any_value(o); - break; - - case MSGPACK_OBJECT_POSITIVE_INTEGER: - case MSGPACK_OBJECT_NEGATIVE_INTEGER: - result = msgpack_integer_to_otlp_any_value(o); - break; - - case MSGPACK_OBJECT_FLOAT32: - case MSGPACK_OBJECT_FLOAT64: - result = msgpack_float_to_otlp_any_value(o); - break; - - case MSGPACK_OBJECT_STR: - result = msgpack_string_to_otlp_any_value(o); - break; - - case MSGPACK_OBJECT_MAP: - result = msgpack_map_to_otlp_any_value(o); - break; - - case MSGPACK_OBJECT_BIN: - result = msgpack_bin_to_otlp_any_value(o); - break; - - case MSGPACK_OBJECT_ARRAY: - result = msgpack_array_to_otlp_any_value(o); - break; - - default: - break; - } - - /* This function will fail if it receives an object with - * type MSGPACK_OBJECT_EXT - */ - - return result; -} - -static int flush_to_otel(struct opentelemetry_context *ctx, - struct flb_event_chunk *event_chunk, - Opentelemetry__Proto__Logs__V1__LogRecord **logs, - size_t log_count) -{ - Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest export_logs; - Opentelemetry__Proto__Logs__V1__ScopeLogs scope_log; - Opentelemetry__Proto__Logs__V1__ResourceLogs resource_log; - Opentelemetry__Proto__Logs__V1__ResourceLogs *resource_logs[1]; - Opentelemetry__Proto__Logs__V1__ScopeLogs *scope_logs[1]; - void *body; - unsigned len; - int res; - - opentelemetry__proto__collector__logs__v1__export_logs_service_request__init(&export_logs); - opentelemetry__proto__logs__v1__resource_logs__init(&resource_log); - opentelemetry__proto__logs__v1__scope_logs__init(&scope_log); - - scope_log.log_records = logs; - scope_log.n_log_records = log_count; - scope_logs[0] = &scope_log; - - resource_log.scope_logs = scope_logs; - resource_log.n_scope_logs = 1; - resource_logs[0] = &resource_log; - - export_logs.resource_logs = resource_logs; - export_logs.n_resource_logs = 1; - - len = opentelemetry__proto__collector__logs__v1__export_logs_service_request__get_packed_size(&export_logs); - body = flb_calloc(len, sizeof(char)); - if (!body) { - flb_errno(); - return FLB_ERROR; - } - - opentelemetry__proto__collector__logs__v1__export_logs_service_request__pack(&export_logs, body); - - // send post request to opentelemetry with content type application/x-protobuf - res = http_post(ctx, body, len, - event_chunk->tag, - flb_sds_len(event_chunk->tag), - ctx->logs_uri); - - flb_free(body); - - return res; -} - -static int process_logs(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *ins, void *out_context, - struct flb_config *config) -{ - size_t log_record_count; - Opentelemetry__Proto__Logs__V1__LogRecord **log_record_list; - Opentelemetry__Proto__Logs__V1__LogRecord *log_records; - Opentelemetry__Proto__Common__V1__AnyValue *log_object; - struct flb_log_event_decoder *decoder; - struct flb_log_event event; - size_t index; - struct opentelemetry_context *ctx; - int res; - - ctx = (struct opentelemetry_context *) out_context; - - log_record_list = (Opentelemetry__Proto__Logs__V1__LogRecord **) \ - flb_calloc(ctx->batch_size, - sizeof(Opentelemetry__Proto__Logs__V1__LogRecord *)); - - if (log_record_list == NULL) { - flb_errno(); - - return -1; - } - - log_records = (Opentelemetry__Proto__Logs__V1__LogRecord *) - flb_calloc(ctx->batch_size, - sizeof(Opentelemetry__Proto__Logs__V1__LogRecord)); - - if (log_records == NULL) { - flb_errno(); - - flb_free(log_record_list); - - return -2; - } - - for(index = 0 ; index < ctx->batch_size ; index++) { - log_record_list[index] = &log_records[index]; - } - - decoder = flb_log_event_decoder_create((char *) event_chunk->data, - event_chunk->size); - - if (decoder == NULL) { - flb_plg_error(ctx->ins, "could not initialize record decoder"); - - flb_free(log_record_list); - flb_free(log_records); - - return -1; - } - - log_record_count = 0; - - res = FLB_OK; - - while (flb_log_event_decoder_next(decoder, &event) == 0 && - res == FLB_OK) { - opentelemetry__proto__logs__v1__log_record__init(&log_records[log_record_count]); - log_records[log_record_count].attributes = \ - msgpack_map_to_otlp_kvarray(event.metadata, - &log_records[log_record_count].n_attributes); - - log_object = msgpack_object_to_otlp_any_value(event.body); - - if (log_object == NULL) { - flb_plg_error(ctx->ins, "log event conversion failure"); - res = FLB_ERROR; - continue; - } - - - log_records[log_record_count].body = log_object; - log_records[log_record_count].time_unix_nano = flb_time_to_nanosec(&event.timestamp); - - log_record_count++; - - if (log_record_count >= ctx->batch_size) { - res = flush_to_otel(ctx, - event_chunk, - log_record_list, - log_record_count); - - clear_array(log_record_list, log_record_count); - - log_record_count = 0; - } - } - - flb_log_event_decoder_destroy(decoder); - - if (log_record_count > 0 && - res == FLB_OK) { - res = flush_to_otel(ctx, - event_chunk, - log_record_list, - log_record_count); - - clear_array(log_record_list, log_record_count); - } - - flb_free(log_record_list); - flb_free(log_records); - - return res; -} - -static int process_metrics(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *ins, void *out_context, - struct flb_config *config) -{ - int c = 0; - int ok; - int ret; - int result; - cfl_sds_t encoded_chunk; - flb_sds_t buf = NULL; - size_t diff = 0; - size_t off = 0; - struct cmt *cmt; - struct opentelemetry_context *ctx = out_context; - - /* Initialize vars */ - ctx = out_context; - ok = CMT_DECODE_MSGPACK_SUCCESS; - result = FLB_OK; - - /* Buffer to concatenate multiple metrics contexts */ - buf = flb_sds_create_size(event_chunk->size); - if (!buf) { - flb_plg_error(ctx->ins, "could not allocate outgoing buffer"); - return FLB_RETRY; - } - - flb_plg_debug(ctx->ins, "cmetrics msgpack size: %lu", - event_chunk->size); - - /* Decode and encode every CMetric context */ - diff = 0; - while ((ret = cmt_decode_msgpack_create(&cmt, - (char *) event_chunk->data, - event_chunk->size, &off)) == ok) { - /* append labels set by config */ - append_labels(ctx, cmt); - - /* Create a OpenTelemetry payload */ - encoded_chunk = cmt_encode_opentelemetry_create(cmt); - if (encoded_chunk == NULL) { - flb_plg_error(ctx->ins, - "Error encoding context as opentelemetry"); - result = FLB_ERROR; - cmt_destroy(cmt); - goto exit; - } - - flb_plg_debug(ctx->ins, "cmetric_id=%i decoded %lu-%lu payload_size=%lu", - c, diff, off, flb_sds_len(encoded_chunk)); - c++; - diff = off; - - /* concat buffer */ - flb_sds_cat_safe(&buf, encoded_chunk, flb_sds_len(encoded_chunk)); - - /* release */ - cmt_encode_opentelemetry_destroy(encoded_chunk); - cmt_destroy(cmt); - } - - if (ret == CMT_DECODE_MSGPACK_INSUFFICIENT_DATA && c > 0) { - flb_plg_debug(ctx->ins, "final payload size: %lu", flb_sds_len(buf)); - if (buf && flb_sds_len(buf) > 0) { - /* Send HTTP request */ - result = http_post(ctx, buf, flb_sds_len(buf), - event_chunk->tag, - flb_sds_len(event_chunk->tag), - ctx->metrics_uri); - - /* Debug http_post() result statuses */ - if (result == FLB_OK) { - flb_plg_debug(ctx->ins, "http_post result FLB_OK"); - } - else if (result == FLB_ERROR) { - flb_plg_debug(ctx->ins, "http_post result FLB_ERROR"); - } - else if (result == FLB_RETRY) { - flb_plg_debug(ctx->ins, "http_post result FLB_RETRY"); - } - } - flb_sds_destroy(buf); - buf = NULL; - return result; - } - else { - flb_plg_error(ctx->ins, "Error decoding msgpack encoded context"); - return FLB_ERROR; - } - -exit: - if (buf) { - flb_sds_destroy(buf); - } - return result; -} - -static int process_traces(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *ins, void *out_context, - struct flb_config *config) -{ - int ret; - int result; - cfl_sds_t encoded_chunk; - flb_sds_t buf = NULL; - size_t off = 0; - struct ctrace *ctr; - struct opentelemetry_context *ctx = out_context; - - /* Initialize vars */ - ctx = out_context; - result = FLB_OK; - - buf = flb_sds_create_size(event_chunk->size); - if (!buf) { - flb_plg_error(ctx->ins, "could not allocate outgoing buffer"); - return FLB_RETRY; - } - - flb_plg_debug(ctx->ins, "ctraces msgpack size: %lu", - event_chunk->size); - - while (ctr_decode_msgpack_create(&ctr, - (char *) event_chunk->data, - event_chunk->size, &off) == 0) { - /* Create a OpenTelemetry payload */ - encoded_chunk = ctr_encode_opentelemetry_create(ctr); - if (encoded_chunk == NULL) { - flb_plg_error(ctx->ins, - "Error encoding context as opentelemetry"); - result = FLB_ERROR; - ctr_destroy(ctr); - goto exit; - } - - /* concat buffer */ - ret = flb_sds_cat_safe(&buf, encoded_chunk, flb_sds_len(encoded_chunk)); - if (ret != 0) { - flb_plg_error(ctx->ins, "Error appending encoded trace to buffer"); - result = FLB_ERROR; - ctr_encode_opentelemetry_destroy(encoded_chunk); - ctr_destroy(ctr); - goto exit; - } - - /* release */ - ctr_encode_opentelemetry_destroy(encoded_chunk); - ctr_destroy(ctr); - } - - flb_plg_debug(ctx->ins, "final payload size: %lu", flb_sds_len(buf)); - if (buf && flb_sds_len(buf) > 0) { - /* Send HTTP request */ - result = http_post(ctx, buf, flb_sds_len(buf), - event_chunk->tag, - flb_sds_len(event_chunk->tag), - ctx->traces_uri); - - /* Debug http_post() result statuses */ - if (result == FLB_OK) { - flb_plg_debug(ctx->ins, "http_post result FLB_OK"); - } - else if (result == FLB_ERROR) { - flb_plg_debug(ctx->ins, "http_post result FLB_ERROR"); - } - else if (result == FLB_RETRY) { - flb_plg_debug(ctx->ins, "http_post result FLB_RETRY"); - } - } - -exit: - if (buf) { - flb_sds_destroy(buf); - } - return result; -} - -static int cb_opentelemetry_exit(void *data, struct flb_config *config) -{ - struct opentelemetry_context *ctx; - - ctx = (struct opentelemetry_context *) data; - - flb_opentelemetry_context_destroy(ctx); - - return 0; -} - -static int cb_opentelemetry_init(struct flb_output_instance *ins, - struct flb_config *config, - void *data) -{ - struct opentelemetry_context *ctx; - - ctx = flb_opentelemetry_context_create(ins, config); - if (!ctx) { - return -1; - } - - if (ctx->batch_size <= 0){ - ctx->batch_size = atoi(DEFAULT_LOG_RECORD_BATCH_SIZE); - } - - flb_output_set_context(ins, ctx); - - return 0; -} - -static void cb_opentelemetry_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *ins, void *out_context, - struct flb_config *config) -{ - int result = FLB_RETRY; - - if (event_chunk->type == FLB_INPUT_METRICS){ - result = process_metrics(event_chunk, out_flush, ins, out_context, config); - } - else if (event_chunk->type == FLB_INPUT_LOGS){ - result = process_logs(event_chunk, out_flush, ins, out_context, config); - } - else if (event_chunk->type == FLB_INPUT_TRACES){ - result = process_traces(event_chunk, out_flush, ins, out_context, config); - } - FLB_OUTPUT_RETURN(result); -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_SLIST_1, "add_label", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct opentelemetry_context, - add_labels), - "Adds a custom label to the metrics use format: 'add_label name value'" - }, - - { - FLB_CONFIG_MAP_STR, "proxy", NULL, - 0, FLB_FALSE, 0, - "Specify an HTTP Proxy. The expected format of this value is http://host:port. " - }, - { - FLB_CONFIG_MAP_STR, "http_user", NULL, - 0, FLB_TRUE, offsetof(struct opentelemetry_context, http_user), - "Set HTTP auth user" - }, - { - FLB_CONFIG_MAP_STR, "http_passwd", "", - 0, FLB_TRUE, offsetof(struct opentelemetry_context, http_passwd), - "Set HTTP auth password" - }, - { - FLB_CONFIG_MAP_SLIST_1, "header", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct opentelemetry_context, headers), - "Add a HTTP header key/value pair. Multiple headers can be set" - }, - { - FLB_CONFIG_MAP_STR, "metrics_uri", "/v1/metrics", - 0, FLB_TRUE, offsetof(struct opentelemetry_context, metrics_uri), - "Specify an optional HTTP URI for the target OTel endpoint." - }, - { - FLB_CONFIG_MAP_STR, "logs_uri", "/v1/logs", - 0, FLB_TRUE, offsetof(struct opentelemetry_context, logs_uri), - "Specify an optional HTTP URI for the target OTel endpoint." - }, - { - FLB_CONFIG_MAP_STR, "traces_uri", "/v1/traces", - 0, FLB_TRUE, offsetof(struct opentelemetry_context, traces_uri), - "Specify an optional HTTP URI for the target OTel endpoint." - }, - { - FLB_CONFIG_MAP_BOOL, "log_response_payload", "true", - 0, FLB_TRUE, offsetof(struct opentelemetry_context, log_response_payload), - "Specify if the response paylod should be logged or not" - }, - { - FLB_CONFIG_MAP_INT, "batch_size", DEFAULT_LOG_RECORD_BATCH_SIZE, - 0, FLB_TRUE, offsetof(struct opentelemetry_context, batch_size), - "Set the maximum number of log records to be flushed at a time" - }, - { - FLB_CONFIG_MAP_STR, "compress", NULL, - 0, FLB_FALSE, 0, - "Set payload compression mechanism. Option available is 'gzip'" - }, - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_output_plugin out_opentelemetry_plugin = { - .name = "opentelemetry", - .description = "OpenTelemetry", - .cb_init = cb_opentelemetry_init, - .cb_flush = cb_opentelemetry_flush, - .cb_exit = cb_opentelemetry_exit, - .config_map = config_map, - .event_type = FLB_OUTPUT_LOGS | FLB_OUTPUT_METRICS | FLB_OUTPUT_TRACES, - .flags = FLB_OUTPUT_NET | FLB_IO_OPT_TLS, -}; diff --git a/fluent-bit/plugins/out_opentelemetry/opentelemetry.h b/fluent-bit/plugins/out_opentelemetry/opentelemetry.h deleted file mode 100644 index 94e424ac7..000000000 --- a/fluent-bit/plugins/out_opentelemetry/opentelemetry.h +++ /dev/null @@ -1,80 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_OPENTELEMETRY_H -#define FLB_OUT_OPENTELEMETRY_H - -#include - -#define FLB_OPENTELEMETRY_CONTENT_TYPE_HEADER_NAME "Content-Type" -#define FLB_OPENTELEMETRY_MIME_PROTOBUF_LITERAL "application/x-protobuf" - -/* - * This lets you send log records in batches instead of a request per log record - * It might be removed in furthur versions since if we have a large number of - * log records, and a later batch fails, Fluent Bit will retry ALL the batches, - * including the ones that succeeded. This is not ideal. - */ -#define DEFAULT_LOG_RECORD_BATCH_SIZE "1000" - -/* Plugin context */ -struct opentelemetry_context { - /* HTTP Auth */ - char *http_user; - char *http_passwd; - - /* Proxy */ - const char *proxy; - char *proxy_host; - int proxy_port; - - /* HTTP URI */ - char *traces_uri; - char *metrics_uri; - char *logs_uri; - char *host; - int port; - - /* Number of logs to flush at a time */ - int batch_size; - - /* Log the response paylod */ - int log_response_payload; - - /* config reader for 'add_label' */ - struct mk_list *add_labels; - - /* internal labels ready to append */ - struct mk_list kv_labels; - - /* Upstream connection to the backend server */ - struct flb_upstream *u; - - /* Arbitrary HTTP headers */ - struct mk_list *headers; - - - /* instance context */ - struct flb_output_instance *ins; - - /* Compression mode (gzip) */ - int compress_gzip; -}; - -#endif diff --git a/fluent-bit/plugins/out_opentelemetry/opentelemetry_conf.c b/fluent-bit/plugins/out_opentelemetry/opentelemetry_conf.c deleted file mode 100644 index 5c9c8f82c..000000000 --- a/fluent-bit/plugins/out_opentelemetry/opentelemetry_conf.c +++ /dev/null @@ -1,262 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "opentelemetry.h" -#include "opentelemetry_conf.h" - -static int config_add_labels(struct flb_output_instance *ins, - struct opentelemetry_context *ctx) -{ - struct mk_list *head; - struct flb_config_map_val *mv; - struct flb_slist_entry *k = NULL; - struct flb_slist_entry *v = NULL; - struct flb_kv *kv; - - if (!ctx->add_labels || mk_list_size(ctx->add_labels) == 0) { - return 0; - } - - /* iterate all 'add_label' definitions */ - flb_config_map_foreach(head, mv, ctx->add_labels) { - if (mk_list_size(mv->val.list) != 2) { - flb_plg_error(ins, "'add_label' expects a key and a value, " - "e.g: 'add_label version 1.8.0'"); - return -1; - } - - k = mk_list_entry_first(mv->val.list, struct flb_slist_entry, _head); - v = mk_list_entry_last(mv->val.list, struct flb_slist_entry, _head); - - kv = flb_kv_item_create(&ctx->kv_labels, k->str, v->str); - if (!kv) { - flb_plg_error(ins, "could not append label %s=%s\n", k->str, v->str); - return -1; - } - } - - return 0; -} - -/* -* Check if a Proxy have been set, if so the Upstream manager will use -* the Proxy end-point and then we let the HTTP client know about it, so -* it can adjust the HTTP requests. -*/ - -static void check_proxy(struct flb_output_instance *ins, - struct opentelemetry_context *ctx, - char *host, char *port, - char *protocol, char *uri){ - - const char *tmp = NULL; - int ret; - tmp = flb_output_get_property("proxy", ins); - if (tmp) { - ret = flb_utils_url_split(tmp, &protocol, &host, &port, &uri); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not parse proxy parameter: '%s'", tmp); - flb_free(ctx); - } - - ctx->proxy_host = host; - ctx->proxy_port = atoi(port); - ctx->proxy = tmp; - flb_free(protocol); - flb_free(port); - flb_free(uri); - uri = NULL; - } - else { - flb_output_net_default("127.0.0.1", 80, ins); - } -} - -static char *sanitize_uri(char *uri){ - char *new_uri; - int uri_len; - - if (uri == NULL) { - uri = flb_strdup("/"); - } - else if (uri[0] != '/') { - uri_len = strlen(uri); - new_uri = flb_calloc(uri_len + 2, sizeof(char)); - - if (new_uri != NULL) { - new_uri[0] = '/'; - - strncat(new_uri, uri, uri_len + 1); - } - - uri = new_uri; - } - - /* This function could return NULL if flb_calloc fails */ - - return uri; -} - -struct opentelemetry_context *flb_opentelemetry_context_create( - struct flb_output_instance *ins, struct flb_config *config) -{ - int ret; - int io_flags = 0; - char *protocol = NULL; - char *host = NULL; - char *port = NULL; - char *metrics_uri = NULL; - char *traces_uri = NULL; - char *logs_uri = NULL; - struct flb_upstream *upstream; - struct opentelemetry_context *ctx = NULL; - const char *tmp = NULL; - - /* Allocate plugin context */ - ctx = flb_calloc(1, sizeof(struct opentelemetry_context)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - mk_list_init(&ctx->kv_labels); - - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return NULL; - } - - /* Parse 'add_label' */ - ret = config_add_labels(ins, ctx); - if (ret == -1) { - return NULL; - } - - check_proxy(ins, ctx, host, port, protocol, metrics_uri); - check_proxy(ins, ctx, host, port, protocol, logs_uri); - - /* Check if SSL/TLS is enabled */ -#ifdef FLB_HAVE_TLS - if (ins->use_tls == FLB_TRUE) { - io_flags = FLB_IO_TLS; - } - else { - io_flags = FLB_IO_TCP; - } -#else - io_flags = FLB_IO_TCP; -#endif - - if (ins->host.ipv6 == FLB_TRUE) { - io_flags |= FLB_IO_IPV6; - } - - if (ctx->proxy) { - flb_plg_trace(ctx->ins, "Upstream Proxy=%s:%i", - ctx->proxy_host, ctx->proxy_port); - upstream = flb_upstream_create(config, - ctx->proxy_host, - ctx->proxy_port, - io_flags, ins->tls); - } - else { - upstream = flb_upstream_create(config, - ins->host.name, - ins->host.port, - io_flags, ins->tls); - } - - if (!upstream) { - flb_free(ctx); - return NULL; - } - - logs_uri = sanitize_uri(ctx->logs_uri); - traces_uri = sanitize_uri(ctx->traces_uri); - metrics_uri = sanitize_uri(ctx->metrics_uri); - - ctx->u = upstream; - ctx->host = ins->host.name; - ctx->port = ins->host.port; - - if (logs_uri == NULL) { - flb_plg_trace(ctx->ins, - "Could not allocate memory for sanitized " - "log endpoint uri"); - } - else { - ctx->logs_uri = logs_uri; - } - - if (traces_uri == NULL) { - flb_plg_trace(ctx->ins, - "Could not allocate memory for sanitized " - "trace endpoint uri"); - } - else { - ctx->traces_uri = traces_uri; - } - - if (metrics_uri == NULL) { - flb_plg_trace(ctx->ins, - "Could not allocate memory for sanitized " - "metric endpoint uri"); - } - else { - ctx->metrics_uri = metrics_uri; - } - - - /* Set instance flags into upstream */ - flb_output_upstream_set(ctx->u, ins); - - tmp = flb_output_get_property("compress", ins); - ctx->compress_gzip = FLB_FALSE; - if (tmp) { - if (strcasecmp(tmp, "gzip") == 0) { - ctx->compress_gzip = FLB_TRUE; - } - } - - return ctx; -} - -void flb_opentelemetry_context_destroy( - struct opentelemetry_context *ctx) -{ - if (!ctx) { - return; - } - - flb_kv_release(&ctx->kv_labels); - - if (ctx->u) { - flb_upstream_destroy(ctx->u); - } - - flb_free(ctx->proxy_host); - flb_free(ctx); -} diff --git a/fluent-bit/plugins/out_opentelemetry/opentelemetry_conf.h b/fluent-bit/plugins/out_opentelemetry/opentelemetry_conf.h deleted file mode 100644 index 974f7fea5..000000000 --- a/fluent-bit/plugins/out_opentelemetry/opentelemetry_conf.h +++ /dev/null @@ -1,33 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_OPENTELEMETRY_CONF_H -#define FLB_OUT_OPENTELEMETRY_CONF_H - -#include -#include - -#include "opentelemetry.h" - -struct opentelemetry_context *flb_opentelemetry_context_create( - struct flb_output_instance *ins, struct flb_config *config); -void flb_opentelemetry_context_destroy( - struct opentelemetry_context *ctx); - -#endif diff --git a/fluent-bit/plugins/out_oracle_log_analytics/CMakeLists.txt b/fluent-bit/plugins/out_oracle_log_analytics/CMakeLists.txt deleted file mode 100644 index 81f971a95..000000000 --- a/fluent-bit/plugins/out_oracle_log_analytics/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(src - oci_logan.c - oci_logan_conf.c - ) - -FLB_PLUGIN(out_oracle_log_analytics "${src}" "") diff --git a/fluent-bit/plugins/out_oracle_log_analytics/oci_logan.c b/fluent-bit/plugins/out_oracle_log_analytics/oci_logan.c deleted file mode 100644 index 630812e2d..000000000 --- a/fluent-bit/plugins/out_oracle_log_analytics/oci_logan.c +++ /dev/null @@ -1,1313 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include - -#include "oci_logan_conf.h" -#include "oci_logan.h" - - -static int check_config_from_record(msgpack_object key, - char *name, int len) -{ - if (key.type != MSGPACK_OBJECT_STR) { - return FLB_FALSE; - } - - if (key.via.str.size != len) { - return FLB_FALSE; - } - - - return memcmp(key.via.str.ptr, name, len) == 0; -} - -/* - * Authorization: Signature version="1",keyId="//", - * algorithm="rsa-sha256",headers="(request-target) date x-content-sha256 content-type content-length", - * signature="signature" - */ -static flb_sds_t create_authorization_header_content(struct flb_oci_logan *ctx, - flb_sds_t signature) -{ - flb_sds_t content; - - content = flb_sds_create_size(512); - flb_sds_cat_safe(&content, FLB_OCI_SIGN_SIGNATURE_VERSION, - sizeof(FLB_OCI_SIGN_SIGNATURE_VERSION) - 1); - flb_sds_cat_safe(&content, ",", 1); - flb_sds_cat_safe(&content, FLB_OCI_SIGN_KEYID, - sizeof(FLB_OCI_SIGN_KEYID) - 1); - flb_sds_cat_safe(&content, "=\"", 2); - flb_sds_cat_safe(&content, ctx->key_id, flb_sds_len(ctx->key_id)); - flb_sds_cat_safe(&content, "\",", 2); - flb_sds_cat_safe(&content, FLB_OCI_SIGN_ALGORITHM, - sizeof(FLB_OCI_SIGN_ALGORITHM) - 1); - flb_sds_cat_safe(&content, ",", 1); - flb_sds_cat_safe(&content, FLB_OCI_SIGN_HEADERS, - sizeof(FLB_OCI_SIGN_HEADERS) - 1); - flb_sds_cat_safe(&content, ",", 1); - flb_sds_cat_safe(&content, FLB_OCI_SIGN_SIGNATURE, - sizeof(FLB_OCI_SIGN_SIGNATURE) - 1); - flb_sds_cat_safe(&content, "=\"", 2); - flb_sds_cat_safe(&content, signature, flb_sds_len(signature)); - flb_sds_cat_safe(&content, "\"", 1); - - return content; -} - -static flb_sds_t create_base64_sha256_signature(struct flb_oci_logan *ctx, - flb_sds_t signing_string) -{ - int len = 0, ret; - size_t outlen; - flb_sds_t signature; - unsigned char sha256_buf[32] = { 0 }; - unsigned char sig[256] = { 0 }; - size_t sig_len = sizeof(sig); - - ret = flb_hash_simple(FLB_HASH_SHA256, - (unsigned char*) signing_string, - flb_sds_len(signing_string), - sha256_buf, sizeof(sha256_buf)); - - if(ret != FLB_CRYPTO_SUCCESS) { - flb_plg_error(ctx->ins, "error generating hash buffer"); - return NULL; - } - - ret = flb_crypto_sign_simple(FLB_CRYPTO_PRIVATE_KEY, - FLB_CRYPTO_PADDING_PKCS1, - FLB_HASH_SHA256, - (unsigned char *) ctx->private_key, - flb_sds_len(ctx->private_key), - sha256_buf, sizeof(sha256_buf), - sig, &sig_len); - - - if (ret != FLB_CRYPTO_SUCCESS) { - flb_plg_error(ctx->ins, "error signing SHA256"); - return NULL; - } - - signature = flb_sds_create_size(512); - if (!signature) { - flb_errno(); - return NULL; - } - - /* base 64 encode */ - len = flb_sds_alloc(signature) - 1; - flb_base64_encode((unsigned char*) signature, len, &outlen, sig, - sizeof(sig)); - signature[outlen] = '\0'; - flb_sds_len_set(signature, outlen); - - return signature; -} - -static flb_sds_t get_date(void) -{ - - flb_sds_t rfc1123date; - time_t t; - size_t size; - struct tm tm = { 0 }; - - /* Format Date */ - rfc1123date = flb_sds_create_size(32); - if (!rfc1123date) { - flb_errno(); - return NULL; - } - - t = time(NULL); - if (!gmtime_r(&t, &tm)) { - flb_errno(); - flb_sds_destroy(rfc1123date); - return NULL; - } - size = strftime(rfc1123date, flb_sds_alloc(rfc1123date) - 1, - "%a, %d %b %Y %H:%M:%S GMT", &tm); - if (size <= 0) { - flb_errno(); - flb_sds_destroy(rfc1123date); - return NULL; - } - flb_sds_len_set(rfc1123date, size); - return rfc1123date; -} - -static flb_sds_t add_header_and_signing(struct flb_http_client *c, - flb_sds_t signing_str, const char *header, int headersize, - const char *val, int val_size) -{ - if (!signing_str) { - return NULL; - } - - flb_http_add_header(c, header, headersize, val, val_size); - - flb_sds_cat_safe(&signing_str, "\n", 1); - flb_sds_cat_safe(&signing_str, header, headersize); - flb_sds_cat_safe(&signing_str, ": ", 2); - flb_sds_cat_safe(&signing_str, val, val_size); - - return signing_str; -} - -static int build_headers(struct flb_http_client *c, struct flb_oci_logan *ctx, - flb_sds_t json, flb_sds_t hostname, int port, flb_sds_t uri) -{ - int ret = -1; - flb_sds_t tmp_sds = NULL; - flb_sds_t signing_str = NULL; - flb_sds_t rfc1123date = NULL; - flb_sds_t encoded_uri = NULL; - flb_sds_t signature = NULL; - flb_sds_t auth_header_str = NULL; - - flb_sds_t tmp_ref = NULL; - - size_t tmp_len = 0; - - unsigned char sha256_buf[32] = { 0 }; - - tmp_sds = flb_sds_create_size(512); - if (!tmp_sds) { - flb_errno(); - goto error_label; - } - - signing_str = flb_sds_create_size(1024); - if (!signing_str) { - flb_errno(); - goto error_label; - } - - /* Add (requeset-target) to signing string */ - encoded_uri = flb_uri_encode(uri, flb_sds_len(uri)); - if (!encoded_uri) { - flb_errno(); - goto error_label; - } - flb_sds_cat_safe(&signing_str, FLB_OCI_HEADER_REQUEST_TARGET, - sizeof(FLB_OCI_HEADER_REQUEST_TARGET) - 1); - flb_sds_cat_safe(&signing_str, ": post ", sizeof(": post ") - 1); - flb_sds_cat_safe(&signing_str, encoded_uri, - flb_sds_len(encoded_uri)); - - /* Add Host to Header */ - if (((c->flags & FLB_IO_TLS) && c->port == 443) - || (!(c->flags & FLB_IO_TLS) && c->port == 80)) { - /* default port */ - tmp_ref = flb_sds_copy(tmp_sds, c->host, strlen(c->host)); - } - else { - tmp_ref = flb_sds_printf(&tmp_sds, "%s:%i", c->host, c->port); - } - if (!tmp_ref) { - flb_plg_error(ctx->ins, "cannot compose temporary host header"); - goto error_label; - } - tmp_sds = tmp_ref; - tmp_ref = NULL; - - signing_str = add_header_and_signing(c, signing_str, FLB_OCI_HEADER_HOST, - sizeof(FLB_OCI_HEADER_HOST) - 1, - tmp_sds, flb_sds_len(tmp_sds)); - if (!signing_str) { - flb_plg_error(ctx->ins, "cannot compose signing string"); - goto error_label; - } - - /* Add Date header */ - rfc1123date = get_date(); - if (!rfc1123date) { - flb_plg_error(ctx->ins, "cannot compose temporary date header"); - goto error_label; - } - signing_str = add_header_and_signing(c, signing_str, FLB_OCI_HEADER_DATE, - sizeof(FLB_OCI_HEADER_DATE) - 1, rfc1123date, - flb_sds_len(rfc1123date)); - if (!signing_str) { - flb_plg_error(ctx->ins, "cannot compose signing string"); - goto error_label; - } - - /* Add x-content-sha256 Header */ - ret = flb_hash_simple(FLB_HASH_SHA256, - (unsigned char*) json, - flb_sds_len(json), - sha256_buf, sizeof(sha256_buf)); - - if (ret != FLB_CRYPTO_SUCCESS) { - flb_plg_error(ctx->ins, "error forming hash buffer for x-content-sha256 Header"); - goto error_label; - } - - flb_base64_encode((unsigned char*) tmp_sds, flb_sds_len(tmp_sds) - 1, - &tmp_len, sha256_buf, sizeof(sha256_buf)); - - tmp_sds[tmp_len] = '\0'; - flb_sds_len_set(tmp_sds, tmp_len); - - signing_str = add_header_and_signing(c, signing_str, - FLB_OCI_HEADER_X_CONTENT_SHA256, - sizeof(FLB_OCI_HEADER_X_CONTENT_SHA256) - 1, tmp_sds, - flb_sds_len(tmp_sds)); - if (!signing_str) { - flb_plg_error(ctx->ins, "cannot compose signing string"); - goto error_label; - } - - /* Add content-Type */ - signing_str = add_header_and_signing(c, signing_str, - FLB_OCI_HEADER_CONTENT_TYPE, sizeof(FLB_OCI_HEADER_CONTENT_TYPE) - 1, - FLB_OCI_HEADER_CONTENT_TYPE_VAL, - sizeof(FLB_OCI_HEADER_CONTENT_TYPE_VAL) - 1); - if (!signing_str) { - flb_plg_error(ctx->ins, "cannot compose signing string"); - goto error_label; - } - - /* Add content-Length */ - tmp_len = snprintf(tmp_sds, flb_sds_alloc(tmp_sds) - 1, "%i", - (int) flb_sds_len(json)); - flb_sds_len_set(tmp_sds, tmp_len); - signing_str = add_header_and_signing(c, signing_str, - FLB_OCI_HEADER_CONTENT_LENGTH, sizeof(FLB_OCI_HEADER_CONTENT_LENGTH) - 1, - tmp_sds, flb_sds_len(tmp_sds)); - if (!signing_str) { - flb_plg_error(ctx->ins, "cannot compose signing string"); - goto error_label; - } - - /* Add Authorization header */ - signature = create_base64_sha256_signature(ctx, signing_str); - if (!signature) { - flb_plg_error(ctx->ins, "cannot compose signing signature"); - goto error_label; - } - - auth_header_str = create_authorization_header_content(ctx, signature); - if (!auth_header_str) { - flb_plg_error(ctx->ins, "cannot compose authorization header"); - goto error_label; - } - - flb_http_add_header(c, FLB_OCI_HEADER_AUTH, sizeof(FLB_OCI_HEADER_AUTH) - 1, - auth_header_str, flb_sds_len(auth_header_str)); - - /* User-Agent */ - flb_http_add_header(c, FLB_OCI_HEADER_USER_AGENT, - sizeof(FLB_OCI_HEADER_USER_AGENT) - 1, - FLB_OCI_HEADER_USER_AGENT_VAL, - sizeof(FLB_OCI_HEADER_USER_AGENT_VAL) - 1); - - /* Accept */ - flb_http_add_header(c, "Accept", 6, "*/*", 3); - - ret = 0; - - error_label: - if (tmp_sds) { - flb_sds_destroy(tmp_sds); - } - if (signing_str) { - flb_sds_destroy(signing_str); - } - if (rfc1123date) { - flb_sds_destroy(rfc1123date); - } - if (encoded_uri) { - flb_sds_destroy(encoded_uri); - } - if (signature) { - flb_sds_destroy(signature); - } - if (auth_header_str) { - flb_sds_destroy(auth_header_str); - } - return ret; -} - -static struct flb_oci_error_response* parse_response_error(struct flb_oci_logan *ctx, - char *response, size_t response_len) -{ - int tok_size = 32, ret, i; - jsmn_parser parser; - jsmntok_t *t; - jsmntok_t *tokens; - char *key; - char *val; - int key_len; - int val_len; - struct flb_oci_error_response *error_response; - - jsmn_init(&parser); - - tokens = flb_calloc(1, sizeof(jsmntok_t) * tok_size); - if (!tokens) { - flb_errno(); - return NULL; - } - - ret = jsmn_parse(&parser, response, response_len, tokens, tok_size); - - if (ret == JSMN_ERROR_INVAL || ret == JSMN_ERROR_PART) { - flb_free(tokens); - flb_plg_info(ctx->ins, - "Unable to parser error response. reponse is not valid json"); - return NULL; - } - tok_size = ret; - - error_response = flb_calloc(1, sizeof(struct flb_oci_error_response)); - if (!error_response) { - flb_errno(); - flb_free(tokens); - return NULL; - } - - /* Parse JSON tokens */ - for (i = 0; i < tok_size; i++) { - t = &tokens[i]; - - if (t->start == -1 || t->end == -1 || (t->start == 0 && t->end == 0)) { - break; - } - - if (t->type != JSMN_STRING) { - continue; - } - - key = response + t->start; - key_len = (t->end - t->start); - - i++; - t = &tokens[i]; - val = response + t->start; - val_len = (t->end - t->start); - - if (val_len < 1) { - continue; - } - - if ((key_len == sizeof(FLB_OCI_ERROR_RESPONSE_CODE) - 1) - && strncasecmp(key, FLB_OCI_ERROR_RESPONSE_CODE, - sizeof(FLB_OCI_ERROR_RESPONSE_CODE) - 1) == 0) { - /* code */ - error_response->code = flb_sds_create_len(val, val_len); - if (!error_response->code) { - flb_free(error_response); - flb_free(tokens); - return NULL; - } - } - else if ((key_len == sizeof(FLB_OCI_ERROR_RESPONSE_MESSAGE) - 1) - && strncasecmp(key, FLB_OCI_ERROR_RESPONSE_MESSAGE, - sizeof(FLB_OCI_ERROR_RESPONSE_MESSAGE) - 1) == 0) { - - /* message */ - error_response->message = flb_sds_create_len(val, val_len); - if (!error_response->message) { - flb_free(error_response); - flb_free(tokens); - return NULL; - } - } - } - - flb_free(tokens); - return error_response; -} - -static int retry_error(struct flb_http_client *c, struct flb_oci_logan *ctx) -{ - struct flb_oci_error_response *error_response = NULL; - int tmp_len; - int ret = FLB_FALSE; - - /* possible retry error message */ - if ( !(c->resp.status == 400 || c->resp.status == 401 - || c->resp.status == 404 || c->resp.status == 409 - || c->resp.status == 429 || c->resp.status == 500)) { - return FLB_FALSE; - } - - /* parse error message */ - error_response = parse_response_error(ctx, c->resp.payload, - c->resp.payload_size); - if (!error_response) { - return FLB_FALSE; - } - - if (error_response->code) { - tmp_len = flb_sds_len(error_response->code); - if (c->resp.status == 400 - && (tmp_len == sizeof(FLB_OCI_ERROR_CODE_RELATED_RESOURCE_NOT_FOUND) - 1) - && strncasecmp(error_response->code, FLB_OCI_ERROR_CODE_RELATED_RESOURCE_NOT_FOUND, tmp_len) == 0) { - ret = FLB_TRUE; - } - else if( c->resp.status == 401 - &&( tmp_len == sizeof(FLB_OCI_ERROR_CODE_NOT_AUTHENTICATED)-1 ) - && strncasecmp(error_response->code, FLB_OCI_ERROR_CODE_NOT_AUTHENTICATED, tmp_len) == 0) { - ret = FLB_TRUE; - } - else if (c->resp.status == 404 - && (tmp_len == sizeof(FLB_OCI_ERROR_CODE_NOT_AUTHENTICATEDORNOTFOUND) - 1) - && strncasecmp(error_response->code, FLB_OCI_ERROR_CODE_NOT_AUTHENTICATEDORNOTFOUND, tmp_len) == 0) { - ret = FLB_TRUE; - } - else if (c->resp.status == 409 - && (tmp_len == sizeof(FLB_OCI_ERROR_CODE_INCORRECTSTATE) - 1) - && strncasecmp(error_response->code, FLB_OCI_ERROR_CODE_INCORRECTSTATE, tmp_len) == 0) { - ret = FLB_TRUE; - } - else if (c->resp.status == 409 - && (tmp_len == sizeof(FLB_OCI_ERROR_CODE_NOT_AUTH_OR_RESOURCE_EXIST) - 1) - && strncasecmp(error_response->code, FLB_OCI_ERROR_CODE_NOT_AUTH_OR_RESOURCE_EXIST, tmp_len) == 0) { - ret = FLB_TRUE; - } - else if (c->resp.status == 429 - && (tmp_len == sizeof(FLB_OCI_ERROR_CODE_TOO_MANY_REQUESTS) - 1) - && strncasecmp(error_response->code, FLB_OCI_ERROR_CODE_TOO_MANY_REQUESTS, tmp_len) == 0) { - ret = FLB_TRUE; - } - else if (c->resp.status == 500 - && (tmp_len == sizeof(FLB_OCI_ERROR_CODE_INTERNAL_SERVER_ERROR) - 1) - && strncasecmp(error_response->code, FLB_OCI_ERROR_CODE_INTERNAL_SERVER_ERROR, tmp_len) == 0) { - ret = FLB_TRUE; - } - } - - if (error_response->code) { - flb_sds_destroy(error_response->code); - } - if (error_response->message) { - flb_sds_destroy(error_response->message); - } - flb_free(error_response); - - return ret; -} - -static int cb_oci_logan_init(struct flb_output_instance *ins, - struct flb_config *config, - void *data) -{ - struct flb_oci_logan *ctx; - ctx = flb_oci_logan_conf_create(ins, config); - if (!ctx) { - flb_plg_error(ins, "cannot initialize plugin"); - return -1; - } - flb_plg_info(ins, "initialized logan plugin"); - flb_output_set_context(ins, ctx); - flb_output_set_http_debug_callbacks(ins); - - return 0; -} - -static flb_sds_t compose_uri(struct flb_oci_logan *ctx, - flb_sds_t log_set, flb_sds_t log_group_id) -{ - flb_sds_t uri_param; - flb_sds_t full_uri; - - uri_param = flb_sds_create_size(512); - if (!uri_param) { - flb_errno(); - return NULL; - } - - /* LogGroupId */ - if (log_group_id) { - if (flb_sds_len(uri_param) > 0) { - flb_sds_cat_safe(&uri_param, "&", 1); - } - flb_sds_cat_safe(&uri_param, FLB_OCI_LOG_GROUP_ID, - FLB_OCI_LOG_GROUP_ID_SIZE); - flb_sds_cat_safe(&uri_param, "=", 1); - flb_sds_cat_safe(&uri_param, log_group_id, - flb_sds_len(log_group_id)); - } - - if (!uri_param) { - return NULL; - } - - /* logSet */ - if (log_set) { - if (flb_sds_len(uri_param) > 0) { - flb_sds_cat_safe(&uri_param, "&", 1); - } - flb_sds_cat_safe(&uri_param, FLB_OCI_LOG_SET, - FLB_OCI_LOG_SET_SIZE); - flb_sds_cat_safe(&uri_param, "=", 1); - flb_sds_cat_safe(&uri_param, log_set, - flb_sds_len(log_set)); - } - - if (!uri_param) { - return NULL; - } - - flb_sds_cat_safe(&uri_param, "&", 1); - flb_sds_cat_safe(&uri_param, FLB_OCI_PAYLOAD_TYPE, - sizeof(FLB_OCI_PAYLOAD_TYPE) - 1); - flb_sds_cat_safe(&uri_param, "=", 1); - flb_sds_cat_safe(&uri_param, "JSON", 4); - - - if (!uri_param) { - return NULL; - } - - - if (flb_sds_len(uri_param) == 0) { - flb_sds_destroy(uri_param); - return flb_sds_create(ctx->uri); - } - - full_uri = flb_sds_create_size( - flb_sds_len(ctx->uri) + 1 + flb_sds_len(uri_param)); - if (!full_uri) { - flb_errno(); - flb_sds_destroy(uri_param); - return NULL; - } - - flb_sds_cat_safe(&full_uri, ctx->uri, flb_sds_len(ctx->uri)); - flb_sds_cat_safe(&full_uri, "?", 1); - flb_sds_cat_safe(&full_uri, uri_param, flb_sds_len(uri_param)); - - flb_sds_destroy(uri_param); - - return full_uri; -} - -static int flush_to_endpoint(struct flb_oci_logan *ctx, - flb_sds_t payload, - flb_sds_t log_group_id, - flb_sds_t log_set_id) -{ - int out_ret = FLB_RETRY; - int http_ret; - size_t b_sent; - flb_sds_t full_uri; - struct flb_http_client *c = NULL; - struct flb_connection *u_conn; - - full_uri = compose_uri(ctx, log_set_id, log_group_id); - if(!full_uri) { - flb_plg_error(ctx->ins, "unable to compose uri for logGroup: %s logSet: %s", - ctx->oci_la_log_group_id, ctx->oci_la_log_set_id); - } - - flb_plg_debug(ctx->ins, "full_uri=%s", full_uri); - - u_conn = flb_upstream_conn_get(ctx->u); - if(!u_conn) { - goto error_label; - } - /* Create HTTP client context */ - c = flb_http_client(u_conn, FLB_HTTP_POST, full_uri, (void*) payload, - flb_sds_len(payload), ctx->ins->host.name, ctx->ins->host.port, ctx->proxy, 0); - if (!c) { - goto error_label; - } - flb_http_allow_duplicated_headers(c, FLB_FALSE); - - flb_plg_debug(ctx->ins, "built client"); - flb_http_buffer_size(c, FLB_HTTP_DATA_SIZE_MAX); - if (build_headers(c, ctx, payload, ctx->ins->host.name, ctx->ins->host.port, full_uri) < 0) { - flb_plg_error(ctx->ins, "failed to build headers"); - goto error_label; - } - flb_plg_debug(ctx->ins, "built request"); - - out_ret = FLB_OK; - - http_ret = flb_http_do(c, &b_sent); - flb_plg_debug(ctx->ins, "placed request"); - - if (http_ret == 0) { - - if (c->resp.status != 200) { - flb_plg_debug(ctx->ins, "request header %s", c->header_buf); - - out_ret = FLB_ERROR; - - if (c->resp.payload && c->resp.payload_size > 0) { - if (retry_error(c, ctx) == FLB_TRUE) { - out_ret = FLB_RETRY; - } - - flb_plg_error(ctx->ins, "%s:%i, retry=%s, HTTP status=%i\n%s", - ctx->ins->host.name, ctx->ins->host.port, - (out_ret == FLB_RETRY ? "true" : "false"), - c->resp.status, c->resp.payload); - } - else { - flb_plg_error(ctx->ins, "%s:%i, retry=%s, HTTP status=%i", - ctx->ins->host.name, ctx->ins->host.port, - (out_ret == FLB_RETRY ? "true" : "false"), - c->resp.status); - } - } - } - else { - out_ret = FLB_RETRY; - flb_plg_error(ctx->ins, "could not flush records to %s:%i (http_do=%i), retry=%s", - ctx->ins->host.name, ctx->ins->host.port, - http_ret, (out_ret == FLB_RETRY ? "true" : "false")); - goto error_label; - } - - - - error_label: - if (full_uri) { - flb_sds_destroy(full_uri); - } - - /* Destroy HTTP client context */ - if (c) { - flb_http_client_destroy(c); - } - - /* Release the TCP connection */ - if (u_conn) { - flb_upstream_conn_release(u_conn); - } - - return out_ret; - -} - -static void pack_oci_fields(msgpack_packer *packer, - struct flb_oci_logan *ctx) -{ - int num_global_meta = 0; - int num_event_meta = 0; - int pck_sz = 2; - struct mk_list *head = NULL; - struct metadata_obj *f; - - - /* number of meta properties */ - if(ctx->oci_la_global_metadata != NULL) { - num_global_meta = mk_list_size(&ctx->global_metadata_fields); - } - if(ctx->oci_la_metadata != NULL) { - num_event_meta = mk_list_size(&ctx->log_event_metadata_fields); - } - - - if (num_global_meta > 0) { - msgpack_pack_map(packer, 2); - msgpack_pack_str(packer, FLB_OCI_LOG_METADATA_SIZE); - msgpack_pack_str_body(packer, FLB_OCI_LOG_METADATA, - FLB_OCI_LOG_METADATA_SIZE); - - msgpack_pack_map(packer, num_global_meta); - /* pack kv list */ - mk_list_foreach(head, &ctx->global_metadata_fields) { - f = mk_list_entry(head, struct metadata_obj, _head); - - msgpack_pack_str(packer, flb_sds_len(f->key)); - msgpack_pack_str_body(packer, f->key, flb_sds_len(f->key)); - - msgpack_pack_str(packer, flb_sds_len(f->val)); - msgpack_pack_str_body(packer, f->val, flb_sds_len(f->val)); - - } - - } - else { - msgpack_pack_map(packer, 1); - } - - /* - *logEvents":[ - { - "entityId":"", - "logSourceName":"LinuxSyslogSource", - "logPath":"/var/log/messages", - "metadata":{ - "Error ID":"1", - "Environment":"prod", - "Client Host Region":"PST" - }, - "logRecords":[ - "May 8 2017 04:02:36 blr00akm syslogd 1.4.1: shutdown.", - "May 8 2017 04:02:37 blr00akm syslogd 1.4.1: restart." - ] - }, - { - - } - ] - */ - msgpack_pack_str(packer, FLB_OCI_LOG_EVENTS_SIZE); - msgpack_pack_str_body(packer, FLB_OCI_LOG_EVENTS, FLB_OCI_LOG_EVENTS_SIZE); - - msgpack_pack_array(packer, 1); - - if (ctx->oci_la_entity_id) { - pck_sz++; - } - if (ctx->oci_la_log_path) { - pck_sz++; - } - if (ctx->oci_la_entity_type) { - pck_sz++; - } - - if (num_event_meta > 0) { - pck_sz++; - } - - msgpack_pack_map(packer, pck_sz); /* entityId, logSourceName, logPath, logRecords */ - - - /* "entityType:"" */ - if (ctx->oci_la_entity_type) { - msgpack_pack_str(packer, FLB_OCI_ENTITY_TYPE_SIZE); - msgpack_pack_str_body(packer, FLB_OCI_ENTITY_TYPE, FLB_OCI_ENTITY_TYPE_SIZE); - msgpack_pack_str(packer, flb_sds_len(ctx->oci_la_entity_type)); - msgpack_pack_str_body(packer, ctx->oci_la_entity_type, - flb_sds_len(ctx->oci_la_entity_type)); - } - - /* "entityId":"", */ - if (ctx->oci_la_entity_id) { - msgpack_pack_str(packer, FLB_OCI_ENTITY_ID_SIZE); - msgpack_pack_str_body(packer, FLB_OCI_ENTITY_ID, FLB_OCI_ENTITY_ID_SIZE); - msgpack_pack_str(packer, flb_sds_len(ctx->oci_la_entity_id)); - msgpack_pack_str_body(packer, ctx->oci_la_entity_id, - flb_sds_len(ctx->oci_la_entity_id)); - } - - - /* "logSourceName":"", */ - msgpack_pack_str(packer, FLB_OCI_LOG_SOURCE_NAME_SIZE); - msgpack_pack_str_body(packer, FLB_OCI_LOG_SOURCE_NAME, - FLB_OCI_LOG_SOURCE_NAME_SIZE); - msgpack_pack_str(packer, flb_sds_len(ctx->oci_la_log_source_name)); - msgpack_pack_str_body(packer, ctx->oci_la_log_source_name, - flb_sds_len(ctx->oci_la_log_source_name)); - - - /* "logPath":"" */ - if (ctx->oci_la_log_path) { - msgpack_pack_str(packer, FLB_OCI_LOG_PATH_SIZE); - msgpack_pack_str_body(packer, FLB_OCI_LOG_PATH, FLB_OCI_LOG_PATH_SIZE); - msgpack_pack_str(packer, flb_sds_len(ctx->oci_la_log_path)); - msgpack_pack_str_body(packer, ctx->oci_la_log_path, - flb_sds_len(ctx->oci_la_log_path)); - } - - - /* Add metadata */ - if (num_event_meta > 0) { - /* - "metadata":{ - "Error ID":"0", - "Environment":"dev", - "Client Host Region":"IST" - }, - */ - msgpack_pack_str(packer, FLB_OCI_LOG_METADATA_SIZE); - msgpack_pack_str_body(packer, FLB_OCI_LOG_METADATA, - FLB_OCI_LOG_METADATA_SIZE); - - msgpack_pack_map(packer, num_event_meta); - /* pack kv list */ - mk_list_foreach(head, &ctx->log_event_metadata_fields) { - f = mk_list_entry(head, struct metadata_obj, _head); - - msgpack_pack_str(packer, flb_sds_len(f->key)); - msgpack_pack_str_body(packer, f->key, flb_sds_len(f->key)); - - msgpack_pack_str(packer, flb_sds_len(f->val)); - msgpack_pack_str_body(packer, f->val, flb_sds_len(f->val)); - - } - - } -} - -static int get_and_pack_oci_fields_from_record(msgpack_packer *packer, - msgpack_object map, - flb_sds_t *lg_id, - flb_sds_t *ls_id, - struct flb_oci_logan *ctx) -{ - int map_size = map.via.map.size; - int pck_size = 1, i; - msgpack_object *log_group_id= NULL; - msgpack_object *log_set_id = NULL; - msgpack_object *entity_id = NULL; - msgpack_object *entity_type = NULL; - msgpack_object *log_path = NULL; - msgpack_object *log_source = NULL; - msgpack_object *global_metadata = NULL; - msgpack_object *metadata = NULL; - - for(i = 0; i < map_size; i++) { - if (check_config_from_record(map.via.map.ptr[i].key, - FLB_OCI_LOG_GROUP_ID_KEY, - FLB_OCI_LOG_GROUP_ID_KEY_SIZE) == FLB_TRUE) { - if (map.via.map.ptr[i].val.type == MSGPACK_OBJECT_STR) { - log_group_id = &map.via.map.ptr[i].val; - } - continue; - } - else if (check_config_from_record(map.via.map.ptr[i].key, - FLB_OCI_LOG_SET_ID_KEY, - FLB_OCI_LOG_SET_ID_KEY_SIZE) == FLB_TRUE) { - if (map.via.map.ptr[i].val.type == MSGPACK_OBJECT_STR) { - log_set_id = &map.via.map.ptr[i].val; - } - continue; - } - else if (check_config_from_record(map.via.map.ptr[i].key, - FLB_OCI_LOG_ENTITY_ID_KEY, - FLB_OCI_LOG_ENTITY_ID_KEY_SIZE) == FLB_TRUE) { - if (map.via.map.ptr[i].val.type == MSGPACK_OBJECT_STR) { - entity_id = &map.via.map.ptr[i].val; - pck_size++; - } - continue; - } - else if (check_config_from_record(map.via.map.ptr[i].key, - FLB_OCI_LOG_ENTITY_TYPE_KEY, - FLB_OCI_LOG_ENTITY_TYPE_KEY_SIZE) == FLB_TRUE) { - if (map.via.map.ptr[i].val.type == MSGPACK_OBJECT_STR) { - entity_type = &map.via.map.ptr[i].val; - pck_size++; - } - continue; - } - else if (check_config_from_record(map.via.map.ptr[i].key, - FLB_OCI_LOG_SOURCE_NAME_KEY, - FLB_OCI_LOG_SOURCE_NAME_KEY_SIZE) == FLB_TRUE) { - if (map.via.map.ptr[i].val.type == MSGPACK_OBJECT_STR) { - log_source = &map.via.map.ptr[i].val; - pck_size++; - } - continue; - } - else if (check_config_from_record(map.via.map.ptr[i].key, - FLB_OCI_LOG_PATH_KEY, - FLB_OCI_LOG_PATH_KEY_SIZE) == FLB_TRUE) { - if (map.via.map.ptr[i].val.type == MSGPACK_OBJECT_STR) { - log_path = &map.via.map.ptr[i].val; - pck_size++; - } - continue; - } - else if (check_config_from_record(map.via.map.ptr[i].key, - FLB_OCI_METADATA_KEY, - FLB_OCI_METADATA_KEY_SIZE) == FLB_TRUE) { - if (map.via.map.ptr[i].val.type == MSGPACK_OBJECT_STR) { - metadata = &map.via.map.ptr[i].val; - pck_size++; - } - continue; - } - else if (check_config_from_record(map.via.map.ptr[i].key, - FLB_OCI_GLOBAL_METADATA_KEY, - FLB_OCI_GLOBAL_METADATA_KEY_SIZE) == FLB_TRUE) { - if (map.via.map.ptr[i].val.type == MSGPACK_OBJECT_STR) { - global_metadata = &map.via.map.ptr[i].val; - } - continue; - } - } - - if (log_group_id == NULL || log_source == NULL) { - flb_plg_error(ctx->ins, - "log source name and log group id are required"); - return -1; - } - if (global_metadata != NULL) { - msgpack_pack_map(packer, 2); - msgpack_pack_str(packer, FLB_OCI_LOG_METADATA_SIZE); - msgpack_pack_str_body(packer, FLB_OCI_LOG_METADATA, - FLB_OCI_LOG_METADATA_SIZE); - - msgpack_pack_object(packer, *global_metadata); - } - else { - msgpack_pack_map(packer, 1); - } - - /* - *logEvents":[ - { - "entityId":"", - "logSourceName":"LinuxSyslogSource", - "logPath":"/var/log/messages", - "metadata":{ - "Error ID":"1", - "Environment":"prod", - "Client Host Region":"PST" - }, - "logRecords":[ - "May 8 2017 04:02:36 blr00akm syslogd 1.4.1: shutdown.", - "May 8 2017 04:02:37 blr00akm syslogd 1.4.1: restart." - ] - }, - { - - } - ] - */ - msgpack_pack_str(packer, FLB_OCI_LOG_EVENTS_SIZE); - msgpack_pack_str_body(packer, FLB_OCI_LOG_EVENTS, FLB_OCI_LOG_EVENTS_SIZE); - - msgpack_pack_array(packer, 1); - - if (metadata != NULL) { - pck_size++; - msgpack_pack_map(packer, pck_size); /* entityType, entityId, logSourceName, logPath, metadata, logRecords */ - msgpack_pack_str(packer, FLB_OCI_LOG_METADATA_SIZE); - msgpack_pack_str_body(packer, FLB_OCI_LOG_METADATA, - FLB_OCI_LOG_METADATA_SIZE); - msgpack_pack_object(packer, *global_metadata); - - } - else { - msgpack_pack_map(packer, pck_size); /* entityType, entityId, logSourceName, logPath, logRecords */ - } - - /* "entityType:"" */ - if (entity_type) { - msgpack_pack_str(packer, FLB_OCI_ENTITY_TYPE_SIZE); - msgpack_pack_str_body(packer, FLB_OCI_ENTITY_TYPE, FLB_OCI_ENTITY_TYPE_SIZE); - msgpack_pack_object(packer, *entity_type); - } - - /* "entityId":"", */ - if (entity_type) { - msgpack_pack_str(packer, FLB_OCI_ENTITY_ID_SIZE); - msgpack_pack_str_body(packer, FLB_OCI_ENTITY_ID, FLB_OCI_ENTITY_ID_SIZE); - msgpack_pack_object(packer, *entity_id); - } - - - - /* "logSourceName":"", */ - msgpack_pack_str(packer, FLB_OCI_LOG_SOURCE_NAME_SIZE); - msgpack_pack_str_body(packer, FLB_OCI_LOG_SOURCE_NAME, - FLB_OCI_LOG_SOURCE_NAME_SIZE); - msgpack_pack_object(packer, *log_source); - - - /* "logPath":"" */ - if (log_path) { - msgpack_pack_str(packer, FLB_OCI_LOG_PATH_SIZE); - msgpack_pack_str_body(packer, FLB_OCI_LOG_PATH, FLB_OCI_LOG_PATH_SIZE); - msgpack_pack_object(packer, *log_path); - } - - *lg_id = flb_sds_create_len(log_group_id->via.str.ptr, log_group_id->via.str.size); - if(!*lg_id) { - return -1; - } - if (log_set_id != NULL) { - *ls_id = flb_sds_create_len(log_set_id->via.str.ptr, log_set_id->via.str.size); - if(!*ls_id) { - return -1; - } - } - return 0; - -} - -static int total_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *ins, void *out_context, - struct flb_config *config) -{ - struct flb_oci_logan *ctx = out_context; - flb_sds_t out_buf = NULL; - int ret = 0, res = FLB_OK, ret1 = 0, i; - msgpack_object map; - int map_size; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - int msg = -1, log = -1; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - int num_records; - flb_sds_t log_group_id = NULL; - flb_sds_t log_set_id = NULL; - int count = 0; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) event_chunk->data, event_chunk->size); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - res = FLB_ERROR; - goto clean_up; - } - - /* Create temporary msgpack buffer */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - /* pack oci fields */ - /* pack_oci_fields(&mp_pck, ctx); */ - - num_records = flb_mp_count(event_chunk->data, event_chunk->size); - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - map = *log_event.body; - map_size = map.via.map.size; - if (count < 1) { - if (ctx->oci_config_in_record == FLB_FALSE) { - pack_oci_fields(&mp_pck, ctx); - log_group_id = ctx->oci_la_log_group_id; - log_set_id = ctx->oci_la_log_set_id; - } else { - ret1 = get_and_pack_oci_fields_from_record(&mp_pck, map, &log_group_id, &log_set_id, ctx); - if (ret1 != 0) { - break; - } - } - msgpack_pack_str(&mp_pck, FLB_OCI_LOG_RECORDS_SIZE); - msgpack_pack_str_body(&mp_pck, FLB_OCI_LOG_RECORDS, - FLB_OCI_LOG_RECORDS_SIZE); - msgpack_pack_array(&mp_pck, num_records); - count++; - } - - for(i = 0; i < map_size; i++) { - if (check_config_from_record(map.via.map.ptr[i].key, - "message", - 7) == FLB_TRUE) { - msg = i; - } - if (check_config_from_record(map.via.map.ptr[i].key, - "log", - 3) == FLB_TRUE) { - log = i; - } - } - if (log >= 0) { - msgpack_pack_str(&mp_pck, map.via.map.ptr[log].val.via.str.size); - msgpack_pack_str_body(&mp_pck, map.via.map.ptr[log].val.via.str.ptr, - map.via.map.ptr[log].val.via.str.size); - } - else if (msg >= 0) { - msgpack_pack_str(&mp_pck, map.via.map.ptr[msg].val.via.str.size); - msgpack_pack_str_body(&mp_pck, map.via.map.ptr[msg].val.via.str.ptr, - map.via.map.ptr[msg].val.via.str.size); - } - log = -1; - msg = -1; - } - - if (ret1 != 0) { - res = FLB_ERROR; - msgpack_sbuffer_destroy(&mp_sbuf); - flb_log_event_decoder_destroy(&log_decoder); - goto clean_up; - } - - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); - msgpack_sbuffer_destroy(&mp_sbuf); - flb_log_event_decoder_destroy(&log_decoder); - - flb_plg_debug(ctx->ins, "payload=%s", out_buf); - flb_plg_debug(ctx->ins, "lg_id=%s", log_group_id); - - ret = flush_to_endpoint(ctx, out_buf, log_group_id, log_set_id); - if(ret != FLB_OK) { - res = FLB_RETRY; - goto clean_up; - } - - clean_up: - if (out_buf != NULL) { - flb_sds_destroy(out_buf); - } - if (log_group_id != NULL && ctx->oci_config_in_record) { - flb_sds_destroy(log_group_id); - } - if (log_set_id != NULL && ctx->oci_config_in_record) { - flb_sds_destroy(log_set_id); - } - return res; -} - -static void cb_oci_logan_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *ins, void *out_context, - struct flb_config *config) -{ - struct flb_oci_logan *ctx = out_context; - int ret = -1; - - ret = total_flush(event_chunk, out_flush, - ins, out_context, - config); - if (ret != FLB_OK) { - flb_oci_logan_conf_destroy(ctx); - FLB_OUTPUT_RETURN(ret); - } - flb_plg_debug(ctx->ins, "success"); - - FLB_OUTPUT_RETURN(FLB_OK); - -} - -static int cb_oci_logan_exit(void *data, struct flb_config *config) -{ - struct flb_oci_logan *ctx = data; - - flb_oci_logan_conf_destroy(ctx); - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "config_file_location", "", - 0, FLB_TRUE, offsetof(struct flb_oci_logan, config_file_location), - "Location of the oci config file for user api key signing" - }, - { - FLB_CONFIG_MAP_STR, "profile_name", "DEFAULT", - 0, FLB_TRUE, offsetof(struct flb_oci_logan, profile_name), - "name of the profile in the config file from which the user configs should be loaded" - }, - { - FLB_CONFIG_MAP_BOOL, "oci_config_in_record", "false", - 0, FLB_TRUE, offsetof(struct flb_oci_logan, oci_config_in_record), - "If true, oci_la_* configs will be read from the record" - }, - { - FLB_CONFIG_MAP_STR, "uri", NULL, - 0, FLB_TRUE, offsetof(struct flb_oci_logan, uri), - "Set the uri for rest api request" - }, - { - FLB_CONFIG_MAP_STR, "oci_la_log_group_id", NULL, - 0, FLB_TRUE, offsetof(struct flb_oci_logan, oci_la_log_group_id), - "log group id" - }, - { - FLB_CONFIG_MAP_STR, "oci_la_log_set_id", NULL, - 0, FLB_TRUE, offsetof(struct flb_oci_logan, oci_la_log_set_id), - "" - }, - { - FLB_CONFIG_MAP_STR, "oci_la_entity_id", NULL, - 0, FLB_TRUE, offsetof(struct flb_oci_logan, oci_la_entity_id), - "" - }, - { - FLB_CONFIG_MAP_STR, "oci_la_entity_type", NULL, - 0, FLB_TRUE, offsetof(struct flb_oci_logan, oci_la_entity_type), - "" - }, - { - FLB_CONFIG_MAP_STR, "oci_la_log_source_name", NULL, - 0, FLB_TRUE, offsetof(struct flb_oci_logan, oci_la_log_source_name), - "" - }, - { - FLB_CONFIG_MAP_STR, "oci_la_log_set_id", NULL, - 0, FLB_TRUE, offsetof(struct flb_oci_logan, oci_la_log_set_id), - "" - }, - { - FLB_CONFIG_MAP_STR, "oci_la_log_path", NULL, - 0, FLB_TRUE, offsetof(struct flb_oci_logan, oci_la_log_path), - "" - }, - { - FLB_CONFIG_MAP_SLIST_2, "oci_la_global_metadata", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct flb_oci_logan, oci_la_global_metadata), - "" - }, - { - FLB_CONFIG_MAP_SLIST_2, "oci_la_metadata", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct flb_oci_logan, oci_la_metadata), - "" - }, - { - FLB_CONFIG_MAP_STR, "namespace", NULL, - 0, FLB_TRUE, offsetof(struct flb_oci_logan, namespace), - "namespace in your tenancy where the log objects reside" - }, - { - FLB_CONFIG_MAP_STR, "proxy", NULL, - 0, FLB_TRUE, offsetof(struct flb_oci_logan, proxy), - "define proxy if required, in http://host:port format, supports only http protocol" - }, - - {0} -}; - -/* Plugin reference */ -struct flb_output_plugin out_oracle_log_analytics_plugin = { - .name = "oracle_log_analytics", - .description = "Oracle log analytics", - .cb_init = cb_oci_logan_init, - .cb_pre_run = NULL, - .cb_flush = cb_oci_logan_flush, - .cb_exit = cb_oci_logan_exit, - - /* Configuration */ - .config_map = config_map, - - /* Events supported */ - .event_type = FLB_OUTPUT_LOGS, - - - /* Plugin flags */ - .flags = FLB_OUTPUT_NET | FLB_IO_OPT_TLS, - .workers = 1, -}; diff --git a/fluent-bit/plugins/out_oracle_log_analytics/oci_logan.h b/fluent-bit/plugins/out_oracle_log_analytics/oci_logan.h deleted file mode 100644 index 7cc9e75f4..000000000 --- a/fluent-bit/plugins/out_oracle_log_analytics/oci_logan.h +++ /dev/null @@ -1,215 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#ifndef FLB_OUT_OCI_LOGAN_H -#define FLB_OUT_OCI_LOGAN_H - -#define FLB_OCI_LOG_ENTITY_ID_KEY "oci_la_entity_id" -#define FLB_OCI_LOG_ENTITY_ID_KEY_SIZE sizeof(FLB_OCI_LOG_ENTITY_ID_KEY) - 1 - -#define FLB_OCI_LOG_ENTITY_TYPE_KEY "oci_la_entity_type" -#define FLB_OCI_LOG_ENTITY_TYPE_KEY_SIZE sizeof(FLB_OCI_LOG_ENTITY_TYPE_KEY) - 1 - -#define FLB_OCI_LOG_GROUP_ID_KEY "oci_la_log_group_id" -#define FLB_OCI_LOG_GROUP_ID_KEY_SIZE sizeof(FLB_OCI_LOG_GROUP_ID_KEY) - 1 - -#define FLB_OCI_LOG_SET_ID_KEY "oci_la_log_set_id" -#define FLB_OCI_LOG_SET_ID_KEY_SIZE sizeof(FLB_OCI_LOG_SET_ID_KEY) - 1 - -#define FLB_OCI_LOG_SOURCE_NAME_KEY "oci_la_log_source_name" -#define FLB_OCI_LOG_SOURCE_NAME_KEY_SIZE sizeof(FLB_OCI_LOG_SOURCE_NAME_KEY) - 1 - -#define FLB_OCI_LOG_PATH_KEY "oci_la_log_path" -#define FLB_OCI_LOG_PATH_KEY_SIZE sizeof(FLB_OCI_LOG_PATH_KEY) - 1 - -#define FLB_OCI_METADATA_KEY "oci_la_metadata" -#define FLB_OCI_METADATA_KEY_SIZE sizeof(FLB_OCI_METADATA_KEY) - 1 - -#define FLB_OCI_GLOBAL_METADATA_KEY "oci_la_global_metadata" -#define FLB_OCI_GLOBAL_METADATA_KEY_SIZE sizeof(FLB_OCI_GLOBAL_METADATA_KEY) - 1 - -#define FLB_OCI_LOG_EVENTS "logEvents" -#define FLB_OCI_LOG_EVENTS_SIZE sizeof(FLB_OCI_LOG_EVENTS)-1 - -#define FLB_OCI_LOG_RECORDS "logRecords" -#define FLB_OCI_LOG_RECORDS_SIZE sizeof(FLB_OCI_LOG_RECORDS)-1 - -#define FLB_OCI_LOG_GROUP_ID "logGroupId" -#define FLB_OCI_LOG_GROUP_ID_SIZE sizeof(FLB_OCI_LOG_GROUP_ID)-1 - -#define FLB_OCI_ENTITY_TYPE "entityType" -#define FLB_OCI_ENTITY_TYPE_SIZE sizeof(FLB_OCI_ENTITY_TYPE) - 1 - -#define FLB_OCI_LOG_SET "logSet" -#define FLB_OCI_LOG_SET_SIZE sizeof(FLB_OCI_LOG_SET)-1 - -#define FLB_OCI_LOG_METADATA "metadata" -#define FLB_OCI_LOG_METADATA_SIZE sizeof(FLB_OCI_LOG_METADATA)-1 - -#define FLB_OCI_ENTITY_ID "entityId" -#define FLB_OCI_ENTITY_ID_SIZE sizeof(FLB_OCI_ENTITY_ID)-1 - -#define FLB_OCI_LOG_SOURCE_NAME "logSourceName" -#define FLB_OCI_LOG_SOURCE_NAME_SIZE sizeof(FLB_OCI_LOG_SOURCE_NAME)-1 - -#define FLB_OCI_LOG_PATH "logPath" -#define FLB_OCI_LOG_PATH_SIZE sizeof(FLB_OCI_LOG_PATH)-1 - -#define FLB_OCI_META_PREFIX "metadata_" -#define FLB_OCI_META_PREFIX_SIZE sizeof(FLB_OCI_META_PREFIX)-1 - -#define FLB_OCI_MATCH_PREFIX "oci_match_" -#define FLB_OCI_MATCH_PREFIX_SIZE sizeof(FLB_OCI_MATCH_PREFIX)-1 - -#ifdef FLB_HAVE_REGEX -#define FLB_OCI_MATCH_REGEX_PREFIX "oci_match_regex_" -#define FLB_OCI_MATCH_REGEX_PREFIX_SIZE sizeof(FLB_OCI_MATCH_REGEX_PREFIX)-1 -#endif - -/* Params */ -#define FLB_OCI_PARAM_SKIP_HTTP_POST "skip_http_post" -#define FLB_OCI_PARAM_URI "uri" -#define FLB_OCI_PARAM_ENABLE_TRACE_OUTPUT "enable_trace" -#define FLB_OCI_PARAM_TRACE_OUTPUT_PATH "trace_file_path" -#define FLB_OCI_PARAM_TRACE_OUTPUT_FILE "trace_file_name" -#define FLB_OCI_PARAM_COLLECT_TIME_FIELD "collect_time_field_name" - -#define FLB_OCI_PARAM_USE_RAW_RECORD "use_raw_record" -#define FLB_OCI_PARAM_USE_RAW_RECORD_SIZE sizeof(FLB_OCI_PARAM_USE_RAW_RECORD)-1 - -#define FLB_OCI_PARAM_INCLUDE_COLLECT_TIME "include_collect_time" -#define FLB_OCI_PARAM_INCLUDE_COLLECT_TIME_SIZE sizeof(FLB_OCI_PARAM_INCLUDE_COLLECT_TIME)-1 - -#define FLB_OCI_MATCH_ID_MAX 1000 // TO avoid too large memory allocation - -#define FLB_OCI_DEFAULT_COLLECT_TIME "oci_collect_time" -#define FLB_OCI_DEFAULT_COLLECT_TIME_SIZE sizeof(FLB_OCI_DEFAULT_COLLECT_TIME)-1 - -/* Http Header */ -#define FLB_OCI_HEADER_REQUEST_TARGET "(request-target)" -#define FLB_OCI_HEADER_USER_AGENT "User-Agent" -#define FLB_OCI_HEADER_USER_AGENT_VAL "Fluent-Bit" -#define FLB_OCI_HEADER_CONTENT_TYPE "content-type" -#define FLB_OCI_HEADER_CONTENT_TYPE_VAL "application/octet-stream" -#define FLB_OCI_HEADER_X_CONTENT_SHA256 "x-content-sha256" -#define FLB_OCI_HEADER_CONTENT_LENGTH "content-length" -#define FLB_OCI_HEADER_HOST "host" -#define FLB_OCI_HEADER_DATE "date" -#define FLB_OCI_HEADER_AUTH "Authorization" -#define FLB_OCI_PAYLOAD_TYPE "payloadType" - - -/* For OCI signing */ -#define FLB_OCI_PARAM_TENANCY "tenancy" -#define FLB_OCI_PARAM_USER "user" -#define FLB_OCI_PARAM_KEY_FINGERPRINT "fingerprint" -#define FLB_OCI_PARAM_KEY_FILE "key_file" -#define FLB_OCI_PARAM_REGION "region" -#define FLB_OCI_PARAM_KEY_FILE_PASSPHRASE "key_file_passphrase" - -#define FLB_OCI_SIGN_SIGNATURE_VERSION "Signature version=\"1\"" -#define FLB_OCI_SIGN_KEYID "keyId" -#define FLB_OCI_SIGN_ALGORITHM "algorithm=\"rsa-sha256\"" - -#define FLB_OCI_SIGN_HEADERS "headers=\"" \ - FLB_OCI_HEADER_REQUEST_TARGET " " \ - FLB_OCI_HEADER_HOST " " \ - FLB_OCI_HEADER_DATE " " \ - FLB_OCI_HEADER_X_CONTENT_SHA256 " " \ - FLB_OCI_HEADER_CONTENT_TYPE " " \ - FLB_OCI_HEADER_CONTENT_LENGTH "\"" - -#define FLB_OCI_SIGN_SIGNATURE "signature" - -/* For error response */ -#define FLB_OCI_ERROR_RESPONSE_CODE "code" -#define FLB_OCI_ERROR_RESPONSE_MESSAGE "message" - -#define FLB_OCI_ERROR_CODE_RELATED_RESOURCE_NOT_FOUND "RelatedResourceNotAuthorizedOrNotFound" -#define FLB_OCI_ERROR_CODE_NOT_AUTHENTICATED "NotAuthenticated" -#define FLB_OCI_ERROR_CODE_NOT_AUTHENTICATEDORNOTFOUND "NotAuthorizedOrNotFound" -#define FLB_OCI_ERROR_CODE_INCORRECTSTATE "IncorrectState" -#define FLB_OCI_ERROR_CODE_NOT_AUTH_OR_RESOURCE_EXIST "NotAuthorizedOrResourceAlreadyExists" -#define FLB_OCI_ERROR_CODE_TOO_MANY_REQUESTS "TooManyRequests" -#define FLB_OCI_ERROR_CODE_INTERNAL_SERVER_ERROR "InternalServerError" - -#include -#include -#include -#include -#include - -struct metadata_obj { - flb_sds_t key; - flb_sds_t val; - struct mk_list _head; - -}; - -struct flb_oci_error_response -{ - flb_sds_t code; - flb_sds_t message; -}; - -struct flb_oci_logan { - flb_sds_t namespace; - flb_sds_t config_file_location; - flb_sds_t profile_name; - int oci_config_in_record; - flb_sds_t uri; - - struct flb_upstream *u; - flb_sds_t proxy; - char *proxy_host; - int proxy_port; - - // oci_la_* configs - flb_sds_t oci_la_entity_id; - - flb_sds_t oci_la_entity_type; - - flb_sds_t oci_la_log_source_name; - - flb_sds_t oci_la_log_path; - - flb_sds_t oci_la_log_group_id; - - flb_sds_t oci_la_log_set_id; - - struct mk_list *oci_la_global_metadata; - struct mk_list global_metadata_fields; - struct mk_list *oci_la_metadata; - struct mk_list log_event_metadata_fields; - - // config_file - flb_sds_t user; - flb_sds_t region; - flb_sds_t tenancy; - flb_sds_t key_fingerprint; - flb_sds_t key_file; - /* For OCI signing */ - flb_sds_t key_id; // tenancy/user/key_fingerprint - flb_sds_t private_key; - - struct flb_output_instance *ins; - -}; -#endif diff --git a/fluent-bit/plugins/out_oracle_log_analytics/oci_logan_conf.c b/fluent-bit/plugins/out_oracle_log_analytics/oci_logan_conf.c deleted file mode 100644 index a39803184..000000000 --- a/fluent-bit/plugins/out_oracle_log_analytics/oci_logan_conf.c +++ /dev/null @@ -1,493 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "oci_logan.h" -#include "oci_logan_conf.h" - -static int create_pk_context(flb_sds_t filepath, const char *key_passphrase, - struct flb_oci_logan *ctx) -{ - int ret; - struct stat st; - struct file_info finfo; - FILE *fp; - flb_sds_t kbuffer; - - - ret = stat(filepath, &st); - if (ret == -1) { - flb_errno(); - flb_plg_error(ctx->ins, "cannot open key file %s", filepath); - return -1; - } - - if (!S_ISREG(st.st_mode) && !S_ISLNK(st.st_mode)) { - flb_plg_error(ctx->ins, "key file is not a valid file: %s", filepath); - return -1; - } - - /* Read file content */ - if (mk_file_get_info(filepath, &finfo, MK_FILE_READ) != 0) { - flb_plg_error(ctx->ins, "error to read key file: %s", filepath); - return -1; - } - - if (!(fp = fopen(filepath, "rb"))) { - flb_plg_error(ctx->ins, "error to open key file: %s", filepath); - return -1; - } - - kbuffer = flb_sds_create_size(finfo.size + 1); - if (!kbuffer) { - flb_errno(); - fclose(fp); - return -1; - } - - ret = fread(kbuffer, finfo.size, 1, fp); - if (ret < 1) { - flb_sds_destroy(kbuffer); - fclose(fp); - flb_plg_error(ctx->ins, "fail to read key file: %s", filepath); - return -1; - } - fclose(fp); - - /* In mbedtls, for PEM, the buffer must contains a null-terminated string */ - kbuffer[finfo.size] = '\0'; - flb_sds_len_set(kbuffer, finfo.size + 1); - - ctx->private_key = kbuffer; - - return 0; -} - -static int load_oci_credentials(struct flb_oci_logan *ctx) -{ - flb_sds_t content; - int found_profile = 0, res = 0; - char *line, *profile = NULL; - int eq_pos = 0; - char* key = NULL; - char* val; - - content = flb_file_read(ctx->config_file_location); - if (content == NULL || flb_sds_len(content) == 0) - { - return -1; - } - flb_plg_debug(ctx->ins, "content = %s", content); - line = strtok(content, "\n"); - while(line != NULL) { - /* process line */ - flb_plg_debug(ctx->ins, "line = %s", line); - if(!found_profile && line[0] == '[') { - profile = mk_string_copy_substr(line, 1, strlen(line) - 1); - if(!strcmp(profile, ctx->profile_name)) { - flb_plg_info(ctx->ins, "found profile"); - found_profile = 1; - goto iterate; - } - mk_mem_free(profile); - } - if(found_profile) { - if(line[0] == '[') { - break; - } - eq_pos = mk_string_char_search(line, '=', strlen(line)); - flb_plg_debug(ctx->ins, "eq_pos %d", eq_pos); - key = mk_string_copy_substr(line, 0, eq_pos); - flb_plg_debug(ctx->ins, "key = %s", key); - val = line + eq_pos + 1; - if (!key || !val) { - res = -1; - break; - } - if (strcmp(key, FLB_OCI_PARAM_USER) == 0) { - ctx->user = flb_sds_create(val); - } - else if (strcmp(key, FLB_OCI_PARAM_TENANCY) == 0) { - ctx->tenancy = flb_sds_create(val); - } - else if (strcmp(key, FLB_OCI_PARAM_KEY_FILE) == 0) { - ctx->key_file = flb_sds_create(val); - } - else if (strcmp(key, FLB_OCI_PARAM_KEY_FINGERPRINT) == 0) { - ctx->key_fingerprint = flb_sds_create(val); - } - else if (strcmp(key, FLB_OCI_PARAM_REGION) == 0) { - ctx->region = flb_sds_create(val); - } - else { - goto iterate; - } - } - iterate: - if (profile) { - mk_mem_free(profile); - profile = NULL; - } - if (key) { - mk_mem_free(key); - key = NULL; - } - line = strtok(NULL, "\n"); - } - if (!found_profile) { - flb_errno(); - res = -1; - } - - flb_sds_destroy(content); - if (profile) { - mk_mem_free(profile); - } - if (key) { - mk_mem_free(key); - } - return res; -} - -static int global_metadata_fields_create(struct flb_oci_logan *ctx) -{ - struct mk_list *head; - struct flb_slist_entry *kname; - struct flb_slist_entry *val; - struct flb_config_map_val *mv; - struct metadata_obj *f; - - if (!ctx->oci_la_global_metadata) { - return 0; - } - - flb_config_map_foreach(head, mv, ctx->oci_la_global_metadata) { - kname = mk_list_entry_first(mv->val.list, struct flb_slist_entry, _head); - val = mk_list_entry_last(mv->val.list, struct flb_slist_entry, _head); - - f = flb_malloc(sizeof(struct metadata_obj)); - if (!f) { - flb_errno(); - return -1; - } - - f->key = flb_sds_create(kname->str); - if (!f->key) { - flb_free(f); - return -1; - } - f->val = flb_sds_create(val->str); - if (!f->val) { - flb_free(f); - return -1; - } - - - mk_list_add(&f->_head, &ctx->global_metadata_fields); - } - - return 0; -} - -static int log_event_metadata_create(struct flb_oci_logan *ctx) -{ - struct mk_list *head; - struct flb_slist_entry *kname; - struct flb_slist_entry *val; - struct flb_config_map_val *mv; - struct metadata_obj *f; - - if (!ctx->oci_la_metadata) { - return 0; - } - - flb_config_map_foreach(head, mv, ctx->oci_la_metadata) { - kname = mk_list_entry_first(mv->val.list, struct flb_slist_entry, _head); - val = mk_list_entry_last(mv->val.list, struct flb_slist_entry, _head); - - f = flb_malloc(sizeof(struct metadata_obj)); - if (!f) { - flb_errno(); - return -1; - } - - f->key = flb_sds_create(kname->str); - if (!f->key) { - flb_free(f); - return -1; - } - f->val = flb_sds_create(val->str); - if (!f->val) { - flb_free(f); - return -1; - } - - - mk_list_add(&f->_head, &ctx->log_event_metadata_fields); - } - - return 0; -} -struct flb_oci_logan *flb_oci_logan_conf_create(struct flb_output_instance *ins, - struct flb_config *config) { - struct flb_oci_logan *ctx; - struct flb_upstream *upstream; - flb_sds_t host = NULL; - int io_flags = 0, default_port; - const char *tmp; - int ret = 0; - char *protocol = NULL; - char *p_host = NULL; - char *p_port = NULL; - char *p_uri = NULL; - - ctx = flb_calloc(1, sizeof(struct flb_oci_logan)); - if (!ctx) { - flb_errno(); - return NULL; - } - - ctx->ins = ins; - - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_plg_error(ctx->ins, "configuration error"); - flb_oci_logan_conf_destroy(ctx); - return NULL; - } - - if (ctx->oci_config_in_record == FLB_FALSE) { - if (ctx->oci_la_log_source_name == NULL || - ctx->oci_la_log_group_id == NULL) { - flb_errno(); - flb_plg_error(ctx->ins, - "log source name and log group id are required"); - flb_oci_logan_conf_destroy(ctx); - return NULL; - } - } - if (ctx->oci_la_global_metadata != NULL) { - mk_list_init(&ctx->global_metadata_fields); - ret = global_metadata_fields_create(ctx); - if (ret != 0) { - flb_errno(); - flb_oci_logan_conf_destroy(ctx); - return NULL; - } - } - - if (ctx->oci_la_metadata != NULL) { - mk_list_init(&ctx->log_event_metadata_fields); - ret = log_event_metadata_create(ctx); - if (ret != 0) { - flb_errno(); - flb_oci_logan_conf_destroy(ctx); - return NULL; - } - } - - if (!ctx->config_file_location) { - flb_errno(); - flb_plg_error(ctx->ins, "config file location is required"); - flb_oci_logan_conf_destroy(ctx); - return NULL; - } - - ret = load_oci_credentials(ctx); - if(ret != 0) { - flb_errno(); - flb_oci_logan_conf_destroy(ctx); - return NULL; - } - - if (ins->host.name) { - host = ins->host.name; - } - else { - if (!ctx->region) { - flb_errno(); - flb_plg_error(ctx->ins, "Region is required"); - flb_oci_logan_conf_destroy(ctx); - return NULL; - } - host = flb_sds_create_size(512); - flb_sds_snprintf(&host, flb_sds_alloc(host), "loganalytics.%s.oci.oraclecloud.com", ctx->region); - } - - if (!ctx->uri) { - if (!ctx->namespace) { - flb_errno(); - flb_plg_error(ctx->ins, "Namespace is required"); - flb_oci_logan_conf_destroy(ctx); - return NULL; - } - ctx->uri = flb_sds_create_size(512); - flb_sds_snprintf(&ctx->uri, flb_sds_alloc(ctx->uri), - "/20200601/namespaces/%s/actions/uploadLogEventsFile", - ctx->namespace); - } - - - - if (create_pk_context(ctx->key_file, NULL, ctx) < 0) { - flb_plg_error(ctx->ins, "failed to create pk context"); - flb_oci_logan_conf_destroy(ctx); - return NULL; - } - - - ctx->key_id = flb_sds_create_size(512); - flb_sds_snprintf(&ctx->key_id, flb_sds_alloc(ctx->key_id), - "%s/%s/%s", ctx->tenancy, ctx->user, ctx->key_fingerprint); - - - /* Check if SSL/TLS is enabled */ - io_flags = FLB_IO_TCP; - default_port = 80; - -#ifdef FLB_HAVE_TLS - if (ins->use_tls == FLB_TRUE) { - io_flags = FLB_IO_TLS; - default_port = 443; - } -#endif - - if (ins->host.ipv6 == FLB_TRUE) { - io_flags |= FLB_IO_IPV6; - } - - flb_output_net_default(host, default_port, ins); - flb_sds_destroy(host); - - if (ctx->proxy) { - ret = flb_utils_url_split(tmp, &protocol, &p_host, &p_port, &p_uri); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not parse proxy parameter: '%s'", tmp); - flb_oci_logan_conf_destroy(ctx); - return NULL; - } - - ctx->proxy_host = p_host; - ctx->proxy_port = atoi(p_port); - flb_free(protocol); - flb_free(p_port); - flb_free(p_uri); - flb_free(p_host); - } - - if (ctx->proxy) { - upstream = flb_upstream_create(config, ctx->proxy_host, ctx->proxy_port, - io_flags, ins->tls); - } - else { - /* Prepare an upstream handler */ - upstream = flb_upstream_create(config, ins->host.name, ins->host.port, - io_flags, ins->tls); - } - - if (!upstream) { - flb_plg_error(ctx->ins, "cannot create Upstream context"); - flb_oci_logan_conf_destroy(ctx); - return NULL; - } - ctx->u = upstream; - - /* Set instance flags into upstream */ - flb_output_upstream_set(ctx->u, ins); - - return ctx; -} - -static void metadata_fields_destroy(struct flb_oci_logan *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - struct metadata_obj *f; - - mk_list_foreach_safe(head, tmp, &ctx->global_metadata_fields) { - f = mk_list_entry(head, struct metadata_obj, _head); - flb_sds_destroy(f->key); - flb_sds_destroy(f->val); - mk_list_del(&f->_head); - flb_free(f); - } - - mk_list_foreach_safe(head, tmp, &ctx->log_event_metadata_fields) { - f = mk_list_entry(head, struct metadata_obj, _head); - flb_sds_destroy(f->key); - flb_sds_destroy(f->val); - mk_list_del(&f->_head); - flb_free(f); - } - -} - -int flb_oci_logan_conf_destroy(struct flb_oci_logan *ctx) { - if(ctx == NULL) { - return 0; - } - - if (ctx->private_key) { - flb_sds_destroy(ctx->private_key); - } - if (ctx->uri) { - flb_sds_destroy(ctx->uri); - } - if (ctx->key_id) { - flb_sds_destroy(ctx->key_id); - } - if (ctx->key_file) { - flb_sds_destroy(ctx->key_file); - } - if(ctx->user) { - flb_sds_destroy(ctx->user); - } - if(ctx->key_fingerprint) { - flb_sds_destroy(ctx->key_fingerprint); - } - if(ctx->tenancy) { - flb_sds_destroy(ctx->tenancy); - } - if(ctx->region) { - flb_sds_destroy(ctx->region); - } - if (ctx->u) { - flb_upstream_destroy(ctx->u); - } - - metadata_fields_destroy(ctx); - - flb_free(ctx); - return 0; -} \ No newline at end of file diff --git a/fluent-bit/plugins/out_oracle_log_analytics/oci_logan_conf.h b/fluent-bit/plugins/out_oracle_log_analytics/oci_logan_conf.h deleted file mode 100644 index a11832b0a..000000000 --- a/fluent-bit/plugins/out_oracle_log_analytics/oci_logan_conf.h +++ /dev/null @@ -1,34 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#ifndef FLB_OUT_OCI_LOGAN_CONF_H -#define FLB_OUT_OCI_LOGAN_CONF_H - -#include -#include -#include - -#include "oci_logan.h" - -struct flb_oci_logan *flb_oci_logan_conf_create(struct flb_output_instance *ins, - struct flb_config *config); -int flb_oci_logan_conf_destroy(struct flb_oci_logan *ctx); - -#endif diff --git a/fluent-bit/plugins/out_pgsql/CMakeLists.txt b/fluent-bit/plugins/out_pgsql/CMakeLists.txt deleted file mode 100644 index 6206c02f9..000000000 --- a/fluent-bit/plugins/out_pgsql/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -set(src - pgsql.c - pgsql_connections.c - ) - -FLB_PLUGIN(out_pgsql "${src}" "") -target_include_directories(flb-plugin-out_pgsql PRIVATE ${PostgreSQL_INCLUDE_DIRS}) -target_link_libraries(flb-plugin-out_pgsql -lpq) diff --git a/fluent-bit/plugins/out_pgsql/pgsql.c b/fluent-bit/plugins/out_pgsql/pgsql.c deleted file mode 100644 index a01090c1a..000000000 --- a/fluent-bit/plugins/out_pgsql/pgsql.c +++ /dev/null @@ -1,389 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include "pgsql.h" -#include "pgsql_connections.h" - -void pgsql_conf_destroy(struct flb_pgsql_config *ctx) -{ - pgsql_destroy_connections(ctx); - - flb_free(ctx->db_hostname); - - if (ctx->db_table != NULL) { - flb_sds_destroy(ctx->db_table); - } - - if (ctx->timestamp_key != NULL) { - flb_sds_destroy(ctx->timestamp_key); - } - - flb_free(ctx); - ctx = NULL; -} - -static int cb_pgsql_init(struct flb_output_instance *ins, - struct flb_config *config, void *data) -{ - - struct flb_pgsql_config *ctx; - size_t str_len; - PGresult *res; - char *query = NULL; - char *temp = NULL; - const char *tmp = NULL; - int ret; - - /* set default network configuration */ - flb_output_net_default(FLB_PGSQL_HOST, FLB_PGSQL_PORT, ins); - - ctx = flb_calloc(1, sizeof(struct flb_pgsql_config)); - if (!ctx) { - flb_errno(); - return -1; - } - - ctx->ins = ins; - - /* Database host */ - ctx->db_hostname = flb_strdup(ins->host.name); - if (!ctx->db_hostname) { - flb_errno(); - pgsql_conf_destroy(ctx); - return -1; - } - - /* Database port */ - snprintf(ctx->db_port, sizeof(ctx->db_port), "%d", ins->host.port); - - /* Database name */ - ctx->db_name = flb_output_get_property("database", ins); - if (!ctx->db_name) { - ctx->db_name = FLB_PGSQL_DBNAME; - } - - /* db table */ - tmp = flb_output_get_property("table", ins); - if (tmp) { - ctx->db_table = flb_sds_create(tmp); - } - else { - ctx->db_table = flb_sds_create(FLB_PGSQL_TABLE); - } - - /* connection options */ - ctx->conn_options = flb_output_get_property("connection_options", ins); - - if (!ctx->db_table) { - flb_errno(); - pgsql_conf_destroy(ctx); - return -1; - } - - /* db user */ - ctx->db_user = flb_output_get_property("user", ins); - if (!ctx->db_user) { - flb_plg_warn(ctx->ins, - "You didn't supply a valid user to connect," - "your current unix user will be used"); - } - - /* db user password */ - ctx->db_passwd = flb_output_get_property("password", ins); - - /* timestamp key */ - tmp = flb_output_get_property("timestamp_key", ins); - if (tmp) { - ctx->timestamp_key = flb_sds_create(tmp); - } - else { - ctx->timestamp_key = flb_sds_create(FLB_PGSQL_TIMESTAMP_KEY); - } - - if (!ctx->timestamp_key) { - flb_errno(); - pgsql_conf_destroy(ctx); - return -1; - } - - /* Pool size */ - tmp = flb_output_get_property("max_pool_size", ins); - if (tmp) { - ctx->max_pool_size = strtol(tmp, NULL, 0); - if (ctx->max_pool_size < 1) - ctx->max_pool_size = 1; - } - else { - ctx->max_pool_size = FLB_PGSQL_POOL_SIZE; - } - - tmp = flb_output_get_property("min_pool_size", ins); - if (tmp) { - ctx->min_pool_size = strtol(tmp, NULL, 0); - if (ctx->min_pool_size < 1 || ctx->min_pool_size > ctx->max_pool_size) - ctx->min_pool_size = ctx->max_pool_size; - } - else { - ctx->min_pool_size = FLB_PGSQL_MIN_POOL_SIZE; - } - - /* Sync Mode */ - tmp = flb_output_get_property("async", ins); - if (tmp && flb_utils_bool(tmp)) { - ctx->async = FLB_TRUE; - } - else { - ctx->async = FLB_FALSE; - } - - if (!ctx->async) { - ctx->min_pool_size = 1; - ctx->max_pool_size = 1; - } - - /* CockroachDB Support */ - tmp = flb_output_get_property("cockroachdb", ins); - if (tmp && flb_utils_bool(tmp)) { - ctx->cockroachdb = FLB_TRUE; - } - else { - ctx->cockroachdb = FLB_FALSE; - } - - ret = pgsql_start_connections(ctx); - if (ret) { - return -1; - } - - flb_plg_info(ctx->ins, "host=%s port=%s dbname=%s OK", - ctx->db_hostname, ctx->db_port, ctx->db_name); - flb_output_set_context(ins, ctx); - - temp = PQescapeIdentifier(ctx->conn_current->conn, ctx->db_table, - flb_sds_len(ctx->db_table)); - - if (temp == NULL) { - flb_plg_error(ctx->ins, "failed to parse table name: %s", - PQerrorMessage(ctx->conn_current->conn)); - pgsql_conf_destroy(ctx); - return -1; - } - - flb_sds_destroy(ctx->db_table); - ctx->db_table = flb_sds_create(temp); - PQfreemem(temp); - - if (!ctx->db_table) { - flb_errno(); - pgsql_conf_destroy(ctx); - return -1; - } - - flb_plg_info(ctx->ins, "we check that the table %s " - "exists, if not we create it", ctx->db_table); - - str_len = 72 + flb_sds_len(ctx->db_table); - - query = flb_malloc(str_len); - if (query == NULL) { - flb_errno(); - pgsql_conf_destroy(ctx); - return -1; - } - - /* Maybe use the timestamp with the TZ specified */ - /* in the postgresql connection? */ - snprintf(query, str_len, - "CREATE TABLE IF NOT EXISTS %s " - "(tag varchar, time timestamp, data jsonb);", - ctx->db_table); - flb_plg_trace(ctx->ins, "%s", query); - res = PQexec(ctx->conn_current->conn, query); - - flb_free(query); - - if (PQresultStatus(res) != PGRES_COMMAND_OK) { - flb_plg_error(ctx->ins, "%s", - PQerrorMessage(ctx->conn_current->conn)); - pgsql_conf_destroy(ctx); - return -1; - } - - PQclear(res); - - return 0; -} - -static void cb_pgsql_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - struct flb_pgsql_config *ctx = out_context; - flb_sds_t json; - char *tmp = NULL; - char *query = NULL; - PGresult *res = NULL; - int send_res; - flb_sds_t tag_escaped = NULL; - size_t str_len; - - - if (pgsql_next_connection(ctx) == 1) { - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* - * PQreset() - * This function will close the connection to the server and attempt to - * reestablish a new connection to the same server, using all the same - * parameters previously used. This might be useful for error recovery - * if a working connection is lost. - */ - if (PQstatus(ctx->conn_current->conn) != CONNECTION_OK) { - PQreset(ctx->conn_current->conn); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - json = flb_pack_msgpack_to_json_format(event_chunk->data, - event_chunk->size, - FLB_PACK_JSON_FORMAT_JSON, - FLB_PACK_JSON_DATE_DOUBLE, - ctx->timestamp_key); - if (json == NULL) { - flb_errno(); - flb_plg_error(ctx->ins, - "Can't parse the msgpack into json"); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - tmp = PQescapeLiteral(ctx->conn_current->conn, json, flb_sds_len(json)); - flb_sds_destroy(json); - if (!tmp) { - flb_errno(); - PQfreemem(tmp); - flb_plg_error(ctx->ins, "Can't escape json string"); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - json = flb_sds_create(tmp); - PQfreemem(tmp); - if (!json) { - flb_errno(); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - tmp = PQescapeLiteral(ctx->conn_current->conn, - event_chunk->tag, - flb_sds_len(event_chunk->tag)); - if (!tmp) { - flb_errno(); - flb_sds_destroy(json); - PQfreemem(tmp); - flb_plg_error(ctx->ins, "Can't escape tag string: %s", - event_chunk->tag); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - tag_escaped = flb_sds_create(tmp); - PQfreemem(tmp); - if (!tag_escaped) { - flb_errno(); - flb_sds_destroy(json); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - str_len = 100 + flb_sds_len(json) - + flb_sds_len(tag_escaped) - + flb_sds_len(ctx->db_table) - + flb_sds_len(ctx->timestamp_key); - query = flb_malloc(str_len); - - if (query == NULL) { - flb_errno(); - flb_sds_destroy(json); - flb_sds_destroy(tag_escaped); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - - snprintf(query, str_len, - ctx->cockroachdb ? FLB_PGSQL_INSERT_COCKROACH : FLB_PGSQL_INSERT, - ctx->db_table, tag_escaped, ctx->timestamp_key, json); - flb_plg_trace(ctx->ins, "query: %s", query); - - if (ctx->async) { - send_res = PQsendQuery(ctx->conn_current->conn, query); - flb_free(query); - flb_sds_destroy(json); - flb_sds_destroy(tag_escaped); - - if (send_res == 0) { - flb_plg_error(ctx->ins, "%s", - PQerrorMessage(ctx->conn_current->conn)); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - PQflush(ctx->conn_current->conn); - } - else { - res = PQexec(ctx->conn_current->conn, query); - flb_free(query); - flb_sds_destroy(json); - flb_sds_destroy(tag_escaped); - - if (PQresultStatus(res) != PGRES_COMMAND_OK) { - flb_plg_error(ctx->ins, "%s", - PQerrorMessage(ctx->conn_current->conn)); - PQclear(res); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - PQclear(res); - } - - FLB_OUTPUT_RETURN(FLB_OK); -} - -static int cb_pgsql_exit(void *data, struct flb_config *config) -{ - struct flb_pgsql_config *ctx = data; - - if (!ctx){ - return 0; - } - - pgsql_conf_destroy(ctx); - - return 0; -} - -struct flb_output_plugin out_pgsql_plugin = { - .name = "pgsql", - .description = "PostgreSQL", - .cb_init = cb_pgsql_init, - .cb_flush = cb_pgsql_flush, - .cb_exit = cb_pgsql_exit, - .flags = 0, -}; diff --git a/fluent-bit/plugins/out_pgsql/pgsql.h b/fluent-bit/plugins/out_pgsql/pgsql.h deleted file mode 100644 index 5190a5a54..000000000 --- a/fluent-bit/plugins/out_pgsql/pgsql.h +++ /dev/null @@ -1,91 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_PGSQL_H -#define FLB_OUT_PGSQL_H - -#include -#include -#include - -#include - -#define FLB_PGSQL_HOST "127.0.0.1" -#define FLB_PGSQL_PORT 5432 -#define FLB_PGSQL_DBNAME "fluentbit" -#define FLB_PGSQL_TABLE "fluentbit" -#define FLB_PGSQL_TIMESTAMP_KEY "date" -#define FLB_PGSQL_POOL_SIZE 4 -#define FLB_PGSQL_MIN_POOL_SIZE 1 -#define FLB_PGSQL_SYNC FLB_FALSE -#define FLB_PGSQL_COCKROACH FLB_FALSE - -#define FLB_PGSQL_INSERT "INSERT INTO %s SELECT %s, " \ - "to_timestamp(CAST(value->>'%s' as FLOAT))," \ - " * FROM json_array_elements(%s);" -#define FLB_PGSQL_INSERT_COCKROACH "INSERT INTO %s SELECT %s," \ - "CAST(value->>'%s' AS INTERVAL) + DATE'1970-01-01'," \ - " * FROM json_array_elements(%s);" - -struct flb_pgsql_conn { - struct mk_list _head; - PGconn *conn; - int number; -}; - -struct flb_pgsql_config { - - /* database */ - char *db_hostname; - char db_port[8]; - const char *db_name; - flb_sds_t db_table; - - /* auth */ - const char *db_user; - const char *db_passwd; - - /* time key */ - flb_sds_t timestamp_key; - - /* instance reference */ - struct flb_output_instance *ins; - - /* connections options */ - const char *conn_options; - - /* connections pool */ - struct mk_list conn_queue; - struct mk_list _head; - - struct flb_pgsql_conn *conn_current; - int max_pool_size; - int min_pool_size; - int active_conn; - - /* async mode or sync mode */ - int async; - - /* cockroachdb */ - int cockroachdb; -}; - -void pgsql_conf_destroy(struct flb_pgsql_config *ctx); - -#endif diff --git a/fluent-bit/plugins/out_pgsql/pgsql_connections.c b/fluent-bit/plugins/out_pgsql/pgsql_connections.c deleted file mode 100644 index 9c4ccfba2..000000000 --- a/fluent-bit/plugins/out_pgsql/pgsql_connections.c +++ /dev/null @@ -1,193 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include "pgsql.h" - -void pgsql_destroy_connections(struct flb_pgsql_config *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_pgsql_conn *conn; - PGresult *res = NULL; - - mk_list_foreach_safe(head, tmp, &ctx->conn_queue) { - conn = mk_list_entry(head, struct flb_pgsql_conn, _head); - if (PQstatus(conn->conn) == CONNECTION_OK) { - while(PQconsumeInput(conn->conn) == 0) { - res = PQgetResult(conn->conn); - if (PQresultStatus(res) != PGRES_COMMAND_OK) { - flb_plg_warn(ctx->ins, "%s", - PQerrorMessage(conn->conn)); - } - PQclear(res); - } - } - PQfinish(conn->conn); - flb_free(conn); - } -} - -void *pgsql_create_connection(struct flb_pgsql_config *ctx) -{ - struct flb_pgsql_conn *conn; - - conn = flb_calloc(1, sizeof(struct flb_pgsql_conn)); - if (!conn) { - flb_errno(); - return NULL; - } - - conn->conn = PQsetdbLogin(ctx->db_hostname, - ctx->db_port, - ctx->conn_options, - NULL, - ctx->db_name, - ctx->db_user, - ctx->db_passwd); - - if (PQstatus(conn->conn) != CONNECTION_OK) { - flb_plg_error(ctx->ins, - "failed connecting to host=%s with error: %s", - ctx->db_hostname, PQerrorMessage(conn->conn)); - PQfinish(conn->conn); - flb_free(conn); - return NULL; - } - - flb_plg_info(ctx->ins, "switching postgresql connection " - "to non-blocking mode"); - - if (PQsetnonblocking(conn->conn, 1) != 0) { - flb_plg_error(ctx->ins, "non-blocking mode not set"); - PQfinish(conn->conn); - flb_free(conn); - return NULL; - } - - return conn; -} - -int pgsql_start_connections(struct flb_pgsql_config *ctx) -{ - int i; - struct flb_pgsql_conn *conn = NULL; - - mk_list_init(&ctx->conn_queue); - ctx->active_conn = 0; - - for(i = 0; i < ctx->min_pool_size; i++) { - flb_plg_info(ctx->ins, "Opening connection: #%d", i); - - conn = (struct flb_pgsql_conn *)pgsql_create_connection(ctx); - if (conn == NULL) { - pgsql_conf_destroy(ctx); - return -1; - } - - conn->number = i; - ctx->active_conn++; - mk_list_add(&conn->_head, &ctx->conn_queue); - } - - ctx->conn_current = mk_list_entry_last(&ctx->conn_queue, - struct flb_pgsql_conn, - _head); - - return 0; -} - -int pgsql_new_connection(struct flb_pgsql_config *ctx) -{ - struct flb_pgsql_conn *conn = NULL; - - if (ctx->active_conn >= ctx->max_pool_size) { - return -1; - } - - conn = (struct flb_pgsql_conn *)pgsql_create_connection(ctx); - if (conn == NULL) { - pgsql_conf_destroy(ctx); - return -1; - } - - conn->number = ctx->active_conn + 1; - ctx->active_conn++; - - mk_list_add(&conn->_head, &ctx->conn_queue); - - return 0; -} - -int pgsql_next_connection(struct flb_pgsql_config *ctx) -{ - struct flb_pgsql_conn *tmp = NULL; - PGresult *res = NULL; - struct mk_list *head; - int ret_conn = 1; - - if (PQconsumeInput(ctx->conn_current->conn) == 1) { - if (PQisBusy(ctx->conn_current->conn) == 0) { - res = PQgetResult(ctx->conn_current->conn); - PQclear(res); - } - } - else { - flb_plg_error(ctx->ins, "%s", - PQerrorMessage(ctx->conn_current->conn)); - } - - mk_list_foreach(head, &ctx->conn_queue) { - tmp = mk_list_entry(head, struct flb_pgsql_conn, _head); - if (ctx->conn_current == NULL) { - ctx->conn_current = tmp; - break; - } - - res = PQgetResult(tmp->conn); - - if (res == NULL) { - flb_plg_debug(ctx->ins, "Connection number %d", - tmp->number); - ctx->conn_current = tmp; - PQclear(res); - return 0; - } - - if (PQresultStatus(res) == PGRES_FATAL_ERROR) { - flb_plg_info(ctx->ins, "%s", - PQerrorMessage(tmp->conn)); - } - - PQclear(res); - } - - if (pgsql_new_connection(ctx) == -1) { - flb_plg_warn(ctx->ins, - "No more free connections." - " Increase max connections"); - } - else { - flb_plg_warn(ctx->ins, "Added new connection"); - ret_conn = pgsql_next_connection(ctx); - } - - return ret_conn; -} diff --git a/fluent-bit/plugins/out_pgsql/pgsql_connections.h b/fluent-bit/plugins/out_pgsql/pgsql_connections.h deleted file mode 100644 index cd8730618..000000000 --- a/fluent-bit/plugins/out_pgsql/pgsql_connections.h +++ /dev/null @@ -1,27 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_PGSQL_CONNECTIONS_H -#define FLB_OUT_PGSQL_CONNECTIONS_H - -void pgsql_destroy_connections(struct flb_pgsql_config *ctx); -int pgsql_start_connections(struct flb_pgsql_config *ctx); -int pgsql_next_connection(struct flb_pgsql_config *ctx); - -#endif diff --git a/fluent-bit/plugins/out_plot/CMakeLists.txt b/fluent-bit/plugins/out_plot/CMakeLists.txt deleted file mode 100644 index e3cd96db5..000000000 --- a/fluent-bit/plugins/out_plot/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - plot.c) - -FLB_PLUGIN(out_plot "${src}" "") diff --git a/fluent-bit/plugins/out_plot/plot.c b/fluent-bit/plugins/out_plot/plot.c deleted file mode 100644 index e5a217f45..000000000 --- a/fluent-bit/plugins/out_plot/plot.c +++ /dev/null @@ -1,242 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -struct flb_plot { - const char *out_file; - flb_sds_t key; - struct flb_output_instance *ins; -}; - -static int cb_plot_init(struct flb_output_instance *ins, - struct flb_config *config, - void *data) -{ - int ret; - (void) config; - (void) data; - struct flb_plot *ctx; - - ctx = flb_calloc(1, sizeof(struct flb_plot)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->ins = ins; - - ret = flb_output_config_map_set(ins, (void *)ctx); - if (ret == -1) { - flb_free(ctx); - return -1; - } - - /* Set the context */ - flb_output_set_context(ins, ctx); - - return 0; -} - -static void cb_plot_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - int i; - int written; - int fd; - const char *out_file; - msgpack_object *map; - msgpack_object *key = NULL; - msgpack_object *val = NULL; - struct flb_plot *ctx = out_context; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - int ret; - - (void) i_ins; - (void) config; - - /* Set the right output */ - if (!ctx->out_file) { - out_file = event_chunk->tag; - } - else { - out_file = ctx->out_file; - } - - /* Open output file with default name as the Tag */ - fd = open(out_file, O_WRONLY | O_CREAT | O_APPEND, 0666); - if (fd == -1) { - flb_errno(); - flb_plg_warn(ctx->ins, "could not open %s, switching to STDOUT", - out_file); - fd = STDOUT_FILENO; - } - - ret = flb_log_event_decoder_init(&log_decoder, - (char *) event_chunk->data, - event_chunk->size); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - close(fd); - - FLB_OUTPUT_RETURN(FLB_ERROR); - } - - /* - * Upon flush, for each array, lookup the time and the first field - * of the map to use as a data point. - */ - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - map = log_event.body; - - /* - * Lookup key, we need to iterate the whole map as sometimes the - * data that gets in can set the keys in different order (e.g: forward, - * tcp, etc). - */ - if (ctx->key) { - for (i = 0; i < map->via.map.size; i++) { - /* Get each key and compare */ - key = &(map->via.map.ptr[i].key); - if (key->type == MSGPACK_OBJECT_BIN) { - if (flb_sds_len(ctx->key) == key->via.bin.size && - memcmp(key->via.bin.ptr, ctx->key, - flb_sds_len(ctx->key)) == 0) { - val = &(map->via.map.ptr[i].val); - break; - } - key = NULL; - val = NULL; - } - else if (key->type == MSGPACK_OBJECT_STR) { - if (flb_sds_len(ctx->key) == key->via.str.size && - memcmp(key->via.str.ptr, ctx->key, - flb_sds_len(ctx->key)) == 0) { - val = &(map->via.map.ptr[i].val); - break; - } - key = NULL; - val = NULL; - } - else { - if (fd != STDOUT_FILENO) { - close(fd); - } - - flb_log_event_decoder_destroy(&log_decoder); - - FLB_OUTPUT_RETURN(FLB_ERROR); - } - } - } - else { - val = &(map->via.map.ptr[0].val); - } - - if (!val) { - flb_plg_error(ctx->ins, "unmatched key '%s'", ctx->key); - - if (fd != STDOUT_FILENO) { - close(fd); - } - - flb_log_event_decoder_destroy(&log_decoder); - - FLB_OUTPUT_RETURN(FLB_ERROR); - } - - if (val->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - written = dprintf(fd, "%f %" PRIu64 "\n", - flb_time_to_double(&log_event.timestamp), val->via.u64); - } - else if (val->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { - written = dprintf(fd, "%f %" PRId64 "\n", - flb_time_to_double(&log_event.timestamp), val->via.i64); - } - else if (val->type == MSGPACK_OBJECT_FLOAT) { - written = dprintf(fd, "%f %lf\n", - flb_time_to_double(&log_event.timestamp), val->via.f64); - } - else { - flb_plg_error(ctx->ins, "value must be integer, negative integer " - "or float"); - written = 0; - } - flb_plg_debug(ctx->ins, "%i bytes written to file '%s'", - written, out_file); - } - - flb_log_event_decoder_destroy(&log_decoder); - - if (fd != STDOUT_FILENO) { - close(fd); - } - - FLB_OUTPUT_RETURN(FLB_OK); -} - -static int cb_plot_exit(void *data, struct flb_config *config) -{ - struct flb_plot *ctx = data; - - flb_free(ctx); - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "key", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_plot, key), - "set a number of times to generate event." - }, - { - FLB_CONFIG_MAP_STR, "file", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_plot, out_file), - "set a number of times to generate event." - }, - /* EOF */ - {0} -}; - -struct flb_output_plugin out_plot_plugin = { - .name = "plot", - .description = "Generate data file for GNU Plot", - .cb_init = cb_plot_init, - .cb_flush = cb_plot_flush, - .cb_exit = cb_plot_exit, - .config_map = config_map, - .flags = 0, -}; diff --git a/fluent-bit/plugins/out_prometheus_exporter/CMakeLists.txt b/fluent-bit/plugins/out_prometheus_exporter/CMakeLists.txt deleted file mode 100644 index f03809e57..000000000 --- a/fluent-bit/plugins/out_prometheus_exporter/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -if(NOT FLB_HTTP_SERVER) - message( - FATAL_ERROR - "Prometheus Exporter output plugin requires built-in HTTP Server be enabled: - Use -DFLB_HTTP_SERVER=On option to enable it" - ) -endif() - -set(src - prom_http.c - prom.c - ) - -FLB_PLUGIN(out_prometheus_exporter "${src}" "") diff --git a/fluent-bit/plugins/out_prometheus_exporter/prom.c b/fluent-bit/plugins/out_prometheus_exporter/prom.c deleted file mode 100644 index d471d2bab..000000000 --- a/fluent-bit/plugins/out_prometheus_exporter/prom.c +++ /dev/null @@ -1,298 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include "prom.h" -#include "prom_http.h" - -static int config_add_labels(struct flb_output_instance *ins, - struct prom_exporter *ctx) -{ - struct mk_list *head; - struct flb_config_map_val *mv; - struct flb_slist_entry *k = NULL; - struct flb_slist_entry *v = NULL; - struct flb_kv *kv; - - if (!ctx->add_labels || mk_list_size(ctx->add_labels) == 0) { - return 0; - } - - /* iterate all 'add_label' definitions */ - flb_config_map_foreach(head, mv, ctx->add_labels) { - if (mk_list_size(mv->val.list) != 2) { - flb_plg_error(ins, "'add_label' expects a key and a value, " - "e.g: 'add_label version 1.8.0'"); - return -1; - } - - k = mk_list_entry_first(mv->val.list, struct flb_slist_entry, _head); - v = mk_list_entry_last(mv->val.list, struct flb_slist_entry, _head); - - kv = flb_kv_item_create(&ctx->kv_labels, k->str, v->str); - if (!kv) { - flb_plg_error(ins, "could not append label %s=%s\n", k->str, v->str); - return -1; - } - } - - return 0; -} - -static int cb_prom_init(struct flb_output_instance *ins, - struct flb_config *config, - void *data) -{ - int ret; - struct prom_exporter *ctx; - - flb_output_net_default("0.0.0.0", 2021 , ins); - - ctx = flb_calloc(1, sizeof(struct prom_exporter)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->ins = ins; - flb_kv_init(&ctx->kv_labels); - flb_output_set_context(ins, ctx); - - /* Load config map */ - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - return -1; - } - - /* Parse 'add_label' */ - ret = config_add_labels(ins, ctx); - if (ret == -1) { - return -1; - } - - /* HTTP Server context */ - ctx->http = prom_http_server_create(ctx, - ins->host.name, ins->host.port, config); - if (!ctx->http) { - flb_plg_error(ctx->ins, "could not initialize HTTP server, aborting"); - return -1; - } - - /* Hash table for metrics */ - ctx->ht_metrics = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, 32, 0); - if (!ctx->ht_metrics) { - flb_plg_error(ctx->ins, "could not initialize hash table for metrics"); - return -1; - } - - /* Start HTTP Server */ - ret = prom_http_server_start(ctx->http); - if (ret == -1) { - return -1; - } - - flb_plg_info(ctx->ins, "listening iface=%s tcp_port=%d", - ins->host.name, ins->host.port); - return 0; -} - -static void append_labels(struct prom_exporter *ctx, struct cmt *cmt) -{ - struct flb_kv *kv; - struct mk_list *head; - - mk_list_foreach(head, &ctx->kv_labels) { - kv = mk_list_entry(head, struct flb_kv, _head); - cmt_label_add(cmt, kv->key, kv->val); - } -} - -static int hash_store(struct prom_exporter *ctx, struct flb_input_instance *ins, - cfl_sds_t buf) -{ - int ret; - int len; - - len = strlen(ins->name); - - /* store/override the content into the hash table */ - ret = flb_hash_table_add(ctx->ht_metrics, ins->name, len, - buf, cfl_sds_len(buf)); - if (ret < 0) { - return -1; - } - - return 0; -} - -static flb_sds_t hash_format_metrics(struct prom_exporter *ctx) -{ - int size = 2048; - flb_sds_t buf; - - struct mk_list *head; - struct flb_hash_table_entry *entry; - - - buf = flb_sds_create_size(size); - if (!buf) { - return NULL; - } - - /* Take every hash entry and compose one buffer with the whole content */ - mk_list_foreach(head, &ctx->ht_metrics->entries) { - entry = mk_list_entry(head, struct flb_hash_table_entry, _head_parent); - flb_sds_cat_safe(&buf, entry->val, entry->val_size); - } - - return buf; -} - -static void cb_prom_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *ins, void *out_context, - struct flb_config *config) -{ - int ret; - int add_ts; - size_t off = 0; - flb_sds_t metrics; - cfl_sds_t text; - struct cmt *cmt; - struct prom_exporter *ctx = out_context; - - /* - * A new set of metrics has arrived, perform decoding, apply labels, - * convert to Prometheus text format and store the output in the - * hash table for metrics. - */ - ret = cmt_decode_msgpack_create(&cmt, - (char *) event_chunk->data, - event_chunk->size, &off); - if (ret != 0) { - FLB_OUTPUT_RETURN(FLB_ERROR); - } - - /* append labels set by config */ - append_labels(ctx, cmt); - - /* add timestamp in the output format ? */ - if (ctx->add_timestamp) { - add_ts = CMT_TRUE; - } - else { - add_ts = CMT_FALSE; - } - - /* convert to text representation */ - text = cmt_encode_prometheus_create(cmt, add_ts); - if (!text) { - cmt_destroy(cmt); - FLB_OUTPUT_RETURN(FLB_ERROR); - } - cmt_destroy(cmt); - - if (cfl_sds_len(text) == 0) { - flb_plg_debug(ctx->ins, "context without metrics (empty)"); - cmt_encode_text_destroy(text); - FLB_OUTPUT_RETURN(FLB_OK); - } - - /* register payload of metrics / override previous one */ - ret = hash_store(ctx, ins, text); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not store metrics coming from: %s", - flb_input_name(ins)); - cmt_encode_prometheus_destroy(text); - cmt_destroy(cmt); - FLB_OUTPUT_RETURN(FLB_ERROR); - } - cmt_encode_prometheus_destroy(text); - - /* retrieve a full copy of all metrics */ - metrics = hash_format_metrics(ctx); - if (!metrics) { - flb_plg_error(ctx->ins, "could not retrieve metrics"); - FLB_OUTPUT_RETURN(FLB_ERROR); - } - - /* push new (full) metrics payload */ - ret = prom_http_server_mq_push_metrics(ctx->http, - (char *) metrics, - flb_sds_len(metrics)); - flb_sds_destroy(metrics); - - if (ret != 0) { - FLB_OUTPUT_RETURN(FLB_ERROR); - } - - FLB_OUTPUT_RETURN(FLB_OK); -} - -static int cb_prom_exit(void *data, struct flb_config *config) -{ - struct prom_exporter *ctx = data; - - if (!ctx) { - return 0; - } - - if (ctx->ht_metrics) { - flb_hash_table_destroy(ctx->ht_metrics); - } - - flb_kv_release(&ctx->kv_labels); - prom_http_server_stop(ctx->http); - prom_http_server_destroy(ctx->http); - flb_free(ctx); - - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_BOOL, "add_timestamp", "false", - 0, FLB_TRUE, offsetof(struct prom_exporter, add_timestamp), - "Add timestamp to every metric honoring collection time." - }, - - { - FLB_CONFIG_MAP_SLIST_1, "add_label", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct prom_exporter, add_labels), - "TCP port for listening for HTTP connections." - }, - - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_output_plugin out_prometheus_exporter_plugin = { - .name = "prometheus_exporter", - .description = "Prometheus Exporter", - .cb_init = cb_prom_init, - .cb_flush = cb_prom_flush, - .cb_exit = cb_prom_exit, - .flags = FLB_OUTPUT_NET, - .event_type = FLB_OUTPUT_METRICS, - .config_map = config_map, -}; diff --git a/fluent-bit/plugins/out_prometheus_exporter/prom.h b/fluent-bit/plugins/out_prometheus_exporter/prom.h deleted file mode 100644 index 1cbab6a59..000000000 --- a/fluent-bit/plugins/out_prometheus_exporter/prom.h +++ /dev/null @@ -1,46 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_PROMETHEUS_EXPORTER_H -#define FLB_PROMETHEUS_EXPORTER_H - -#include -#include - -/* Plugin context */ -struct prom_exporter { - void *http; - - /* hash table for metrics reported */ - struct flb_hash_table *ht_metrics; - - /* add timestamp to every metric */ - int add_timestamp; - - /* config reader for 'add_label' */ - struct mk_list *add_labels; - - /* internal labels ready to append */ - struct mk_list kv_labels; - - /* instance context */ - struct flb_output_instance *ins; -}; - -#endif diff --git a/fluent-bit/plugins/out_prometheus_exporter/prom_http.c b/fluent-bit/plugins/out_prometheus_exporter/prom_http.c deleted file mode 100644 index 7ff3f8200..000000000 --- a/fluent-bit/plugins/out_prometheus_exporter/prom_http.c +++ /dev/null @@ -1,268 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include "prom.h" -#include "prom_http.h" - -pthread_key_t ph_metrics_key; - -/* Return the newest storage metrics buffer */ -static struct prom_http_buf *metrics_get_latest() -{ - struct prom_http_buf *buf; - struct mk_list *metrics_list; - - metrics_list = pthread_getspecific(ph_metrics_key); - if (!metrics_list) { - return NULL; - } - - if (mk_list_size(metrics_list) == 0) { - return NULL; - } - - buf = mk_list_entry_last(metrics_list, struct prom_http_buf, _head); - return buf; -} - -/* Delete unused metrics, note that we only care about the latest node */ -static int cleanup_metrics() -{ - int c = 0; - struct mk_list *tmp; - struct mk_list *head; - struct mk_list *metrics_list; - struct prom_http_buf *last; - struct prom_http_buf *entry; - - metrics_list = pthread_getspecific(ph_metrics_key); - if (!metrics_list) { - return -1; - } - - last = metrics_get_latest(); - if (!last) { - return -1; - } - - mk_list_foreach_safe(head, tmp, metrics_list) { - entry = mk_list_entry(head, struct prom_http_buf, _head); - if (entry != last && entry->users == 0) { - mk_list_del(&entry->_head); - flb_free(entry->buf_data); - flb_free(entry); - c++; - } - } - - return c; -} - -/* destructor callback */ -static void destruct_metrics(void *data) -{ - struct mk_list *tmp; - struct mk_list *head; - struct mk_list *metrics_list = (struct mk_list*)data; - struct prom_http_buf *entry; - - if (!metrics_list) { - return; - } - - mk_list_foreach_safe(head, tmp, metrics_list) { - entry = mk_list_entry(head, struct prom_http_buf, _head); - mk_list_del(&entry->_head); - flb_free(entry->buf_data); - flb_free(entry); - } - - flb_free(metrics_list); -} - -/* - * Callback invoked every time a new payload of Metrics is received from - * Fluent Bit engine through Message Queue channel. - * - * This function runs in a Monkey HTTP thread worker and it purpose is - * to take the metrics data and store it locally for every thread, so then - * it can be available on 'cb_metrics()' to serve it as a response. - */ -static void cb_mq_metrics(mk_mq_t *queue, void *data, size_t size) -{ - struct prom_http_buf *buf; - struct mk_list *metrics_list = NULL; - - metrics_list = pthread_getspecific(ph_metrics_key); - if (!metrics_list) { - metrics_list = flb_malloc(sizeof(struct mk_list)); - if (!metrics_list) { - flb_errno(); - return; - } - mk_list_init(metrics_list); - pthread_setspecific(ph_metrics_key, metrics_list); - } - - /* FIXME: convert data ? */ - buf = flb_malloc(sizeof(struct prom_http_buf)); - if (!buf) { - flb_errno(); - return; - } - buf->users = 0; - buf->buf_data = flb_malloc(size); - if (!buf->buf_data) { - flb_errno(); - flb_free(buf); - return; - } - memcpy(buf->buf_data, data, size); - buf->buf_size = size; - - mk_list_add(&buf->_head, metrics_list); - cleanup_metrics(); -} - -/* Create message queue to receive Metrics payload from the engine */ -static int http_server_mq_create(struct prom_http *ph) -{ - int ret; - - pthread_key_create(&ph_metrics_key, destruct_metrics); - - ret = mk_mq_create(ph->ctx, "/metrics", cb_mq_metrics, NULL); - if (ret == -1) { - return -1; - } - ph->qid_metrics = ret; - return 0; -} - -/* HTTP endpoint: /metrics */ -static void cb_metrics(mk_request_t *request, void *data) -{ - struct prom_http_buf *buf; - (void) data; - - buf = metrics_get_latest(); - if (!buf) { - mk_http_status(request, 404); - mk_http_done(request); - return; - } - - buf->users++; - - mk_http_status(request, 200); - flb_hs_add_content_type_to_req(request, FLB_HS_CONTENT_TYPE_PROMETHEUS); - mk_http_send(request, buf->buf_data, buf->buf_size, NULL); - mk_http_done(request); - - buf->users--; -} - -/* HTTP endpoint: / (root) */ -static void cb_root(mk_request_t *request, void *data) -{ - (void) data; - - mk_http_status(request, 200); - mk_http_send(request, "Fluent Bit Prometheus Exporter\n", 31, NULL); - mk_http_done(request); -} - -struct prom_http *prom_http_server_create(struct prom_exporter *ctx, - const char *listen, - int tcp_port, - struct flb_config *config) -{ - int ret; - int vid; - char tmp[32]; - struct prom_http *ph; - - ph = flb_malloc(sizeof(struct prom_http)); - if (!ph) { - flb_errno(); - return NULL; - } - ph->config = config; - - /* HTTP Server context */ - ph->ctx = mk_create(); - if (!ph->ctx) { - flb_free(ph); - return NULL; - } - - /* Compose listen address */ - snprintf(tmp, sizeof(tmp) -1, "%s:%d", listen, tcp_port); - mk_config_set(ph->ctx, - "Listen", tmp, - "Workers", "1", - NULL); - - /* Virtual host */ - vid = mk_vhost_create(ph->ctx, NULL); - ph->vid = vid; - - /* Set HTTP URI callbacks */ - mk_vhost_handler(ph->ctx, vid, "/metrics", cb_metrics, NULL); - mk_vhost_handler(ph->ctx, vid, "/", cb_root, NULL); - - /* Create a Message Queue to push 'metrics' to HTTP workers */ - ret = http_server_mq_create(ph); - if (ret == -1) { - mk_destroy(ph->ctx); - flb_free(ph); - return NULL; - } - - return ph; -} - -void prom_http_server_destroy(struct prom_http *ph) -{ - if (ph) { - /* TODO: release mk_vhost */ - if (ph->ctx) { - mk_destroy(ph->ctx); - } - flb_free(ph); - } -} - -int prom_http_server_start(struct prom_http *ph) -{ - return mk_start(ph->ctx); -} - -int prom_http_server_stop(struct prom_http *ph) -{ - return mk_stop(ph->ctx); -} - -int prom_http_server_mq_push_metrics(struct prom_http *ph, - void *data, size_t size) -{ - return mk_mq_send(ph->ctx, ph->qid_metrics, data, size); -} diff --git a/fluent-bit/plugins/out_prometheus_exporter/prom_http.h b/fluent-bit/plugins/out_prometheus_exporter/prom_http.h deleted file mode 100644 index 79b4c87bb..000000000 --- a/fluent-bit/plugins/out_prometheus_exporter/prom_http.h +++ /dev/null @@ -1,56 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_PROMETHEUS_EXPORTER_HTTP_H -#define FLB_PROMETHEUS_EXPORTER_HTTP_H - -#include -#include - -#include "prom.h" - -/* HTTP response payload received through a Message Queue */ -struct prom_http_buf { - int users; - char *buf_data; - size_t buf_size; - struct mk_list _head; -}; - -/* Prom HTTP Server context */ -struct prom_http { - mk_ctx_t *ctx; /* Monkey HTTP Context */ - int vid; /* Virtual host ID */ - int qid_metrics; /* Queue ID for Metrics buffer */ - struct flb_config *config; /* Fluent Bit context */ -}; - -struct prom_http *prom_http_server_create(struct prom_exporter *ctx, - const char *listen, - int tcp_port, - struct flb_config *config); -void prom_http_server_destroy(struct prom_http *ph); - -int prom_http_server_start(struct prom_http *ph); -int prom_http_server_stop(struct prom_http *ph); - -int prom_http_server_mq_push_metrics(struct prom_http *ph, - void *data, size_t size); - -#endif diff --git a/fluent-bit/plugins/out_prometheus_remote_write/CMakeLists.txt b/fluent-bit/plugins/out_prometheus_remote_write/CMakeLists.txt deleted file mode 100644 index 71779cb9d..000000000 --- a/fluent-bit/plugins/out_prometheus_remote_write/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(src - remote_write.c - remote_write_conf.c - ) - -FLB_PLUGIN(out_prometheus_remote_write "${src}" "") diff --git a/fluent-bit/plugins/out_prometheus_remote_write/remote_write.c b/fluent-bit/plugins/out_prometheus_remote_write/remote_write.c deleted file mode 100644 index 2349afd61..000000000 --- a/fluent-bit/plugins/out_prometheus_remote_write/remote_write.c +++ /dev/null @@ -1,466 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#ifdef FLB_HAVE_SIGNV4 -#ifdef FLB_HAVE_AWS -#include -#include -#endif -#endif - -#include "remote_write.h" -#include "remote_write_conf.h" - -static int http_post(struct prometheus_remote_write_context *ctx, - const void *body, size_t body_len, - const char *tag, int tag_len) -{ - int ret; - int out_ret = FLB_OK; - size_t b_sent; - void *payload_buf = NULL; - size_t payload_size = 0; - struct flb_upstream *u; - struct flb_connection *u_conn; - struct flb_http_client *c; - struct mk_list *head; - struct flb_config_map_val *mv; - struct flb_slist_entry *key = NULL; - struct flb_slist_entry *val = NULL; - flb_sds_t signature = NULL; - - /* Get upstream context and connection */ - u = ctx->u; - u_conn = flb_upstream_conn_get(u); - if (!u_conn) { - flb_plg_error(ctx->ins, "no upstream connections available to %s:%i", - u->tcp_host, u->tcp_port); - return FLB_RETRY; - } - - /* Map payload */ - - if (strcasecmp(ctx->compression, "snappy") == 0) { - ret = flb_snappy_compress((void *) body, body_len, - (char **) &payload_buf, - &payload_size); - } - else if (strcasecmp(ctx->compression, "gzip") == 0) { - ret = flb_gzip_compress((void *) body, body_len, - &payload_buf, &payload_size); - } - else { - payload_buf = (void *) body; - payload_size = body_len; - - ret = 0; - } - - if (ret != 0) { - flb_upstream_conn_release(u_conn); - - flb_plg_error(ctx->ins, - "cannot compress payload, aborting"); - - return FLB_ERROR; - } - - /* Create HTTP client context */ - c = flb_http_client(u_conn, FLB_HTTP_POST, ctx->uri, - payload_buf, payload_size, - ctx->host, ctx->port, - ctx->proxy, 0); - - - if (c->proxy.host) { - flb_plg_debug(ctx->ins, "[http_client] proxy host: %s port: %i", - c->proxy.host, c->proxy.port); - } - - /* Allow duplicated headers ? */ - flb_http_allow_duplicated_headers(c, FLB_FALSE); - - /* - * Direct assignment of the callback context to the HTTP client context. - * This needs to be improved through a more clean API. - */ - c->cb_ctx = ctx->ins->callback; - - flb_http_add_header(c, - FLB_PROMETHEUS_REMOTE_WRITE_CONTENT_TYPE_HEADER_NAME, - sizeof(FLB_PROMETHEUS_REMOTE_WRITE_CONTENT_TYPE_HEADER_NAME) - 1, - FLB_PROMETHEUS_REMOTE_WRITE_MIME_PROTOBUF_LITERAL, - sizeof(FLB_PROMETHEUS_REMOTE_WRITE_MIME_PROTOBUF_LITERAL) - 1); - - flb_http_add_header(c, - FLB_PROMETHEUS_REMOTE_WRITE_VERSION_HEADER_NAME, - sizeof(FLB_PROMETHEUS_REMOTE_WRITE_VERSION_HEADER_NAME) - 1, - FLB_PROMETHEUS_REMOTE_WRITE_VERSION_LITERAL, - sizeof(FLB_PROMETHEUS_REMOTE_WRITE_VERSION_LITERAL) - 1); - - if (strcasecmp(ctx->compression, "snappy") == 0) { - flb_http_add_header(c, - "Content-Encoding", - strlen("Content-Encoding"), - "snappy", - strlen("snappy")); - } - else if (strcasecmp(ctx->compression, "gzip") == 0) { - flb_http_add_header(c, - "Content-Encoding", - strlen("Content-Encoding"), - "gzip", - strlen("gzip")); - } - - /* Basic Auth headers */ - if (ctx->http_user && ctx->http_passwd) { - flb_http_basic_auth(c, ctx->http_user, ctx->http_passwd); - } - - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - - flb_config_map_foreach(head, mv, ctx->headers) { - key = mk_list_entry_first(mv->val.list, struct flb_slist_entry, _head); - val = mk_list_entry_last(mv->val.list, struct flb_slist_entry, _head); - - flb_http_add_header(c, - key->str, flb_sds_len(key->str), - val->str, flb_sds_len(val->str)); - } - -#ifdef FLB_HAVE_SIGNV4 -#ifdef FLB_HAVE_AWS - /* AWS SigV4 headers */ - if (ctx->has_aws_auth == FLB_TRUE) { - flb_plg_debug(ctx->ins, "signing request with AWS Sigv4"); - signature = flb_signv4_do(c, - FLB_TRUE, /* normalize URI ? */ - FLB_TRUE, /* add x-amz-date header ? */ - time(NULL), - (char *) ctx->aws_region, - (char *) ctx->aws_service, - 0, NULL, - ctx->aws_provider); - - if (!signature) { - flb_plg_error(ctx->ins, "could not sign request with sigv4"); - out_ret = FLB_RETRY; - goto cleanup; - } - flb_sds_destroy(signature); - } -#endif -#endif - - ret = flb_http_do(c, &b_sent); - if (ret == 0) { - /* - * Only allow the following HTTP status: - * - * - 200: OK - * - 201: Created - * - 202: Accepted - * - 203: no authorative resp - * - 204: No Content - * - 205: Reset content - * - */ - if ((c->resp.status < 200 || c->resp.status > 205) && - c->resp.status != 400) { - if (ctx->log_response_payload && - c->resp.payload && c->resp.payload_size > 0) { - flb_plg_error(ctx->ins, "%s:%i, HTTP status=%i\n%s", - ctx->host, ctx->port, - c->resp.status, c->resp.payload); - } - else { - flb_plg_error(ctx->ins, "%s:%i, HTTP status=%i", - ctx->host, ctx->port, c->resp.status); - } - out_ret = FLB_RETRY; - } - else if (c->resp.status == 400) { - /* Returned 400 status means unrecoverable. Immidiately - * returning as a error. */ - if (ctx->log_response_payload && - c->resp.payload && c->resp.payload_size > 0) { - flb_plg_error(ctx->ins, "%s:%i, HTTP status=%i\n%s", - ctx->host, ctx->port, - c->resp.status, c->resp.payload); - } - else { - flb_plg_error(ctx->ins, "%s:%i, HTTP status=%i", - ctx->host, ctx->port, c->resp.status); - } - out_ret = FLB_ERROR; - } - else { - if (ctx->log_response_payload && - c->resp.payload && c->resp.payload_size > 0) { - flb_plg_debug(ctx->ins, "%s:%i, HTTP status=%i\n%s", - ctx->host, ctx->port, - c->resp.status, c->resp.payload); - } - else { - flb_plg_debug(ctx->ins, "%s:%i, HTTP status=%i", - ctx->host, ctx->port, - c->resp.status); - } - } - } - else { - flb_plg_error(ctx->ins, "could not flush records to %s:%i (http_do=%i)", - ctx->host, ctx->port, ret); - out_ret = FLB_RETRY; - } - -cleanup: - /* - * If the payload buffer is different than incoming records in body, means - * we generated a different payload and must be freed. - */ - if (payload_buf != body) { - flb_free(payload_buf); - } - - /* Destroy HTTP client context */ - flb_http_client_destroy(c); - - /* Release the TCP connection */ - flb_upstream_conn_release(u_conn); - - return out_ret; -} - -static int cb_prom_init(struct flb_output_instance *ins, - struct flb_config *config, - void *data) -{ - struct prometheus_remote_write_context *ctx; - - ctx = flb_prometheus_remote_write_context_create(ins, config); - if (!ctx) { - return -1; - } - - flb_output_set_context(ins, ctx); - - return 0; -} - -static void append_labels(struct prometheus_remote_write_context *ctx, - struct cmt *cmt) -{ - struct flb_kv *kv; - struct mk_list *head; - - mk_list_foreach(head, &ctx->kv_labels) { - kv = mk_list_entry(head, struct flb_kv, _head); - cmt_label_add(cmt, kv->key, kv->val); - } -} - -static void cb_prom_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *ins, void *out_context, - struct flb_config *config) -{ - int c = 0; - int ok; - int ret; - int result; - cfl_sds_t encoded_chunk; - flb_sds_t buf = NULL; - size_t diff = 0; - size_t off = 0; - struct cmt *cmt; - struct prometheus_remote_write_context *ctx = out_context; - - /* Initialize vars */ - ctx = out_context; - ok = CMT_DECODE_MSGPACK_SUCCESS; - result = FLB_OK; - - /* Buffer to concatenate multiple metrics contexts */ - buf = flb_sds_create_size(event_chunk->size); - if (!buf) { - flb_plg_error(ctx->ins, "could not allocate outgoing buffer"); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - flb_plg_debug(ctx->ins, "cmetrics msgpack size: %lu", - event_chunk->size); - - /* Decode and encode every CMetric context */ - diff = 0; - while ((ret = cmt_decode_msgpack_create(&cmt, - (char *) event_chunk->data, - event_chunk->size, &off)) == ok) { - /* append labels set by config */ - append_labels(ctx, cmt); - - /* Create a Prometheus Remote Write payload */ - encoded_chunk = cmt_encode_prometheus_remote_write_create(cmt); - if (encoded_chunk == NULL) { - flb_plg_error(ctx->ins, - "Error encoding context as prometheus remote write"); - result = FLB_ERROR; - goto exit; - } - - flb_plg_debug(ctx->ins, "cmetric_id=%i decoded %lu-%lu payload_size=%lu", - c, diff, off, flb_sds_len(encoded_chunk)); - c++; - diff = off; - - /* concat buffer */ - flb_sds_cat_safe(&buf, encoded_chunk, flb_sds_len(encoded_chunk)); - - /* release */ - cmt_encode_prometheus_remote_write_destroy(encoded_chunk); - cmt_destroy(cmt); - } - - if (ret == CMT_DECODE_MSGPACK_INSUFFICIENT_DATA && c > 0) { - flb_plg_debug(ctx->ins, "final payload size: %lu", flb_sds_len(buf)); - if (buf && flb_sds_len(buf) > 0) { - /* Send HTTP request */ - result = http_post(ctx, buf, flb_sds_len(buf), - event_chunk->tag, - flb_sds_len(event_chunk->tag)); - - /* Debug http_post() result statuses */ - if (result == FLB_OK) { - flb_plg_debug(ctx->ins, "http_post result FLB_OK"); - } - else if (result == FLB_ERROR) { - flb_plg_debug(ctx->ins, "http_post result FLB_ERROR"); - } - else if (result == FLB_RETRY) { - flb_plg_debug(ctx->ins, "http_post result FLB_RETRY"); - } - } - flb_sds_destroy(buf); - buf = NULL; - } - else { - flb_plg_error(ctx->ins, "Error decoding msgpack encoded context"); - } - -exit: - if (buf) { - flb_sds_destroy(buf); - } - FLB_OUTPUT_RETURN(result); -} - -static int cb_prom_exit(void *data, struct flb_config *config) -{ - struct prometheus_remote_write_context *ctx; - - ctx = (struct prometheus_remote_write_context *) data; - - flb_prometheus_remote_write_context_destroy(ctx); - - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_SLIST_1, "add_label", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct prometheus_remote_write_context, - add_labels), - "Adds a custom label to the metrics use format: 'add_label name value'" - }, - - { - FLB_CONFIG_MAP_STR, "proxy", NULL, - 0, FLB_FALSE, 0, - "Specify an HTTP Proxy. The expected format of this value is http://host:port. " - }, - { - FLB_CONFIG_MAP_STR, "http_user", NULL, - 0, FLB_TRUE, offsetof(struct prometheus_remote_write_context, http_user), - "Set HTTP auth user" - }, - { - FLB_CONFIG_MAP_STR, "http_passwd", "", - 0, FLB_TRUE, offsetof(struct prometheus_remote_write_context, http_passwd), - "Set HTTP auth password" - }, - { - FLB_CONFIG_MAP_STR, "compression", "snappy", - 0, FLB_TRUE, offsetof(struct prometheus_remote_write_context, compression), - "Compress the payload with either snappy, gzip if set" - }, - -#ifdef FLB_HAVE_SIGNV4 -#ifdef FLB_HAVE_AWS - { - FLB_CONFIG_MAP_BOOL, "aws_auth", "false", - 0, FLB_TRUE, offsetof(struct prometheus_remote_write_context, has_aws_auth), - "Enable AWS SigV4 authentication" - }, - { - FLB_CONFIG_MAP_STR, "aws_service", "aps", - 0, FLB_TRUE, offsetof(struct prometheus_remote_write_context, aws_service), - "AWS destination service code, used by SigV4 authentication" - }, - FLB_AWS_CREDENTIAL_BASE_CONFIG_MAP(FLB_PROMETHEUS_REMOTE_WRITE_CREDENTIAL_PREFIX), -#endif -#endif - { - FLB_CONFIG_MAP_SLIST_1, "header", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct prometheus_remote_write_context, headers), - "Add a HTTP header key/value pair. Multiple headers can be set" - }, - { - FLB_CONFIG_MAP_STR, "uri", NULL, - 0, FLB_TRUE, offsetof(struct prometheus_remote_write_context, uri), - "Specify an optional HTTP URI for the target web server, e.g: /something" - }, - { - FLB_CONFIG_MAP_BOOL, "log_response_payload", "true", - 0, FLB_TRUE, offsetof(struct prometheus_remote_write_context, log_response_payload), - "Specify if the response paylod should be logged or not" - }, - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_output_plugin out_prometheus_remote_write_plugin = { - .name = "prometheus_remote_write", - .description = "Prometheus remote write", - .cb_init = cb_prom_init, - .cb_flush = cb_prom_flush, - .cb_exit = cb_prom_exit, - .config_map = config_map, - .event_type = FLB_OUTPUT_METRICS, - .workers = 2, - .flags = FLB_OUTPUT_NET | FLB_IO_OPT_TLS, -}; diff --git a/fluent-bit/plugins/out_prometheus_remote_write/remote_write.h b/fluent-bit/plugins/out_prometheus_remote_write/remote_write.h deleted file mode 100644 index 349c7a7b0..000000000 --- a/fluent-bit/plugins/out_prometheus_remote_write/remote_write.h +++ /dev/null @@ -1,83 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_PROMETHEUS_REMOTE_WRITE_H -#define FLB_PROMETHEUS_REMOTE_WRITE_H - -#include - -#define FLB_PROMETHEUS_REMOTE_WRITE_CONTENT_TYPE_HEADER_NAME "Content-Type" -#define FLB_PROMETHEUS_REMOTE_WRITE_MIME_PROTOBUF_LITERAL "application/x-protobuf" -#define FLB_PROMETHEUS_REMOTE_WRITE_VERSION_HEADER_NAME "X-Prometheus-Remote-Write-Version" -#define FLB_PROMETHEUS_REMOTE_WRITE_VERSION_LITERAL "0.1.0" -#ifdef FLB_HAVE_SIGNV4 -#ifdef FLB_HAVE_AWS -#define FLB_PROMETHEUS_REMOTE_WRITE_CREDENTIAL_PREFIX "aws_" -#endif -#endif - -/* Plugin context */ -struct prometheus_remote_write_context { - /* HTTP Auth */ - char *http_user; - char *http_passwd; - - /* AWS Auth */ -#ifdef FLB_HAVE_SIGNV4 -#ifdef FLB_HAVE_AWS - int has_aws_auth; - struct flb_aws_provider *aws_provider; - const char *aws_region; - const char *aws_service; -#endif -#endif - - /* Proxy */ - const char *proxy; - char *proxy_host; - int proxy_port; - - /* HTTP URI */ - char *uri; - char *host; - int port; - - const char *compression; - - /* Log the response paylod */ - int log_response_payload; - - /* config reader for 'add_label' */ - struct mk_list *add_labels; - - /* internal labels ready to append */ - struct mk_list kv_labels; - - /* Upstream connection to the backend server */ - struct flb_upstream *u; - - /* Arbitrary HTTP headers */ - struct mk_list *headers; - - - /* instance context */ - struct flb_output_instance *ins; -}; - -#endif diff --git a/fluent-bit/plugins/out_prometheus_remote_write/remote_write_conf.c b/fluent-bit/plugins/out_prometheus_remote_write/remote_write_conf.c deleted file mode 100644 index 09c7e0d52..000000000 --- a/fluent-bit/plugins/out_prometheus_remote_write/remote_write_conf.c +++ /dev/null @@ -1,254 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#ifdef FLB_HAVE_SIGNV4 -#ifdef FLB_HAVE_AWS -#include -#endif -#endif -#include "remote_write.h" -#include "remote_write_conf.h" - -static int config_add_labels(struct flb_output_instance *ins, - struct prometheus_remote_write_context *ctx) -{ - struct mk_list *head; - struct flb_config_map_val *mv; - struct flb_slist_entry *k = NULL; - struct flb_slist_entry *v = NULL; - struct flb_kv *kv; - - if (!ctx->add_labels || mk_list_size(ctx->add_labels) == 0) { - return 0; - } - - /* iterate all 'add_label' definitions */ - flb_config_map_foreach(head, mv, ctx->add_labels) { - if (mk_list_size(mv->val.list) != 2) { - flb_plg_error(ins, "'add_label' expects a key and a value, " - "e.g: 'add_label version 1.8.0'"); - return -1; - } - - k = mk_list_entry_first(mv->val.list, struct flb_slist_entry, _head); - v = mk_list_entry_last(mv->val.list, struct flb_slist_entry, _head); - - kv = flb_kv_item_create(&ctx->kv_labels, k->str, v->str); - if (!kv) { - flb_plg_error(ins, "could not append label %s=%s\n", k->str, v->str); - return -1; - } - } - - return 0; -} - -struct prometheus_remote_write_context *flb_prometheus_remote_write_context_create( - struct flb_output_instance *ins, struct flb_config *config) -{ - int ret; - int ulen; - int io_flags = 0; - char *protocol = NULL; - char *host = NULL; - char *port = NULL; - char *uri = NULL; - char *tmp_uri = NULL; - const char *tmp; - struct flb_upstream *upstream; - struct prometheus_remote_write_context *ctx = NULL; - - /* Allocate plugin context */ - ctx = flb_calloc(1, sizeof(struct prometheus_remote_write_context)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - mk_list_init(&ctx->kv_labels); - - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return NULL; - } - - /* Parse 'add_label' */ - ret = config_add_labels(ins, ctx); - if (ret == -1) { - return NULL; - } - - /* - * Check if a Proxy have been set, if so the Upstream manager will use - * the Proxy end-point and then we let the HTTP client know about it, so - * it can adjust the HTTP requests. - */ - tmp = flb_output_get_property("proxy", ins); - if (tmp) { - ret = flb_utils_url_split(tmp, &protocol, &host, &port, &uri); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not parse proxy parameter: '%s'", tmp); - flb_free(ctx); - return NULL; - } - - ctx->proxy_host = host; - ctx->proxy_port = atoi(port); - ctx->proxy = tmp; - flb_free(protocol); - flb_free(port); - flb_free(uri); - uri = NULL; - } - else { - flb_output_net_default("127.0.0.1", 80, ins); - } - - /* Check if AWS SigV4 authentication is enabled */ -#ifdef FLB_HAVE_SIGNV4 -#ifdef FLB_HAVE_AWS - if (ctx->has_aws_auth) { - if (!ctx->aws_service) { - flb_plg_error(ins, "aws_auth option requires " FLB_PROMETHEUS_REMOTE_WRITE_CREDENTIAL_PREFIX - "service to be set"); - flb_free(ctx); - return NULL; - } - - ctx->aws_provider = flb_managed_chain_provider_create( - ins, - config, - FLB_PROMETHEUS_REMOTE_WRITE_CREDENTIAL_PREFIX, - NULL, - flb_aws_client_generator() - ); - if (!ctx->aws_provider) { - flb_plg_error(ins, "failed to create aws credential provider for sigv4 auth"); - flb_free(ctx); - return NULL; - } - - /* If managed provider creation succeeds, then region key is present */ - ctx->aws_region = flb_output_get_property(FLB_PROMETHEUS_REMOTE_WRITE_CREDENTIAL_PREFIX - "region", ctx->ins); - } -#endif /* !FLB_HAVE_AWS */ -#endif /* !FLB_HAVE_SIGNV4 */ - - /* Check if SSL/TLS is enabled */ -#ifdef FLB_HAVE_TLS - if (ins->use_tls == FLB_TRUE) { - io_flags = FLB_IO_TLS; - } - else { - io_flags = FLB_IO_TCP; - } -#else - io_flags = FLB_IO_TCP; -#endif - - if (ins->host.ipv6 == FLB_TRUE) { - io_flags |= FLB_IO_IPV6; - } - - if (ctx->proxy) { - flb_plg_trace(ctx->ins, "Upstream Proxy=%s:%i", - ctx->proxy_host, ctx->proxy_port); - upstream = flb_upstream_create(config, - ctx->proxy_host, - ctx->proxy_port, - io_flags, ins->tls); - } - else { - upstream = flb_upstream_create(config, - ins->host.name, - ins->host.port, - io_flags, ins->tls); - } - - if (!upstream) { - flb_free(ctx); - return NULL; - } - - if (ins->host.uri) { - uri = flb_strdup(ins->host.uri->full); - } - else { - tmp = flb_output_get_property("uri", ins); - if (tmp) { - uri = flb_strdup(tmp); - } - } - - if (!uri) { - uri = flb_strdup("/"); - } - else if (uri[0] != '/') { - ulen = strlen(uri); - tmp_uri = flb_malloc(ulen + 2); - tmp_uri[0] = '/'; - memcpy(tmp_uri + 1, uri, ulen); - tmp_uri[ulen + 1] = '\0'; - flb_free(uri); - uri = tmp_uri; - } - - ctx->u = upstream; - ctx->uri = uri; - ctx->host = ins->host.name; - ctx->port = ins->host.port; - - /* Set instance flags into upstream */ - flb_output_upstream_set(ctx->u, ins); - - return ctx; -} - -void flb_prometheus_remote_write_context_destroy( - struct prometheus_remote_write_context *ctx) -{ - if (!ctx) { - return; - } - - flb_kv_release(&ctx->kv_labels); - - if (ctx->u) { - flb_upstream_destroy(ctx->u); - } - -#ifdef FLB_HAVE_SIGNV4 -#ifdef FLB_HAVE_AWS - if (ctx->aws_provider) { - flb_aws_provider_destroy(ctx->aws_provider); - } -#endif -#endif - - flb_free(ctx->proxy_host); - flb_free(ctx->uri); - flb_free(ctx); -} diff --git a/fluent-bit/plugins/out_prometheus_remote_write/remote_write_conf.h b/fluent-bit/plugins/out_prometheus_remote_write/remote_write_conf.h deleted file mode 100644 index e6268fc54..000000000 --- a/fluent-bit/plugins/out_prometheus_remote_write/remote_write_conf.h +++ /dev/null @@ -1,33 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_PROMETHEUS_REMOTE_WRITE_CONF_H -#define FLB_OUT_PROMETHEUS_REMOTE_WRITE_CONF_H - -#include -#include - -#include "remote_write.h" - -struct prometheus_remote_write_context *flb_prometheus_remote_write_context_create( - struct flb_output_instance *ins, struct flb_config *config); -void flb_prometheus_remote_write_context_destroy( - struct prometheus_remote_write_context *ctx); - -#endif diff --git a/fluent-bit/plugins/out_retry/CMakeLists.txt b/fluent-bit/plugins/out_retry/CMakeLists.txt deleted file mode 100644 index e206ff719..000000000 --- a/fluent-bit/plugins/out_retry/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - retry.c) - -FLB_PLUGIN(out_retry "${src}" "") diff --git a/fluent-bit/plugins/out_retry/retry.c b/fluent-bit/plugins/out_retry/retry.c deleted file mode 100644 index cb8f4da8c..000000000 --- a/fluent-bit/plugins/out_retry/retry.c +++ /dev/null @@ -1,116 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include -#include -#include - - -/* Retry context, only works with one instance */ -struct retry_ctx { - int n_retry; /* max retries before real flush (OK) */ - int count; /* number of retries done */ - struct flb_output_instance *ins; /* plugin instance */ -}; - - -static int cb_retry_init(struct flb_output_instance *ins, - struct flb_config *config, - void *data) -{ - (void) config; - (void) data; - struct retry_ctx *ctx; - int ret; - - ctx = flb_calloc(1, sizeof(struct retry_ctx)); - if (!ctx) { - return -1; - } - ctx->ins = ins; - ctx->count = 0; - - ret = flb_output_config_map_set(ins, ctx); - if (ret == -1) { - flb_plg_error(ins, "unable to load configuration"); - return -1; - } - - flb_output_set_context(ins, ctx); - return 0; -} - -static void cb_retry_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - (void) i_ins; - (void) out_context; - (void) config; - struct retry_ctx *ctx; - - ctx = out_context; - ctx->count++; - - if (ctx->count <= ctx->n_retry) { - flb_plg_debug(ctx->ins, "retry %i/%i", ctx->count, ctx->n_retry); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - else { - flb_plg_debug(ctx->ins, "flush", ctx->count, ctx->n_retry); - ctx->count = 0; - } - - flb_pack_print(event_chunk->data, event_chunk->size); - FLB_OUTPUT_RETURN(FLB_OK); -} - -static int cb_retry_exit(void *data, struct flb_config *config) -{ - struct retry_ctx *ctx = data; - (void) config; - - flb_free(ctx); - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_INT, "retry", "3", - 0, FLB_TRUE, offsetof(struct retry_ctx, n_retry), - "Number of retries." - }, - {0} -}; - -struct flb_output_plugin out_retry_plugin = { - .name = "retry", - .description = "Issue a retry upon flush request", - .cb_init = cb_retry_init, - .cb_flush = cb_retry_flush, - .cb_exit = cb_retry_exit, - .config_map = config_map, - .flags = 0, -}; diff --git a/fluent-bit/plugins/out_s3/CMakeLists.txt b/fluent-bit/plugins/out_s3/CMakeLists.txt deleted file mode 100644 index 94e048617..000000000 --- a/fluent-bit/plugins/out_s3/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(src - s3.c - s3_store.c - s3_multipart.c) - -FLB_PLUGIN(out_s3 "${src}" "") diff --git a/fluent-bit/plugins/out_s3/s3.c b/fluent-bit/plugins/out_s3/s3.c deleted file mode 100644 index 57e68e6ef..000000000 --- a/fluent-bit/plugins/out_s3/s3.c +++ /dev/null @@ -1,2500 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "s3.h" -#include "s3_store.h" - -#define DEFAULT_S3_PORT 443 -#define DEFAULT_S3_INSECURE_PORT 80 - -static int construct_request_buffer(struct flb_s3 *ctx, flb_sds_t new_data, - struct s3_file *chunk, - char **out_buf, size_t *out_size); - -static int s3_put_object(struct flb_s3 *ctx, const char *tag, time_t file_first_log_time, - char *body, size_t body_size); - -static int put_all_chunks(struct flb_s3 *ctx); - -static void cb_s3_upload(struct flb_config *ctx, void *data); - -static struct multipart_upload *get_upload(struct flb_s3 *ctx, - const char *tag, int tag_len); - -static struct multipart_upload *create_upload(struct flb_s3 *ctx, - const char *tag, int tag_len, - time_t file_first_log_time); - -static void remove_from_queue(struct upload_queue *entry); - -static struct flb_aws_header content_encoding_header = { - .key = "Content-Encoding", - .key_len = 16, - .val = "gzip", - .val_len = 4, -}; - -static struct flb_aws_header content_type_header = { - .key = "Content-Type", - .key_len = 12, - .val = "", - .val_len = 0, -}; - -static struct flb_aws_header canned_acl_header = { - .key = "x-amz-acl", - .key_len = 9, - .val = "", - .val_len = 0, -}; - -static struct flb_aws_header content_md5_header = { - .key = "Content-MD5", - .key_len = 11, - .val = "", - .val_len = 0, -}; - -static struct flb_aws_header storage_class_header = { - .key = "x-amz-storage-class", - .key_len = 19, - .val = "", - .val_len = 0, -}; - -static char *mock_error_response(char *error_env_var) -{ - char *err_val = NULL; - char *error = NULL; - int len = 0; - - err_val = getenv(error_env_var); - if (err_val != NULL && strlen(err_val) > 0) { - error = flb_calloc(strlen(err_val) + 1, sizeof(char)); - if (error == NULL) { - flb_errno(); - return NULL; - } - - len = strlen(err_val); - memcpy(error, err_val, len); - error[len] = '\0'; - return error; - } - - return NULL; -} - -int s3_plugin_under_test() -{ - if (getenv("FLB_S3_PLUGIN_UNDER_TEST") != NULL) { - return FLB_TRUE; - } - - return FLB_FALSE; -} - -int create_headers(struct flb_s3 *ctx, char *body_md5, - struct flb_aws_header **headers, int *num_headers, - int multipart_upload) -{ - int n = 0; - int headers_len = 0; - struct flb_aws_header *s3_headers = NULL; - - if (ctx->content_type != NULL) { - headers_len++; - } - if (ctx->compression == FLB_AWS_COMPRESS_GZIP) { - headers_len++; - } - if (ctx->canned_acl != NULL) { - headers_len++; - } - if (body_md5 != NULL && strlen(body_md5) && multipart_upload == FLB_FALSE) { - headers_len++; - } - if (ctx->storage_class != NULL) { - headers_len++; - } - if (headers_len == 0) { - *num_headers = headers_len; - *headers = s3_headers; - return 0; - } - - s3_headers = flb_calloc(headers_len, sizeof(struct flb_aws_header)); - if (s3_headers == NULL) { - flb_errno(); - return -1; - } - - if (ctx->content_type != NULL) { - s3_headers[n] = content_type_header; - s3_headers[n].val = ctx->content_type; - s3_headers[n].val_len = strlen(ctx->content_type); - n++; - } - if (ctx->compression == FLB_AWS_COMPRESS_GZIP) { - s3_headers[n] = content_encoding_header; - n++; - } - if (ctx->canned_acl != NULL) { - s3_headers[n] = canned_acl_header; - s3_headers[n].val = ctx->canned_acl; - s3_headers[n].val_len = strlen(ctx->canned_acl); - n++; - } - if (body_md5 != NULL && strlen(body_md5) && multipart_upload == FLB_FALSE) { - s3_headers[n] = content_md5_header; - s3_headers[n].val = body_md5; - s3_headers[n].val_len = strlen(body_md5); - n++; - } - if (ctx->storage_class != NULL) { - s3_headers[n] = storage_class_header; - s3_headers[n].val = ctx->storage_class; - s3_headers[n].val_len = strlen(ctx->storage_class); - } - - *num_headers = headers_len; - *headers = s3_headers; - return 0; -}; - -struct flb_http_client *mock_s3_call(char *error_env_var, char *api) -{ - /* create an http client so that we can set the response */ - struct flb_http_client *c = NULL; - char *error = mock_error_response(error_env_var); - char *resp; - int len; - - c = flb_calloc(1, sizeof(struct flb_http_client)); - if (!c) { - flb_errno(); - flb_free(error); - return NULL; - } - mk_list_init(&c->headers); - - if (error != NULL) { - c->resp.status = 400; - /* resp.data is freed on destroy, payload is supposed to reference it */ - c->resp.data = error; - c->resp.payload = c->resp.data; - c->resp.payload_size = strlen(error); - } - else { - c->resp.status = 200; - c->resp.payload = ""; - c->resp.payload_size = 0; - if (strcmp(api, "CreateMultipartUpload") == 0) { - /* mocked success response */ - c->resp.payload = "\n" - "\n" - "example-bucket\n" - "example-object\n" - "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA\n" - ""; - c->resp.payload_size = strlen(c->resp.payload); - } - else if (strcmp(api, "UploadPart") == 0) { - /* mocked success response */ - resp = "Date: Mon, 1 Nov 2010 20:34:56 GMT\n" - "ETag: \"b54357faf0632cce46e942fa68356b38\"\n" - "Content-Length: 0\n" - "Connection: keep-alive\n" - "Server: AmazonS3"; - /* since etag is in the headers, this code uses resp.data */ - len = strlen(resp); - c->resp.data = flb_calloc(len + 1, sizeof(char)); - if (!c->resp.data) { - flb_errno(); - return NULL; - } - memcpy(c->resp.data, resp, len); - c->resp.data[len] = '\0'; - c->resp.data_size = len; - } - else { - c->resp.payload = ""; - c->resp.payload_size = 0; - } - } - - return c; -} - -static flb_sds_t concat_path(char *p1, char *p2) -{ - flb_sds_t dir; - flb_sds_t tmp; - - dir = flb_sds_create_size(64); - - tmp = flb_sds_printf(&dir, "%s/%s", p1, p2); - if (!tmp) { - flb_errno(); - flb_sds_destroy(dir); - return NULL; - } - dir = tmp; - - return dir; -} - -/* Reads in index value from metadata file and sets seq_index to value */ -static int read_seq_index(char *seq_index_file, uint64_t *seq_index) -{ - FILE *fp; - int ret; - - fp = fopen(seq_index_file, "r"); - if (fp == NULL) { - flb_errno(); - return -1; - } - - ret = fscanf(fp, "%"PRIu64, seq_index); - if (ret != 1) { - fclose(fp); - flb_errno(); - return -1; - } - - fclose(fp); - return 0; -} - -/* Writes index value to metadata file */ -static int write_seq_index(char *seq_index_file, uint64_t seq_index) -{ - FILE *fp; - int ret; - - fp = fopen(seq_index_file, "w+"); - if (fp == NULL) { - flb_errno(); - return -1; - } - - ret = fprintf(fp, "%"PRIu64, seq_index); - if (ret < 0) { - fclose(fp); - flb_errno(); - return -1; - } - - fclose(fp); - return 0; -} - -static int init_seq_index(void *context) { - int ret; - const char *tmp; - char tmp_buf[1024]; - struct flb_s3 *ctx = context; - - ctx->key_fmt_has_seq_index = FLB_TRUE; - - ctx->stream_metadata = flb_fstore_stream_create(ctx->fs, "sequence"); - if (!ctx->stream_metadata) { - flb_plg_error(ctx->ins, "could not initialize metadata stream"); - flb_fstore_destroy(ctx->fs); - ctx->fs = NULL; - return -1; - } - - /* Construct directories and file path names */ - ctx->metadata_dir = flb_sds_create(ctx->stream_metadata->path); - if (ctx->metadata_dir == NULL) { - flb_plg_error(ctx->ins, "Failed to create metadata path"); - flb_errno(); - return -1; - } - tmp = "/index_metadata"; - ret = flb_sds_cat_safe(&ctx->metadata_dir, tmp, strlen(tmp)); - if (ret < 0) { - flb_plg_error(ctx->ins, "Failed to create metadata path"); - flb_errno(); - return -1; - } - - ctx->seq_index_file = flb_sds_create(ctx->metadata_dir); - if (ctx->seq_index_file == NULL) { - flb_plg_error(ctx->ins, "Failed to create sequential index file path"); - flb_errno(); - return -1; - } - tmp = "/seq_index_"; - ret = flb_sds_cat_safe(&ctx->seq_index_file, tmp, strlen(tmp)); - if (ret < 0) { - flb_plg_error(ctx->ins, "Failed to create sequential index file path"); - flb_errno(); - return -1; - } - - sprintf(tmp_buf, "%d", ctx->ins->id); - ret = flb_sds_cat_safe(&ctx->seq_index_file, tmp_buf, strlen(tmp_buf)); - if (ret < 0) { - flb_plg_error(ctx->ins, "Failed to create sequential index file path"); - flb_errno(); - return -1; - } - - /* Create directory path if it doesn't exist */ - ret = mkdir(ctx->metadata_dir, 0700); - if (ret < 0 && errno != EEXIST) { - flb_plg_error(ctx->ins, "Failed to create metadata directory"); - return -1; - } - - /* Check if index file doesn't exist and set index value */ - if (access(ctx->seq_index_file, F_OK) != 0) { - ctx->seq_index = 0; - ret = write_seq_index(ctx->seq_index_file, ctx->seq_index); - if (ret < 0) { - flb_plg_error(ctx->ins, "Failed to write to sequential index metadata file"); - return -1; - } - } - else { - ret = read_seq_index(ctx->seq_index_file, &ctx->seq_index); - if (ret < 0) { - flb_plg_error(ctx->ins, "Failed to read from sequential index " - "metadata file"); - return -1; - } - flb_plg_info(ctx->ins, "Successfully recovered index. " - "Continuing at index=%"PRIu64, ctx->seq_index); - } - return 0; -} - -void multipart_upload_destroy(struct multipart_upload *m_upload) -{ - int i; - flb_sds_t etag; - - if (!m_upload) { - return; - } - - if (m_upload->s3_key) { - flb_sds_destroy(m_upload->s3_key); - } - if (m_upload->tag) { - flb_sds_destroy(m_upload->tag); - } - if (m_upload->upload_id) { - flb_sds_destroy(m_upload->upload_id); - } - - for (i = 0; i < m_upload->part_number; i++) { - etag = m_upload->etags[i]; - if (etag) { - flb_sds_destroy(etag); - } - } - - flb_free(m_upload); -} - -static void s3_context_destroy(struct flb_s3 *ctx) -{ - struct mk_list *head; - struct mk_list *tmp; - struct multipart_upload *m_upload; - struct upload_queue *upload_contents; - - if (!ctx) { - return; - } - - if (ctx->base_provider) { - flb_aws_provider_destroy(ctx->base_provider); - } - - if (ctx->provider) { - flb_aws_provider_destroy(ctx->provider); - } - - if (ctx->provider_tls) { - flb_tls_destroy(ctx->provider_tls); - } - - if (ctx->sts_provider_tls) { - flb_tls_destroy(ctx->sts_provider_tls); - } - - if (ctx->s3_client) { - flb_aws_client_destroy(ctx->s3_client); - } - - if (ctx->client_tls) { - flb_tls_destroy(ctx->client_tls); - } - - if (ctx->free_endpoint == FLB_TRUE) { - flb_free(ctx->endpoint); - } - - if (ctx->buffer_dir) { - flb_sds_destroy(ctx->buffer_dir); - } - - if (ctx->metadata_dir) { - flb_sds_destroy(ctx->metadata_dir); - } - - if (ctx->seq_index_file) { - flb_sds_destroy(ctx->seq_index_file); - } - - /* Remove uploads */ - mk_list_foreach_safe(head, tmp, &ctx->uploads) { - m_upload = mk_list_entry(head, struct multipart_upload, _head); - mk_list_del(&m_upload->_head); - multipart_upload_destroy(m_upload); - } - - mk_list_foreach_safe(head, tmp, &ctx->upload_queue) { - upload_contents = mk_list_entry(head, struct upload_queue, _head); - s3_store_file_delete(ctx, upload_contents->upload_file); - multipart_upload_destroy(upload_contents->m_upload_file); - remove_from_queue(upload_contents); - } - - flb_free(ctx); -} - -static int cb_s3_init(struct flb_output_instance *ins, - struct flb_config *config, void *data) -{ - int ret; - flb_sds_t tmp_sds; - char *role_arn = NULL; - char *session_name; - const char *tmp; - struct flb_s3 *ctx = NULL; - struct flb_aws_client_generator *generator; - (void) config; - (void) data; - char *ep; - struct flb_split_entry *tok; - struct mk_list *split; - int list_size; - - ctx = flb_calloc(1, sizeof(struct flb_s3)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->ins = ins; - mk_list_init(&ctx->uploads); - mk_list_init(&ctx->upload_queue); - - ctx->retry_time = 0; - ctx->upload_queue_success = FLB_FALSE; - - /* Export context */ - flb_output_set_context(ins, ctx); - - /* initialize config map */ - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - return -1; - } - - /* the check against -1 is works here because size_t is unsigned - * and (int) -1 == unsigned max value - * Fluent Bit uses -1 (which becomes max value) to indicate undefined - */ - if (ctx->ins->total_limit_size != -1) { - flb_plg_warn(ctx->ins, "Please use 'store_dir_limit_size' with s3 output instead of 'storage.total_limit_size'. " - "S3 has its own buffer files located in the store_dir."); - } - - /* Date key */ - ctx->date_key = ctx->json_date_key; - tmp = flb_output_get_property("json_date_key", ins); - if (tmp) { - /* Just check if we have to disable it */ - if (flb_utils_bool(tmp) == FLB_FALSE) { - ctx->date_key = NULL; - } - } - - /* Date format for JSON output */ - ctx->json_date_format = FLB_PACK_JSON_DATE_ISO8601; - tmp = flb_output_get_property("json_date_format", ins); - if (tmp) { - ret = flb_pack_to_json_date_type(tmp); - if (ret == -1) { - flb_plg_error(ctx->ins, "invalid json_date_format '%s'. ", tmp); - return -1; - } - else { - ctx->json_date_format = ret; - } - } - - tmp = flb_output_get_property("bucket", ins); - if (!tmp) { - flb_plg_error(ctx->ins, "'bucket' is a required parameter"); - return -1; - } - - /* - * store_dir is the user input, buffer_dir is what the code uses - * We append the bucket name to the dir, to support multiple instances - * of this plugin using the same buffer dir - */ - tmp_sds = concat_path(ctx->store_dir, ctx->bucket); - if (!tmp_sds) { - flb_plg_error(ctx->ins, "Could not construct buffer path"); - return -1; - } - ctx->buffer_dir = tmp_sds; - - /* Initialize local storage */ - ret = s3_store_init(ctx); - if (ret == -1) { - flb_plg_error(ctx->ins, "Failed to initialize S3 storage: %s", - ctx->store_dir); - return -1; - } - - tmp = flb_output_get_property("s3_key_format", ins); - if (tmp) { - if (tmp[0] != '/') { - flb_plg_error(ctx->ins, "'s3_key_format' must start with a '/'"); - return -1; - } - if (strstr((char *) tmp, "$INDEX")) { - ret = init_seq_index(ctx); - if (ret < 0) { - return -1; - } - } - if (strstr((char *) tmp, "$UUID")) { - ctx->key_fmt_has_uuid = FLB_TRUE; - } - } - - /* validate 'total_file_size' */ - if (ctx->file_size <= 0) { - flb_plg_error(ctx->ins, "Failed to parse total_file_size %s", tmp); - return -1; - } - if (ctx->file_size < 1000000) { - flb_plg_error(ctx->ins, "total_file_size must be at least 1MB"); - return -1; - } - if (ctx->file_size > MAX_FILE_SIZE) { - flb_plg_error(ctx->ins, "Max total_file_size is %s bytes", MAX_FILE_SIZE_STR); - return -1; - } - flb_plg_info(ctx->ins, "Using upload size %lu bytes", ctx->file_size); - - if (ctx->use_put_object == FLB_FALSE && ctx->file_size < 2 * MIN_CHUNKED_UPLOAD_SIZE) { - flb_plg_info(ctx->ins, - "total_file_size is less than 10 MB, will use PutObject API"); - ctx->use_put_object = FLB_TRUE; - } - - tmp = flb_output_get_property("compression", ins); - if (tmp) { - ret = flb_aws_compression_get_type(tmp); - if (ret == -1) { - flb_plg_error(ctx->ins, "unknown compression: %s", tmp); - return -1; - } - if (ctx->use_put_object == FLB_FALSE && ctx->compression == FLB_AWS_COMPRESS_ARROW) { - flb_plg_error(ctx->ins, - "use_put_object must be enabled when Apache Arrow is enabled"); - return -1; - } - ctx->compression = ret; - } - - tmp = flb_output_get_property("content_type", ins); - if (tmp) { - ctx->content_type = (char *) tmp; - } - if (ctx->use_put_object == FLB_FALSE) { - /* upload_chunk_size */ - if (ctx->upload_chunk_size <= 0) { - flb_plg_error(ctx->ins, "Failed to parse upload_chunk_size %s", tmp); - return -1; - } - if (ctx->upload_chunk_size > ctx->file_size) { - flb_plg_error(ctx->ins, - "upload_chunk_size can not be larger than total_file_size"); - return -1; - } - if (ctx->upload_chunk_size < MIN_CHUNKED_UPLOAD_SIZE) { - flb_plg_error(ctx->ins, "upload_chunk_size must be at least 5,242,880 bytes"); - return -1; - } - if (ctx->compression == FLB_AWS_COMPRESS_GZIP) { - if(ctx->upload_chunk_size > MAX_CHUNKED_UPLOAD_COMPRESS_SIZE) { - flb_plg_error(ctx->ins, "upload_chunk_size in compressed multipart upload cannot exceed 5GB"); - return -1; - } - } else { - if (ctx->upload_chunk_size > MAX_CHUNKED_UPLOAD_SIZE) { - flb_plg_error(ctx->ins, "Max upload_chunk_size is 50MB"); - return -1; - } - } - } - - if (ctx->upload_chunk_size != MIN_CHUNKED_UPLOAD_SIZE && - (ctx->upload_chunk_size * 2) > ctx->file_size) { - flb_plg_error(ctx->ins, "total_file_size is less than 2x upload_chunk_size"); - return -1; - } - - if (ctx->use_put_object == FLB_TRUE) { - /* - * code internally uses 'upload_chunk_size' as the unit for each Put, - * regardless of which API is used to send data - */ - ctx->upload_chunk_size = ctx->file_size; - if (ctx->file_size > MAX_FILE_SIZE_PUT_OBJECT) { - flb_plg_error(ctx->ins, "Max total_file_size is 50M when use_put_object is enabled"); - return -1; - } - } - - tmp = flb_output_get_property("endpoint", ins); - if (tmp) { - ctx->insecure = strncmp(tmp, "http://", 7) == 0 ? FLB_TRUE : FLB_FALSE; - if (ctx->insecure == FLB_TRUE) { - ep = removeProtocol((char *) tmp, "http://"); - } - else { - ep = removeProtocol((char *) tmp, "https://"); - } - - split = flb_utils_split((const char *)ep, ':', 1); - if (!split) { - flb_errno(); - return -1; - } - list_size = mk_list_size(split); - if (list_size > 2) { - flb_plg_error(ctx->ins, "Failed to split endpoint"); - flb_utils_split_free(split); - return -1; - } - - tok = mk_list_entry_first(split, struct flb_split_entry, _head); - ctx->endpoint = flb_strndup(tok->value, tok->len); - if (!ctx->endpoint) { - flb_errno(); - flb_utils_split_free(split); - return -1; - } - ctx->free_endpoint = FLB_TRUE; - if (list_size == 2) { - tok = mk_list_entry_next(&tok->_head, struct flb_split_entry, _head, split); - ctx->port = atoi(tok->value); - } - else { - ctx->port = ctx->insecure == FLB_TRUE ? DEFAULT_S3_INSECURE_PORT : DEFAULT_S3_PORT; - } - flb_utils_split_free(split); - } - else { - /* default endpoint for the given region */ - ctx->endpoint = flb_aws_endpoint("s3", ctx->region); - ctx->insecure = FLB_FALSE; - ctx->port = DEFAULT_S3_PORT; - ctx->free_endpoint = FLB_TRUE; - if (!ctx->endpoint) { - flb_plg_error(ctx->ins, "Could not construct S3 endpoint"); - return -1; - } - } - - tmp = flb_output_get_property("sts_endpoint", ins); - if (tmp) { - ctx->sts_endpoint = (char *) tmp; - } - - tmp = flb_output_get_property("canned_acl", ins); - if (tmp) { - ctx->canned_acl = (char *) tmp; - } - - tmp = flb_output_get_property("storage_class", ins); - if (tmp) { - ctx->storage_class = (char *) tmp; - } - - if (ctx->insecure == FLB_FALSE) { - ctx->client_tls = flb_tls_create(FLB_TLS_CLIENT_MODE, - ins->tls_verify, - ins->tls_debug, - ins->tls_vhost, - ins->tls_ca_path, - ins->tls_ca_file, - ins->tls_crt_file, - ins->tls_key_file, - ins->tls_key_passwd); - if (!ctx->client_tls) { - flb_plg_error(ctx->ins, "Failed to create tls context"); - return -1; - } - } - - /* AWS provider needs a separate TLS instance */ - ctx->provider_tls = flb_tls_create(FLB_TLS_CLIENT_MODE, - FLB_TRUE, - ins->tls_debug, - ins->tls_vhost, - ins->tls_ca_path, - ins->tls_ca_file, - ins->tls_crt_file, - ins->tls_key_file, - ins->tls_key_passwd); - if (!ctx->provider_tls) { - flb_errno(); - return -1; - } - - ctx->provider = flb_standard_chain_provider_create(config, - ctx->provider_tls, - ctx->region, - ctx->sts_endpoint, - NULL, - flb_aws_client_generator(), - ctx->profile); - - if (!ctx->provider) { - flb_plg_error(ctx->ins, "Failed to create AWS Credential Provider"); - return -1; - } - - tmp = flb_output_get_property("role_arn", ins); - if (tmp) { - /* Use the STS Provider */ - ctx->base_provider = ctx->provider; - role_arn = (char *) tmp; - - /* STS provider needs yet another separate TLS instance */ - ctx->sts_provider_tls = flb_tls_create(FLB_TLS_CLIENT_MODE, - FLB_TRUE, - ins->tls_debug, - ins->tls_vhost, - ins->tls_ca_path, - ins->tls_ca_file, - ins->tls_crt_file, - ins->tls_key_file, - ins->tls_key_passwd); - - if (!ctx->sts_provider_tls) { - flb_errno(); - return -1; - } - - session_name = flb_sts_session_name(); - if (!session_name) { - flb_plg_error(ctx->ins, "Failed to create aws iam role " - "session name"); - flb_errno(); - return -1; - } - - ctx->provider = flb_sts_provider_create(config, - ctx->sts_provider_tls, - ctx->base_provider, - ctx->external_id, - role_arn, - session_name, - ctx->region, - ctx->sts_endpoint, - NULL, - flb_aws_client_generator()); - flb_free(session_name); - if (!ctx->provider) { - flb_plg_error(ctx->ins, "Failed to create AWS STS Credential " - "Provider"); - return -1; - } - } - - /* read any remaining buffers from previous (failed) executions */ - ctx->has_old_buffers = s3_store_has_data(ctx); - ctx->has_old_uploads = s3_store_has_uploads(ctx); - - /* Multipart */ - multipart_read_uploads_from_fs(ctx); - - if (mk_list_size(&ctx->uploads) > 0) { - /* note that these should be sent */ - ctx->has_old_uploads = FLB_TRUE; - } - - /* create S3 client */ - generator = flb_aws_client_generator(); - ctx->s3_client = generator->create(); - if (!ctx->s3_client) { - return -1; - } - ctx->s3_client->name = "s3_client"; - ctx->s3_client->has_auth = FLB_TRUE; - ctx->s3_client->provider = ctx->provider; - ctx->s3_client->region = ctx->region; - ctx->s3_client->service = "s3"; - ctx->s3_client->port = ctx->port; - ctx->s3_client->flags = 0; - ctx->s3_client->proxy = NULL; - ctx->s3_client->s3_mode = S3_MODE_SIGNED_PAYLOAD; - ctx->s3_client->retry_requests = ctx->retry_requests; - - if (ctx->insecure == FLB_TRUE) { - ctx->s3_client->upstream = flb_upstream_create(config, ctx->endpoint, ctx->port, - FLB_IO_TCP, NULL); - } else { - ctx->s3_client->upstream = flb_upstream_create(config, ctx->endpoint, ctx->port, - FLB_IO_TLS, ctx->client_tls); - } - if (!ctx->s3_client->upstream) { - flb_plg_error(ctx->ins, "Connection initialization error"); - return -1; - } - - flb_output_upstream_set(ctx->s3_client->upstream, ctx->ins); - - ctx->s3_client->host = ctx->endpoint; - - /* set to sync mode and initialize credentials */ - ctx->provider->provider_vtable->sync(ctx->provider); - ctx->provider->provider_vtable->init(ctx->provider); - - ctx->timer_created = FLB_FALSE; - ctx->timer_ms = (int) (ctx->upload_timeout / 6) * 1000; - if (ctx->timer_ms > UPLOAD_TIMER_MAX_WAIT) { - ctx->timer_ms = UPLOAD_TIMER_MAX_WAIT; - } - else if (ctx->timer_ms < UPLOAD_TIMER_MIN_WAIT) { - ctx->timer_ms = UPLOAD_TIMER_MIN_WAIT; - } - - /* - * S3 must ALWAYS use sync mode - * In the timer thread we do a mk_list_foreach_safe on the queue of uplaods and chunks - * Iterating over those lists is not concurrent safe. If a flush call ran at the same time - * And deleted an item from the list, this could cause a crash/corruption. - */ - flb_stream_disable_async_mode(&ctx->s3_client->upstream->base); - - /* clean up any old buffers found on startup */ - if (ctx->has_old_buffers == FLB_TRUE) { - flb_plg_info(ctx->ins, - "Sending locally buffered data from previous " - "executions to S3; buffer=%s", - ctx->fs->root_path); - ctx->has_old_buffers = FLB_FALSE; - ret = put_all_chunks(ctx); - if (ret < 0) { - ctx->has_old_buffers = FLB_TRUE; - flb_plg_error(ctx->ins, - "Failed to send locally buffered data left over " - "from previous executions; will retry. Buffer=%s", - ctx->fs->root_path); - } - } - - /* clean up any old uploads found on start up */ - if (ctx->has_old_uploads == FLB_TRUE) { - flb_plg_info(ctx->ins, - "Completing multipart uploads from previous " - "executions to S3; buffer=%s", - ctx->stream_upload->path); - ctx->has_old_uploads = FLB_FALSE; - - /* - * we don't need to worry if this fails; it will retry each - * time the upload callback is called - */ - cb_s3_upload(config, ctx); - } - - /* this is done last since in the previous block we make calls to AWS */ - ctx->provider->provider_vtable->upstream_set(ctx->provider, ctx->ins); - - return 0; -} - -/* - * return value is one of FLB_OK, FLB_RETRY, FLB_ERROR - * - * Chunk is allowed to be NULL - */ -static int upload_data(struct flb_s3 *ctx, struct s3_file *chunk, - struct multipart_upload *m_upload, - char *body, size_t body_size, - const char *tag, int tag_len) -{ - int init_upload = FLB_FALSE; - int complete_upload = FLB_FALSE; - int size_check = FLB_FALSE; - int part_num_check = FLB_FALSE; - int timeout_check = FLB_FALSE; - int ret; - void *payload_buf = NULL; - size_t payload_size = 0; - size_t preCompress_size = 0; - time_t file_first_log_time = time(NULL); - - /* - * When chunk does not exist, file_first_log_time will be the current time. - * This is only for unit tests and prevents unit tests from segfaulting when chunk is - * NULL because if so chunk->first_log_time will be NULl either and will cause - * segfault during the process of put_object upload or mutipart upload. - */ - if (chunk != NULL) { - file_first_log_time = chunk->first_log_time; - } - - if (ctx->compression == FLB_AWS_COMPRESS_GZIP) { - /* Map payload */ - ret = flb_aws_compression_compress(ctx->compression, body, body_size, &payload_buf, &payload_size); - if (ret == -1) { - flb_plg_error(ctx->ins, "Failed to compress data"); - return FLB_RETRY; - } else { - preCompress_size = body_size; - body = (void *) payload_buf; - body_size = payload_size; - } - } - - if (ctx->use_put_object == FLB_TRUE) { - goto put_object; - } - - if (s3_plugin_under_test() == FLB_TRUE) { - init_upload = FLB_TRUE; - complete_upload = FLB_TRUE; - if (ctx->use_put_object == FLB_TRUE) { - goto put_object; - } - else { - goto multipart; - } - } - - if (m_upload == NULL) { - if (chunk != NULL && time(NULL) > - (chunk->create_time + ctx->upload_timeout + ctx->retry_time)) { - /* timeout already reached, just PutObject */ - goto put_object; - } - else if (body_size >= ctx->file_size) { - /* already big enough, just use PutObject API */ - goto put_object; - } - else if(body_size > MIN_CHUNKED_UPLOAD_SIZE) { - init_upload = FLB_TRUE; - goto multipart; - } - else { - if (ctx->use_put_object == FLB_FALSE && ctx->compression == FLB_AWS_COMPRESS_GZIP) { - flb_plg_info(ctx->ins, "Pre-compression upload_chunk_size= %zu, After compression, chunk is only %zu bytes, " - "the chunk was too small, using PutObject to upload", preCompress_size, body_size); - } - goto put_object; - } - } - else { - /* existing upload */ - if (body_size < MIN_CHUNKED_UPLOAD_SIZE) { - complete_upload = FLB_TRUE; - } - - goto multipart; - } - -put_object: - - /* - * remove chunk from buffer list - */ - ret = s3_put_object(ctx, tag, file_first_log_time, body, body_size); - if (ctx->compression == FLB_AWS_COMPRESS_GZIP) { - flb_free(payload_buf); - } - if (ret < 0) { - /* re-add chunk to list */ - if (chunk) { - s3_store_file_unlock(chunk); - chunk->failures += 1; - } - return FLB_RETRY; - } - - /* data was sent successfully- delete the local buffer */ - if (chunk) { - s3_store_file_delete(ctx, chunk); - } - return FLB_OK; - -multipart: - - if (init_upload == FLB_TRUE) { - m_upload = create_upload(ctx, tag, tag_len, file_first_log_time); - if (!m_upload) { - flb_plg_error(ctx->ins, "Could not find or create upload for tag %s", tag); - if (chunk) { - s3_store_file_unlock(chunk); - } - if (ctx->compression == FLB_AWS_COMPRESS_GZIP) { - flb_free(payload_buf); - } - return FLB_RETRY; - } - } - - if (m_upload->upload_state == MULTIPART_UPLOAD_STATE_NOT_CREATED) { - ret = create_multipart_upload(ctx, m_upload); - if (ret < 0) { - flb_plg_error(ctx->ins, "Could not initiate multipart upload"); - if (chunk) { - s3_store_file_unlock(chunk); - } - if (ctx->compression == FLB_AWS_COMPRESS_GZIP) { - flb_free(payload_buf); - } - return FLB_RETRY; - } - m_upload->upload_state = MULTIPART_UPLOAD_STATE_CREATED; - } - - ret = upload_part(ctx, m_upload, body, body_size); - if (ret < 0) { - if (ctx->compression == FLB_AWS_COMPRESS_GZIP) { - flb_free(payload_buf); - } - m_upload->upload_errors += 1; - /* re-add chunk to list */ - if (chunk) { - s3_store_file_unlock(chunk); - chunk->failures += 1; - } - return FLB_RETRY; - } - m_upload->part_number += 1; - /* data was sent successfully- delete the local buffer */ - if (chunk) { - s3_store_file_delete(ctx, chunk); - chunk = NULL; - } - if (ctx->compression == FLB_AWS_COMPRESS_GZIP) { - flb_free(payload_buf); - } - if (m_upload->bytes >= ctx->file_size) { - size_check = FLB_TRUE; - flb_plg_info(ctx->ins, "Will complete upload for %s because uploaded data is greater" - " than size set by total_file_size", m_upload->s3_key); - } - if (m_upload->part_number >= 10000) { - part_num_check = FLB_TRUE; - flb_plg_info(ctx->ins, "Will complete upload for %s because 10,000 chunks " - "(the API limit) have been uploaded", m_upload->s3_key); - } - if (time(NULL) > - (m_upload->init_time + ctx->upload_timeout + ctx->retry_time)) { - timeout_check = FLB_TRUE; - flb_plg_info(ctx->ins, "Will complete upload for %s because upload_timeout" - " has elapsed", m_upload->s3_key); - } - if (size_check || part_num_check || timeout_check) { - complete_upload = FLB_TRUE; - } - - if (complete_upload == FLB_TRUE) { - /* mark for completion- the upload timer will handle actual completion */ - m_upload->upload_state = MULTIPART_UPLOAD_STATE_COMPLETE_IN_PROGRESS; - } - - return FLB_OK; -} - - -/* - * Attempts to send all chunks to S3 using PutObject - * Used on shut down to try to send all buffered data - * Used on start up to try to send any leftover buffers from previous executions - */ -static int put_all_chunks(struct flb_s3 *ctx) -{ - struct s3_file *chunk; - struct mk_list *tmp; - struct mk_list *head; - struct mk_list *f_head; - struct flb_fstore_file *fsf; - struct flb_fstore_stream *fs_stream; - void *payload_buf = NULL; - size_t payload_size = 0; - char *buffer = NULL; - size_t buffer_size; - int ret; - - mk_list_foreach(head, &ctx->fs->streams) { - /* skip multi upload stream */ - fs_stream = mk_list_entry(head, struct flb_fstore_stream, _head); - if (fs_stream == ctx->stream_upload) { - continue; - } - /* skip metadata stream */ - if (fs_stream == ctx->stream_metadata) { - continue; - } - - mk_list_foreach_safe(f_head, tmp, &fs_stream->files) { - fsf = mk_list_entry(f_head, struct flb_fstore_file, _head); - chunk = fsf->data; - - /* Locked chunks are being processed, skip */ - if (chunk->locked == FLB_TRUE) { - continue; - } - - if (chunk->failures >= MAX_UPLOAD_ERRORS) { - flb_plg_warn(ctx->ins, - "Chunk for tag %s failed to send %i times, " - "will not retry", - (char *) fsf->meta_buf, MAX_UPLOAD_ERRORS); - flb_fstore_file_inactive(ctx->fs, fsf); - continue; - } - - ret = construct_request_buffer(ctx, NULL, chunk, - &buffer, &buffer_size); - if (ret < 0) { - flb_plg_error(ctx->ins, - "Could not construct request buffer for %s", - chunk->file_path); - return -1; - } - - if (ctx->compression != FLB_AWS_COMPRESS_NONE) { - /* Map payload */ - ret = flb_aws_compression_compress(ctx->compression, buffer, buffer_size, &payload_buf, &payload_size); - if (ret == -1) { - flb_plg_error(ctx->ins, "Failed to compress data, uploading uncompressed data instead to prevent data loss"); - } else { - flb_plg_info(ctx->ins, "Pre-compression chunk size is %zu, After compression, chunk is %zu bytes", buffer_size, payload_size); - buffer = (void *) payload_buf; - buffer_size = payload_size; - } - } - - ret = s3_put_object(ctx, (const char *) - fsf->meta_buf, - chunk->create_time, buffer, buffer_size); - flb_free(buffer); - if (ret < 0) { - s3_store_file_unlock(chunk); - chunk->failures += 1; - return -1; - } - - /* data was sent successfully- delete the local buffer */ - s3_store_file_delete(ctx, chunk); - } - } - - return 0; -} - -/* - * Either new_data or chunk can be NULL, but not both - */ -static int construct_request_buffer(struct flb_s3 *ctx, flb_sds_t new_data, - struct s3_file *chunk, - char **out_buf, size_t *out_size) -{ - char *body; - char *tmp; - size_t body_size = 0; - char *buffered_data = NULL; - size_t buffer_size = 0; - int ret; - - if (new_data == NULL && chunk == NULL) { - flb_plg_error(ctx->ins, "[construct_request_buffer] Something went wrong" - " both chunk and new_data are NULL"); - return -1; - } - - if (chunk) { - ret = s3_store_file_read(ctx, chunk, &buffered_data, &buffer_size); - if (ret < 0) { - flb_plg_error(ctx->ins, "Could not read locally buffered data %s", - chunk->file_path); - return -1; - } - - /* - * lock the chunk from buffer list - */ - s3_store_file_lock(chunk); - body = buffered_data; - body_size = buffer_size; - } - - /* - * If new data is arriving, increase the original 'buffered_data' size - * to append the new one. - */ - if (new_data) { - body_size += flb_sds_len(new_data); - - tmp = flb_realloc(buffered_data, body_size + 1); - if (!tmp) { - flb_errno(); - flb_free(buffered_data); - if (chunk) { - s3_store_file_unlock(chunk); - } - return -1; - } - body = buffered_data = tmp; - memcpy(body + buffer_size, new_data, flb_sds_len(new_data)); - body[body_size] = '\0'; - } - - *out_buf = body; - *out_size = body_size; - - return 0; -} - -static int s3_put_object(struct flb_s3 *ctx, const char *tag, time_t file_first_log_time, - char *body, size_t body_size) -{ - flb_sds_t s3_key = NULL; - struct flb_http_client *c = NULL; - struct flb_aws_client *s3_client; - struct flb_aws_header *headers = NULL; - char *random_alphanumeric; - int append_random = FLB_FALSE; - int len; - int ret; - int num_headers = 0; - char *final_key; - flb_sds_t uri; - flb_sds_t tmp; - char final_body_md5[25]; - - s3_key = flb_get_s3_key(ctx->s3_key_format, file_first_log_time, tag, - ctx->tag_delimiters, ctx->seq_index); - if (!s3_key) { - flb_plg_error(ctx->ins, "Failed to construct S3 Object Key for %s", tag); - return -1; - } - - len = strlen(s3_key); - if ((len + 16) <= 1024 && !ctx->key_fmt_has_uuid && !ctx->static_file_path && - !ctx->key_fmt_has_seq_index) { - append_random = FLB_TRUE; - len += 16; - } - len += strlen(ctx->bucket + 1); - - uri = flb_sds_create_size(len); - - if (append_random == FLB_TRUE) { - random_alphanumeric = flb_sts_session_name(); - if (!random_alphanumeric) { - flb_sds_destroy(s3_key); - flb_sds_destroy(uri); - flb_plg_error(ctx->ins, "Failed to create randomness for S3 key %s", tag); - return -1; - } - /* only use 8 chars of the random string */ - random_alphanumeric[8] = '\0'; - - tmp = flb_sds_printf(&uri, "/%s%s-object%s", ctx->bucket, s3_key, - random_alphanumeric); - flb_free(random_alphanumeric); - } - else { - tmp = flb_sds_printf(&uri, "/%s%s", ctx->bucket, s3_key); - } - - if (!tmp) { - flb_sds_destroy(s3_key); - flb_plg_error(ctx->ins, "Failed to create PutObject URI"); - return -1; - } - flb_sds_destroy(s3_key); - uri = tmp; - - memset(final_body_md5, 0, sizeof(final_body_md5)); - if (ctx->send_content_md5 == FLB_TRUE) { - ret = get_md5_base64(body, body_size, - final_body_md5, sizeof(final_body_md5)); - if (ret != 0) { - flb_plg_error(ctx->ins, "Failed to create Content-MD5 header"); - flb_sds_destroy(uri); - return -1; - } - } - - /* Update file and increment index value right before request */ - if (ctx->key_fmt_has_seq_index) { - ctx->seq_index++; - - ret = write_seq_index(ctx->seq_index_file, ctx->seq_index); - if (ret < 0 && access(ctx->seq_index_file, F_OK) == 0) { - ctx->seq_index--; - flb_sds_destroy(s3_key); - flb_plg_error(ctx->ins, "Failed to update sequential index metadata file"); - return -1; - } - } - - s3_client = ctx->s3_client; - if (s3_plugin_under_test() == FLB_TRUE) { - c = mock_s3_call("TEST_PUT_OBJECT_ERROR", "PutObject"); - } - else { - ret = create_headers(ctx, final_body_md5, &headers, &num_headers, FLB_FALSE); - if (ret == -1) { - flb_plg_error(ctx->ins, "Failed to create headers"); - flb_sds_destroy(uri); - goto decrement_index; - } - c = s3_client->client_vtable->request(s3_client, FLB_HTTP_PUT, - uri, body, body_size, - headers, num_headers); - flb_free(headers); - } - if (c) { - flb_plg_debug(ctx->ins, "PutObject http status=%d", c->resp.status); - if (c->resp.status == 200) { - /* - * URI contains bucket name, so we must advance over it - * to print the object key - */ - final_key = uri + strlen(ctx->bucket) + 1; - flb_plg_info(ctx->ins, "Successfully uploaded object %s", final_key); - flb_sds_destroy(uri); - flb_http_client_destroy(c); - - return 0; - } - flb_aws_print_xml_error(c->resp.payload, c->resp.payload_size, - "PutObject", ctx->ins); - if (c->resp.data != NULL) { - flb_plg_error(ctx->ins, "Raw PutObject response: %s", c->resp.data); - } - flb_http_client_destroy(c); - } - - flb_plg_error(ctx->ins, "PutObject request failed"); - flb_sds_destroy(uri); - goto decrement_index; - -decrement_index: - if (ctx->key_fmt_has_seq_index) { - ctx->seq_index--; - - ret = write_seq_index(ctx->seq_index_file, ctx->seq_index); - if (ret < 0) { - flb_plg_error(ctx->ins, "Failed to decrement index after request error"); - return -1; - } - } - return -1; -} - -int get_md5_base64(char *buf, size_t buf_size, char *md5_str, size_t md5_str_size) -{ - unsigned char md5_bin[16]; - size_t olen; - int ret; - - ret = flb_hash_simple(FLB_HASH_MD5, - (unsigned char *) buf, buf_size, - md5_bin, sizeof(md5_bin)); - - if (ret != FLB_CRYPTO_SUCCESS) { - return -1; - } - - ret = flb_base64_encode((unsigned char*) md5_str, md5_str_size, - &olen, md5_bin, sizeof(md5_bin)); - if (ret != 0) { - return ret; - } - - return 0; -} - -static struct multipart_upload *get_upload(struct flb_s3 *ctx, - const char *tag, int tag_len) -{ - struct multipart_upload *m_upload = NULL; - struct multipart_upload *tmp_upload = NULL; - struct mk_list *tmp; - struct mk_list *head; - - mk_list_foreach_safe(head, tmp, &ctx->uploads) { - tmp_upload = mk_list_entry(head, struct multipart_upload, _head); - - if (tmp_upload->upload_state == MULTIPART_UPLOAD_STATE_COMPLETE_IN_PROGRESS) { - continue; - } - if (tmp_upload->upload_errors >= MAX_UPLOAD_ERRORS) { - tmp_upload->upload_state = MULTIPART_UPLOAD_STATE_COMPLETE_IN_PROGRESS; - flb_plg_error(ctx->ins, "Upload for %s has reached max upload errors", - tmp_upload->s3_key); - continue; - } - if (strcmp(tmp_upload->tag, tag) == 0) { - m_upload = tmp_upload; - break; - } - } - - return m_upload; -} - -static struct multipart_upload *create_upload(struct flb_s3 *ctx, const char *tag, - int tag_len, time_t file_first_log_time) -{ - int ret; - struct multipart_upload *m_upload = NULL; - flb_sds_t s3_key = NULL; - flb_sds_t tmp_sds = NULL; - - /* create new upload for this key */ - m_upload = flb_calloc(1, sizeof(struct multipart_upload)); - if (!m_upload) { - flb_errno(); - return NULL; - } - s3_key = flb_get_s3_key(ctx->s3_key_format, file_first_log_time, tag, - ctx->tag_delimiters, ctx->seq_index); - if (!s3_key) { - flb_plg_error(ctx->ins, "Failed to construct S3 Object Key for %s", tag); - flb_free(m_upload); - return NULL; - } - m_upload->s3_key = s3_key; - tmp_sds = flb_sds_create_len(tag, tag_len); - if (!tmp_sds) { - flb_errno(); - flb_free(m_upload); - return NULL; - } - m_upload->tag = tmp_sds; - m_upload->upload_state = MULTIPART_UPLOAD_STATE_NOT_CREATED; - m_upload->part_number = 1; - m_upload->init_time = time(NULL); - mk_list_add(&m_upload->_head, &ctx->uploads); - - /* Update file and increment index value right before request */ - if (ctx->key_fmt_has_seq_index) { - ctx->seq_index++; - - ret = write_seq_index(ctx->seq_index_file, ctx->seq_index); - if (ret < 0) { - ctx->seq_index--; - flb_sds_destroy(s3_key); - flb_plg_error(ctx->ins, "Failed to write to sequential index metadata file"); - return NULL; - } - } - - return m_upload; -} - -/* Adds an entry to upload queue */ -static int add_to_queue(struct flb_s3 *ctx, struct s3_file *upload_file, - struct multipart_upload *m_upload_file, const char *tag, int tag_len) -{ - struct upload_queue *upload_contents; - flb_sds_t tag_cpy; - - /* Create upload contents object and add to upload queue */ - upload_contents = flb_calloc(1, sizeof(struct upload_queue)); - if (upload_contents == NULL) { - flb_plg_error(ctx->ins, "Error allocating memory for upload_queue entry"); - flb_errno(); - return -1; - } - upload_contents->upload_file = upload_file; - upload_contents->m_upload_file = m_upload_file; - upload_contents->tag_len = tag_len; - upload_contents->retry_counter = 0; - upload_contents->upload_time = -1; - - /* Necessary to create separate string for tag to prevent corruption */ - tag_cpy = flb_sds_create_len(tag, tag_len); - if (!tag_cpy) { - flb_errno(); - flb_free(upload_contents); - return -1; - } - upload_contents->tag = tag_cpy; - - - /* Add entry to upload queue */ - mk_list_add(&upload_contents->_head, &ctx->upload_queue); - return 0; -} - -/* Removes an entry from upload_queue */ -void remove_from_queue(struct upload_queue *entry) -{ - mk_list_del(&entry->_head); - flb_sds_destroy(entry->tag); - flb_free(entry); - return; -} - -/* Validity check for upload queue object */ -static int upload_queue_valid(struct upload_queue *upload_contents, time_t now, - void *out_context) -{ - struct flb_s3 *ctx = out_context; - - if (upload_contents == NULL) { - flb_plg_error(ctx->ins, "Error getting entry from upload_queue"); - return -1; - } - if (upload_contents->_head.next == NULL || upload_contents->_head.prev == NULL) { - flb_plg_debug(ctx->ins, "Encountered previously deleted entry in " - "upload_queue. Deleting invalid entry"); - mk_list_del(&upload_contents->_head); - return -1; - } - if (upload_contents->upload_file->locked == FLB_FALSE) { - flb_plg_debug(ctx->ins, "Encountered unlocked file in upload_queue. " - "Exiting"); - return -1; - } - if (upload_contents->upload_file->size <= 0) { - flb_plg_debug(ctx->ins, "Encountered empty chunk file in upload_queue. " - "Deleting empty chunk file"); - remove_from_queue(upload_contents); - return -1; - } - if (now < upload_contents->upload_time) { - flb_plg_debug(ctx->ins, "Found valid chunk file but not ready to upload"); - return -1; - } - return 0; -} - -static int send_upload_request(void *out_context, flb_sds_t chunk, - struct s3_file *upload_file, - struct multipart_upload *m_upload_file, - const char *tag, int tag_len) -{ - int ret; - char *buffer; - size_t buffer_size; - struct flb_s3 *ctx = out_context; - - /* Create buffer to upload to S3 */ - ret = construct_request_buffer(ctx, chunk, upload_file, &buffer, &buffer_size); - flb_sds_destroy(chunk); - if (ret < 0) { - flb_plg_error(ctx->ins, "Could not construct request buffer for %s", - upload_file->file_path); - return -1; - } - - /* Upload to S3 */ - ret = upload_data(ctx, upload_file, m_upload_file, buffer, buffer_size, tag, tag_len); - flb_free(buffer); - - return ret; -} - -static int buffer_chunk(void *out_context, struct s3_file *upload_file, - flb_sds_t chunk, int chunk_size, - const char *tag, int tag_len, - time_t file_first_log_time) -{ - int ret; - struct flb_s3 *ctx = out_context; - - ret = s3_store_buffer_put(ctx, upload_file, tag, - tag_len, chunk, (size_t) chunk_size, file_first_log_time); - flb_sds_destroy(chunk); - if (ret < 0) { - flb_plg_warn(ctx->ins, "Could not buffer chunk. Data order preservation " - "will be compromised"); - return -1; - } - return 0; -} - -/* Uploads all chunk files in queue synchronously */ -static void s3_upload_queue(struct flb_config *config, void *out_context) -{ - int ret; - time_t now; - struct upload_queue *upload_contents; - struct flb_s3 *ctx = out_context; - struct mk_list *tmp; - struct mk_list *head; - - flb_plg_debug(ctx->ins, "Running upload timer callback (upload_queue).."); - - /* No chunks in upload queue. Scan for timed out chunks. */ - if (mk_list_size(&ctx->upload_queue) == 0) { - flb_plg_debug(ctx->ins, "No files found in upload_queue. Scanning for timed " - "out chunks"); - cb_s3_upload(config, out_context); - } - - /* Iterate through each file in upload queue */ - mk_list_foreach_safe(head, tmp, &ctx->upload_queue) { - upload_contents = mk_list_entry(head, struct upload_queue, _head); - - now = time(NULL); - - /* Checks if upload_contents is valid */ - ret = upload_queue_valid(upload_contents, now, ctx); - if (ret < 0) { - goto exit; - } - - /* Try to upload file. Return value can be -1, FLB_OK, FLB_ERROR, FLB_RETRY. */ - ret = send_upload_request(ctx, NULL, upload_contents->upload_file, - upload_contents->m_upload_file, - upload_contents->tag, upload_contents->tag_len); - if (ret < 0) { - goto exit; - } - else if (ret == FLB_OK) { - remove_from_queue(upload_contents); - ctx->retry_time = 0; - ctx->upload_queue_success = FLB_TRUE; - } - else { - s3_store_file_lock(upload_contents->upload_file); - ctx->upload_queue_success = FLB_FALSE; - - /* If retry limit was reached, discard file and remove file from queue */ - upload_contents->retry_counter++; - if (upload_contents->retry_counter >= MAX_UPLOAD_ERRORS) { - flb_plg_warn(ctx->ins, "Chunk file failed to send %d times, will not " - "retry", upload_contents->retry_counter); - s3_store_file_inactive(ctx, upload_contents->upload_file); - multipart_upload_destroy(upload_contents->m_upload_file); - remove_from_queue(upload_contents); - continue; - } - - /* Retry in N seconds */ - upload_contents->upload_time = now + 2 * upload_contents->retry_counter; - ctx->retry_time += 2 * upload_contents->retry_counter; - flb_plg_debug(ctx->ins, "Failed to upload file in upload_queue. Will not " - "retry for %d seconds", 2 * upload_contents->retry_counter); - break; - } - } - -exit: - return; -} - -static void cb_s3_upload(struct flb_config *config, void *data) -{ - struct flb_s3 *ctx = data; - struct s3_file *chunk = NULL; - struct multipart_upload *m_upload = NULL; - struct flb_fstore_file *fsf; - char *buffer = NULL; - size_t buffer_size = 0; - struct mk_list *tmp; - struct mk_list *head; - int complete; - int ret; - time_t now; - - flb_plg_debug(ctx->ins, "Running upload timer callback (cb_s3_upload).."); - - now = time(NULL); - - /* Check all chunks and see if any have timed out */ - mk_list_foreach_safe(head, tmp, &ctx->stream_active->files) { - fsf = mk_list_entry(head, struct flb_fstore_file, _head); - chunk = fsf->data; - - if (now < (chunk->create_time + ctx->upload_timeout + ctx->retry_time)) { - continue; /* Only send chunks which have timed out */ - } - - /* Locked chunks are being processed, skip */ - if (chunk->locked == FLB_TRUE) { - continue; - } - - m_upload = get_upload(ctx, (const char *) fsf->meta_buf, fsf->meta_size); - - ret = construct_request_buffer(ctx, NULL, chunk, &buffer, &buffer_size); - if (ret < 0) { - flb_plg_error(ctx->ins, "Could not construct request buffer for %s", - chunk->file_path); - continue; - } - - /* FYI: if construct_request_buffer() succeedeed, the s3_file is locked */ - ret = upload_data(ctx, chunk, m_upload, buffer, buffer_size, - (const char *) fsf->meta_buf, fsf->meta_size); - flb_free(buffer); - if (ret != FLB_OK) { - flb_plg_error(ctx->ins, "Could not send chunk with tag %s", - (char *) fsf->meta_buf); - } - } - - /* Check all uploads and see if any need completion */ - mk_list_foreach_safe(head, tmp, &ctx->uploads) { - m_upload = mk_list_entry(head, struct multipart_upload, _head); - complete = FLB_FALSE; - - if (m_upload->complete_errors >= MAX_UPLOAD_ERRORS) { - flb_plg_error(ctx->ins, - "Upload for %s has reached max completion errors, " - "plugin will give up", m_upload->s3_key); - mk_list_del(&m_upload->_head); - continue; - } - - if (m_upload->upload_state == MULTIPART_UPLOAD_STATE_NOT_CREATED) { - continue; - } - - if (m_upload->upload_state == MULTIPART_UPLOAD_STATE_COMPLETE_IN_PROGRESS) { - complete = FLB_TRUE; - } - if (time(NULL) > (m_upload->init_time + ctx->upload_timeout + ctx->retry_time)) { - flb_plg_info(ctx->ins, "Completing upload for %s because upload_timeout" - " has passed", m_upload->s3_key); - complete = FLB_TRUE; - } - if (complete == FLB_TRUE) { - m_upload->upload_state = MULTIPART_UPLOAD_STATE_COMPLETE_IN_PROGRESS; - mk_list_del(&m_upload->_head); - ret = complete_multipart_upload(ctx, m_upload); - if (ret == 0) { - multipart_upload_destroy(m_upload); - } - else { - mk_list_add(&m_upload->_head, &ctx->uploads); - /* data was persisted, this can be retried */ - m_upload->complete_errors += 1; - flb_plg_error(ctx->ins, "Could not complete upload %s, will retry..", - m_upload->s3_key); - } - } - } - -} - -static flb_sds_t flb_pack_msgpack_extract_log_key(void *out_context, const char *data, - uint64_t bytes) -{ - int i; - int records = 0; - int map_size; - int check = FLB_FALSE; - int found = FLB_FALSE; - int log_key_missing = 0; - int ret; - int alloc_error = 0; - struct flb_s3 *ctx = out_context; - char *val_buf; - char *key_str = NULL; - size_t key_str_size = 0; - size_t msgpack_size = bytes + bytes / 4; - size_t val_offset = 0; - flb_sds_t out_buf; - msgpack_object map; - msgpack_object key; - msgpack_object val; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - /* Iterate the original buffer and perform adjustments */ - records = flb_mp_count(data, bytes); - if (records <= 0) { - return NULL; - } - - /* Allocate buffer to store log_key contents */ - val_buf = flb_calloc(1, msgpack_size); - if (val_buf == NULL) { - flb_plg_error(ctx->ins, "Could not allocate enough " - "memory to read record"); - flb_errno(); - return NULL; - } - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - flb_free(val_buf); - - return NULL; - } - - - while (!alloc_error && - (ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - - /* Get the record/map */ - map = *log_event.body; - - if (map.type != MSGPACK_OBJECT_MAP) { - continue; - } - - map_size = map.via.map.size; - - /* Reset variables for found log_key and correct type */ - found = FLB_FALSE; - check = FLB_FALSE; - - /* Extract log_key from record and append to output buffer */ - for (i = 0; i < map_size; i++) { - key = map.via.map.ptr[i].key; - val = map.via.map.ptr[i].val; - - if (key.type == MSGPACK_OBJECT_BIN) { - key_str = (char *) key.via.bin.ptr; - key_str_size = key.via.bin.size; - check = FLB_TRUE; - } - if (key.type == MSGPACK_OBJECT_STR) { - key_str = (char *) key.via.str.ptr; - key_str_size = key.via.str.size; - check = FLB_TRUE; - } - - if (check == FLB_TRUE) { - if (strncmp(ctx->log_key, key_str, key_str_size) == 0) { - found = FLB_TRUE; - - /* - * Copy contents of value into buffer. Necessary to copy - * strings because flb_msgpack_to_json does not handle nested - * JSON gracefully and double escapes them. - */ - if (val.type == MSGPACK_OBJECT_BIN) { - memcpy(val_buf + val_offset, val.via.bin.ptr, val.via.bin.size); - val_offset += val.via.bin.size; - val_buf[val_offset] = '\n'; - val_offset++; - } - else if (val.type == MSGPACK_OBJECT_STR) { - memcpy(val_buf + val_offset, val.via.str.ptr, val.via.str.size); - val_offset += val.via.str.size; - val_buf[val_offset] = '\n'; - val_offset++; - } - else { - ret = flb_msgpack_to_json(val_buf + val_offset, - msgpack_size - val_offset, &val); - if (ret < 0) { - break; - } - val_offset += ret; - val_buf[val_offset] = '\n'; - val_offset++; - } - /* Exit early once log_key has been found for current record */ - break; - } - } - } - - /* If log_key was not found in the current record, mark log key as missing */ - if (found == FLB_FALSE) { - log_key_missing++; - } - } - - /* Throw error once per chunk if at least one log key was not found */ - if (log_key_missing > 0) { - flb_plg_error(ctx->ins, "Could not find log_key '%s' in %d records", - ctx->log_key, log_key_missing); - } - - flb_log_event_decoder_destroy(&log_decoder); - - /* If nothing was read, destroy buffer */ - if (val_offset == 0) { - flb_free(val_buf); - return NULL; - } - val_buf[val_offset] = '\0'; - - /* Create output buffer to store contents */ - out_buf = flb_sds_create(val_buf); - if (out_buf == NULL) { - flb_plg_error(ctx->ins, "Error creating buffer to store log_key contents."); - flb_errno(); - } - flb_free(val_buf); - - return out_buf; -} - -static void unit_test_flush(void *out_context, struct s3_file *upload_file, - const char *tag, int tag_len, flb_sds_t chunk, - int chunk_size, struct multipart_upload *m_upload_file, - time_t file_first_log_time) -{ - int ret; - char *buffer; - size_t buffer_size; - struct flb_s3 *ctx = out_context; - - s3_store_buffer_put(ctx, upload_file, tag, tag_len, - chunk, (size_t) chunk_size, file_first_log_time); - ret = construct_request_buffer(ctx, chunk, upload_file, &buffer, &buffer_size); - if (ret < 0) { - flb_plg_error(ctx->ins, "Could not construct request buffer for %s", - upload_file->file_path); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - ret = upload_data(ctx, upload_file, m_upload_file, buffer, buffer_size, tag, tag_len); - flb_free(buffer); - - FLB_OUTPUT_RETURN(ret); -} - -static void flush_init(void *out_context) -{ - int ret; - struct flb_s3 *ctx = out_context; - struct flb_sched *sched; - - /* clean up any old buffers found on startup */ - if (ctx->has_old_buffers == FLB_TRUE) { - flb_plg_info(ctx->ins, - "Sending locally buffered data from previous " - "executions to S3; buffer=%s", - ctx->fs->root_path); - ctx->has_old_buffers = FLB_FALSE; - ret = put_all_chunks(ctx); - if (ret < 0) { - ctx->has_old_buffers = FLB_TRUE; - flb_plg_error(ctx->ins, - "Failed to send locally buffered data left over " - "from previous executions; will retry. Buffer=%s", - ctx->fs->root_path); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - } - - /* - * create a timer that will run periodically and check if uploads - * are ready for completion - * this is created once on the first flush - */ - if (ctx->timer_created == FLB_FALSE) { - flb_plg_debug(ctx->ins, - "Creating upload timer with frequency %ds", - ctx->timer_ms / 1000); - - sched = flb_sched_ctx_get(); - - if (ctx->preserve_data_ordering) { - ret = flb_sched_timer_cb_create(sched, FLB_SCHED_TIMER_CB_PERM, - ctx->timer_ms, s3_upload_queue, ctx, NULL); - } - else { - ret = flb_sched_timer_cb_create(sched, FLB_SCHED_TIMER_CB_PERM, - ctx->timer_ms, cb_s3_upload, ctx, NULL); - } - if (ret == -1) { - flb_plg_error(ctx->ins, "Failed to create upload timer"); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - ctx->timer_created = FLB_TRUE; - } -} - -static void cb_s3_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - int ret; - int chunk_size; - int upload_timeout_check = FLB_FALSE; - int total_file_size_check = FLB_FALSE; - flb_sds_t chunk = NULL; - struct s3_file *upload_file = NULL; - struct flb_s3 *ctx = out_context; - struct multipart_upload *m_upload_file = NULL; - time_t file_first_log_time = 0; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - /* Cleanup old buffers and initialize upload timer */ - flush_init(ctx); - - /* Process chunk */ - if (ctx->log_key) { - chunk = flb_pack_msgpack_extract_log_key(ctx, - event_chunk->data, - event_chunk->size); - } - else { - chunk = flb_pack_msgpack_to_json_format(event_chunk->data, - event_chunk->size, - FLB_PACK_JSON_FORMAT_LINES, - ctx->json_date_format, - ctx->date_key); - } - if (chunk == NULL) { - flb_plg_error(ctx->ins, "Could not marshal msgpack to output string"); - FLB_OUTPUT_RETURN(FLB_ERROR); - } - chunk_size = flb_sds_len(chunk); - - /* Get a file candidate matching the given 'tag' */ - upload_file = s3_store_file_get(ctx, - event_chunk->tag, - flb_sds_len(event_chunk->tag)); - - if (upload_file == NULL) { - ret = flb_log_event_decoder_init(&log_decoder, - (char *) event_chunk->data, - event_chunk->size); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - flb_sds_destroy(chunk); - - FLB_OUTPUT_RETURN(FLB_ERROR); - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - if (log_event.timestamp.tm.tv_sec != 0) { - file_first_log_time = log_event.timestamp.tm.tv_sec; - break; - } - } - - flb_log_event_decoder_destroy(&log_decoder); - } - else { - /* Get file_first_log_time from upload_file */ - file_first_log_time = upload_file->first_log_time; - } - - if (file_first_log_time == 0) { - file_first_log_time = time(NULL); - } - - /* Specific to unit tests, will not get called normally */ - if (s3_plugin_under_test() == FLB_TRUE) { - unit_test_flush(ctx, upload_file, - event_chunk->tag, flb_sds_len(event_chunk->tag), - chunk, chunk_size, - m_upload_file, file_first_log_time); - } - - /* Discard upload_file if it has failed to upload MAX_UPLOAD_ERRORS times */ - if (upload_file != NULL && upload_file->failures >= MAX_UPLOAD_ERRORS) { - flb_plg_warn(ctx->ins, "File with tag %s failed to send %d times, will not " - "retry", event_chunk->tag, MAX_UPLOAD_ERRORS); - s3_store_file_inactive(ctx, upload_file); - upload_file = NULL; - } - - /* If upload_timeout has elapsed, upload file */ - if (upload_file != NULL && time(NULL) > - (upload_file->create_time + ctx->upload_timeout)) { - upload_timeout_check = FLB_TRUE; - flb_plg_info(ctx->ins, "upload_timeout reached for %s", - event_chunk->tag); - } - - m_upload_file = get_upload(ctx, - event_chunk->tag, flb_sds_len(event_chunk->tag)); - - if (m_upload_file != NULL && time(NULL) > - (m_upload_file->init_time + ctx->upload_timeout)) { - upload_timeout_check = FLB_TRUE; - flb_plg_info(ctx->ins, "upload_timeout reached for %s", event_chunk->tag); - } - - /* If total_file_size has been reached, upload file */ - if ((upload_file && upload_file->size + chunk_size > ctx->upload_chunk_size) || - (m_upload_file && m_upload_file->bytes + chunk_size > ctx->file_size)) { - total_file_size_check = FLB_TRUE; - } - - /* File is ready for upload, upload_file != NULL prevents from segfaulting. */ - if ((upload_file != NULL) && (upload_timeout_check == FLB_TRUE || total_file_size_check == FLB_TRUE)) { - if (ctx->preserve_data_ordering == FLB_TRUE) { - /* Buffer last chunk in file and lock file to prevent further changes */ - ret = buffer_chunk(ctx, upload_file, chunk, chunk_size, - event_chunk->tag, flb_sds_len(event_chunk->tag), - file_first_log_time); - - if (ret < 0) { - FLB_OUTPUT_RETURN(FLB_RETRY); - } - s3_store_file_lock(upload_file); - - /* Add chunk file to upload queue */ - ret = add_to_queue(ctx, upload_file, m_upload_file, - event_chunk->tag, flb_sds_len(event_chunk->tag)); - if (ret < 0) { - FLB_OUTPUT_RETURN(FLB_ERROR); - } - - /* Go through upload queue and return error if something went wrong */ - s3_upload_queue(config, ctx); - if (ctx->upload_queue_success == FLB_FALSE) { - ctx->upload_queue_success = FLB_TRUE; - FLB_OUTPUT_RETURN(FLB_ERROR); - } - FLB_OUTPUT_RETURN(FLB_OK); - } - else { - /* Send upload directly without upload queue */ - ret = send_upload_request(ctx, chunk, upload_file, m_upload_file, - event_chunk->tag, - flb_sds_len(event_chunk->tag)); - if (ret < 0) { - FLB_OUTPUT_RETURN(FLB_ERROR); - } - FLB_OUTPUT_RETURN(ret); - } - } - - /* Buffer current chunk in filesystem and wait for next chunk from engine */ - ret = buffer_chunk(ctx, upload_file, chunk, chunk_size, - event_chunk->tag, flb_sds_len(event_chunk->tag), - file_first_log_time); - - if (ret < 0) { - FLB_OUTPUT_RETURN(FLB_RETRY); - } - FLB_OUTPUT_RETURN(FLB_OK); -} - -static int cb_s3_exit(void *data, struct flb_config *config) -{ - int ret; - struct flb_s3 *ctx = data; - struct multipart_upload *m_upload = NULL; - struct mk_list *tmp; - struct mk_list *head; - - if (!ctx) { - return 0; - } - - if (s3_store_has_data(ctx) == FLB_TRUE) { - flb_plg_info(ctx->ins, "Sending all locally buffered data to S3"); - ret = put_all_chunks(ctx); - if (ret < 0) { - flb_plg_error(ctx->ins, "Could not send all chunks on exit"); - } - } - - if (s3_store_has_uploads(ctx) == FLB_TRUE) { - mk_list_foreach_safe(head, tmp, &ctx->uploads) { - m_upload = mk_list_entry(head, struct multipart_upload, _head); - - if (m_upload->upload_state == MULTIPART_UPLOAD_STATE_NOT_CREATED) { - continue; - } - - if (m_upload->bytes > 0) { - m_upload->upload_state = MULTIPART_UPLOAD_STATE_COMPLETE_IN_PROGRESS; - mk_list_del(&m_upload->_head); - ret = complete_multipart_upload(ctx, m_upload); - if (ret == 0) { - multipart_upload_destroy(m_upload); - } - else { - mk_list_add(&m_upload->_head, &ctx->uploads); - flb_plg_error(ctx->ins, "Could not complete upload %s", - m_upload->s3_key); - } - } - } - } - - s3_store_exit(ctx); - s3_context_destroy(ctx); - - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "json_date_format", NULL, - 0, FLB_FALSE, 0, - FBL_PACK_JSON_DATE_FORMAT_DESCRIPTION - }, - { - FLB_CONFIG_MAP_STR, "json_date_key", "date", - 0, FLB_TRUE, offsetof(struct flb_s3, json_date_key), - "Specifies the name of the date field in output." - }, - { - FLB_CONFIG_MAP_SIZE, "total_file_size", "100000000", - 0, FLB_TRUE, offsetof(struct flb_s3, file_size), - "Specifies the size of files in S3. Maximum size is 50GB, minimum is 1MB" - }, - { - FLB_CONFIG_MAP_SIZE, "upload_chunk_size", "5242880", - 0, FLB_TRUE, offsetof(struct flb_s3, upload_chunk_size), - "This plugin uses the S3 Multipart Upload API to stream data to S3, " - "ensuring your data gets-off-the-box as quickly as possible. " - "This parameter configures the size of each “part” in the upload. " - "The total_file_size option configures the size of the file you will see " - "in S3; this option determines the size of chunks uploaded until that " - "size is reached. These chunks are temporarily stored in chunk_buffer_path " - "until their size reaches upload_chunk_size, which point the chunk is " - "uploaded to S3. Default: 5M, Max: 50M, Min: 5M." - }, - - { - FLB_CONFIG_MAP_TIME, "upload_timeout", "10m", - 0, FLB_TRUE, offsetof(struct flb_s3, upload_timeout), - "Optionally specify a timeout for uploads. " - "Whenever this amount of time has elapsed, Fluent Bit will complete an " - "upload and create a new file in S3. For example, set this value to 60m " - "and you will get a new file in S3 every hour. Default is 10m." - }, - { - FLB_CONFIG_MAP_STR, "bucket", NULL, - 0, FLB_TRUE, offsetof(struct flb_s3, bucket), - "S3 bucket name." - }, - { - FLB_CONFIG_MAP_STR, "region", "us-east-1", - 0, FLB_TRUE, offsetof(struct flb_s3, region), - "AWS region." - }, - { - FLB_CONFIG_MAP_STR, "role_arn", NULL, - 0, FLB_FALSE, 0, - "ARN of an IAM role to assume (ex. for cross account access)." - }, - { - FLB_CONFIG_MAP_STR, "endpoint", NULL, - 0, FLB_TRUE, offsetof(struct flb_s3, endpoint), - "Custom endpoint for the S3 API." - }, - { - FLB_CONFIG_MAP_STR, "sts_endpoint", NULL, - 0, FLB_TRUE, offsetof(struct flb_s3, sts_endpoint), - "Custom endpoint for the STS API." - }, - { - FLB_CONFIG_MAP_STR, "canned_acl", NULL, - 0, FLB_FALSE, 0, - "Predefined Canned ACL policy for S3 objects." - }, - { - FLB_CONFIG_MAP_STR, "compression", NULL, - 0, FLB_FALSE, 0, - "Compression type for S3 objects. 'gzip' and 'arrow' are the supported values. " - "'arrow' is only an available if Apache Arrow was enabled at compile time. " - "Defaults to no compression. " - "If 'gzip' is selected, the Content-Encoding HTTP Header will be set to 'gzip'." - }, - { - FLB_CONFIG_MAP_STR, "content_type", NULL, - 0, FLB_FALSE, 0, - "A standard MIME type for the S3 object; this will be set " - "as the Content-Type HTTP header." - }, - - { - FLB_CONFIG_MAP_STR, "store_dir", "/tmp/fluent-bit/s3", - 0, FLB_TRUE, offsetof(struct flb_s3, store_dir), - "Directory to locally buffer data before sending. Plugin uses the S3 Multipart " - "upload API to send data in chunks of 5 MB at a time- only a small amount of" - " data will be locally buffered at any given point in time." - }, - - { - FLB_CONFIG_MAP_SIZE, "store_dir_limit_size", (char *) NULL, - 0, FLB_TRUE, offsetof(struct flb_s3, store_dir_limit_size), - "S3 plugin has its own buffering system with files in the `store_dir`. " - "Use the `store_dir_limit_size` to limit the amount of data S3 buffers in " - "the `store_dir` to limit disk usage. If the limit is reached, " - "data will be discarded. Default is 0 which means unlimited." - }, - - { - FLB_CONFIG_MAP_STR, "s3_key_format", "/fluent-bit-logs/$TAG/%Y/%m/%d/%H/%M/%S", - 0, FLB_TRUE, offsetof(struct flb_s3, s3_key_format), - "Format string for keys in S3. This option supports strftime time formatters " - "and a syntax for selecting parts of the Fluent log tag using a syntax inspired " - "by the rewrite_tag filter. Add $TAG in the format string to insert the full " - "log tag; add $TAG[0] to insert the first part of the tag in the s3 key. " - "The tag is split into “parts” using the characters specified with the " - "s3_key_format_tag_delimiters option. Add $INDEX to enable sequential indexing " - "for file names. Adding $INDEX will prevent random string being added to end of key" - "when $UUID is not provided. See the in depth examples and tutorial in the " - "documentation." - }, - - { - FLB_CONFIG_MAP_STR, "s3_key_format_tag_delimiters", ".", - 0, FLB_TRUE, offsetof(struct flb_s3, tag_delimiters), - "A series of characters which will be used to split the tag into “parts” for " - "use with the s3_key_format option. See the in depth examples and tutorial in " - "the documentation." - }, - - { - FLB_CONFIG_MAP_BOOL, "auto_retry_requests", "true", - 0, FLB_TRUE, offsetof(struct flb_s3, retry_requests), - "Immediately retry failed requests to AWS services once. This option " - "does not affect the normal Fluent Bit retry mechanism with backoff. " - "Instead, it enables an immediate retry with no delay for networking " - "errors, which may help improve throughput when there are transient/random " - "networking issues." - }, - - { - FLB_CONFIG_MAP_BOOL, "use_put_object", "false", - 0, FLB_TRUE, offsetof(struct flb_s3, use_put_object), - "Use the S3 PutObject API, instead of the multipart upload API" - }, - - { - FLB_CONFIG_MAP_BOOL, "send_content_md5", "false", - 0, FLB_TRUE, offsetof(struct flb_s3, send_content_md5), - "Send the Content-MD5 header with object uploads, as is required when Object Lock is enabled" - }, - - { - FLB_CONFIG_MAP_BOOL, "preserve_data_ordering", "true", - 0, FLB_TRUE, offsetof(struct flb_s3, preserve_data_ordering), - "Normally, when an upload request fails, there is a high chance for the last " - "received chunk to be swapped with a later chunk, resulting in data shuffling. " - "This feature prevents this shuffling by using a queue logic for uploads." - }, - - { - FLB_CONFIG_MAP_STR, "log_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_s3, log_key), - "By default, the whole log record will be sent to S3. " - "If you specify a key name with this option, then only the value of " - "that key will be sent to S3." - }, - - { - FLB_CONFIG_MAP_STR, "external_id", NULL, - 0, FLB_TRUE, offsetof(struct flb_s3, external_id), - "Specify an external ID for the STS API, can be used with the role_arn parameter if your role " - "requires an external ID." - }, - - { - FLB_CONFIG_MAP_BOOL, "static_file_path", "false", - 0, FLB_TRUE, offsetof(struct flb_s3, static_file_path), - "Disables behavior where UUID string is automatically appended to end of S3 key name when " - "$UUID is not provided in s3_key_format. $UUID, time formatters, $TAG, and other dynamic " - "key formatters all work as expected while this feature is set to true." - }, - - { - FLB_CONFIG_MAP_STR, "storage_class", NULL, - 0, FLB_FALSE, 0, - "Specify the storage class for S3 objects. If this option is not specified, objects " - "will be stored with the default 'STANDARD' storage class." - }, - - { - FLB_CONFIG_MAP_STR, "profile", NULL, - 0, FLB_TRUE, offsetof(struct flb_s3, profile), - "AWS Profile name. AWS Profiles can be configured with AWS CLI and are usually stored in " - "$HOME/.aws/ directory." - }, - - /* EOF */ - {0} -}; - -/* Plugin registration */ -struct flb_output_plugin out_s3_plugin = { - .name = "s3", - .description = "Send to S3", - .cb_init = cb_s3_init, - .cb_flush = cb_s3_flush, - .cb_exit = cb_s3_exit, - .workers = 1, - .flags = FLB_OUTPUT_NET | FLB_IO_TLS, - .config_map = config_map -}; diff --git a/fluent-bit/plugins/out_s3/s3.h b/fluent-bit/plugins/out_s3/s3.h deleted file mode 100644 index e145b1ad6..000000000 --- a/fluent-bit/plugins/out_s3/s3.h +++ /dev/null @@ -1,203 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_S3 -#define FLB_OUT_S3 - -#include -#include -#include -#include -#include - -/* Upload data to S3 in 5MB chunks */ -#define MIN_CHUNKED_UPLOAD_SIZE 5242880 -#define MAX_CHUNKED_UPLOAD_SIZE 50000000 -#define MAX_CHUNKED_UPLOAD_COMPRESS_SIZE 5000000000 - -#define UPLOAD_TIMER_MAX_WAIT 60000 -#define UPLOAD_TIMER_MIN_WAIT 6000 - -#define MULTIPART_UPLOAD_STATE_NOT_CREATED 0 -#define MULTIPART_UPLOAD_STATE_CREATED 1 -#define MULTIPART_UPLOAD_STATE_COMPLETE_IN_PROGRESS 2 - -#define DEFAULT_FILE_SIZE 100000000 -#define MAX_FILE_SIZE 50000000000 -#define MAX_FILE_SIZE_STR "50,000,000,000" - -/* Allowed max file size 1 GB for publishing to S3 */ -#define MAX_FILE_SIZE_PUT_OBJECT 1000000000 - -#define DEFAULT_UPLOAD_TIMEOUT 3600 - -/* - * If we see repeated errors on an upload/chunk, we will discard it - * This saves us from scenarios where something goes wrong and an upload can - * not proceed (may be some other process completed it or deleted the upload) - * instead of erroring out forever, we eventually discard the upload. - * - * The same is done for chunks, just to be safe, even though realistically - * I can't think of a reason why a chunk could become unsendable. - */ -#define MAX_UPLOAD_ERRORS 5 - -struct upload_queue { - struct s3_file *upload_file; - struct multipart_upload *m_upload_file; - flb_sds_t tag; - int tag_len; - - int retry_counter; - time_t upload_time; - - struct mk_list _head; -}; - -struct multipart_upload { - flb_sds_t s3_key; - flb_sds_t tag; - flb_sds_t upload_id; - int upload_state; - time_t init_time; - - /* - * maximum of 10,000 parts in an upload, for each we need to store mapping - * of Part Number to ETag - */ - flb_sds_t etags[10000]; - int part_number; - - /* - * we use async http, so we need to check that all part requests have - * completed before we complete the upload - */ - int parts_uploaded; - - /* ongoing tracker of how much data has been sent for this upload */ - size_t bytes; - - struct mk_list _head; - - /* see note for MAX_UPLOAD_ERRORS */ - int upload_errors; - int complete_errors; -}; - -struct flb_s3 { - char *bucket; - char *region; - char *s3_key_format; - char *tag_delimiters; - char *endpoint; - char *sts_endpoint; - char *canned_acl; - char *content_type; - char *storage_class; - char *log_key; - char *external_id; - char *profile; - int free_endpoint; - int retry_requests; - int use_put_object; - int send_content_md5; - int static_file_path; - int compression; - int port; - int insecure; - size_t store_dir_limit_size; - - /* track the total amount of buffered data */ - size_t current_buffer_size; - - struct flb_aws_provider *provider; - struct flb_aws_provider *base_provider; - /* tls instances can't be re-used; aws provider requires a separate one */ - struct flb_tls *provider_tls; - /* one for the standard chain provider, one for sts assume role */ - struct flb_tls *sts_provider_tls; - struct flb_tls *client_tls; - - struct flb_aws_client *s3_client; - int json_date_format; - flb_sds_t json_date_key; - flb_sds_t date_key; - - flb_sds_t buffer_dir; - - char *store_dir; - struct flb_fstore *fs; - struct flb_fstore_stream *stream_active; /* default active stream */ - struct flb_fstore_stream *stream_upload; /* multipart upload stream */ - struct flb_fstore_stream *stream_metadata; /* s3 metadata stream */ - - /* - * used to track that unset buffers were found on startup that have not - * been sent - */ - int has_old_buffers; - /* old multipart uploads read on start up */ - int has_old_uploads; - - struct mk_list uploads; - - int preserve_data_ordering; - int upload_queue_success; - struct mk_list upload_queue; - - size_t file_size; - size_t upload_chunk_size; - time_t upload_timeout; - time_t retry_time; - - int timer_created; - int timer_ms; - int key_fmt_has_uuid; - - uint64_t seq_index; - int key_fmt_has_seq_index; - flb_sds_t metadata_dir; - flb_sds_t seq_index_file; - - struct flb_output_instance *ins; -}; - -int upload_part(struct flb_s3 *ctx, struct multipart_upload *m_upload, - char *body, size_t body_size); - -int create_multipart_upload(struct flb_s3 *ctx, - struct multipart_upload *m_upload); - -int complete_multipart_upload(struct flb_s3 *ctx, - struct multipart_upload *m_upload); - -void multipart_read_uploads_from_fs(struct flb_s3 *ctx); - -void multipart_upload_destroy(struct multipart_upload *m_upload); - -struct flb_http_client *mock_s3_call(char *error_env_var, char *api); -int s3_plugin_under_test(); - -int get_md5_base64(char *buf, size_t buf_size, char *md5_str, size_t md5_str_size); - -int create_headers(struct flb_s3 *ctx, char *body_md5, - struct flb_aws_header **headers, int *num_headers, - int multipart_upload); - -#endif diff --git a/fluent-bit/plugins/out_s3/s3_multipart.c b/fluent-bit/plugins/out_s3/s3_multipart.c deleted file mode 100644 index 1eb2a1061..000000000 --- a/fluent-bit/plugins/out_s3/s3_multipart.c +++ /dev/null @@ -1,707 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "s3.h" -#include "s3_store.h" - -#define COMPLETE_MULTIPART_UPLOAD_BASE_LEN 100 -#define COMPLETE_MULTIPART_UPLOAD_PART_LEN 124 - -flb_sds_t get_etag(char *response, size_t size); - -static inline int try_to_write(char *buf, int *off, size_t left, - const char *str, size_t str_len) -{ - if (str_len <= 0){ - str_len = strlen(str); - } - if (left <= *off+str_len) { - return FLB_FALSE; - } - memcpy(buf+*off, str, str_len); - *off += str_len; - return FLB_TRUE; -} - - -/* the 'tag' or key in the upload_dir is s3_key + \n + upload_id */ -static flb_sds_t upload_key(struct multipart_upload *m_upload) -{ - flb_sds_t key; - flb_sds_t tmp; - - key = flb_sds_create_size(64); - - tmp = flb_sds_printf(&key, "%s\n%s", m_upload->s3_key, m_upload->upload_id); - if (!tmp) { - flb_errno(); - flb_sds_destroy(key); - return NULL; - } - key = tmp; - - return key; -} - -/* the 'tag' or key in the upload_dir is s3_key + \n + upload_id */ -static int upload_data_from_key(struct multipart_upload *m_upload, char *key) -{ - flb_sds_t tmp_sds; - int len = 0; - int original_len; - char *tmp; - - original_len = strlen(key); - - tmp = strchr(key, '\n'); - if (!tmp) { - return -1; - } - - len = tmp - key; - tmp_sds = flb_sds_create_len(key, len); - if (!tmp_sds) { - flb_errno(); - return -1; - } - m_upload->s3_key = tmp_sds; - - tmp++; - original_len -= (len + 1); - - tmp_sds = flb_sds_create_len(tmp, original_len); - if (!tmp_sds) { - flb_errno(); - return -1; - } - m_upload->upload_id = tmp_sds; - - return 0; -} - -/* parse etags from file data */ -static void parse_etags(struct multipart_upload *m_upload, char *data) -{ - char *line = data; - char *start; - char *end; - flb_sds_t etag; - int part_num; - int len; - - if (!data) { - return; - } - - line = strtok(data, "\n"); - - if (!line) { - return; - } - - do { - start = strstr(line, "part_number="); - if (!start) { - return; - } - start += 12; - end = strchr(start, '\t'); - if (!end) { - flb_debug("[s3 restart parser] Did not find tab separator in line %s", start); - return; - } - *end = '\0'; - part_num = atoi(start); - if (part_num <= 0) { - flb_debug("[s3 restart parser] Could not parse part_number from %s", start); - return; - } - m_upload->part_number = part_num; - *end = '\t'; - - start = strstr(line, "tag="); - if (!start) { - flb_debug("[s3 restart parser] Could not find 'etag=' %s", line); - return; - } - - start += 4; - len = strlen(start); - - if (len <= 0) { - flb_debug("[s3 restart parser] Could not find etag %s", line); - return; - } - - etag = flb_sds_create_len(start, len); - if (!etag) { - flb_debug("[s3 restart parser] Could create etag"); - return; - } - flb_debug("[s3 restart parser] found part number %d=%s", part_num, etag); - m_upload->etags[part_num - 1] = etag; - - line = strtok(NULL, "\n"); - } while (line != NULL); -} - -static struct multipart_upload *upload_from_file(struct flb_s3 *ctx, - struct flb_fstore_file *fsf) -{ - struct multipart_upload *m_upload = NULL; - char *buffered_data = NULL; - size_t buffer_size = 0; - int ret; - - ret = s3_store_file_upload_read(ctx, fsf, &buffered_data, &buffer_size); - if (ret < 0) { - flb_plg_error(ctx->ins, "Could not read locally buffered data %s", - fsf->name); - return NULL; - } - - /* always make sure we have a fresh copy of metadata */ - ret = s3_store_file_meta_get(ctx, fsf); - if (ret == -1) { - flb_plg_error(ctx->ins, "Could not read file metadata: %s", - fsf->name); - return NULL; - } - - m_upload = flb_calloc(1, sizeof(struct multipart_upload)); - if (!m_upload) { - flb_errno(); - flb_free(buffered_data); - return NULL; - } - m_upload->init_time = time(NULL); - m_upload->upload_state = MULTIPART_UPLOAD_STATE_COMPLETE_IN_PROGRESS; - - ret = upload_data_from_key(m_upload, fsf->meta_buf); - if (ret < 0) { - flb_plg_error(ctx->ins, "Could not extract upload data from: %s", - fsf->name); - flb_free(buffered_data); - multipart_upload_destroy(m_upload); - return NULL; - } - - parse_etags(m_upload, buffered_data); - flb_free(buffered_data); - if (m_upload->part_number == 0) { - flb_plg_error(ctx->ins, "Could not extract upload data from %s", - fsf->name); - multipart_upload_destroy(m_upload); - return NULL; - } - - /* code expects it to be 1 more than the last part read */ - m_upload->part_number++; - - return m_upload; -} - -void multipart_read_uploads_from_fs(struct flb_s3 *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - struct multipart_upload *m_upload = NULL; - struct flb_fstore_file *fsf; - - mk_list_foreach_safe(head, tmp, &ctx->stream_upload->files) { - fsf = mk_list_entry(head, struct flb_fstore_file, _head); - m_upload = upload_from_file(ctx, fsf); - if (!m_upload) { - flb_plg_error(ctx->ins, - "Could not process multipart upload data in %s", - fsf->name); - continue; - } - mk_list_add(&m_upload->_head, &ctx->uploads); - flb_plg_info(ctx->ins, - "Successfully read existing upload from file system, s3_key=%s", - m_upload->s3_key); - } -} - -/* store list of part number and etag */ -static flb_sds_t upload_data(flb_sds_t etag, int part_num) -{ - flb_sds_t data; - flb_sds_t tmp; - - data = flb_sds_create_size(64); - - tmp = flb_sds_printf(&data, "part_number=%d\tetag=%s\n", part_num, etag); - if (!tmp) { - flb_errno(); - flb_sds_destroy(data); - return NULL; - } - data = tmp; - - return data; -} - -/* persists upload data to the file system */ -static int save_upload(struct flb_s3 *ctx, struct multipart_upload *m_upload, - flb_sds_t etag) -{ - int ret; - flb_sds_t key; - flb_sds_t data; - struct flb_fstore_file *fsf; - - key = upload_key(m_upload); - if (!key) { - flb_plg_debug(ctx->ins, "Could not constuct upload key for buffer dir"); - return -1; - } - - data = upload_data(etag, m_upload->part_number); - if (!data) { - flb_plg_debug(ctx->ins, "Could not constuct upload key for buffer dir"); - return -1; - } - - fsf = s3_store_file_upload_get(ctx, key, flb_sds_len(key)); - - /* Write the key to the file */ - ret = s3_store_file_upload_put(ctx, fsf, key, data); - - flb_sds_destroy(key); - flb_sds_destroy(data); - - return ret; -} - -static int remove_upload_from_fs(struct flb_s3 *ctx, struct multipart_upload *m_upload) -{ - flb_sds_t key; - struct flb_fstore_file *fsf; - - key = upload_key(m_upload); - if (!key) { - flb_plg_debug(ctx->ins, "Could not construct upload key"); - return -1; - } - - fsf = s3_store_file_upload_get(ctx, key, flb_sds_len(key)); - if (fsf) { - s3_store_file_upload_delete(ctx, fsf); - } - flb_sds_destroy(key); - return 0; -} - -/* - * https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html - */ -static int complete_multipart_upload_payload(struct flb_s3 *ctx, - struct multipart_upload *m_upload, - char **out_buf, size_t *out_size) -{ - char *buf; - int i; - int offset = 0; - flb_sds_t etag; - size_t size = COMPLETE_MULTIPART_UPLOAD_BASE_LEN; - char part_num[7]; - - size = size + (COMPLETE_MULTIPART_UPLOAD_PART_LEN * m_upload->part_number); - - buf = flb_malloc(size + 1); - if (!buf) { - flb_errno(); - return -1; - } - - if (!try_to_write(buf, &offset, size, - "", 73)) { - goto error; - } - - for (i = 0; i < m_upload->part_number; i++) { - etag = m_upload->etags[i]; - if (etag == NULL) { - continue; - } - if (!try_to_write(buf, &offset, size, - "", 12)) { - goto error; - } - - if (!try_to_write(buf, &offset, size, - etag, 0)) { - goto error; - } - - if (!try_to_write(buf, &offset, size, - "", 19)) { - goto error; - } - - if (!sprintf(part_num, "%d", i + 1)) { - goto error; - } - - if (!try_to_write(buf, &offset, size, - part_num, 0)) { - goto error; - } - - if (!try_to_write(buf, &offset, size, - "", 20)) { - goto error; - } - } - - if (!try_to_write(buf, &offset, size, - "", 26)) { - goto error; - } - - buf[offset] = '\0'; - - *out_buf = buf; - *out_size = offset; - return 0; - -error: - flb_free(buf); - flb_plg_error(ctx->ins, "Failed to construct CompleteMultipartUpload " - "request body"); - return -1; -} - -int complete_multipart_upload(struct flb_s3 *ctx, - struct multipart_upload *m_upload) -{ - char *body; - size_t size; - flb_sds_t uri = NULL; - flb_sds_t tmp; - int ret; - struct flb_http_client *c = NULL; - struct flb_aws_client *s3_client; - - if (!m_upload->upload_id) { - flb_plg_error(ctx->ins, "Cannot complete multipart upload for key %s: " - "upload ID is unset ", m_upload->s3_key); - return -1; - } - - uri = flb_sds_create_size(flb_sds_len(m_upload->s3_key) + 11 + - flb_sds_len(m_upload->upload_id)); - if (!uri) { - flb_errno(); - return -1; - } - - tmp = flb_sds_printf(&uri, "/%s%s?uploadId=%s", ctx->bucket, - m_upload->s3_key, m_upload->upload_id); - if (!tmp) { - flb_sds_destroy(uri); - return -1; - } - uri = tmp; - - ret = complete_multipart_upload_payload(ctx, m_upload, &body, &size); - if (ret < 0) { - flb_sds_destroy(uri); - return -1; - } - - s3_client = ctx->s3_client; - if (s3_plugin_under_test() == FLB_TRUE) { - c = mock_s3_call("TEST_COMPLETE_MULTIPART_UPLOAD_ERROR", "CompleteMultipartUpload"); - } - else { - c = s3_client->client_vtable->request(s3_client, FLB_HTTP_POST, - uri, body, size, - NULL, 0); - } - flb_sds_destroy(uri); - flb_free(body); - if (c) { - flb_plg_debug(ctx->ins, "CompleteMultipartUpload http status=%d", - c->resp.status); - if (c->resp.status == 200) { - flb_plg_info(ctx->ins, "Successfully completed multipart upload " - "for %s, UploadId=%s", m_upload->s3_key, - m_upload->upload_id); - flb_http_client_destroy(c); - /* remove this upload from the file system */ - remove_upload_from_fs(ctx, m_upload); - return 0; - } - flb_aws_print_xml_error(c->resp.payload, c->resp.payload_size, - "CompleteMultipartUpload", ctx->ins); - if (c->resp.payload != NULL) { - flb_plg_debug(ctx->ins, "Raw CompleteMultipartUpload response: %s", - c->resp.payload); - } - flb_http_client_destroy(c); - } - - flb_plg_error(ctx->ins, "CompleteMultipartUpload request failed"); - return -1; -} - - -int create_multipart_upload(struct flb_s3 *ctx, - struct multipart_upload *m_upload) -{ - flb_sds_t uri = NULL; - flb_sds_t tmp; - struct flb_http_client *c = NULL; - struct flb_aws_client *s3_client; - struct flb_aws_header *headers = NULL; - int num_headers = 0; - int ret; - - uri = flb_sds_create_size(flb_sds_len(m_upload->s3_key) + 8); - if (!uri) { - flb_errno(); - return -1; - } - - tmp = flb_sds_printf(&uri, "/%s%s?uploads=", ctx->bucket, m_upload->s3_key); - if (!tmp) { - flb_sds_destroy(uri); - return -1; - } - uri = tmp; - - s3_client = ctx->s3_client; - if (s3_plugin_under_test() == FLB_TRUE) { - c = mock_s3_call("TEST_CREATE_MULTIPART_UPLOAD_ERROR", "CreateMultipartUpload"); - } - else { - ret = create_headers(ctx, NULL, &headers, &num_headers, FLB_TRUE); - if (ret == -1) { - flb_plg_error(ctx->ins, "Failed to create headers"); - flb_sds_destroy(uri); - return -1; - } - c = s3_client->client_vtable->request(s3_client, FLB_HTTP_POST, - uri, NULL, 0, headers, num_headers); - if (headers) { - flb_free(headers); - } - } - flb_sds_destroy(uri); - if (c) { - flb_plg_debug(ctx->ins, "CreateMultipartUpload http status=%d", - c->resp.status); - if (c->resp.status == 200) { - tmp = flb_aws_xml_get_val(c->resp.payload, c->resp.payload_size, - "", ""); - if (!tmp) { - flb_plg_error(ctx->ins, "Could not find upload ID in " - "CreateMultipartUpload response"); - flb_plg_debug(ctx->ins, "Raw CreateMultipartUpload response: %s", - c->resp.payload); - flb_http_client_destroy(c); - return -1; - } - m_upload->upload_id = tmp; - flb_plg_info(ctx->ins, "Successfully initiated multipart upload " - "for %s, UploadId=%s", m_upload->s3_key, - m_upload->upload_id); - flb_http_client_destroy(c); - return 0; - } - flb_aws_print_xml_error(c->resp.payload, c->resp.payload_size, - "CreateMultipartUpload", ctx->ins); - if (c->resp.payload != NULL) { - flb_plg_debug(ctx->ins, "Raw CreateMultipartUpload response: %s", - c->resp.payload); - } - flb_http_client_destroy(c); - } - - flb_plg_error(ctx->ins, "CreateMultipartUpload request failed"); - return -1; -} - -/* gets the ETag value from response headers */ -flb_sds_t get_etag(char *response, size_t size) -{ - char *tmp; - int start; - int end; - int len; - int i = 0; - flb_sds_t etag; - - if (response == NULL) { - return NULL; - } - - tmp = strstr(response, "ETag:"); - if (!tmp) { - return NULL; - } - i = tmp - response; - - /* advance to end of ETag key */ - i += 5; - - /* advance across any whitespace and the opening quote */ - while (i < size && (response[i] == '\"' || isspace(response[i]) != 0)) { - i++; - } - start = i; - /* advance until we hit whitespace or the end quote */ - while (i < size && (response[i] != '\"' && isspace(response[i]) == 0)) { - i++; - } - end = i; - len = end - start; - - etag = flb_sds_create_len(response + start, len); - if (!etag) { - flb_errno(); - return NULL; - } - - return etag; -} - -int upload_part(struct flb_s3 *ctx, struct multipart_upload *m_upload, - char *body, size_t body_size) -{ - flb_sds_t uri = NULL; - flb_sds_t tmp; - int ret; - struct flb_http_client *c = NULL; - struct flb_aws_client *s3_client; - struct flb_aws_header *headers = NULL; - int num_headers = 0; - char body_md5[25]; - - uri = flb_sds_create_size(flb_sds_len(m_upload->s3_key) + 8); - if (!uri) { - flb_errno(); - return -1; - } - - tmp = flb_sds_printf(&uri, "/%s%s?partNumber=%d&uploadId=%s", - ctx->bucket, m_upload->s3_key, m_upload->part_number, - m_upload->upload_id); - if (!tmp) { - flb_errno(); - flb_sds_destroy(uri); - return -1; - } - uri = tmp; - - memset(body_md5, 0, sizeof(body_md5)); - if (ctx->send_content_md5 == FLB_TRUE) { - ret = get_md5_base64(body, body_size, body_md5, sizeof(body_md5)); - if (ret != 0) { - flb_plg_error(ctx->ins, "Failed to create Content-MD5 header"); - flb_sds_destroy(uri); - return -1; - } - - num_headers = 1; - headers = flb_malloc(sizeof(struct flb_aws_header) * num_headers); - if (headers == NULL) { - flb_errno(); - flb_sds_destroy(uri); - return -1; - } - - headers[0].key = "Content-MD5"; - headers[0].key_len = 11; - headers[0].val = body_md5; - headers[0].val_len = strlen(body_md5); - } - - s3_client = ctx->s3_client; - if (s3_plugin_under_test() == FLB_TRUE) { - c = mock_s3_call("TEST_UPLOAD_PART_ERROR", "UploadPart"); - } - else { - c = s3_client->client_vtable->request(s3_client, FLB_HTTP_PUT, - uri, body, body_size, - headers, num_headers); - } - flb_free(headers); - flb_sds_destroy(uri); - if (c) { - flb_plg_info(ctx->ins, "UploadPart http status=%d", - c->resp.status); - if (c->resp.status == 200) { - tmp = get_etag(c->resp.data, c->resp.data_size); - if (!tmp) { - flb_plg_error(ctx->ins, "Could not find ETag in " - "UploadPart response"); - flb_plg_debug(ctx->ins, "Raw UploadPart response: %s", - c->resp.payload); - flb_http_client_destroy(c); - return -1; - } - m_upload->etags[m_upload->part_number - 1] = tmp; - flb_plg_info(ctx->ins, "Successfully uploaded part #%d " - "for %s, UploadId=%s, ETag=%s", m_upload->part_number, - m_upload->s3_key, m_upload->upload_id, tmp); - flb_http_client_destroy(c); - /* track how many bytes are have gone toward this upload */ - m_upload->bytes += body_size; - - /* finally, attempt to persist the data for this upload */ - ret = save_upload(ctx, m_upload, tmp); - if (ret == 0) { - flb_plg_debug(ctx->ins, "Successfully persisted upload data, UploadId=%s", - m_upload->upload_id); - } - else { - flb_plg_warn(ctx->ins, "Was not able to persisted upload data to disk; " - "if fluent bit dies without completing this upload the part " - "could be lost, UploadId=%s, ETag=%s", - m_upload->upload_id, tmp); - } - return 0; - } - flb_aws_print_xml_error(c->resp.payload, c->resp.payload_size, - "UploadPart", ctx->ins); - if (c->resp.payload != NULL) { - flb_plg_debug(ctx->ins, "Raw UploadPart response: %s", - c->resp.payload); - } - flb_http_client_destroy(c); - } - - flb_plg_error(ctx->ins, "UploadPart request failed"); - return -1; -} diff --git a/fluent-bit/plugins/out_s3/s3_store.c b/fluent-bit/plugins/out_s3/s3_store.c deleted file mode 100644 index 8a9640633..000000000 --- a/fluent-bit/plugins/out_s3/s3_store.c +++ /dev/null @@ -1,543 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include "s3.h" -#include "s3_store.h" - -static int s3_store_under_travis_ci() -{ - - if (getenv("CI") != NULL && getenv("TRAVIS") != NULL) { - return FLB_TRUE; - } - - return FLB_FALSE; -} - -/* - * Simple and fast hashing algorithm to create keys in the local buffer - */ -static flb_sds_t gen_store_filename(const char *tag) -{ - int c; - unsigned long hash = 5381; - unsigned long hash2 = 5381; - flb_sds_t hash_str; - flb_sds_t tmp; - struct flb_time tm; - - /* get current time */ - flb_time_get(&tm); - - /* compose hash */ - while ((c = *tag++)) { - hash = ((hash << 5) + hash) + c; /* hash * 33 + c */ - } - hash2 = (unsigned long) hash2 * tm.tm.tv_sec * tm.tm.tv_nsec; - - /* flb_sds_printf allocs if the incoming sds is not at least 64 bytes */ - hash_str = flb_sds_create_size(64); - if (!hash_str) { - flb_errno(); - return NULL; - } - tmp = flb_sds_printf(&hash_str, "%lu-%lu", hash, hash2); - if (!tmp) { - flb_errno(); - flb_sds_destroy(hash_str); - return NULL; - } - hash_str = tmp; - - return hash_str; -} - -/* Retrieve a candidate s3 local file using the tag */ -struct s3_file *s3_store_file_get(struct flb_s3 *ctx, const char *tag, - int tag_len) -{ - struct mk_list *head; - struct mk_list *tmp; - struct flb_fstore_file *fsf = NULL; - struct s3_file *s3_file; - - /* - * Based in the current ctx->stream_name, locate a candidate file to - * store the incoming data using as a lookup pattern the content Tag. - */ - mk_list_foreach_safe(head, tmp, &ctx->stream_active->files) { - fsf = mk_list_entry(head, struct flb_fstore_file, _head); - - /* skip and warn on partially initialized chunks */ - if (fsf->data == NULL) { - flb_plg_warn(ctx->ins, "BAD: found flb_fstore_file with NULL data reference, tag=%s, file=%s, will try to delete", tag, fsf->name); - flb_fstore_file_delete(ctx->fs, fsf); - } - - if (fsf->meta_size != tag_len) { - fsf = NULL; - continue; - } - - /* skip locked chunks */ - s3_file = fsf->data; - if (s3_file->locked == FLB_TRUE) { - fsf = NULL; - continue; - } - - /* compare meta and tag */ - if (strncmp((char *) fsf->meta_buf, tag, tag_len) == 0) { - break; - } - - /* not found, invalidate the reference */ - fsf = NULL; - } - - if (!fsf) { - return NULL; - } - - return fsf->data; -} - -/* Append data to a new or existing fstore file */ -int s3_store_buffer_put(struct flb_s3 *ctx, struct s3_file *s3_file, - const char *tag, int tag_len, - char *data, size_t bytes, - time_t file_first_log_time) -{ - int ret; - flb_sds_t name; - struct flb_fstore_file *fsf; - size_t space_remaining; - - if (ctx->store_dir_limit_size > 0 && ctx->current_buffer_size + bytes >= ctx->store_dir_limit_size) { - flb_plg_error(ctx->ins, "Buffer is full: current_buffer_size=%zu, new_data=%zu, store_dir_limit_size=%zu bytes", - ctx->current_buffer_size, bytes, ctx->store_dir_limit_size); - return -1; - } - - /* If no target file was found, create a new one */ - if (!s3_file) { - name = gen_store_filename(tag); - if (!name) { - flb_plg_error(ctx->ins, "could not generate chunk file name"); - return -1; - } - - /* Create the file */ - fsf = flb_fstore_file_create(ctx->fs, ctx->stream_active, name, bytes); - if (!fsf) { - flb_plg_error(ctx->ins, "could not create the file '%s' in the store", - name); - flb_sds_destroy(name); - return -1; - } - flb_sds_destroy(name); - - /* Write tag as metadata */ - ret = flb_fstore_file_meta_set(ctx->fs, fsf, (char *) tag, tag_len); - if (ret == -1) { - flb_plg_error(ctx->ins, "error writing tag metadata"); - flb_plg_warn(ctx->ins, "Deleting buffer file because metadata could not be written"); - flb_fstore_file_delete(ctx->fs, fsf); - return -1; - } - - /* Allocate local context */ - s3_file = flb_calloc(1, sizeof(struct s3_file)); - if (!s3_file) { - flb_errno(); - flb_plg_error(ctx->ins, "cannot allocate s3 file context"); - flb_plg_warn(ctx->ins, "Deleting buffer file because S3 context creation failed"); - flb_fstore_file_delete(ctx->fs, fsf); - return -1; - } - s3_file->fsf = fsf; - s3_file->first_log_time = file_first_log_time; - s3_file->create_time = time(NULL); - - /* Use fstore opaque 'data' reference to keep our context */ - fsf->data = s3_file; - } - else { - fsf = s3_file->fsf; - } - - /* Append data to the target file */ - ret = flb_fstore_file_append(fsf, data, bytes); - if (ret != 0) { - flb_plg_error(ctx->ins, "error writing data to local s3 file"); - return -1; - } - s3_file->size += bytes; - ctx->current_buffer_size += bytes; - - /* if buffer is 95% full, warn user */ - if (ctx->store_dir_limit_size > 0) { - space_remaining = ctx->store_dir_limit_size - ctx->current_buffer_size; - if ((space_remaining * 20) < ctx->store_dir_limit_size) { - flb_plg_warn(ctx->ins, "Buffer is almost full: current_buffer_size=%zu, store_dir_limit_size=%zu bytes", - ctx->current_buffer_size, ctx->store_dir_limit_size); - return -1; - } - } - - return 0; -} - -static int set_files_context(struct flb_s3 *ctx) -{ - struct mk_list *head; - struct mk_list *f_head; - struct flb_fstore_stream *fs_stream; - struct flb_fstore_file *fsf; - struct s3_file *s3_file; - - mk_list_foreach(head, &ctx->fs->streams) { - fs_stream = mk_list_entry(head, struct flb_fstore_stream, _head); - - /* skip current stream since it's new */ - if (fs_stream == ctx->stream_active) { - continue; - } - - /* skip multi-upload */ - if (fs_stream == ctx->stream_upload) { - continue; - } - - mk_list_foreach(f_head, &fs_stream->files) { - fsf = mk_list_entry(f_head, struct flb_fstore_file, _head); - if (fsf->data) { - continue; - } - - /* Allocate local context */ - s3_file = flb_calloc(1, sizeof(struct s3_file)); - if (!s3_file) { - flb_errno(); - flb_plg_error(ctx->ins, "cannot allocate s3 file context"); - continue; - } - s3_file->fsf = fsf; - s3_file->first_log_time = time(NULL); - s3_file->create_time = time(NULL); - - /* Use fstore opaque 'data' reference to keep our context */ - fsf->data = s3_file; - } - } - - return 0; -} - -/* Initialize filesystem storage for S3 plugin */ -int s3_store_init(struct flb_s3 *ctx) -{ - int type; - time_t now; - char tmp[64]; - struct tm *tm; - struct flb_fstore *fs; - struct flb_fstore_stream *fs_stream; - - if (s3_store_under_travis_ci() == FLB_TRUE) { - type = FLB_FSTORE_MEM; - flb_plg_warn(ctx->ins, "Travis CI test, using s3 store memory backend"); - } - else { - type = FLB_FSTORE_FS; - } - - /* Initialize the storage context */ - fs = flb_fstore_create(ctx->buffer_dir, type); - if (!fs) { - return -1; - } - ctx->fs = fs; - - /* - * On every start we create a new stream, this stream in the file system - * is directory with the name using the date like '2020-10-03T13:00:02'. So - * all the 'new' data that is generated on this process is stored there. - * - * Note that previous data in similar directories from previous runs is - * considered backlog data, in the S3 plugin we need to differenciate the - * new v/s the older buffered data. - * - * Compose a stream name... - */ - now = time(NULL); - tm = localtime(&now); - -#ifdef FLB_SYSTEM_WINDOWS - /* Windows does not allow ':' in directory names */ - strftime(tmp, sizeof(tmp) - 1, "%Y-%m-%dT%H-%M-%S", tm); -#else - strftime(tmp, sizeof(tmp) - 1, "%Y-%m-%dT%H:%M:%S", tm); -#endif - - /* Create the stream */ - fs_stream = flb_fstore_stream_create(ctx->fs, tmp); - if (!fs_stream) { - /* Upon exception abort */ - flb_plg_error(ctx->ins, "could not initialize active stream: %s", tmp); - flb_fstore_destroy(fs); - ctx->fs = NULL; - return -1; - } - ctx->stream_active = fs_stream; - - /* Multipart upload stream */ - fs_stream = flb_fstore_stream_create(ctx->fs, "multipart_upload_metadata"); - if (!fs_stream) { - flb_plg_error(ctx->ins, "could not initialize multipart_upload stream"); - flb_fstore_destroy(fs); - ctx->fs = NULL; - return -1; - } - ctx->stream_upload = fs_stream; - - set_files_context(ctx); - return 0; -} - -int s3_store_exit(struct flb_s3 *ctx) -{ - struct mk_list *head; - struct mk_list *f_head; - struct flb_fstore_stream *fs_stream; - struct flb_fstore_file *fsf; - struct s3_file *s3_file; - - if (!ctx->fs) { - return 0; - } - - /* release local context on non-multi upload files */ - mk_list_foreach(head, &ctx->fs->streams) { - fs_stream = mk_list_entry(head, struct flb_fstore_stream, _head); - if (fs_stream == ctx->stream_upload) { - continue; - } - - mk_list_foreach(f_head, &fs_stream->files) { - fsf = mk_list_entry(f_head, struct flb_fstore_file, _head); - if (fsf->data != NULL) { - s3_file = fsf->data; - flb_sds_destroy(s3_file->file_path); - flb_free(s3_file); - } - } - } - - if (ctx->fs) { - flb_fstore_destroy(ctx->fs); - } - return 0; -} - -/* - * Check if the store has data. This function is only used on plugin - * initialization - */ -int s3_store_has_data(struct flb_s3 *ctx) -{ - struct mk_list *head; - struct flb_fstore_stream *fs_stream; - - if (!ctx->fs) { - return FLB_FALSE; - } - - mk_list_foreach(head, &ctx->fs->streams) { - /* skip multi upload stream */ - fs_stream = mk_list_entry(head, struct flb_fstore_stream, _head); - if (fs_stream == ctx->stream_upload) { - continue; - } - - if (mk_list_size(&fs_stream->files) > 0) { - return FLB_TRUE; - } - } - - return FLB_FALSE; -} - -int s3_store_has_uploads(struct flb_s3 *ctx) -{ - if (!ctx || !ctx->stream_upload) { - return FLB_FALSE; - } - - if (mk_list_size(&ctx->stream_upload->files) > 0) { - return FLB_TRUE; - } - - return FLB_FALSE; -} - -int s3_store_file_inactive(struct flb_s3 *ctx, struct s3_file *s3_file) -{ - int ret; - struct flb_fstore_file *fsf; - - fsf = s3_file->fsf; - flb_free(s3_file); - ret = flb_fstore_file_inactive(ctx->fs, fsf); - - return ret; -} - -int s3_store_file_delete(struct flb_s3 *ctx, struct s3_file *s3_file) -{ - struct flb_fstore_file *fsf; - - fsf = s3_file->fsf; - ctx->current_buffer_size -= s3_file->size; - - /* permanent deletion */ - flb_fstore_file_delete(ctx->fs, fsf); - flb_free(s3_file); - - return 0; -} - -int s3_store_file_read(struct flb_s3 *ctx, struct s3_file *s3_file, - char **out_buf, size_t *out_size) -{ - int ret; - - ret = flb_fstore_file_content_copy(ctx->fs, s3_file->fsf, - (void **) out_buf, out_size); - return ret; -} - -int s3_store_file_upload_read(struct flb_s3 *ctx, struct flb_fstore_file *fsf, - char **out_buf, size_t *out_size) -{ - int ret; - - ret = flb_fstore_file_content_copy(ctx->fs, fsf, - (void **) out_buf, out_size); - return ret; -} - -struct flb_fstore_file *s3_store_file_upload_get(struct flb_s3 *ctx, - char *key, int key_len) -{ - struct mk_list *head; - struct flb_fstore_file *fsf = NULL; - - mk_list_foreach(head, &ctx->stream_upload->files) { - fsf = mk_list_entry(head, struct flb_fstore_file, _head); - if (fsf->meta_buf == NULL) { - continue; - } - - if (fsf->meta_size != key_len ){ - continue; - } - - if (strncmp(fsf->meta_buf, key, key_len) == 0) { - break; - } - fsf = NULL; - } - - return fsf; -} - -/* param fsf can NULL if the file has not yet been created */ -int s3_store_file_upload_put(struct flb_s3 *ctx, - struct flb_fstore_file *fsf, flb_sds_t key, - flb_sds_t data) -{ - int ret; - flb_sds_t name; - - /* If no target file was found, create a new one */ - if (!fsf) { - name = gen_store_filename(key); - if (!name) { - flb_plg_error(ctx->ins, "could not generate chunk file name"); - return -1; - } - - /* Create the file */ - fsf = flb_fstore_file_create(ctx->fs, ctx->stream_upload, name, flb_sds_len(data)); - if (!fsf) { - flb_plg_error(ctx->ins, "could not create the file '%s' in the upload store", - name); - flb_sds_destroy(name); - return -1; - } - flb_sds_destroy(name); - - /* Write key as metadata */ - ret = flb_fstore_file_meta_set(ctx->fs, fsf, - key, flb_sds_len(key)); - if (ret == -1) { - flb_plg_error(ctx->ins, "error writing upload metadata"); - flb_plg_warn(ctx->ins, "Deleting s3 upload cache file because metadata could not be written"); - flb_fstore_file_delete(ctx->fs, fsf); - return -1; - } - } - - /* Append data to the target file */ - ret = flb_fstore_file_append(fsf, data, flb_sds_len(data)); - if (ret != 0) { - flb_plg_error(ctx->ins, "error writing data to local s3 file"); - return -1; - } - - return 0; -} - -int s3_store_file_upload_delete(struct flb_s3 *ctx, struct flb_fstore_file *fsf) -{ - /* permanent deletion */ - flb_fstore_file_delete(ctx->fs, fsf); - return 0; -} - -/* Always set an updated copy of metadata into the fs_store_file entry */ -int s3_store_file_meta_get(struct flb_s3 *ctx, struct flb_fstore_file *fsf) -{ - return flb_fstore_file_meta_get(ctx->fs, fsf); -} - -void s3_store_file_lock(struct s3_file *s3_file) -{ - s3_file->locked = FLB_TRUE; -} - -void s3_store_file_unlock(struct s3_file *s3_file) -{ - s3_file->locked = FLB_FALSE; -} diff --git a/fluent-bit/plugins/out_s3/s3_store.h b/fluent-bit/plugins/out_s3/s3_store.h deleted file mode 100644 index 9caa7bdf4..000000000 --- a/fluent-bit/plugins/out_s3/s3_store.h +++ /dev/null @@ -1,68 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_S3_STORE_H -#define FLB_S3_STORE_H - -#include -#include - -struct s3_file { - int locked; /* locked chunk is busy, cannot write to it */ - int failures; /* delivery failures */ - size_t size; /* file size */ - time_t create_time; /* creation time */ - time_t first_log_time; /* first log time */ - flb_sds_t file_path; /* file path */ - struct flb_fstore_file *fsf; /* reference to parent flb_fstore_file */ -}; - -int s3_store_buffer_put(struct flb_s3 *ctx, struct s3_file *s3_file, - const char *tag, int tag_len, - char *data, size_t bytes, - time_t file_first_log_time); - -int s3_store_init(struct flb_s3 *ctx); -int s3_store_exit(struct flb_s3 *ctx); - -int s3_store_has_data(struct flb_s3 *ctx); -int s3_store_has_uploads(struct flb_s3 *ctx); - -int s3_store_file_inactive(struct flb_s3 *ctx, struct s3_file *s3_file); -struct s3_file *s3_store_file_get(struct flb_s3 *ctx, const char *tag, - int tag_len); -int s3_store_file_delete(struct flb_s3 *ctx, struct s3_file *s3_file); -int s3_store_file_read(struct flb_s3 *ctx, struct s3_file *s3_file, - char **out_buf, size_t *out_size); -int s3_store_file_upload_read(struct flb_s3 *ctx, struct flb_fstore_file *fsf, - char **out_buf, size_t *out_size); -struct flb_fstore_file *s3_store_file_upload_get(struct flb_s3 *ctx, - char *key, int key_len); - -int s3_store_file_upload_put(struct flb_s3 *ctx, - struct flb_fstore_file *fsf, flb_sds_t key, - flb_sds_t data); -int s3_store_file_upload_delete(struct flb_s3 *ctx, struct flb_fstore_file *fsf); - -int s3_store_file_meta_get(struct flb_s3 *ctx, struct flb_fstore_file *fsf); - -void s3_store_file_lock(struct s3_file *s3_file); -void s3_store_file_unlock(struct s3_file *s3_file); - -#endif diff --git a/fluent-bit/plugins/out_skywalking/CMakeLists.txt b/fluent-bit/plugins/out_skywalking/CMakeLists.txt deleted file mode 100644 index ad5a5845e..000000000 --- a/fluent-bit/plugins/out_skywalking/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - skywalking.c) - -FLB_PLUGIN(out_skywalking "${src}" "") diff --git a/fluent-bit/plugins/out_skywalking/skywalking.c b/fluent-bit/plugins/out_skywalking/skywalking.c deleted file mode 100644 index c5a9a1e2d..000000000 --- a/fluent-bit/plugins/out_skywalking/skywalking.c +++ /dev/null @@ -1,427 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "skywalking.h" - -#define DEFAULT_SW_OAP_HOST "127.0.0.1" -#define DEFAULT_SW_OAP_PORT 12800 -#define DEFAULT_SW_SVC_NAME "sw-service" -#define DEFAULT_SW_INS_NAME "fluent-bit" -#define DEFAULT_SW_LOG_PATH "/v3/logs" - -static void sw_output_ctx_destroy(struct flb_output_sw* ctx) { - if (!ctx) { - return; - } - - if (ctx->u) { - flb_upstream_destroy(ctx->u); - } - - flb_sds_destroy(ctx->http_scheme); - flb_sds_destroy(ctx->uri); - flb_free(ctx); -} - -static int cb_sw_init(struct flb_output_instance *ins, - struct flb_config *config, void *data) -{ - int ret; - int io_flags; - struct flb_output_sw *ctx; - - /* Allocate plugin context */ - ctx = flb_calloc(1, sizeof(struct flb_output_sw)); - if (!ctx) { - flb_errno(); - return -1; - } - - ctx->ins = ins; - - ret = flb_output_config_map_set(ins, (void *)ctx); - if (ret == -1) { - sw_output_ctx_destroy(ctx); - return -1; - } - - flb_output_net_default(DEFAULT_SW_OAP_HOST, DEFAULT_SW_OAP_PORT, ctx->ins); - - ctx->uri = flb_sds_create(DEFAULT_SW_LOG_PATH); - if (!ctx->uri) { - flb_plg_error(ctx->ins, "failed to configure endpoint"); - sw_output_ctx_destroy(ctx); - return -1; - } - - if (!ctx->svc_name) { - flb_plg_error(ctx->ins, "failed to configure service name"); - sw_output_ctx_destroy(ctx); - return -1; - } - - if (!ctx->svc_inst_name) { - flb_plg_error(ctx->ins, "failed to configure instance name"); - sw_output_ctx_destroy(ctx); - return -1; - } - - flb_plg_debug(ctx->ins, "configured %s/%s", ctx->svc_name, ctx->svc_inst_name); - flb_plg_debug(ctx->ins, "OAP address is %s:%d", ins->host.name, ins->host.port); - - /* scheme configuration */ - if (ins->use_tls == FLB_TRUE) { - io_flags = FLB_IO_TLS; - ctx->http_scheme = flb_sds_create("https://"); - } - else { - io_flags = FLB_IO_TCP; - ctx->http_scheme = flb_sds_create("http://"); - } - - /* configure upstream instance */ - ctx->u = flb_upstream_create(config, ins->host.name, ins->host.port, io_flags, ins->tls); - if (!ctx->u) { - flb_plg_error(ctx->ins, "failed to create upstream context"); - sw_output_ctx_destroy(ctx); - return -1; - } - - flb_output_upstream_set(ctx->u, ins); - - /* Set the plugin context */ - flb_output_set_context(ins, ctx); - flb_output_set_http_debug_callbacks(ins); - - return 0; -} - -static int64_t timestamp_format(const struct flb_time* tms) -{ - int64_t timestamp = 0; - - /* Format the time, use milliseconds precision not nanoseconds */ - timestamp = tms->tm.tv_sec * 1000; - timestamp += tms->tm.tv_nsec / 1000000; - - /* round up if necessary */ - if (tms->tm.tv_nsec % 1000000 >= 500000) { - ++timestamp; - } - return timestamp; -} - -static void sw_msgpack_pack_kv_str(msgpack_packer* pk, const char* key, - size_t key_len, const char *value, - size_t value_len) -{ - msgpack_pack_str(pk, key_len); - msgpack_pack_str_body(pk, key, key_len); - msgpack_pack_str(pk, value_len); - msgpack_pack_str_body(pk, value, value_len); -} - -static void sw_msgpack_pack_kv_int64_t(msgpack_packer* pk, const char* key, - size_t key_len, int64_t value) -{ - msgpack_pack_str(pk, key_len); - msgpack_pack_str_body(pk, key, key_len); - msgpack_pack_int64(pk, value); -} - -static void sw_msgpack_pack_log_body(msgpack_packer* pk, - msgpack_object* obj, size_t obj_size) -{ - int i, j = 0; - int log_entry_num = 0; - msgpack_sbuffer sbuf; - msgpack_packer body_pk; - msgpack_object key; - msgpack_object value; - flb_sds_t out_body_str; - size_t out_body_str_len; - int* valid_log_entry = NULL; - - valid_log_entry = (int*)flb_malloc(obj_size * sizeof(int)); - if (!valid_log_entry) { - flb_errno(); - return; - } - - msgpack_sbuffer_init(&sbuf); - msgpack_packer_init(&body_pk, &sbuf, msgpack_sbuffer_write); - - for (i = 0; i < obj_size; ++i) { - key = obj->via.map.ptr[i].key; - value = obj->via.map.ptr[i].val; - - if (key.type != MSGPACK_OBJECT_STR || - value.type != MSGPACK_OBJECT_STR) { - continue; - } - - valid_log_entry[j] = i; - ++j; - ++log_entry_num; - } - - msgpack_pack_map(&body_pk, log_entry_num); - - for (i = 0; i < log_entry_num; ++i) { - key = obj->via.map.ptr[valid_log_entry[i]].key; - value = obj->via.map.ptr[valid_log_entry[i]].val; - sw_msgpack_pack_kv_str(&body_pk, key.via.str.ptr, key.via.str.size, - value.via.str.ptr, value.via.str.size); - } - - out_body_str = flb_msgpack_raw_to_json_sds(sbuf.data, sbuf.size); - if (!out_body_str) { - msgpack_sbuffer_destroy(&sbuf); - flb_free(valid_log_entry); - return; - } - out_body_str_len = flb_sds_len(out_body_str); - - msgpack_pack_str(pk, 4); - msgpack_pack_str_body(pk, "body", 4); - msgpack_pack_map(pk, 1); - - /* body['json'] */ - msgpack_pack_str(pk, 4); - msgpack_pack_str_body(pk, "json", 4); - msgpack_pack_map(pk, 1); - - /* body['json']['json'] */ - msgpack_pack_str(pk, 4); - msgpack_pack_str_body(pk, "json", 4); - msgpack_pack_str(pk, out_body_str_len); - msgpack_pack_str_body(pk, out_body_str, out_body_str_len); - - flb_sds_destroy(out_body_str); - msgpack_sbuffer_destroy(&sbuf); - flb_free(valid_log_entry); -} - -static int sw_format(struct flb_output_sw* ctx, const void *data, size_t bytes, - void** buf, size_t* buf_len) -{ - int ret = 0; - int chunk_size = 0; - uint32_t map_size; - msgpack_sbuffer sbuf; - msgpack_packer pk; - msgpack_object map; - int64_t timestamp; - flb_sds_t out_str; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return -1; - } - - msgpack_sbuffer_init(&sbuf); - msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); - - chunk_size = flb_mp_count(data, bytes); - flb_plg_debug(ctx->ins, "%i messages flushed", chunk_size); - - msgpack_pack_array(&pk, chunk_size); - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - timestamp = timestamp_format(&log_event.timestamp); - - map = *log_event.body; - map_size = map.via.map.size; - - msgpack_pack_map(&pk, 4); - - sw_msgpack_pack_kv_int64_t(&pk, "timestamp", 9, timestamp); - sw_msgpack_pack_kv_str(&pk, "service", 7, ctx->svc_name, - flb_sds_len(ctx->svc_name)); - sw_msgpack_pack_kv_str(&pk, "serviceInstance", 15, - ctx->svc_inst_name, flb_sds_len(ctx->svc_inst_name)); - sw_msgpack_pack_log_body(&pk, &map, map_size); - } - - out_str = flb_msgpack_raw_to_json_sds(sbuf.data, sbuf.size); - if (!out_str) { - ret = -1; - goto done; - } - else { - ret = 0; - } - - *buf = out_str; - *buf_len = flb_sds_len(out_str); - -done: - msgpack_sbuffer_destroy(&sbuf); - flb_log_event_decoder_destroy(&log_decoder); - - return ret; -} - -static int mock_oap_request(struct flb_http_client* client, int mock_status) -{ - client->resp.status = mock_status; - return 0; -} - -static bool check_sw_under_test() -{ - if (getenv("FLB_SW_PLUGIN_UNDER_TEST") != NULL) { - return FLB_TRUE; - } - return FLB_FALSE; -} - -static void cb_sw_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, struct flb_config *config) -{ - int flush_ret = -1; - int tmp_ret = -1; - struct flb_output_sw *ctx = out_context; - struct flb_connection *conn = NULL; - struct flb_http_client *client = NULL; - void* buf = NULL; - size_t buf_len; - size_t sent_size; - - tmp_ret = sw_format(ctx, - event_chunk->data, - event_chunk->size, - &buf, &buf_len); - if (tmp_ret != 0) { - flb_plg_error(ctx->ins, "failed to create buffer"); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - conn = flb_upstream_conn_get(ctx->u); - if (!conn) { - flb_plg_error(ctx->ins, "failed to establish connection to %s:%i", - ctx->ins->host.name, ctx->ins->host.port); - flb_sds_destroy(buf); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - client = flb_http_client(conn, FLB_HTTP_POST, ctx->uri, - (const char*)buf, buf_len, ctx->ins->host.name, ctx->ins->host.port, - NULL, 0); - if (!client) { - flb_plg_error(ctx->ins, "failed to create HTTP client"); - flb_sds_destroy(buf); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - if (ctx->auth_token && flb_sds_len(ctx->auth_token) != 0) { - flb_http_add_header(client, "Authentication", 14, - ctx->auth_token, strlen(ctx->auth_token)); - } - - flb_http_add_header(client, "Content-Type", 12, - "application/json", 16); - flb_http_add_header(client, "User-Agent", 10, - "Fluent-Bit", 10); - - if (check_sw_under_test() == FLB_TRUE) { - tmp_ret = mock_oap_request(client, 200); - } - else { - tmp_ret = flb_http_do(client, &sent_size); - } - - if (tmp_ret == 0) { - flb_plg_debug(ctx->ins, "%s:%i, HTTP status=%i", ctx->ins->host.name, - ctx->ins->host.port, client->resp.status); - - if (client->resp.status < 200 || client->resp.status > 205) { - flush_ret = FLB_RETRY; - } - else { - flush_ret = FLB_OK; - } - } - else { - flb_plg_error(ctx->ins, "failed to flush buffer to %s:%i", - ctx->ins->host.name, ctx->ins->host.port); - flush_ret = FLB_RETRY; - } - - flb_sds_destroy(buf); - flb_http_client_destroy(client); - flb_upstream_conn_release(conn); - - FLB_OUTPUT_RETURN(flush_ret); -} - -static int cb_sw_exit(void *data, struct flb_config *config) -{ - struct flb_output_sw *ctx; - - ctx = (struct flb_output_sw*)data; - sw_output_ctx_destroy(ctx); - - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "auth_token", NULL, - 0, FLB_TRUE, offsetof(struct flb_output_sw, auth_token), - "Auth token for SkyWalking OAP" - }, - { - FLB_CONFIG_MAP_STR, "svc_name", DEFAULT_SW_SVC_NAME, - 0, FLB_TRUE, offsetof(struct flb_output_sw, svc_name), - "Service name" - }, - { - FLB_CONFIG_MAP_STR, "svc_inst_name", DEFAULT_SW_INS_NAME, - 0, FLB_TRUE, offsetof(struct flb_output_sw, svc_inst_name), - "Instance name" - }, - {0} -}; - -struct flb_output_plugin out_skywalking_plugin = { - .name = "skywalking", - .description = "Send logs into log collector on SkyWalking OAP", - .cb_init = cb_sw_init, - .cb_flush = cb_sw_flush, - .cb_exit = cb_sw_exit, - .flags = FLB_OUTPUT_NET | FLB_IO_OPT_TLS, - .config_map = config_map -}; diff --git a/fluent-bit/plugins/out_skywalking/skywalking.h b/fluent-bit/plugins/out_skywalking/skywalking.h deleted file mode 100644 index 554ab6912..000000000 --- a/fluent-bit/plugins/out_skywalking/skywalking.h +++ /dev/null @@ -1,41 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_SKYWALKING_H -#define FLB_OUT_SKYWALKING_H - -#include - -struct flb_output_sw { - /* Configured by user */ - flb_sds_t auth_token; - flb_sds_t svc_name; - flb_sds_t svc_inst_name; - - /* Upstream log collector context */ - struct flb_upstream *u; - - /* Output plugin instance */ - struct flb_output_instance *ins; - - flb_sds_t http_scheme; - flb_sds_t uri; -}; - -#endif diff --git a/fluent-bit/plugins/out_slack/CMakeLists.txt b/fluent-bit/plugins/out_slack/CMakeLists.txt deleted file mode 100644 index b62a70472..000000000 --- a/fluent-bit/plugins/out_slack/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -set(src - slack.c - ) - -FLB_PLUGIN(out_slack "${src}" "") diff --git a/fluent-bit/plugins/out_slack/slack.c b/fluent-bit/plugins/out_slack/slack.c deleted file mode 100644 index f014228e9..000000000 --- a/fluent-bit/plugins/out_slack/slack.c +++ /dev/null @@ -1,336 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "slack.h" - -#define FLB_HTTP_CONTENT_TYPE "Content-Type" -#define FLB_HTTP_MIME_JSON "application/json" - -static int cb_slack_init(struct flb_output_instance *ins, - struct flb_config *config, void *data) -{ - int ret; - char *protocol = NULL; - char *host = NULL; - char *port = NULL; - char *uri = NULL; - struct flb_slack *ctx; - (void) config; - (void) data; - - /* Allocate context */ - ctx = flb_calloc(1, sizeof(struct flb_slack)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->ins = ins; - - /* Set the plugin context */ - flb_output_set_context(ins, ctx); - - /* Create config map and validate expected parameters */ - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return -1; - } - - /* Validate if the slack webhook is defined */ - if (!ctx->webhook) { - flb_plg_error(ctx->ins, "the 'webhook' address has not been defined"); - return -1; - } - - /* Split the address */ - ret = flb_utils_url_split(ctx->webhook, &protocol, &host, &port, &uri); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not process 'webhook' address"); - return -1; - } - - if (strcasecmp(protocol, "https") != 0) { - flb_plg_error(ctx->ins, "invalid protocol '%s', we expected 'https'", - protocol); - goto error; - } - - if (!host) { - flb_plg_error(ctx->ins, "invalid slack host"); - goto error; - } - - if (!uri) { - flb_plg_error(ctx->ins, "slack webhook uri has not been defined"); - goto error; - } - - ctx->host = flb_sds_create(host); - ctx->uri = flb_sds_create(uri); - - if (port) { - ctx->port = atoi(port); - } - else { - ctx->port = 443; - } - - /* Create upstream context */ - ctx->u = flb_upstream_create(config, - ctx->host, - ctx->port, - FLB_IO_TLS, ins->tls); - if (!ctx->u) { - flb_plg_error(ctx->ins, "error creating upstream context"); - goto error; - } - - /* Set instance flags into upstream */ - flb_output_upstream_set(ctx->u, ins); - - /* Cleanup */ - if (protocol) { - flb_free(protocol); - } - if (host) { - flb_free(host); - } - if (port) { - flb_free(port); - } - if (uri) { - flb_free(uri); - } - - return 0; - -error: - if (protocol) { - flb_free(protocol); - } - if (host) { - flb_free(host); - } - if (port) { - flb_free(port); - } - if (uri) { - flb_free(uri); - } - - return -1; -} - -static void cb_slack_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - int len; - int ret; - int out_ret = FLB_OK; - size_t size; - size_t printed = 0; - size_t b_sent; - flb_sds_t json; - flb_sds_t out_buf; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - struct flb_http_client *c; - struct flb_connection *u_conn; - struct flb_slack *ctx = out_context; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - size = event_chunk->size * 4; - json = flb_sds_create_size(size); - if (!json) { - FLB_OUTPUT_RETURN(FLB_RETRY); - } - memset(json, '\0', size); - - ret = flb_log_event_decoder_init(&log_decoder, - (char *) event_chunk->data, - event_chunk->size); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - flb_sds_destroy(json); - - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - - ret = snprintf(json + printed, size - printed, - "[\"timestamp\": %" PRIu32 ".%09lu, ", - (uint32_t) log_event.timestamp.tm.tv_sec, - log_event.timestamp.tm.tv_nsec); - printed += ret; - - ret = msgpack_object_print_buffer(json + printed, - size - printed, - *log_event.body); - if (ret < 0) { - flb_plg_error(ctx->ins, "error formatting payload"); - flb_sds_destroy(json); - flb_log_event_decoder_destroy(&log_decoder); - - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* the previous call returns the remaining available space in the buffer */ - printed += ret; - json[printed++] = ']'; - json[printed++] = '\n'; - } - - flb_log_event_decoder_destroy(&log_decoder); - - /* Take formatted message and convert it to msgpack */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - len = strlen(json); - - msgpack_pack_map(&mp_pck, 1); - msgpack_pack_str(&mp_pck, 4); - msgpack_pack_str_body(&mp_pck, "text", 4); - msgpack_pack_str(&mp_pck, len); - msgpack_pack_str_body(&mp_pck, json, len); - - /* Release buffer */ - flb_sds_destroy(json); - - /* Re-format mspgack as JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); - if (!out_buf) { - msgpack_sbuffer_destroy(&mp_sbuf); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - msgpack_sbuffer_destroy(&mp_sbuf); - - /* Create upstream connection */ - u_conn = flb_upstream_conn_get(ctx->u); - if (!u_conn) { - flb_sds_destroy(out_buf); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Create HTTP client context */ - c = flb_http_client(u_conn, FLB_HTTP_POST, ctx->uri, - out_buf, flb_sds_len(out_buf), - ctx->host, ctx->port, - NULL, 0); - flb_http_add_header(c, - FLB_HTTP_CONTENT_TYPE, - sizeof(FLB_HTTP_CONTENT_TYPE) - 1, - FLB_HTTP_MIME_JSON, - sizeof(FLB_HTTP_MIME_JSON) - 1); - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - - ret = flb_http_do(c, &b_sent); - if (ret == 0) { - if (c->resp.status < 200 || c->resp.status > 205) { - flb_plg_error(ctx->ins, "%s:%i, HTTP status=%i", - ctx->host, ctx->port, c->resp.status); - out_ret = FLB_RETRY; - } - else { - if (c->resp.payload) { - flb_plg_info(ctx->ins, "%s:%i, HTTP status=%i\n%s", - ctx->host, ctx->port, - c->resp.status, c->resp.payload); - } - else { - flb_plg_info(ctx->ins, "%s:%i, HTTP status=%i", - ctx->host, ctx->port, - c->resp.status); - } - } - } - else { - flb_plg_error(ctx->ins, "could not flush records to %s:%i (http_do=%i)", - ctx->host, ctx->port, ret); - out_ret = FLB_RETRY; - } - - flb_upstream_conn_release(u_conn); - flb_http_client_destroy(c); - flb_sds_destroy(out_buf); - FLB_OUTPUT_RETURN(out_ret); -} - -static int cb_slack_exit(void *data, struct flb_config *config) -{ - struct flb_slack *ctx; - - ctx = (struct flb_slack *) data; - if (!ctx) { - return 0; - } - - if (ctx->host) { - flb_sds_destroy(ctx->host); - } - if (ctx->uri) { - flb_sds_destroy(ctx->uri); - } - - if (ctx->u) { - flb_upstream_destroy(ctx->u); - } - - flb_free(ctx); - - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "webhook", NULL, - 0, FLB_TRUE, offsetof(struct flb_slack, webhook), - NULL - }, - - /* EOF */ - {0} -}; - -struct flb_output_plugin out_slack_plugin = { - .name = "slack", - .description = "Send events to a Slack channel", - .cb_init = cb_slack_init, - .cb_flush = cb_slack_flush, - .cb_exit = cb_slack_exit, - .flags = FLB_OUTPUT_NET | FLB_IO_TLS, - .config_map = config_map -}; diff --git a/fluent-bit/plugins/out_slack/slack.h b/fluent-bit/plugins/out_slack/slack.h deleted file mode 100644 index e7babc263..000000000 --- a/fluent-bit/plugins/out_slack/slack.h +++ /dev/null @@ -1,43 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_SLACK_H -#define FLB_OUT_SLACK_H - -#include -#include -#include - -struct flb_slack { - /* full webhook address */ - flb_sds_t webhook; - - /* processed webhook */ - flb_sds_t host; - int port; - flb_sds_t uri; - - /* upstream context */ - struct flb_upstream *u; - - /* Plugin instance */ - struct flb_output_instance *ins; -}; - -#endif diff --git a/fluent-bit/plugins/out_splunk/CMakeLists.txt b/fluent-bit/plugins/out_splunk/CMakeLists.txt deleted file mode 100644 index da66bca70..000000000 --- a/fluent-bit/plugins/out_splunk/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(src - splunk_conf.c - splunk.c - ) - -FLB_PLUGIN(out_splunk "${src}" "") diff --git a/fluent-bit/plugins/out_splunk/splunk.c b/fluent-bit/plugins/out_splunk/splunk.c deleted file mode 100644 index d9c28380a..000000000 --- a/fluent-bit/plugins/out_splunk/splunk.c +++ /dev/null @@ -1,873 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include "splunk.h" -#include "splunk_conf.h" - -static int cb_splunk_init(struct flb_output_instance *ins, - struct flb_config *config, void *data) -{ - struct flb_splunk *ctx; - - ctx = flb_splunk_conf_create(ins, config); - if (!ctx) { - flb_plg_error(ins, "configuration failed"); - return -1; - } - - flb_output_set_context(ins, ctx); - - /* - * This plugin instance uses the HTTP client interface, let's register - * it debugging callbacks. - */ - flb_output_set_http_debug_callbacks(ins); - return 0; -} - -static int pack_map_meta(struct flb_splunk *ctx, - struct flb_mp_map_header *mh, - msgpack_packer *mp_pck, - msgpack_object map, - char *tag, int tag_len) -{ - int index_key_set = FLB_FALSE; - int sourcetype_key_set = FLB_FALSE; - flb_sds_t str; - struct mk_list *head; - struct flb_splunk_field *f; - struct flb_mp_map_header mh_fields; - struct flb_ra_value *rval; - - /* event host */ - if (ctx->event_host) { - str = flb_ra_translate(ctx->ra_event_host, tag, tag_len, - map, NULL); - if (str) { - if (flb_sds_len(str) > 0) { - flb_mp_map_header_append(mh); - msgpack_pack_str(mp_pck, sizeof(FLB_SPLUNK_DEFAULT_EVENT_HOST) -1); - msgpack_pack_str_body(mp_pck, - FLB_SPLUNK_DEFAULT_EVENT_HOST, - sizeof(FLB_SPLUNK_DEFAULT_EVENT_HOST) - 1); - msgpack_pack_str(mp_pck, flb_sds_len(str)); - msgpack_pack_str_body(mp_pck, str, flb_sds_len(str)); - } - flb_sds_destroy(str); - } - } - - /* event source */ - if (ctx->event_source) { - str = flb_ra_translate(ctx->ra_event_source, tag, tag_len, - map, NULL); - if (str) { - if (flb_sds_len(str) > 0) { - flb_mp_map_header_append(mh); - msgpack_pack_str(mp_pck, sizeof(FLB_SPLUNK_DEFAULT_EVENT_SOURCE) -1); - msgpack_pack_str_body(mp_pck, - FLB_SPLUNK_DEFAULT_EVENT_SOURCE, - sizeof(FLB_SPLUNK_DEFAULT_EVENT_SOURCE) - 1); - msgpack_pack_str(mp_pck, flb_sds_len(str)); - msgpack_pack_str_body(mp_pck, str, flb_sds_len(str)); - } - flb_sds_destroy(str); - } - } - - /* event sourcetype (key lookup) */ - if (ctx->event_sourcetype_key) { - str = flb_ra_translate(ctx->ra_event_sourcetype_key, tag, tag_len, - map, NULL); - if (str) { - /* sourcetype_key was found */ - if (flb_sds_len(str) > 0) { - flb_mp_map_header_append(mh); - msgpack_pack_str(mp_pck, sizeof(FLB_SPLUNK_DEFAULT_EVENT_SOURCET) -1); - msgpack_pack_str_body(mp_pck, - FLB_SPLUNK_DEFAULT_EVENT_SOURCET, - sizeof(FLB_SPLUNK_DEFAULT_EVENT_SOURCET) - 1); - msgpack_pack_str(mp_pck, flb_sds_len(str)); - msgpack_pack_str_body(mp_pck, str, flb_sds_len(str)); - sourcetype_key_set = FLB_TRUE; - } - flb_sds_destroy(str); - } - /* If not found, it will fallback to the value set in event_sourcetype */ - } - - if (sourcetype_key_set == FLB_FALSE && ctx->event_sourcetype) { - flb_mp_map_header_append(mh); - msgpack_pack_str(mp_pck, sizeof(FLB_SPLUNK_DEFAULT_EVENT_SOURCET) -1); - msgpack_pack_str_body(mp_pck, - FLB_SPLUNK_DEFAULT_EVENT_SOURCET, - sizeof(FLB_SPLUNK_DEFAULT_EVENT_SOURCET) - 1); - msgpack_pack_str(mp_pck, flb_sds_len(ctx->event_sourcetype)); - msgpack_pack_str_body(mp_pck, - ctx->event_sourcetype, flb_sds_len(ctx->event_sourcetype)); - } - - /* event index (key lookup) */ - if (ctx->event_index_key) { - str = flb_ra_translate(ctx->ra_event_index_key, tag, tag_len, - map, NULL); - if (str) { - /* sourcetype_key was found */ - if (flb_sds_len(str) > 0) { - flb_mp_map_header_append(mh); - msgpack_pack_str(mp_pck, sizeof(FLB_SPLUNK_DEFAULT_EVENT_INDEX) -1); - msgpack_pack_str_body(mp_pck, - FLB_SPLUNK_DEFAULT_EVENT_INDEX, - sizeof(FLB_SPLUNK_DEFAULT_EVENT_INDEX) - 1); - msgpack_pack_str(mp_pck, flb_sds_len(str)); - msgpack_pack_str_body(mp_pck, str, flb_sds_len(str)); - index_key_set = FLB_TRUE; - } - flb_sds_destroy(str); - } - /* If not found, it will fallback to the value set in event_index */ - } - - if (index_key_set == FLB_FALSE && ctx->event_index) { - flb_mp_map_header_append(mh); - msgpack_pack_str(mp_pck, sizeof(FLB_SPLUNK_DEFAULT_EVENT_INDEX) -1); - msgpack_pack_str_body(mp_pck, - FLB_SPLUNK_DEFAULT_EVENT_INDEX, - sizeof(FLB_SPLUNK_DEFAULT_EVENT_INDEX) - 1); - msgpack_pack_str(mp_pck, flb_sds_len(ctx->event_index)); - msgpack_pack_str_body(mp_pck, - ctx->event_index, flb_sds_len(ctx->event_index)); - } - - /* event 'fields' */ - if (mk_list_size(&ctx->fields) > 0) { - flb_mp_map_header_append(mh); - msgpack_pack_str(mp_pck, sizeof(FLB_SPLUNK_DEFAULT_EVENT_FIELDS) -1); - msgpack_pack_str_body(mp_pck, - FLB_SPLUNK_DEFAULT_EVENT_FIELDS, - sizeof(FLB_SPLUNK_DEFAULT_EVENT_FIELDS) - 1); - - /* Pack map */ - flb_mp_map_header_init(&mh_fields, mp_pck); - - mk_list_foreach(head, &ctx->fields) { - f = mk_list_entry(head, struct flb_splunk_field, _head); - rval = flb_ra_get_value_object(f->ra, map); - if (!rval) { - continue; - } - - flb_mp_map_header_append(&mh_fields); - - /* key */ - msgpack_pack_str(mp_pck, flb_sds_len(f->key_name)); - msgpack_pack_str_body(mp_pck, f->key_name, flb_sds_len(f->key_name)); - - /* value */ - msgpack_pack_object(mp_pck, rval->o); - flb_ra_key_value_destroy(rval); - } - flb_mp_map_header_end(&mh_fields); - } - - return 0; -} - -static int pack_map(struct flb_splunk *ctx, msgpack_packer *mp_pck, - struct flb_time *tm, msgpack_object map, - char *tag, int tag_len) -{ - int i; - double t; - int map_size; - msgpack_object k; - msgpack_object v; - struct flb_mp_map_header mh; - - t = flb_time_to_double(tm); - map_size = map.via.map.size; - - if (ctx->splunk_send_raw == FLB_TRUE) { - msgpack_pack_map(mp_pck, map_size /* all k/v */); - } - else { - flb_mp_map_header_init(&mh, mp_pck); - - /* Append the time key */ - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, sizeof(FLB_SPLUNK_DEFAULT_TIME) -1); - msgpack_pack_str_body(mp_pck, - FLB_SPLUNK_DEFAULT_TIME, - sizeof(FLB_SPLUNK_DEFAULT_TIME) - 1); - msgpack_pack_double(mp_pck, t); - - /* Pack Splunk metadata */ - pack_map_meta(ctx, &mh, mp_pck, map, tag, tag_len); - - /* Add k/v pairs under the key 'event' instead of to the top level object */ - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, sizeof(FLB_SPLUNK_DEFAULT_EVENT) -1); - msgpack_pack_str_body(mp_pck, - FLB_SPLUNK_DEFAULT_EVENT, - sizeof(FLB_SPLUNK_DEFAULT_EVENT) - 1); - - flb_mp_map_header_end(&mh); - - msgpack_pack_map(mp_pck, map_size); - } - - /* Append k/v */ - for (i = 0; i < map_size; i++) { - k = map.via.map.ptr[i].key; - v = map.via.map.ptr[i].val; - - msgpack_pack_object(mp_pck, k); - msgpack_pack_object(mp_pck, v); - } - - return 0; -} - - -static inline int pack_event_key(struct flb_splunk *ctx, msgpack_packer *mp_pck, - struct flb_time *tm, msgpack_object map, - char *tag, int tag_len) -{ - double t; - struct flb_mp_map_header mh; - flb_sds_t val; - - t = flb_time_to_double(tm); - val = flb_ra_translate(ctx->ra_event_key, tag, tag_len, map, NULL); - if (!val || flb_sds_len(val) == 0) { - if (val != NULL) { - flb_sds_destroy(val); - } - - return -1; - } - - if (ctx->splunk_send_raw == FLB_FALSE) { - flb_mp_map_header_init(&mh, mp_pck); - - /* Append the time key */ - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, sizeof(FLB_SPLUNK_DEFAULT_TIME) -1); - msgpack_pack_str_body(mp_pck, - FLB_SPLUNK_DEFAULT_TIME, - sizeof(FLB_SPLUNK_DEFAULT_TIME) - 1); - msgpack_pack_double(mp_pck, t); - - /* Pack Splunk metadata */ - pack_map_meta(ctx, &mh, mp_pck, map, tag, tag_len); - - /* Add k/v pairs under the key 'event' instead of to the top level object */ - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, sizeof(FLB_SPLUNK_DEFAULT_EVENT) -1); - msgpack_pack_str_body(mp_pck, - FLB_SPLUNK_DEFAULT_EVENT, - sizeof(FLB_SPLUNK_DEFAULT_EVENT) - 1); - - flb_mp_map_header_end(&mh); - } - - msgpack_pack_str(mp_pck, flb_sds_len(val)); - msgpack_pack_str_body(mp_pck, val, flb_sds_len(val)); - flb_sds_destroy(val); - - return 0; -} - -#ifdef FLB_HAVE_METRICS -static inline int splunk_metrics_format(struct flb_output_instance *ins, - const void *in_buf, size_t in_bytes, - char **out_buf, size_t *out_size, - struct flb_splunk *ctx) -{ - int ret; - size_t off = 0; - cfl_sds_t text; - cfl_sds_t host; - struct cmt *cmt = NULL; - - if (ctx->event_host != NULL) { - host = ctx->event_host; - } - else { - host = "localhost"; - } - - /* get cmetrics context */ - ret = cmt_decode_msgpack_create(&cmt, (char *) in_buf, in_bytes, &off); - if (ret != 0) { - flb_plg_error(ins, "could not process metrics payload"); - return -1; - } - - /* convert to text representation */ - text = cmt_encode_splunk_hec_create(cmt, host, ctx->event_index, ctx->event_source, ctx->event_sourcetype); - - /* destroy cmt context */ - cmt_destroy(cmt); - - *out_buf = text; - *out_size = flb_sds_len(text); - - return 0; -} -#endif - -static inline int splunk_format(const void *in_buf, size_t in_bytes, - char *tag, int tag_len, - char **out_buf, size_t *out_size, - struct flb_splunk *ctx) -{ - int ret; - msgpack_object map; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - char *err; - flb_sds_t tmp; - flb_sds_t record; - flb_sds_t json_out; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - json_out = flb_sds_create_size(in_bytes * 1.5); - if (!json_out) { - flb_errno(); - return -1; - } - - ret = flb_log_event_decoder_init(&log_decoder, (char *) in_buf, in_bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - flb_sds_destroy(json_out); - - return -1; - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - - /* Create temporary msgpack buffer */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - map = *log_event.body; - - if (ctx->event_key) { - /* Pack the value of a event key */ - ret = pack_event_key(ctx, &mp_pck, &log_event.timestamp, map, tag, tag_len); - if (ret != 0) { - /* - * if pack_event_key fails due to missing content in the - * record, we just warn the user and try to pack it - * as a normal map. - */ - ret = pack_map(ctx, &mp_pck, &log_event.timestamp, map, tag, tag_len); - } - } - else { - /* Pack as a map */ - ret = pack_map(ctx, &mp_pck, &log_event.timestamp, map, tag, tag_len); - } - - /* Validate packaging */ - if (ret != 0) { - /* Format invalid record */ - err = flb_msgpack_to_json_str(2048, &map); - if (err) { - /* Print error and continue processing other records */ - flb_plg_warn(ctx->ins, "could not process or pack record: %s", err); - msgpack_sbuffer_destroy(&mp_sbuf); - flb_free(err); - } - continue; - } - - /* Format as JSON */ - record = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); - if (!record) { - flb_errno(); - msgpack_sbuffer_destroy(&mp_sbuf); - flb_log_event_decoder_destroy(&log_decoder); - flb_sds_destroy(json_out); - return -1; - } - - /* On raw mode, append a breakline to every record */ - if (ctx->splunk_send_raw) { - tmp = flb_sds_cat(record, "\n", 1); - if (tmp) { - record = tmp; - } - } - - tmp = flb_sds_cat(json_out, record, flb_sds_len(record)); - flb_sds_destroy(record); - if (tmp) { - json_out = tmp; - } - else { - flb_errno(); - msgpack_sbuffer_destroy(&mp_sbuf); - flb_log_event_decoder_destroy(&log_decoder); - flb_sds_destroy(json_out); - return -1; - } - msgpack_sbuffer_destroy(&mp_sbuf); - } - - *out_buf = json_out; - *out_size = flb_sds_len(json_out); - - flb_log_event_decoder_destroy(&log_decoder); - - return 0; -} - -static void debug_request_response(struct flb_splunk *ctx, - struct flb_http_client *c) -{ - int ret; - int uncompressed = FLB_FALSE; - time_t now; - void *tmp_buf = NULL; - size_t tmp_size; - size_t req_size; - char *req_buf = NULL; - struct tm result; - struct tm *current; - unsigned char *ptr; - flb_sds_t req_headers = NULL; - flb_sds_t req_body = NULL; - - if (c->body_len > 3) { - ptr = (unsigned char *) c->body_buf; - if (ptr[0] == 0x1F && ptr[1] == 0x8B && ptr[2] == 0x08) { - /* uncompress payload */ - ret = flb_gzip_uncompress((void *) c->body_buf, c->body_len, - &tmp_buf, &tmp_size); - if (ret == -1) { - fprintf(stdout, "[out_splunk] could not uncompress data\n"); - } - else { - req_buf = (char *) tmp_buf; - req_size = tmp_size; - uncompressed = FLB_TRUE; - } - } - else { - req_buf = (char *) c->body_buf; - req_size = c->body_len; - } - - /* create a safe buffer */ - if (req_buf) { - req_body = flb_sds_create_len(req_buf, req_size); - } - } - - req_headers = flb_sds_create_len(c->header_buf, c->header_len); - - if (c->resp.data) - now = time(NULL); - current = localtime_r(&now, &result); - - fprintf(stdout, - "[%i/%02i/%02i %02i:%02i:%02i] " - "[out_splunk] debug HTTP 400 (bad request)\n" - ">>> request\n" - "%s%s\n\n" - "<<< response\n" - "%s\n\n", - - current->tm_year + 1900, - current->tm_mon + 1, - current->tm_mday, - current->tm_hour, - current->tm_min, - current->tm_sec, - - req_headers, - req_body, - c->resp.data); - - if (uncompressed) { - flb_free(tmp_buf); - } - - if (req_headers) { - flb_sds_destroy(req_headers); - } - if (req_body) { - flb_sds_destroy(req_body); - } -} - -static void cb_splunk_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - int ret; - int compressed = FLB_FALSE; - size_t b_sent; - flb_sds_t buf_data; - size_t resp_size; - size_t buf_size; - char *endpoint; - struct flb_splunk *ctx = out_context; - struct flb_connection *u_conn; - struct flb_http_client *c; - void *payload_buf; - size_t payload_size; - (void) i_ins; - (void) config; - - /* Get upstream connection */ - u_conn = flb_upstream_conn_get(ctx->u); - if (!u_conn) { - FLB_OUTPUT_RETURN(FLB_RETRY); - } - -#ifdef FLB_HAVE_METRICS - /* Check if the event type is metrics, handle the payload differently */ - if (event_chunk->type == FLB_EVENT_TYPE_METRICS) { - ret = splunk_metrics_format(ctx->ins, - event_chunk->data, - event_chunk->size, - &buf_data, &buf_size, ctx); - } -#endif - if (event_chunk->type == FLB_EVENT_TYPE_LOGS) { - /* Convert binary logs into a JSON payload */ - ret = splunk_format(event_chunk->data, - event_chunk->size, - (char *) event_chunk->tag, - flb_sds_len(event_chunk->tag), - &buf_data, &buf_size, ctx); - } - - if (ret == -1) { - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_ERROR); - } - - /* Map buffer */ - payload_buf = buf_data; - payload_size = buf_size; - - /* Should we compress the payload ? */ - if (ctx->compress_gzip == FLB_TRUE) { - ret = flb_gzip_compress((void *) buf_data, buf_size, - &payload_buf, &payload_size); - if (ret == -1) { - flb_plg_error(ctx->ins, - "cannot gzip payload, disabling compression"); - } - else { - compressed = FLB_TRUE; - - /* JSON buffer is not longer needed */ - flb_sds_destroy(buf_data); - } - } - - /* Splunk URI endpoint */ - if (ctx->splunk_send_raw) { - endpoint = FLB_SPLUNK_DEFAULT_URI_RAW; - } - else { - endpoint = FLB_SPLUNK_DEFAULT_URI_EVENT; - } - - /* Compose HTTP Client request */ - c = flb_http_client(u_conn, FLB_HTTP_POST, endpoint, - payload_buf, payload_size, NULL, 0, NULL, 0); - - /* HTTP Response buffer size, honor value set by the user */ - if (ctx->buffer_size > 0) { - flb_http_buffer_size(c, ctx->buffer_size); - } - else { - /* - * If no value was set, we try to accomodate by using our post - * payload size * 1.5, on that way we make room for large responses - * if something goes wrong, so we don't get a partial response. - */ - resp_size = payload_size * 1.5; - if (resp_size < 4096) { - resp_size = 4096; - } - flb_http_buffer_size(c, resp_size); - } - - /* HTTP Client */ - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - - /* Try to use http_user and http_passwd if not, fallback to auth_header */ - if (ctx->http_user && ctx->http_passwd) { - flb_http_basic_auth(c, ctx->http_user, ctx->http_passwd); - } - else if (ctx->auth_header) { - flb_http_add_header(c, "Authorization", 13, - ctx->auth_header, flb_sds_len(ctx->auth_header)); - } - - /* Append Channel identifier header */ - if (ctx->channel) { - flb_http_add_header(c, FLB_SPLUNK_CHANNEL_IDENTIFIER_HEADER, - strlen(FLB_SPLUNK_CHANNEL_IDENTIFIER_HEADER), - ctx->channel, ctx->channel_len); - } - - /* Content Encoding: gzip */ - if (compressed == FLB_TRUE) { - flb_http_set_content_encoding_gzip(c); - } - - /* Map debug callbacks */ - flb_http_client_debug(c, ctx->ins->callback); - - /* Perform HTTP request */ - ret = flb_http_do(c, &b_sent); - if (ret != 0) { - flb_plg_warn(ctx->ins, "http_do=%i", ret); - ret = FLB_RETRY; - } - else { - if (c->resp.status != 200) { - if (c->resp.payload_size > 0) { - flb_plg_warn(ctx->ins, "http_status=%i:\n%s", - c->resp.status, c->resp.payload); - } - else { - flb_plg_warn(ctx->ins, "http_status=%i", c->resp.status); - } - /* - * Requests that get 4xx responses from the Splunk HTTP Event - * Collector will 'always' fail, so there is no point in retrying - * them: - * - * https://docs.splunk.com/Documentation/Splunk/8.0.5/Data/TroubleshootHTTPEventCollector#Possible_error_codes - */ - ret = (c->resp.status < 400 || c->resp.status >= 500) ? - FLB_RETRY : FLB_ERROR; - - - if (c->resp.status == 400 && ctx->http_debug_bad_request) { - debug_request_response(ctx, c); - } - } - else { - ret = FLB_OK; - } - } - - /* - * If the payload buffer is different than incoming records in body, means - * we generated a different payload and must be freed. - */ - if (compressed == FLB_TRUE) { - flb_free(payload_buf); - } - else { - flb_sds_destroy(buf_data); - } - - /* Cleanup */ - flb_http_client_destroy(c); - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(ret); -} - -static int cb_splunk_exit(void *data, struct flb_config *config) -{ - struct flb_splunk *ctx = data; - - flb_splunk_conf_destroy(ctx); - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "compress", NULL, - 0, FLB_FALSE, 0, - "Set payload compression mechanism. Option available is 'gzip'" - }, - - { - FLB_CONFIG_MAP_STR, "http_user", NULL, - 0, FLB_TRUE, offsetof(struct flb_splunk, http_user), - "Set HTTP auth user" - }, - - { - FLB_CONFIG_MAP_STR, "http_passwd", "", - 0, FLB_TRUE, offsetof(struct flb_splunk, http_passwd), - "Set HTTP auth password" - }, - - { - FLB_CONFIG_MAP_SIZE, "http_buffer_size", NULL, - 0, FLB_FALSE, 0, - "Specify the buffer size used to read the response from the Splunk HTTP " - "service. This option is useful for debugging purposes where is required to read " - "full responses, note that response size grows depending of the number of records " - "inserted. To set an unlimited amount of memory set this value to 'false', " - "otherwise the value must be according to the Unit Size specification" - }, - - { - FLB_CONFIG_MAP_BOOL, "http_debug_bad_request", "false", - 0, FLB_TRUE, offsetof(struct flb_splunk, http_debug_bad_request), - "If the HTTP server response code is 400 (bad request) and this flag is " - "enabled, it will print the full HTTP request and response to the stdout " - "interface. This feature is available for debugging purposes." - }, - - { - FLB_CONFIG_MAP_STR, "event_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_splunk, event_key), - "Specify the key name that will be used to send a single value as part of the record." - }, - - { - FLB_CONFIG_MAP_STR, "event_host", NULL, - 0, FLB_TRUE, offsetof(struct flb_splunk, event_host), - "Set the host value to the event data. The value allows a record accessor " - "pattern." - }, - - { - FLB_CONFIG_MAP_STR, "event_source", NULL, - 0, FLB_TRUE, offsetof(struct flb_splunk, event_source), - "Set the source value to assign to the event data." - }, - - { - FLB_CONFIG_MAP_STR, "event_sourcetype", NULL, - 0, FLB_TRUE, offsetof(struct flb_splunk, event_sourcetype), - "Set the sourcetype value to assign to the event data." - }, - - { - FLB_CONFIG_MAP_STR, "event_sourcetype_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_splunk, event_sourcetype_key), - "Set a record key that will populate 'sourcetype'. If the key is found, it will " - "have precedence over the value set in 'event_sourcetype'." - }, - - { - FLB_CONFIG_MAP_STR, "event_index", NULL, - 0, FLB_TRUE, offsetof(struct flb_splunk, event_index), - "The name of the index by which the event data is to be indexed." - }, - - { - FLB_CONFIG_MAP_STR, "event_index_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_splunk, event_index_key), - "Set a record key that will populate the 'index' field. If the key is found, " - "it will have precedence over the value set in 'event_index'." - }, - - { - FLB_CONFIG_MAP_SLIST_2, "event_field", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct flb_splunk, event_fields), - "Set event fields for the record. This option can be set multiple times and " - "the format is 'key_name record_accessor_pattern'." - }, - - { - FLB_CONFIG_MAP_STR, "splunk_token", NULL, - 0, FLB_FALSE, 0, - "Specify the Authentication Token for the HTTP Event Collector interface." - }, - - { - FLB_CONFIG_MAP_BOOL, "splunk_send_raw", "off", - 0, FLB_TRUE, offsetof(struct flb_splunk, splunk_send_raw), - "When enabled, the record keys and values are set in the top level of the " - "map instead of under the event key. Refer to the Sending Raw Events section " - "from the docs for more details to make this option work properly." - }, - - { - FLB_CONFIG_MAP_STR, "channel", NULL, - 0, FLB_TRUE, offsetof(struct flb_splunk, channel), - "Specify X-Splunk-Request-Channel Header for the HTTP Event Collector interface." - }, - - /* EOF */ - {0} -}; - - -static int cb_splunk_format_test(struct flb_config *config, - struct flb_input_instance *ins, - void *plugin_context, - void *flush_ctx, - int event_type, - const char *tag, int tag_len, - const void *data, size_t bytes, - void **out_data, size_t *out_size) -{ - struct flb_splunk *ctx = plugin_context; - - return splunk_format(data, bytes, (char *) tag, tag_len, - (char**) out_data, out_size,ctx); -} - -struct flb_output_plugin out_splunk_plugin = { - .name = "splunk", - .description = "Send events to Splunk HTTP Event Collector", - .cb_init = cb_splunk_init, - .cb_flush = cb_splunk_flush, - .cb_exit = cb_splunk_exit, - .config_map = config_map, - .workers = 2, -#ifdef FLB_HAVE_METRICS - .event_type = FLB_OUTPUT_LOGS | FLB_OUTPUT_METRICS, -#endif - - /* for testing */ - .test_formatter.callback = cb_splunk_format_test, - /* Plugin flags */ - .flags = FLB_OUTPUT_NET | FLB_IO_OPT_TLS, -}; diff --git a/fluent-bit/plugins/out_splunk/splunk.h b/fluent-bit/plugins/out_splunk/splunk.h deleted file mode 100644 index eef8fa8b0..000000000 --- a/fluent-bit/plugins/out_splunk/splunk.h +++ /dev/null @@ -1,119 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_SPLUNK -#define FLB_OUT_SPLUNK - -#define FLB_SPLUNK_DEFAULT_HOST "127.0.0.1" -#define FLB_SPLUNK_DEFAULT_PORT 8088 -#define FLB_SPLUNK_DEFAULT_URI_RAW "/services/collector/raw" -#define FLB_SPLUNK_DEFAULT_URI_EVENT "/services/collector/event" -#define FLB_SPLUNK_DEFAULT_TIME "time" -#define FLB_SPLUNK_DEFAULT_EVENT_HOST "host" -#define FLB_SPLUNK_DEFAULT_EVENT_SOURCE "source" -#define FLB_SPLUNK_DEFAULT_EVENT_SOURCET "sourcetype" -#define FLB_SPLUNK_DEFAULT_EVENT_INDEX "index" -#define FLB_SPLUNK_DEFAULT_EVENT_FIELDS "fields" -#define FLB_SPLUNK_DEFAULT_EVENT "event" -#define FLB_SPLUNK_DEFAULT_HTTP_MAX "2M" - -#define FLB_SPLUNK_CHANNEL_IDENTIFIER_HEADER "X-Splunk-Request-Channel" - -#include -#include -#include - -struct flb_splunk_field { - flb_sds_t key_name; - struct flb_record_accessor *ra; - struct mk_list _head; -}; - -struct flb_splunk { - /* Payload compression */ - int compress_gzip; - - /* HTTP Auth */ - char *http_user; - char *http_passwd; - - /* Event key */ - flb_sds_t event_key; - struct flb_record_accessor *ra_event_key; - - /* Event host */ - flb_sds_t event_host; - struct flb_record_accessor *ra_event_host; - - /* Event source */ - flb_sds_t event_source; - struct flb_record_accessor *ra_event_source; - - /* - * NOTE: EVENT SOURCE - * ------------------- - * we use two separate variables since we aim to specify a default in case - * a record accessor pattern is given but not found. The event_sourcetype_key - * has precedence over th the 'event_sourcetype' variable. - */ - - /* Event sourcetype */ - flb_sds_t event_sourcetype; - - /* Event sourcetype record key */ - flb_sds_t event_sourcetype_key; - struct flb_record_accessor *ra_event_sourcetype_key; - - /* Event index */ - flb_sds_t event_index; - - /* Event sourcetype record key */ - flb_sds_t event_index_key; - struct flb_record_accessor *ra_event_index_key; - - /* Event fields */ - struct mk_list *event_fields; - - /* Internal/processed event fields */ - struct mk_list fields; - - /* Token Auth */ - flb_sds_t auth_header; - - /* Channel identifier */ - flb_sds_t channel; - size_t channel_len; - - /* Send fields directly or pack data into "event" object */ - int splunk_send_raw; - - /* HTTP Client Setup */ - size_t buffer_size; - - /* HTTP: Debug bad requests (HTTP status 400) to stdout */ - int http_debug_bad_request; - - /* Upstream connection to the backend server */ - struct flb_upstream *u; - - /* Plugin instance */ - struct flb_output_instance *ins; -}; - -#endif diff --git a/fluent-bit/plugins/out_splunk/splunk_conf.c b/fluent-bit/plugins/out_splunk/splunk_conf.c deleted file mode 100644 index cc911cbeb..000000000 --- a/fluent-bit/plugins/out_splunk/splunk_conf.c +++ /dev/null @@ -1,313 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include "splunk.h" -#include "splunk_conf.h" - -static int event_fields_create(struct flb_splunk *ctx) -{ - int i = 0; - struct mk_list *head; - struct flb_slist_entry *kname; - struct flb_slist_entry *pattern; - struct flb_config_map_val *mv; - struct flb_splunk_field *f; - - if (!ctx->event_fields) { - return 0; - } - - flb_config_map_foreach(head, mv, ctx->event_fields) { - kname = mk_list_entry_first(mv->val.list, struct flb_slist_entry, _head); - pattern = mk_list_entry_last(mv->val.list, struct flb_slist_entry, _head); - - f = flb_malloc(sizeof(struct flb_splunk_field)); - if (!f) { - flb_errno(); - return -1; - } - - f->key_name = flb_sds_create(kname->str); - if (!f->key_name) { - flb_free(f); - return -1; - } - - f->ra = flb_ra_create(pattern->str, FLB_TRUE); - if (!f->ra) { - flb_plg_error(ctx->ins, - "could not process event_field number #%i with " - "pattern '%s'", - i, pattern->str); - flb_sds_destroy(f->key_name); - flb_free(f); - return -1; - } - - mk_list_add(&f->_head, &ctx->fields); - } - - return 0; -} - -static void event_fields_destroy(struct flb_splunk *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - struct flb_splunk_field *f; - - mk_list_foreach_safe(head, tmp, &ctx->fields) { - f = mk_list_entry(head, struct flb_splunk_field, _head); - flb_sds_destroy(f->key_name); - flb_ra_destroy(f->ra); - mk_list_del(&f->_head); - flb_free(f); - } -} - -struct flb_splunk *flb_splunk_conf_create(struct flb_output_instance *ins, - struct flb_config *config) -{ - int ret; - int io_flags = 0; - size_t size; - flb_sds_t t; - const char *tmp; - struct flb_upstream *upstream; - struct flb_splunk *ctx; - - ctx = flb_calloc(1, sizeof(struct flb_splunk)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - mk_list_init(&ctx->fields); - - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return NULL; - } - - /* Set default network configuration */ - flb_output_net_default(FLB_SPLUNK_DEFAULT_HOST, FLB_SPLUNK_DEFAULT_PORT, ins); - - /* use TLS ? */ - if (ins->use_tls == FLB_TRUE) { - io_flags = FLB_IO_TLS; - } - else { - io_flags = FLB_IO_TCP; - } - - if (ins->host.ipv6 == FLB_TRUE) { - io_flags |= FLB_IO_IPV6; - } - - /* Prepare an upstream handler */ - upstream = flb_upstream_create(config, - ins->host.name, - ins->host.port, - io_flags, - ins->tls); - if (!upstream) { - flb_plg_error(ctx->ins, "cannot create Upstream context"); - flb_splunk_conf_destroy(ctx); - return NULL; - } - - /* Set manual Index and Type */ - ctx->u = upstream; - - tmp = flb_output_get_property("http_buffer_size", ins); - if (!tmp) { - ctx->buffer_size = 0; - } - else { - size = flb_utils_size_to_bytes(tmp); - if (size == -1) { - flb_plg_error(ctx->ins, "invalid 'buffer_size' value"); - flb_splunk_conf_destroy(ctx); - return NULL; - } - if (size < 4 *1024) { - size = 4 * 1024; - } - ctx->buffer_size = size; - } - - /* Compress (gzip) */ - tmp = flb_output_get_property("compress", ins); - ctx->compress_gzip = FLB_FALSE; - if (tmp) { - if (strcasecmp(tmp, "gzip") == 0) { - ctx->compress_gzip = FLB_TRUE; - } - } - - /* Event key */ - if (ctx->event_key) { - if (ctx->event_key[0] != '$') { - flb_plg_error(ctx->ins, - "invalid event_key pattern, it must start with '$'"); - flb_splunk_conf_destroy(ctx); - return NULL; - } - ctx->ra_event_key = flb_ra_create(ctx->event_key, FLB_TRUE); - if (!ctx->ra_event_key) { - flb_plg_error(ctx->ins, - "cannot create record accessor for event_key pattern: '%s'", - ctx->event_key); - flb_splunk_conf_destroy(ctx); - return NULL; - } - } - - /* Event host */ - if (ctx->event_host) { - ctx->ra_event_host = flb_ra_create(ctx->event_host, FLB_TRUE); - if (!ctx->ra_event_host) { - flb_plg_error(ctx->ins, - "cannot create record accessor for event_key pattern: '%s'", - ctx->event_host); - flb_splunk_conf_destroy(ctx); - return NULL; - } - } - - /* Event source */ - if (ctx->event_source) { - ctx->ra_event_source = flb_ra_create(ctx->event_source, FLB_TRUE); - if (!ctx->ra_event_source) { - flb_plg_error(ctx->ins, - "cannot create record accessor for event_source pattern: '%s'", - ctx->event_host); - flb_splunk_conf_destroy(ctx); - return NULL; - } - } - - /* Event source (key lookup) */ - if (ctx->event_sourcetype_key) { - ctx->ra_event_sourcetype_key = flb_ra_create(ctx->event_sourcetype_key, FLB_TRUE); - if (!ctx->ra_event_sourcetype_key) { - flb_plg_error(ctx->ins, - "cannot create record accessor for " - "event_sourcetype_key pattern: '%s'", - ctx->event_host); - flb_splunk_conf_destroy(ctx); - return NULL; - } - } - - /* Event index (key lookup) */ - if (ctx->event_index_key) { - ctx->ra_event_index_key = flb_ra_create(ctx->event_index_key, FLB_TRUE); - if (!ctx->ra_event_index_key) { - flb_plg_error(ctx->ins, - "cannot create record accessor for " - "event_index_key pattern: '%s'", - ctx->event_host); - flb_splunk_conf_destroy(ctx); - return NULL; - } - } - - /* Event fields */ - ret = event_fields_create(ctx); - if (ret == -1) { - flb_splunk_conf_destroy(ctx); - return NULL; - } - - /* No http_user is set, fallback to splunk_token, if splunk_token is unset, fail. */ - if (!ctx->http_user) { - /* Splunk Auth Token */ - tmp = flb_output_get_property("splunk_token", ins); - if(!tmp) { - flb_plg_error(ctx->ins, "either splunk_token or http_user should be set"); - flb_splunk_conf_destroy(ctx); - return NULL; - } - ctx->auth_header = flb_sds_create("Splunk "); - t = flb_sds_cat(ctx->auth_header, tmp, strlen(tmp)); - if (t) { - ctx->auth_header = t; - } - else { - flb_plg_error(ctx->ins, "error on token generation"); - flb_splunk_conf_destroy(ctx); - return NULL; - } - } - - /* channel */ - if (ctx->channel != NULL) { - ctx->channel_len = flb_sds_len(ctx->channel); - } - - /* Set instance flags into upstream */ - flb_output_upstream_set(ctx->u, ins); - - return ctx; -} - -int flb_splunk_conf_destroy(struct flb_splunk *ctx) -{ - if (!ctx) { - return -1; - } - - if (ctx->auth_header) { - flb_sds_destroy(ctx->auth_header); - } - if (ctx->u) { - flb_upstream_destroy(ctx->u); - } - - if (ctx->ra_event_key) { - flb_ra_destroy(ctx->ra_event_key); - } - - if (ctx->ra_event_host) { - flb_ra_destroy(ctx->ra_event_host); - } - - if (ctx->ra_event_source) { - flb_ra_destroy(ctx->ra_event_source); - } - - if (ctx->ra_event_sourcetype_key) { - flb_ra_destroy(ctx->ra_event_sourcetype_key); - } - - if (ctx->ra_event_index_key) { - flb_ra_destroy(ctx->ra_event_index_key); - } - - event_fields_destroy(ctx); - - flb_free(ctx); - - return 0; -} diff --git a/fluent-bit/plugins/out_splunk/splunk_conf.h b/fluent-bit/plugins/out_splunk/splunk_conf.h deleted file mode 100644 index c5114b1f9..000000000 --- a/fluent-bit/plugins/out_splunk/splunk_conf.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_SPLUNK_CONF_H -#define FLB_OUT_SPLUNK_CONF_H - -#include "splunk.h" - -struct flb_splunk *flb_splunk_conf_create(struct flb_output_instance *ins, - struct flb_config *config); -int flb_splunk_conf_destroy(struct flb_splunk *ctx); - -#endif diff --git a/fluent-bit/plugins/out_stackdriver/CMakeLists.txt b/fluent-bit/plugins/out_stackdriver/CMakeLists.txt deleted file mode 100644 index 2d7fa71bb..000000000 --- a/fluent-bit/plugins/out_stackdriver/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -set(src - gce_metadata.c - stackdriver_conf.c - stackdriver.c - stackdriver_operation.c - stackdriver_source_location.c - stackdriver_http_request.c - stackdriver_timestamp.c - stackdriver_helper.c - stackdriver_resource_types.c - ) - -FLB_PLUGIN(out_stackdriver "${src}" "") diff --git a/fluent-bit/plugins/out_stackdriver/gce_metadata.c b/fluent-bit/plugins/out_stackdriver/gce_metadata.c deleted file mode 100644 index fb942213b..000000000 --- a/fluent-bit/plugins/out_stackdriver/gce_metadata.c +++ /dev/null @@ -1,222 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include - -#include - -#include "gce_metadata.h" -#include "stackdriver.h" -#include "stackdriver_conf.h" - - -static int fetch_metadata(struct flb_stackdriver *ctx, - struct flb_upstream *upstream, char *uri, - char *payload) -{ - int ret; - int ret_code; - size_t b_sent; - struct flb_connection *metadata_conn; - struct flb_http_client *c; - - /* If runtime test mode is enabled, add test data */ - if (ctx->ins->test_mode == FLB_TRUE) { - if (strcmp(uri, FLB_STD_METADATA_PROJECT_ID_URI) == 0) { - flb_sds_cat(payload, "fluent-bit-test", 15); - return 0; - } - else if (strcmp(uri, FLB_STD_METADATA_ZONE_URI) == 0) { - flb_sds_cat(payload, "projects/0123456789/zones/fluent", 32); - return 0; - } - else if (strcmp(uri, FLB_STD_METADATA_INSTANCE_ID_URI) == 0) { - flb_sds_cat(payload, "333222111", 9); - return 0; - } - return -1; - } - - /* Get metadata connection */ - metadata_conn = flb_upstream_conn_get(upstream); - if (!metadata_conn) { - flb_plg_error(ctx->ins, "failed to create metadata connection"); - return -1; - } - - /* Compose HTTP Client request */ - c = flb_http_client(metadata_conn, FLB_HTTP_GET, uri, - "", 0, NULL, 0, NULL, 0); - - flb_http_buffer_size(c, FLB_STD_METADATA_TOKEN_SIZE_MAX); - - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - flb_http_add_header(c, "Content-Type", 12, "application/text", 16); - flb_http_add_header(c, "Metadata-Flavor", 15, "Google", 6); - - /* Send HTTP request */ - ret = flb_http_do(c, &b_sent); - - /* validate response */ - if (ret != 0) { - flb_plg_warn(ctx->ins, "http_do=%i", ret); - ret_code = -1; - } - else { - /* The request was issued successfully, validate the 'error' field */ - flb_plg_debug(ctx->ins, "HTTP Status=%i", c->resp.status); - if (c->resp.status == 200) { - ret_code = 0; - flb_sds_copy(payload, c->resp.payload, c->resp.payload_size); - } - else { - if (c->resp.payload_size > 0) { - /* we got an error */ - flb_plg_warn(ctx->ins, "error\n%s", c->resp.payload); - } - else { - flb_plg_debug(ctx->ins, "response\n%s", c->resp.payload); - } - ret_code = -1; - } - } - - /* Cleanup */ - flb_http_client_destroy(c); - flb_upstream_conn_release(metadata_conn); - - return ret_code; -} - -int gce_metadata_read_token(struct flb_stackdriver *ctx) -{ - int ret; - flb_sds_t uri = flb_sds_create(FLB_STD_METADATA_SERVICE_ACCOUNT_URI); - flb_sds_t payload = flb_sds_create_size(FLB_STD_METADATA_TOKEN_SIZE_MAX); - - uri = flb_sds_cat(uri, ctx->client_email, flb_sds_len(ctx->client_email)); - uri = flb_sds_cat(uri, "/token", 6); - ret = fetch_metadata(ctx, ctx->metadata_u, uri, payload); - if (ret != 0) { - flb_plg_error(ctx->ins, "can't fetch token from the metadata server"); - flb_sds_destroy(payload); - flb_sds_destroy(uri); - return -1; - } - - ret = flb_oauth2_parse_json_response(payload, flb_sds_len(payload), ctx->o); - flb_sds_destroy(payload); - flb_sds_destroy(uri); - - if (ret != 0) { - flb_plg_error(ctx->ins, "unable to parse token body"); - return -1; - } - ctx->o->expires = time(NULL) + ctx->o->expires_in; - return 0; -} - -int gce_metadata_read_zone(struct flb_stackdriver *ctx) -{ - int ret; - int i; - int j; - int part = 0; - flb_sds_t payload = flb_sds_create_size(4096); - flb_sds_t zone = NULL; - - ret = fetch_metadata(ctx, ctx->metadata_u, FLB_STD_METADATA_ZONE_URI, - payload); - if (ret != 0) { - flb_plg_error(ctx->ins, "can't fetch zone from the metadata server"); - flb_sds_destroy(payload); - return -1; - } - - /* Data returned in the format projects/{project-id}/zones/{name} */ - for (i = 0; i < flb_sds_len(payload); ++i) { - if (payload[i] == '/') { - part++; - } - if (part == 3) { - i++; - break; - } - } - - if (part != 3) { - flb_plg_error(ctx->ins, "wrong format of zone response"); - flb_sds_destroy(payload); - return -1; - } - - zone = flb_sds_create_size(flb_sds_len(payload) - i); - - j = 0; - while (i != flb_sds_len(payload)) { - zone[j] = payload[i]; - i++; - j++; - } - zone[j] = '\0'; - ctx->zone = flb_sds_create(zone); - flb_sds_destroy(zone); - flb_sds_destroy(payload); - - return 0; -} - -int gce_metadata_read_project_id(struct flb_stackdriver *ctx) -{ - int ret; - flb_sds_t payload = flb_sds_create_size(4096); - - ret = fetch_metadata(ctx, ctx->metadata_u, - FLB_STD_METADATA_PROJECT_ID_URI, payload); - if (ret != 0) { - flb_plg_error(ctx->ins, "can't fetch project id from the metadata server"); - flb_sds_destroy(payload); - return -1; - } - ctx->project_id = flb_sds_create(payload); - flb_sds_destroy(payload); - return 0; -} - -int gce_metadata_read_instance_id(struct flb_stackdriver *ctx) -{ - int ret; - flb_sds_t payload = flb_sds_create_size(4096); - - ret = fetch_metadata(ctx, ctx->metadata_u, - FLB_STD_METADATA_INSTANCE_ID_URI, payload); - if (ret != 0) { - flb_plg_error(ctx->ins, "can't fetch instance id from the metadata server"); - flb_sds_destroy(payload); - return -1; - } - ctx->instance_id = flb_sds_create(payload); - flb_sds_destroy(payload); - return 0; -} diff --git a/fluent-bit/plugins/out_stackdriver/gce_metadata.h b/fluent-bit/plugins/out_stackdriver/gce_metadata.h deleted file mode 100644 index 65009588d..000000000 --- a/fluent-bit/plugins/out_stackdriver/gce_metadata.h +++ /dev/null @@ -1,48 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLUENT_BIT_GCE_METADATA_H -#define FLUENT_BIT_GCE_METADATA_H - -#include "stackdriver.h" - -/* Metadata server URL */ -#define FLB_STD_METADATA_SERVER "http://metadata.google.internal" - -/* Project ID metadata URI */ -#define FLB_STD_METADATA_PROJECT_ID_URI "/computeMetadata/v1/project/project-id" - -/* Zone metadata URI */ -#define FLB_STD_METADATA_ZONE_URI "/computeMetadata/v1/instance/zone" - -/* Instance ID metadata URI */ -#define FLB_STD_METADATA_INSTANCE_ID_URI "/computeMetadata/v1/instance/id" - -/* Service account metadata URI */ -#define FLB_STD_METADATA_SERVICE_ACCOUNT_URI "/computeMetadata/v1/instance/service-accounts/" - -/* Max size of token response from metadata server */ -#define FLB_STD_METADATA_TOKEN_SIZE_MAX 14336 - -int gce_metadata_read_token(struct flb_stackdriver *ctx); -int gce_metadata_read_zone(struct flb_stackdriver *ctx); -int gce_metadata_read_project_id(struct flb_stackdriver *ctx); -int gce_metadata_read_instance_id(struct flb_stackdriver *ctx); - -#endif //FLUENT_BIT_GCE_METADATA_H diff --git a/fluent-bit/plugins/out_stackdriver/stackdriver.c b/fluent-bit/plugins/out_stackdriver/stackdriver.c deleted file mode 100644 index 5c48a338b..000000000 --- a/fluent-bit/plugins/out_stackdriver/stackdriver.c +++ /dev/null @@ -1,2867 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "gce_metadata.h" -#include "stackdriver.h" -#include "stackdriver_conf.h" -#include "stackdriver_operation.h" -#include "stackdriver_source_location.h" -#include "stackdriver_http_request.h" -#include "stackdriver_timestamp.h" -#include "stackdriver_helper.h" -#include "stackdriver_resource_types.h" - -pthread_key_t oauth2_type; -pthread_key_t oauth2_token; -pthread_key_t oauth2_token_expires; - -static void oauth2_cache_exit(void *ptr) -{ - if (ptr) { - flb_sds_destroy(ptr); - } -} - -static void oauth2_cache_free_expiration(void *ptr) -{ - if (ptr) { - flb_free(ptr); - } -} - -static void oauth2_cache_init() -{ - /* oauth2 pthread key */ - pthread_key_create(&oauth2_type, oauth2_cache_exit); - pthread_key_create(&oauth2_token, oauth2_cache_exit); - pthread_key_create(&oauth2_token_expires, oauth2_cache_free_expiration); -} - -/* Set oauth2 type and token in pthread keys */ -static void oauth2_cache_set(char *type, char *token, time_t expires) -{ - flb_sds_t tmp; - time_t *tmp_expires; - - /* oauth2 type */ - tmp = pthread_getspecific(oauth2_type); - if (tmp) { - flb_sds_destroy(tmp); - } - tmp = flb_sds_create(type); - pthread_setspecific(oauth2_type, tmp); - - /* oauth2 access token */ - tmp = pthread_getspecific(oauth2_token); - if (tmp) { - flb_sds_destroy(tmp); - } - tmp = flb_sds_create(token); - pthread_setspecific(oauth2_token, tmp); - - /* oauth2 access token expiration */ - tmp_expires = pthread_getspecific(oauth2_token_expires); - if (tmp_expires) { - flb_free(tmp_expires); - } - tmp_expires = flb_calloc(1, sizeof(time_t)); - if (!tmp_expires) { - flb_errno(); - return; - } - *tmp_expires = expires; - pthread_setspecific(oauth2_token_expires, tmp_expires); -} - -/* By using pthread keys cached values, compose the authorizatoin token */ -static time_t oauth2_cache_get_expiration() -{ - time_t *expires = pthread_getspecific(oauth2_token_expires); - if (expires) { - return *expires; - } - return 0; -} - -/* By using pthread keys cached values, compose the authorizatoin token */ -static flb_sds_t oauth2_cache_to_token() -{ - flb_sds_t type; - flb_sds_t token; - flb_sds_t output; - - type = pthread_getspecific(oauth2_type); - if (!type) { - return NULL; - } - - output = flb_sds_create(type); - if (!output) { - return NULL; - } - - token = pthread_getspecific(oauth2_token); - flb_sds_printf(&output, " %s", token); - return output; -} - -/* - * Base64 Encoding in JWT must: - * - * - remove any trailing padding '=' character - * - replace '+' with '-' - * - replace '/' with '_' - * - * ref: https://www.rfc-editor.org/rfc/rfc7515.txt Appendix C - */ -int jwt_base64_url_encode(unsigned char *out_buf, size_t out_size, - unsigned char *in_buf, size_t in_size, - size_t *olen) - -{ - int i; - size_t len; - int result; - - - /* do normal base64 encoding */ - result = flb_base64_encode((unsigned char *) out_buf, out_size - 1, - &len, in_buf, in_size); - if (result != 0) { - return -1; - } - - /* Replace '+' and '/' characters */ - for (i = 0; i < len && out_buf[i] != '='; i++) { - if (out_buf[i] == '+') { - out_buf[i] = '-'; - } - else if (out_buf[i] == '/') { - out_buf[i] = '_'; - } - } - - /* Now 'i' becomes the new length */ - *olen = i; - return 0; -} - -static int jwt_encode(char *payload, char *secret, - char **out_signature, size_t *out_size, - struct flb_stackdriver *ctx) -{ - int ret; - int len; - int buf_size; - size_t olen; - char *buf; - char *sigd; - char *headers = "{\"alg\": \"RS256\", \"typ\": \"JWT\"}"; - unsigned char sha256_buf[32] = {0}; - flb_sds_t out; - unsigned char sig[256] = {0}; - size_t sig_len; - - buf_size = (strlen(payload) + strlen(secret)) * 2; - buf = flb_malloc(buf_size); - if (!buf) { - flb_errno(); - return -1; - } - - /* Encode header */ - len = strlen(headers); - ret = flb_base64_encode((unsigned char *) buf, buf_size - 1, - &olen, (unsigned char *) headers, len); - if (ret != 0) { - flb_free(buf); - - return ret; - } - - /* Create buffer to store JWT */ - out = flb_sds_create_size(2048); - if (!out) { - flb_errno(); - flb_free(buf); - return -1; - } - - /* Append header */ - flb_sds_cat(out, buf, olen); - flb_sds_cat(out, ".", 1); - - /* Encode Payload */ - len = strlen(payload); - jwt_base64_url_encode((unsigned char *) buf, buf_size, - (unsigned char *) payload, len, &olen); - - /* Append Payload */ - flb_sds_cat(out, buf, olen); - - /* do sha256() of base64(header).base64(payload) */ - ret = flb_hash_simple(FLB_HASH_SHA256, - (unsigned char *) out, flb_sds_len(out), - sha256_buf, sizeof(sha256_buf)); - - if (ret != FLB_CRYPTO_SUCCESS) { - flb_plg_error(ctx->ins, "error hashing token"); - flb_free(buf); - flb_sds_destroy(out); - return -1; - } - - len = strlen(secret); - sig_len = sizeof(sig); - - ret = flb_crypto_sign_simple(FLB_CRYPTO_PRIVATE_KEY, - FLB_CRYPTO_PADDING_PKCS1, - FLB_HASH_SHA256, - (unsigned char *) secret, len, - sha256_buf, sizeof(sha256_buf), - sig, &sig_len); - - if (ret != FLB_CRYPTO_SUCCESS) { - flb_plg_error(ctx->ins, "error creating RSA context"); - flb_free(buf); - flb_sds_destroy(out); - return -1; - } - - sigd = flb_malloc(2048); - if (!sigd) { - flb_errno(); - flb_free(buf); - flb_sds_destroy(out); - return -1; - } - - jwt_base64_url_encode((unsigned char *) sigd, 2048, sig, 256, &olen); - - flb_sds_cat(out, ".", 1); - flb_sds_cat(out, sigd, olen); - - *out_signature = out; - *out_size = flb_sds_len(out); - - flb_free(buf); - flb_free(sigd); - - return 0; -} - -/* Create a new oauth2 context and get a oauth2 token */ -static int get_oauth2_token(struct flb_stackdriver *ctx) -{ - int ret; - char *token; - char *sig_data; - size_t sig_size; - time_t issued; - time_t expires; - char payload[1024]; - - flb_oauth2_payload_clear(ctx->o); - - /* In case of using metadata server, fetch token from there */ - if (ctx->metadata_server_auth) { - return gce_metadata_read_token(ctx); - } - - /* JWT encode for oauth2 */ - issued = time(NULL); - expires = issued + FLB_STD_TOKEN_REFRESH; - - snprintf(payload, sizeof(payload) - 1, - "{\"iss\": \"%s\", \"scope\": \"%s\", " - "\"aud\": \"%s\", \"exp\": %lu, \"iat\": %lu}", - ctx->client_email, FLB_STD_SCOPE, - FLB_STD_AUTH_URL, - expires, issued); - - /* Compose JWT signature */ - ret = jwt_encode(payload, ctx->private_key, &sig_data, &sig_size, ctx); - if (ret != 0) { - flb_plg_error(ctx->ins, "JWT signature generation failed"); - return -1; - } - flb_plg_debug(ctx->ins, "JWT signature:\n%s", sig_data); - - ret = flb_oauth2_payload_append(ctx->o, - "grant_type", -1, - "urn%3Aietf%3Aparams%3Aoauth%3A" - "grant-type%3Ajwt-bearer", -1); - if (ret == -1) { - flb_plg_error(ctx->ins, "error appending oauth2 params"); - flb_sds_destroy(sig_data); - return -1; - } - - ret = flb_oauth2_payload_append(ctx->o, - "assertion", -1, - sig_data, sig_size); - if (ret == -1) { - flb_plg_error(ctx->ins, "error appending oauth2 params"); - flb_sds_destroy(sig_data); - return -1; - } - flb_sds_destroy(sig_data); - - /* Retrieve access token */ - token = flb_oauth2_token_get(ctx->o); - if (!token) { - flb_plg_error(ctx->ins, "error retrieving oauth2 access token"); - return -1; - } - - return 0; -} - -static flb_sds_t get_google_token(struct flb_stackdriver *ctx) -{ - int ret = 0; - flb_sds_t output = NULL; - time_t cached_expiration = 0; - - ret = pthread_mutex_trylock(&ctx->token_mutex); - if (ret == EBUSY) { - /* - * If the routine is locked we just use our pre-cached values and - * compose the expected authorization value. - * - * If the routine fails it will return NULL and the caller will just - * issue a FLB_RETRY. - */ - output = oauth2_cache_to_token(); - cached_expiration = oauth2_cache_get_expiration(); - if (time(NULL) >= cached_expiration) { - return output; - } else { - /* - * Cached token is expired. Wait on lock to use up-to-date token - * by either waiting for it to be refreshed or refresh it ourselves. - */ - flb_plg_info(ctx->ins, "Cached token is expired. Waiting on lock."); - ret = pthread_mutex_lock(&ctx->token_mutex); - } - } - - if (ret != 0) { - flb_plg_error(ctx->ins, "error locking mutex"); - return NULL; - } - - if (flb_oauth2_token_expired(ctx->o) == FLB_TRUE) { - ret = get_oauth2_token(ctx); - } - - /* Copy string to prevent race conditions (get_oauth2 can free the string) */ - if (ret == 0) { - /* Update pthread keys cached values */ - oauth2_cache_set(ctx->o->token_type, ctx->o->access_token, ctx->o->expires); - - /* Compose outgoing buffer using cached values */ - output = oauth2_cache_to_token(); - } - - if (pthread_mutex_unlock(&ctx->token_mutex)){ - flb_plg_error(ctx->ins, "error unlocking mutex"); - if (output) { - flb_sds_destroy(output); - } - return NULL; - } - - - return output; -} - -void replace_prefix_dot(flb_sds_t s, int tag_prefix_len) -{ - int i; - int str_len; - char c; - - if (!s) { - return; - } - - str_len = flb_sds_len(s); - if (tag_prefix_len > str_len) { - flb_error("[output] tag_prefix shouldn't be longer than local_resource_id"); - return; - } - - for (i = 0; i < tag_prefix_len; i++) { - c = s[i]; - - if (c == '.') { - s[i] = '_'; - } - } -} - -static flb_sds_t get_str_value_from_msgpack_map(msgpack_object_map map, - const char *key, int key_size) -{ - int i; - msgpack_object k; - msgpack_object v; - flb_sds_t ptr = NULL; - - for (i = 0; i < map.size; i++) { - k = map.ptr[i].key; - v = map.ptr[i].val; - - if (k.type != MSGPACK_OBJECT_STR) { - continue; - } - - if (k.via.str.size == key_size && - strncmp(key, (char *) k.via.str.ptr, k.via.str.size) == 0) { - /* make sure to free it after use */ - ptr = flb_sds_create_len(v.via.str.ptr, v.via.str.size); - break; - } - } - - return ptr; -} - -/* parse_monitored_resource is to extract the monitoired resource labels - * from "logging.googleapis.com/monitored_resource" in log data - * and append to 'resource'/'labels' in log entry. - * Monitored resource type is already read from resource field in stackdriver - * output plugin configuration parameters. - * - * The structure of monitored_resource is: - * { - * "logging.googleapis.com/monitored_resource": { - * "labels": { - * "resource_label": , - * } - * } - * } - * See https://cloud.google.com/logging/docs/api/v2/resource-list#resource-types - * for required labels for each monitored resource. - */ - -static int parse_monitored_resource(struct flb_stackdriver *ctx, const void *data, size_t bytes, msgpack_packer *mp_pck) -{ - int ret = -1; - msgpack_object *obj; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return -1; - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - obj = log_event.body; - - msgpack_object_kv *kv = obj->via.map.ptr; - msgpack_object_kv *const kvend = obj->via.map.ptr + obj->via.map.size; - for (; kv < kvend; ++kv) { - if (kv->val.type == MSGPACK_OBJECT_MAP && kv->key.type == MSGPACK_OBJECT_STR - && strncmp (MONITORED_RESOURCE_KEY, kv->key.via.str.ptr, kv->key.via.str.size) == 0) { - msgpack_object subobj = kv->val; - msgpack_object_kv *p = subobj.via.map.ptr; - msgpack_object_kv *pend = subobj.via.map.ptr + subobj.via.map.size; - for (; p < pend; ++p) { - if (p->key.type != MSGPACK_OBJECT_STR || p->val.type != MSGPACK_OBJECT_MAP) { - continue; - } - if (strncmp("labels", p->key.via.str.ptr, p->key.via.str.size) == 0) { - msgpack_object labels = p->val; - msgpack_object_kv *q = labels.via.map.ptr; - msgpack_object_kv *qend = labels.via.map.ptr + labels.via.map.size; - int fields = 0; - for (; q < qend; ++q) { - if (q->key.type != MSGPACK_OBJECT_STR || q->val.type != MSGPACK_OBJECT_STR) { - flb_plg_error(ctx->ins, "Key and value should be string in the %s/labels", MONITORED_RESOURCE_KEY); - } - ++fields; - } - if (fields > 0) { - msgpack_pack_map(mp_pck, fields); - q = labels.via.map.ptr; - for (; q < qend; ++q) { - if (q->key.type != MSGPACK_OBJECT_STR || q->val.type != MSGPACK_OBJECT_STR) { - continue; - } - flb_plg_debug(ctx->ins, "[%s] found in the payload", MONITORED_RESOURCE_KEY); - msgpack_pack_str(mp_pck, q->key.via.str.size); - msgpack_pack_str_body(mp_pck, q->key.via.str.ptr, q->key.via.str.size); - msgpack_pack_str(mp_pck, q->val.via.str.size); - msgpack_pack_str_body(mp_pck, q->val.via.str.ptr, q->val.via.str.size); - } - - flb_log_event_decoder_destroy(&log_decoder); - - return 0; - } - } - } - } - } - } - - flb_log_event_decoder_destroy(&log_decoder); - - flb_plg_debug(ctx->ins, "[%s] not found in the payload", MONITORED_RESOURCE_KEY); - - return ret; -} - -/* - * Given a local_resource_id, split the content using the proper separator generating - * a linked list to store the spliited string - */ -static struct mk_list *parse_local_resource_id_to_list(char *local_resource_id, char *type) -{ - int ret = -1; - int max_split = -1; - int len_k8s_container; - int len_k8s_node; - int len_k8s_pod; - struct mk_list *list; - - len_k8s_container = sizeof(K8S_CONTAINER) - 1; - len_k8s_node = sizeof(K8S_NODE) - 1; - len_k8s_pod = sizeof(K8S_POD) - 1; - - /* Allocate list head */ - list = flb_malloc(sizeof(struct mk_list)); - if (!list) { - flb_errno(); - return NULL; - } - mk_list_init(list); - - /* Determinate the max split value based on type */ - if (strncmp(type, K8S_CONTAINER, len_k8s_container) == 0) { - /* including the prefix of tag */ - max_split = 4; - } - else if (strncmp(type, K8S_NODE, len_k8s_node) == 0) { - max_split = 2; - } - else if (strncmp(type, K8S_POD, len_k8s_pod) == 0) { - max_split = 3; - } - - /* The local_resource_id is splitted by '.' */ - ret = flb_slist_split_string(list, local_resource_id, '.', max_split); - - if (ret == -1 || mk_list_size(list) != max_split) { - flb_error("error parsing local_resource_id [%s] for type %s", local_resource_id, type); - flb_slist_destroy(list); - flb_free(list); - return NULL; - } - - return list; -} - -/* - * extract_local_resource_id(): - * - extract the value from "logging.googleapis.com/local_resource_id" field - * - if local_resource_id is missing from the payLoad, use the tag of the log - */ -static int extract_local_resource_id(const void *data, size_t bytes, - struct flb_stackdriver *ctx, const char *tag) { - msgpack_object_map map; - flb_sds_t local_resource_id; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - int ret; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return -1; - } - - if ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - map = log_event.body->via.map; - local_resource_id = get_str_value_from_msgpack_map(map, LOCAL_RESOURCE_ID_KEY, - LEN_LOCAL_RESOURCE_ID_KEY); - - if (local_resource_id == NULL) { - /* if local_resource_id is not found, use the tag of the log */ - flb_plg_debug(ctx->ins, "local_resource_id not found, " - "tag [%s] is assigned for local_resource_id", tag); - local_resource_id = flb_sds_create(tag); - } - - /* we need to create up the local_resource_id from previous log */ - if (ctx->local_resource_id) { - flb_sds_destroy(ctx->local_resource_id); - } - - ctx->local_resource_id = flb_sds_create(local_resource_id); - - flb_sds_destroy(local_resource_id); - - ret = 0; - } - else { - flb_plg_error(ctx->ins, "failed to unpack data"); - - ret = -1; - } - - flb_log_event_decoder_destroy(&log_decoder); - - return ret; -} - -/* - * set_monitored_resource_labels(): - * - use the extracted local_resource_id to assign the label keys for different - * resource types that are specified in the configuration of stackdriver_out plugin - */ -static int set_monitored_resource_labels(struct flb_stackdriver *ctx, char *type) -{ - int ret = -1; - int first = FLB_TRUE; - int counter = 0; - int len_k8s_container; - int len_k8s_node; - int len_k8s_pod; - size_t prefix_len = 0; - struct local_resource_id_list *ptr; - struct mk_list *list = NULL; - struct mk_list *head; - flb_sds_t new_local_resource_id; - - if (!ctx->local_resource_id) { - flb_plg_error(ctx->ins, "local_resource_is is not assigned"); - return -1; - } - - len_k8s_container = sizeof(K8S_CONTAINER) - 1; - len_k8s_node = sizeof(K8S_NODE) - 1; - len_k8s_pod = sizeof(K8S_POD) - 1; - - prefix_len = flb_sds_len(ctx->tag_prefix); - if (flb_sds_casecmp(ctx->tag_prefix, ctx->local_resource_id, prefix_len) != 0) { - flb_plg_error(ctx->ins, "tag_prefix [%s] doesn't match the prefix of" - " local_resource_id [%s]", ctx->tag_prefix, - ctx->local_resource_id); - return -1; - } - - new_local_resource_id = flb_sds_create_len(ctx->local_resource_id, - flb_sds_len(ctx->local_resource_id)); - replace_prefix_dot(new_local_resource_id, prefix_len - 1); - - if (strncmp(type, K8S_CONTAINER, len_k8s_container) == 0) { - list = parse_local_resource_id_to_list(new_local_resource_id, K8S_CONTAINER); - if (!list) { - goto error; - } - - /* iterate through the list */ - mk_list_foreach(head, list) { - ptr = mk_list_entry(head, struct local_resource_id_list, _head); - if (first) { - first = FLB_FALSE; - continue; - } - - /* Follow the order of fields in local_resource_id */ - if (counter == 0) { - if (ctx->namespace_name) { - flb_sds_destroy(ctx->namespace_name); - } - ctx->namespace_name = flb_sds_create(ptr->val); - } - else if (counter == 1) { - if (ctx->pod_name) { - flb_sds_destroy(ctx->pod_name); - } - ctx->pod_name = flb_sds_create(ptr->val); - } - else if (counter == 2) { - if (ctx->container_name) { - flb_sds_destroy(ctx->container_name); - } - ctx->container_name = flb_sds_create(ptr->val); - } - - counter++; - } - - if (!ctx->namespace_name || !ctx->pod_name || !ctx->container_name) { - goto error; - } - } - else if (strncmp(type, K8S_NODE, len_k8s_node) == 0) { - list = parse_local_resource_id_to_list(new_local_resource_id, K8S_NODE); - if (!list) { - goto error; - } - - mk_list_foreach(head, list) { - ptr = mk_list_entry(head, struct local_resource_id_list, _head); - if (first) { - first = FLB_FALSE; - continue; - } - - if (ptr != NULL) { - if (ctx->node_name) { - flb_sds_destroy(ctx->node_name); - } - ctx->node_name = flb_sds_create(ptr->val); - } - } - - if (!ctx->node_name) { - goto error; - } - } - else if (strncmp(type, K8S_POD, len_k8s_pod) == 0) { - list = parse_local_resource_id_to_list(new_local_resource_id, K8S_POD); - if (!list) { - goto error; - } - - mk_list_foreach(head, list) { - ptr = mk_list_entry(head, struct local_resource_id_list, _head); - if (first) { - first = FLB_FALSE; - continue; - } - - /* Follow the order of fields in local_resource_id */ - if (counter == 0) { - if (ctx->namespace_name) { - flb_sds_destroy(ctx->namespace_name); - } - ctx->namespace_name = flb_sds_create(ptr->val); - } - else if (counter == 1) { - if (ctx->pod_name) { - flb_sds_destroy(ctx->pod_name); - } - ctx->pod_name = flb_sds_create(ptr->val); - } - - counter++; - } - - if (!ctx->namespace_name || !ctx->pod_name) { - goto error; - } - } - - ret = 0; - - if (list) { - flb_slist_destroy(list); - flb_free(list); - } - flb_sds_destroy(new_local_resource_id); - - return ret; - - error: - if (list) { - flb_slist_destroy(list); - flb_free(list); - } - - if (strncmp(type, K8S_CONTAINER, len_k8s_container) == 0) { - if (ctx->namespace_name) { - flb_sds_destroy(ctx->namespace_name); - } - - if (ctx->pod_name) { - flb_sds_destroy(ctx->pod_name); - } - - if (ctx->container_name) { - flb_sds_destroy(ctx->container_name); - } - } - else if (strncmp(type, K8S_NODE, len_k8s_node) == 0) { - if (ctx->node_name) { - flb_sds_destroy(ctx->node_name); - } - } - else if (strncmp(type, K8S_POD, len_k8s_pod) == 0) { - if (ctx->namespace_name) { - flb_sds_destroy(ctx->namespace_name); - } - - if (ctx->pod_name) { - flb_sds_destroy(ctx->pod_name); - } - } - - flb_sds_destroy(new_local_resource_id); - return -1; -} - -static int is_tag_match_regex(struct flb_stackdriver *ctx, - const char *tag, int tag_len) -{ - int ret; - int tag_prefix_len; - int len_to_be_matched; - const char *tag_str_to_be_matcheds; - - tag_prefix_len = flb_sds_len(ctx->tag_prefix); - if (tag_len > tag_prefix_len && - flb_sds_cmp(ctx->tag_prefix, tag, tag_prefix_len) != 0) { - return 0; - } - - tag_str_to_be_matcheds = tag + tag_prefix_len; - len_to_be_matched = tag_len - tag_prefix_len; - ret = flb_regex_match(ctx->regex, - (unsigned char *) tag_str_to_be_matcheds, - len_to_be_matched); - - /* 1 -> match; 0 -> doesn't match; < 0 -> error */ - return ret; -} - -static int is_local_resource_id_match_regex(struct flb_stackdriver *ctx) -{ - int ret; - int prefix_len; - int len_to_be_matched; - const char *str_to_be_matcheds; - - if (!ctx->local_resource_id) { - flb_plg_warn(ctx->ins, "local_resource_id not found in the payload"); - return -1; - } - - prefix_len = flb_sds_len(ctx->tag_prefix); - str_to_be_matcheds = ctx->local_resource_id + prefix_len; - len_to_be_matched = flb_sds_len(ctx->local_resource_id) - prefix_len; - - ret = flb_regex_match(ctx->regex, - (unsigned char *) str_to_be_matcheds, - len_to_be_matched); - - /* 1 -> match; 0 -> doesn't match; < 0 -> error */ - return ret; -} - -static void cb_results(const char *name, const char *value, - size_t vlen, void *data); -/* - * extract_resource_labels_from_regex(4) will only be called if the - * tag or local_resource_id field matches the regex rule - */ -static int extract_resource_labels_from_regex(struct flb_stackdriver *ctx, - const char *tag, int tag_len, - int from_tag) -{ - int ret = 1; - int prefix_len; - int len_to_be_matched; - int local_resource_id_len; - const char *str_to_be_matcheds; - struct flb_regex_search result; - - prefix_len = flb_sds_len(ctx->tag_prefix); - if (from_tag == FLB_TRUE) { - local_resource_id_len = tag_len; - str_to_be_matcheds = tag + prefix_len; - } - else { - // this will be called only if the payload contains local_resource_id - local_resource_id_len = flb_sds_len(ctx->local_resource_id); - str_to_be_matcheds = ctx->local_resource_id + prefix_len; - } - - len_to_be_matched = local_resource_id_len - prefix_len; - ret = flb_regex_do(ctx->regex, str_to_be_matcheds, len_to_be_matched, &result); - if (ret <= 0) { - flb_plg_warn(ctx->ins, "invalid pattern for given value %s when" - " extracting resource labels", str_to_be_matcheds); - return -1; - } - - flb_regex_parse(ctx->regex, &result, cb_results, ctx); - - return ret; -} - -static int process_local_resource_id(struct flb_stackdriver *ctx, - const char *tag, int tag_len, char *type) -{ - int ret; - - // parsing local_resource_id from tag takes higher priority - if (is_tag_match_regex(ctx, tag, tag_len) > 0) { - ret = extract_resource_labels_from_regex(ctx, tag, tag_len, FLB_TRUE); - } - else if (is_local_resource_id_match_regex(ctx) > 0) { - ret = extract_resource_labels_from_regex(ctx, tag, tag_len, FLB_FALSE); - } - else { - ret = set_monitored_resource_labels(ctx, type); - } - - return ret; -} - -/* - * get_payload_labels - * - Iterate throught the original payload (obj) and find out the entry that matches - * the labels_key - * - Used to convert all labels under labels_key to root-level `labels` field - */ -static msgpack_object *get_payload_labels(struct flb_stackdriver *ctx, msgpack_object *obj) -{ - int i; - int len; - msgpack_object_kv *kv = NULL; - - if (!obj || obj->type != MSGPACK_OBJECT_MAP) { - return NULL; - } - - len = flb_sds_len(ctx->labels_key); - for (i = 0; i < obj->via.map.size; i++) { - kv = &obj->via.map.ptr[i]; - if (flb_sds_casecmp(ctx->labels_key, kv->key.via.str.ptr, len) == 0) { - /* only the first matching entry will be returned */ - return &kv->val; - } - } - - //flb_plg_debug(ctx->ins, "labels_key [%s] not found in the payload", - // ctx->labels_key); - return NULL; -} - -/* - * pack_resource_labels(): - * - Looks through the resource_labels parameter and appends new key value - * pair to the log entry. - * - Supports field access, plaintext assignment and environment variables. - */ -static int pack_resource_labels(struct flb_stackdriver *ctx, - struct flb_mp_map_header *mh, - msgpack_packer *mp_pck, - const void *data, - size_t bytes) -{ - struct mk_list *head; - struct flb_kv *label_kv; - struct flb_record_accessor *ra; - struct flb_ra_value *rval; - int len; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - int ret; - - if (ctx->should_skip_resource_labels_api == FLB_TRUE) { - return -1; - } - - len = mk_list_size(&ctx->resource_labels_kvs); - if (len == 0) { - return -1; - } - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return -1; - } - - if ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - - flb_mp_map_header_init(mh, mp_pck); - mk_list_foreach(head, &ctx->resource_labels_kvs) { - label_kv = mk_list_entry(head, struct flb_kv, _head); - /* - * KVs have the form destination=original, so the original key is the value. - * If the value starts with '$', it will be processed using record accessor. - * Otherwise, it will be treated as a plaintext assignment. - */ - if (label_kv->val[0] == '$') { - ra = flb_ra_create(label_kv->val, FLB_TRUE); - rval = flb_ra_get_value_object(ra, *log_event.body); - - if (rval != NULL && rval->o.type == MSGPACK_OBJECT_STR) { - flb_mp_map_header_append(mh); - msgpack_pack_str(mp_pck, flb_sds_len(label_kv->key)); - msgpack_pack_str_body(mp_pck, label_kv->key, - flb_sds_len(label_kv->key)); - msgpack_pack_str(mp_pck, flb_sds_len(rval->val.string)); - msgpack_pack_str_body(mp_pck, rval->val.string, - flb_sds_len(rval->val.string)); - flb_ra_key_value_destroy(rval); - } else { - flb_plg_warn(ctx->ins, "failed to find a corresponding entry for " - "resource label entry [%s=%s]", label_kv->key, label_kv->val); - } - flb_ra_destroy(ra); - } else { - flb_mp_map_header_append(mh); - msgpack_pack_str(mp_pck, flb_sds_len(label_kv->key)); - msgpack_pack_str_body(mp_pck, label_kv->key, - flb_sds_len(label_kv->key)); - msgpack_pack_str(mp_pck, flb_sds_len(label_kv->val)); - msgpack_pack_str_body(mp_pck, label_kv->val, - flb_sds_len(label_kv->val)); - } - } - } - else { - flb_plg_error(ctx->ins, "failed to unpack data"); - - flb_log_event_decoder_destroy(&log_decoder); - - return -1; - } - - /* project_id should always be packed from config parameter */ - flb_mp_map_header_append(mh); - msgpack_pack_str(mp_pck, 10); - msgpack_pack_str_body(mp_pck, "project_id", 10); - msgpack_pack_str(mp_pck, flb_sds_len(ctx->project_id)); - msgpack_pack_str_body(mp_pck, - ctx->project_id, flb_sds_len(ctx->project_id)); - - flb_log_event_decoder_destroy(&log_decoder); - flb_mp_map_header_end(mh); - - return 0; -} - -static void pack_labels(struct flb_stackdriver *ctx, - msgpack_packer *mp_pck, - msgpack_object *payload_labels_ptr) -{ - int i; - int labels_size = 0; - struct mk_list *head; - struct flb_kv *list_kv; - msgpack_object_kv *obj_kv = NULL; - - /* Determine size of labels map */ - labels_size = mk_list_size(&ctx->config_labels); - if (payload_labels_ptr != NULL && - payload_labels_ptr->type == MSGPACK_OBJECT_MAP) { - labels_size += payload_labels_ptr->via.map.size; - } - - msgpack_pack_map(mp_pck, labels_size); - - /* pack labels from the payload */ - if (payload_labels_ptr != NULL && - payload_labels_ptr->type == MSGPACK_OBJECT_MAP) { - - for (i = 0; i < payload_labels_ptr->via.map.size; i++) { - obj_kv = &payload_labels_ptr->via.map.ptr[i]; - msgpack_pack_object(mp_pck, obj_kv->key); - msgpack_pack_object(mp_pck, obj_kv->val); - } - } - - /* pack labels set in configuration */ - /* in msgpack duplicate keys are overriden by the last set */ - /* static label keys override payload labels */ - mk_list_foreach(head, &ctx->config_labels){ - list_kv = mk_list_entry(head, struct flb_kv, _head); - msgpack_pack_str(mp_pck, flb_sds_len(list_kv->key)); - msgpack_pack_str_body(mp_pck, list_kv->key, flb_sds_len(list_kv->key)); - msgpack_pack_str(mp_pck, flb_sds_len(list_kv->val)); - msgpack_pack_str_body(mp_pck, list_kv->val, flb_sds_len(list_kv->val)); - } -} - -static void cb_results(const char *name, const char *value, - size_t vlen, void *data) -{ - struct flb_stackdriver *ctx = data; - - if (vlen == 0) { - return; - } - - if (strcmp(name, "pod_name") == 0) { - if (ctx->pod_name != NULL) { - flb_sds_destroy(ctx->pod_name); - } - ctx->pod_name = flb_sds_create_len(value, vlen); - } - else if (strcmp(name, "namespace_name") == 0) { - if (ctx->namespace_name != NULL) { - flb_sds_destroy(ctx->namespace_name); - } - ctx->namespace_name = flb_sds_create_len(value, vlen); - } - else if (strcmp(name, "container_name") == 0) { - if (ctx->container_name != NULL) { - flb_sds_destroy(ctx->container_name); - } - ctx->container_name = flb_sds_create_len(value, vlen); - } - else if (strcmp(name, "node_name") == 0) { - if (ctx->node_name != NULL) { - flb_sds_destroy(ctx->node_name); - } - ctx->node_name = flb_sds_create_len(value, vlen); - } - - return; -} - -int flb_stackdriver_regex_init(struct flb_stackdriver *ctx) -{ - /* If a custom regex is not set, use the defaults */ - ctx->regex = flb_regex_create(ctx->custom_k8s_regex); - if (!ctx->regex) { - return -1; - } - - return 0; -} - -static int cb_stackdriver_init(struct flb_output_instance *ins, - struct flb_config *config, void *data) -{ - int ret; - int io_flags = FLB_IO_TLS; - char *token; - struct flb_stackdriver *ctx; - - /* Create config context */ - ctx = flb_stackdriver_conf_create(ins, config); - if (!ctx) { - flb_plg_error(ins, "configuration failed"); - return -1; - } - - /* Load config map */ - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - return -1; - } - - /* Set context */ - flb_output_set_context(ins, ctx); - - /* Network mode IPv6 */ - if (ins->host.ipv6 == FLB_TRUE) { - io_flags |= FLB_IO_IPV6; - } - - /* Initialize oauth2 cache pthread keys */ - oauth2_cache_init(); - - /* Create mutex for acquiring oauth tokens (they are shared across flush coroutines) */ - pthread_mutex_init(&ctx->token_mutex, NULL); - - /* Create Upstream context for Stackdriver Logging (no oauth2 service) */ - ctx->u = flb_upstream_create_url(config, FLB_STD_WRITE_URL, - io_flags, ins->tls); - ctx->metadata_u = flb_upstream_create_url(config, ctx->metadata_server, - FLB_IO_TCP, NULL); - - /* Create oauth2 context */ - ctx->o = flb_oauth2_create(ctx->config, FLB_STD_AUTH_URL, 3000); - - if (!ctx->u) { - flb_plg_error(ctx->ins, "upstream creation failed"); - return -1; - } - if (!ctx->metadata_u) { - flb_plg_error(ctx->ins, "metadata upstream creation failed"); - return -1; - } - if (!ctx->o) { - flb_plg_error(ctx->ins, "cannot create oauth2 context"); - return -1; - } - flb_output_upstream_set(ctx->u, ins); - - /* Metadata Upstream Sync flags */ - flb_stream_disable_async_mode(&ctx->metadata_u->base); - - if (ins->test_mode == FLB_FALSE) { - /* Retrieve oauth2 token */ - token = get_google_token(ctx); - if (!token) { - flb_plg_warn(ctx->ins, "token retrieval failed"); - } - else { - flb_sds_destroy(token); - } - } - - if (ctx->metadata_server_auth) { - ret = gce_metadata_read_project_id(ctx); - if (ret == -1) { - return -1; - } - - if (ctx->resource_type != RESOURCE_TYPE_GENERIC_NODE - && ctx->resource_type != RESOURCE_TYPE_GENERIC_TASK) { - ret = gce_metadata_read_zone(ctx); - if (ret == -1) { - return -1; - } - - ret = gce_metadata_read_instance_id(ctx); - if (ret == -1) { - return -1; - } - } - } - - /* Validate project_id */ - if (!ctx->project_id) { - flb_plg_error(ctx->ins, "property 'project_id' is not set"); - return -1; - } - - if (!ctx->export_to_project_id) { - ctx->export_to_project_id = ctx->project_id; - } - - ret = flb_stackdriver_regex_init(ctx); - if (ret == -1) { - flb_plg_error(ctx->ins, "failed to init stackdriver custom regex"); - return -1; - } - - return 0; -} - -static int validate_severity_level(severity_t * s, - const char * str, - const unsigned int str_size) -{ - int i = 0; - - const static struct { - severity_t s; - const unsigned int str_size; - const char * str; - } enum_mapping[] = { - {FLB_STD_EMERGENCY, 9, "EMERGENCY"}, - {FLB_STD_EMERGENCY, 5, "EMERG" }, - - {FLB_STD_ALERT , 1, "A" }, - {FLB_STD_ALERT , 5, "ALERT" }, - - {FLB_STD_CRITICAL , 1, "C" }, - {FLB_STD_CRITICAL , 1, "F" }, - {FLB_STD_CRITICAL , 4, "CRIT" }, - {FLB_STD_CRITICAL , 5, "FATAL" }, - {FLB_STD_CRITICAL , 8, "CRITICAL" }, - - {FLB_STD_ERROR , 1, "E" }, - {FLB_STD_ERROR , 3, "ERR" }, - {FLB_STD_ERROR , 5, "ERROR" }, - {FLB_STD_ERROR , 6, "SEVERE" }, - - {FLB_STD_WARNING , 1, "W" }, - {FLB_STD_WARNING , 4, "WARN" }, - {FLB_STD_WARNING , 7, "WARNING" }, - - {FLB_STD_NOTICE , 1, "N" }, - {FLB_STD_NOTICE , 6, "NOTICE" }, - - {FLB_STD_INFO , 1, "I" }, - {FLB_STD_INFO , 4, "INFO" }, - - {FLB_STD_DEBUG , 1, "D" }, - {FLB_STD_DEBUG , 5, "DEBUG" }, - {FLB_STD_DEBUG , 5, "TRACE" }, - {FLB_STD_DEBUG , 9, "TRACE_INT"}, - {FLB_STD_DEBUG , 4, "FINE" }, - {FLB_STD_DEBUG , 5, "FINER" }, - {FLB_STD_DEBUG , 6, "FINEST" }, - {FLB_STD_DEBUG , 6, "CONFIG" }, - - {FLB_STD_DEFAULT , 7, "DEFAULT" } - }; - - for (i = 0; i < sizeof (enum_mapping) / sizeof (enum_mapping[0]); ++i) { - if (enum_mapping[i].str_size != str_size) { - continue; - } - - if (strncasecmp(str, enum_mapping[i].str, str_size) == 0) { - *s = enum_mapping[i].s; - return 0; - } - } - return -1; -} - -static int get_msgpack_obj(msgpack_object * subobj, const msgpack_object * o, - const flb_sds_t key, const int key_size, - msgpack_object_type type) -{ - int i = 0; - msgpack_object_kv * p = NULL; - - if (o == NULL || subobj == NULL) { - return -1; - } - - for (i = 0; i < o->via.map.size; i++) { - p = &o->via.map.ptr[i]; - if (p->val.type != type) { - continue; - } - - if (flb_sds_cmp(key, p->key.via.str.ptr, p->key.via.str.size) == 0) { - *subobj = p->val; - return 0; - } - } - return -1; -} - -static int get_string(flb_sds_t * s, const msgpack_object * o, const flb_sds_t key) -{ - msgpack_object tmp; - if (get_msgpack_obj(&tmp, o, key, flb_sds_len(key), MSGPACK_OBJECT_STR) == 0) { - *s = flb_sds_create_len(tmp.via.str.ptr, tmp.via.str.size); - return 0; - } - - *s = 0; - return -1; -} - -static int get_severity_level(severity_t * s, const msgpack_object * o, - const flb_sds_t key) -{ - msgpack_object tmp; - if (get_msgpack_obj(&tmp, o, key, flb_sds_len(key), MSGPACK_OBJECT_STR) == 0 - && validate_severity_level(s, tmp.via.str.ptr, tmp.via.str.size) == 0) { - return 0; - } - *s = 0; - return -1; -} - -static int get_trace_sampled(int * trace_sampled_value, const msgpack_object * src_obj, - const flb_sds_t key) -{ - msgpack_object tmp; - int ret = get_msgpack_obj(&tmp, src_obj, key, flb_sds_len(key), MSGPACK_OBJECT_BOOLEAN); - - if (ret == 0 && tmp.via.boolean == true) { - *trace_sampled_value = FLB_TRUE; - return 0; - } else if (ret == 0 && tmp.via.boolean == false) { - *trace_sampled_value = FLB_FALSE; - return 0; - } - - return -1; -} - -static insert_id_status validate_insert_id(msgpack_object * insert_id_value, - const msgpack_object * obj) -{ - int i = 0; - msgpack_object_kv * p = NULL; - insert_id_status ret = INSERTID_NOT_PRESENT; - - if (obj == NULL) { - return ret; - } - - for (i = 0; i < obj->via.map.size; i++) { - p = &obj->via.map.ptr[i]; - if (p->key.type != MSGPACK_OBJECT_STR) { - continue; - } - if (validate_key(p->key, DEFAULT_INSERT_ID_KEY, INSERT_ID_SIZE)) { - if (p->val.type == MSGPACK_OBJECT_STR && p->val.via.str.size > 0) { - *insert_id_value = p->val; - ret = INSERTID_VALID; - } - else { - ret = INSERTID_INVALID; - } - break; - } - } - return ret; -} - -static int pack_json_payload(int insert_id_extracted, - int operation_extracted, int operation_extra_size, - int source_location_extracted, - int source_location_extra_size, - int http_request_extracted, - int http_request_extra_size, - timestamp_status tms_status, - msgpack_packer *mp_pck, msgpack_object *obj, - struct flb_stackdriver *ctx) -{ - /* Specified fields include local_resource_id, operation, sourceLocation ... */ - int i, j; - int to_remove = 0; - int ret; - int map_size; - int new_map_size; - int len; - int len_to_be_removed; - int key_not_found; - flb_sds_t removed; - flb_sds_t monitored_resource_key; - flb_sds_t local_resource_id_key; - flb_sds_t stream; - msgpack_object_kv *kv = obj->via.map.ptr; - msgpack_object_kv *const kvend = obj->via.map.ptr + obj->via.map.size; - - monitored_resource_key = flb_sds_create(MONITORED_RESOURCE_KEY); - local_resource_id_key = flb_sds_create(LOCAL_RESOURCE_ID_KEY); - stream = flb_sds_create("stream"); - /* - * array of elements that need to be removed from payload, - * special field 'operation' will be processed individually - */ - flb_sds_t to_be_removed[] = - { - monitored_resource_key, - local_resource_id_key, - ctx->labels_key, - ctx->severity_key, - ctx->trace_key, - ctx->span_id_key, - ctx->trace_sampled_key, - ctx->log_name_key, - stream - /* more special fields are required to be added, but, if this grows with more - than a few records, it might need to be converted to flb_hash - */ - }; - - if (insert_id_extracted == FLB_TRUE) { - to_remove += 1; - } - if (operation_extracted == FLB_TRUE && operation_extra_size == 0) { - to_remove += 1; - } - if (source_location_extracted == FLB_TRUE && source_location_extra_size == 0) { - to_remove += 1; - } - if (http_request_extracted == FLB_TRUE && http_request_extra_size == 0) { - to_remove += 1; - } - if (tms_status == FORMAT_TIMESTAMP_OBJECT) { - to_remove += 1; - } - if (tms_status == FORMAT_TIMESTAMP_DUO_FIELDS) { - to_remove += 2; - } - - map_size = obj->via.map.size; - len_to_be_removed = sizeof(to_be_removed) / sizeof(to_be_removed[0]); - for (i = 0; i < map_size; i++) { - kv = &obj->via.map.ptr[i]; - len = kv->key.via.str.size; - for (j = 0; j < len_to_be_removed; j++) { - removed = to_be_removed[j]; - /* - * check length of key to avoid partial matching - * e.g. labels key = labels && kv->key = labelss - */ - if (removed && flb_sds_cmp(removed, kv->key.via.str.ptr, len) == 0) { - to_remove += 1; - break; - } - } - } - - new_map_size = map_size - to_remove; - - ret = msgpack_pack_map(mp_pck, new_map_size); - if (ret < 0) { - goto error; - } - - /* points back to the beginning of map */ - kv = obj->via.map.ptr; - for(; kv != kvend; ++kv ) { - key_not_found = 1; - - /* processing logging.googleapis.com/insertId */ - if (insert_id_extracted == FLB_TRUE - && validate_key(kv->key, DEFAULT_INSERT_ID_KEY, INSERT_ID_SIZE)) { - continue; - } - - /* processing logging.googleapis.com/operation */ - if (validate_key(kv->key, OPERATION_FIELD_IN_JSON, - OPERATION_KEY_SIZE) - && kv->val.type == MSGPACK_OBJECT_MAP) { - if (operation_extra_size > 0) { - msgpack_pack_object(mp_pck, kv->key); - pack_extra_operation_subfields(mp_pck, &kv->val, operation_extra_size); - } - continue; - } - - if (validate_key(kv->key, SOURCELOCATION_FIELD_IN_JSON, - SOURCE_LOCATION_SIZE) - && kv->val.type == MSGPACK_OBJECT_MAP) { - - if (source_location_extra_size > 0) { - msgpack_pack_object(mp_pck, kv->key); - pack_extra_source_location_subfields(mp_pck, &kv->val, - source_location_extra_size); - } - continue; - } - - if (validate_key(kv->key, ctx->http_request_key, - ctx->http_request_key_size) - && kv->val.type == MSGPACK_OBJECT_MAP) { - - if(http_request_extra_size > 0) { - msgpack_pack_object(mp_pck, kv->key); - pack_extra_http_request_subfields(mp_pck, &kv->val, - http_request_extra_size); - } - continue; - } - - if (validate_key(kv->key, "timestamp", 9) - && tms_status == FORMAT_TIMESTAMP_OBJECT) { - continue; - } - - if (validate_key(kv->key, "timestampSeconds", 16) - && tms_status == FORMAT_TIMESTAMP_DUO_FIELDS) { - continue; - } - if (validate_key(kv->key, "timestampNanos", 14) - && tms_status == FORMAT_TIMESTAMP_DUO_FIELDS) { - continue; - } - - len = kv->key.via.str.size; - for (j = 0; j < len_to_be_removed; j++) { - removed = to_be_removed[j]; - if (removed && flb_sds_cmp(removed, kv->key.via.str.ptr, len) == 0) { - key_not_found = 0; - break; - } - } - - if (key_not_found) { - ret = msgpack_pack_object(mp_pck, kv->key); - if (ret < 0) { - goto error; - } - ret = msgpack_pack_object(mp_pck, kv->val); - if (ret < 0) { - goto error; - } - } - } - - flb_sds_destroy(monitored_resource_key); - flb_sds_destroy(local_resource_id_key); - flb_sds_destroy(stream); - return 0; - - error: - flb_sds_destroy(monitored_resource_key); - flb_sds_destroy(local_resource_id_key); - flb_sds_destroy(stream); - return ret; -} - -static flb_sds_t stackdriver_format(struct flb_stackdriver *ctx, - int total_records, - const char *tag, int tag_len, - const void *data, size_t bytes) -{ - int len; - int ret; - int array_size = 0; - /* The default value is 3: timestamp, jsonPayload, logName. */ - int entry_size = 3; - size_t s; - // size_t off = 0; - char path[PATH_MAX]; - char time_formatted[255]; - const char *newtag; - const char *new_log_name; - msgpack_object *obj; - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; - flb_sds_t out_buf; - struct flb_mp_map_header mh; - - /* Parameters for severity */ - int severity_extracted = FLB_FALSE; - severity_t severity; - - /* Parameters for trace */ - int trace_extracted = FLB_FALSE; - flb_sds_t trace; - char stackdriver_trace[PATH_MAX]; - const char *new_trace; - - /* Parameters for span id */ - int span_id_extracted = FLB_FALSE; - flb_sds_t span_id; - - /* Parameters for trace sampled */ - int trace_sampled_extracted = FLB_FALSE; - int trace_sampled = FLB_FALSE; - - /* Parameters for log name */ - int log_name_extracted = FLB_FALSE; - flb_sds_t log_name = NULL; - flb_sds_t stream = NULL; - flb_sds_t stream_key; - - /* Parameters for insertId */ - msgpack_object insert_id_obj; - insert_id_status in_status; - int insert_id_extracted; - - /* Parameters in Operation */ - flb_sds_t operation_id; - flb_sds_t operation_producer; - int operation_first = FLB_FALSE; - int operation_last = FLB_FALSE; - int operation_extracted = FLB_FALSE; - int operation_extra_size = 0; - - /* Parameters for sourceLocation */ - flb_sds_t source_location_file; - int64_t source_location_line = 0; - flb_sds_t source_location_function; - int source_location_extracted = FLB_FALSE; - int source_location_extra_size = 0; - - /* Parameters for httpRequest */ - struct http_request_field http_request; - int http_request_extracted = FLB_FALSE; - int http_request_extra_size = 0; - - /* Parameters for Timestamp */ - struct tm tm; - // struct flb_time tms; - timestamp_status tms_status; - /* Count number of records */ - array_size = total_records; - - /* Parameters for labels */ - msgpack_object *payload_labels_ptr; - int labels_size = 0; - - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return NULL; - } - - /* - * Search each entry and validate insertId. - * Reject the entry if insertId is invalid. - * If all the entries are rejected, stop formatting. - * - */ - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - /* Extract insertId */ - in_status = validate_insert_id(&insert_id_obj, log_event.body); - - if (in_status == INSERTID_INVALID) { - flb_plg_error(ctx->ins, - "Incorrect insertId received. InsertId should be non-empty string."); - array_size -= 1; - } - } - - flb_log_event_decoder_destroy(&log_decoder); - - /* Sounds like this should compare to -1 instead of zero */ - if (array_size == 0) { - return NULL; - } - - /* Create temporal msgpack buffer */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - /* - * Pack root map (resource & entries): - * - * {"resource": {"type": "...", "labels": {...}, - * "entries": [] - */ - msgpack_pack_map(&mp_pck, 2); - - msgpack_pack_str(&mp_pck, 8); - msgpack_pack_str_body(&mp_pck, "resource", 8); - - /* type & labels */ - msgpack_pack_map(&mp_pck, 2); - - /* type */ - msgpack_pack_str(&mp_pck, 4); - msgpack_pack_str_body(&mp_pck, "type", 4); - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->resource)); - msgpack_pack_str_body(&mp_pck, ctx->resource, - flb_sds_len(ctx->resource)); - - msgpack_pack_str(&mp_pck, 6); - msgpack_pack_str_body(&mp_pck, "labels", 6); - - ret = pack_resource_labels(ctx, &mh, &mp_pck, data, bytes); - if (ret != 0) { - if (ctx->resource_type == RESOURCE_TYPE_K8S) { - ret = extract_local_resource_id(data, bytes, ctx, tag); - if (ret != 0) { - flb_plg_error(ctx->ins, "fail to construct local_resource_id"); - msgpack_sbuffer_destroy(&mp_sbuf); - return NULL; - } - } - ret = parse_monitored_resource(ctx, data, bytes, &mp_pck); - if (ret != 0) { - if (strcmp(ctx->resource, "global") == 0) { - /* global resource has field project_id */ - msgpack_pack_map(&mp_pck, 1); - msgpack_pack_str(&mp_pck, 10); - msgpack_pack_str_body(&mp_pck, "project_id", 10); - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->project_id)); - msgpack_pack_str_body(&mp_pck, - ctx->project_id, flb_sds_len(ctx->project_id)); - } - else if (ctx->resource_type == RESOURCE_TYPE_GENERIC_NODE - || ctx->resource_type == RESOURCE_TYPE_GENERIC_TASK) { - flb_mp_map_header_init(&mh, &mp_pck); - - if (ctx->resource_type == RESOURCE_TYPE_GENERIC_NODE && ctx->node_id) { - /* generic_node has fields project_id, location, namespace, node_id */ - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 7); - msgpack_pack_str_body(&mp_pck, "node_id", 7); - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->node_id)); - msgpack_pack_str_body(&mp_pck, - ctx->node_id, flb_sds_len(ctx->node_id)); - } - else { - /* generic_task has fields project_id, location, namespace, job, task_id */ - if (ctx->job) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 3); - msgpack_pack_str_body(&mp_pck, "job", 3); - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->job)); - msgpack_pack_str_body(&mp_pck, - ctx->job, flb_sds_len(ctx->job)); - } - - if (ctx->task_id) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 7); - msgpack_pack_str_body(&mp_pck, "task_id", 7); - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->task_id)); - msgpack_pack_str_body(&mp_pck, - ctx->task_id, flb_sds_len(ctx->task_id)); - } - } - - if (ctx->project_id) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 10); - msgpack_pack_str_body(&mp_pck, "project_id", 10); - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->project_id)); - msgpack_pack_str_body(&mp_pck, - ctx->project_id, flb_sds_len(ctx->project_id)); - } - - if (ctx->location) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 8); - msgpack_pack_str_body(&mp_pck, "location", 8); - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->location)); - msgpack_pack_str_body(&mp_pck, - ctx->location, flb_sds_len(ctx->location)); - } - - if (ctx->namespace_id) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 9); - msgpack_pack_str_body(&mp_pck, "namespace", 9); - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->namespace_id)); - msgpack_pack_str_body(&mp_pck, - ctx->namespace_id, flb_sds_len(ctx->namespace_id)); - } - - flb_mp_map_header_end(&mh); - } - else if (strcmp(ctx->resource, "gce_instance") == 0) { - /* gce_instance resource has fields project_id, zone, instance_id */ - flb_mp_map_header_init(&mh, &mp_pck); - - if (ctx->project_id) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 10); - msgpack_pack_str_body(&mp_pck, "project_id", 10); - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->project_id)); - msgpack_pack_str_body(&mp_pck, - ctx->project_id, flb_sds_len(ctx->project_id)); - } - - if (ctx->zone) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 4); - msgpack_pack_str_body(&mp_pck, "zone", 4); - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->zone)); - msgpack_pack_str_body(&mp_pck, ctx->zone, flb_sds_len(ctx->zone)); - } - - if (ctx->instance_id) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 11); - msgpack_pack_str_body(&mp_pck, "instance_id", 11); - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->instance_id)); - msgpack_pack_str_body(&mp_pck, - ctx->instance_id, flb_sds_len(ctx->instance_id)); - } - flb_mp_map_header_end(&mh); - } - else if (strcmp(ctx->resource, K8S_CONTAINER) == 0) { - /* k8s_container resource has fields project_id, location, cluster_name, - * namespace_name, pod_name, container_name - * - * The local_resource_id for k8s_container is in format: - * k8s_container... - */ - - ret = process_local_resource_id(ctx, tag, tag_len, K8S_CONTAINER); - if (ret == -1) { - flb_plg_error(ctx->ins, "fail to extract resource labels " - "for k8s_container resource type"); - msgpack_sbuffer_destroy(&mp_sbuf); - return NULL; - } - - flb_mp_map_header_init(&mh, &mp_pck); - - if (ctx->project_id) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 10); - msgpack_pack_str_body(&mp_pck, "project_id", 10); - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->project_id)); - msgpack_pack_str_body(&mp_pck, - ctx->project_id, flb_sds_len(ctx->project_id)); - } - - if (ctx->cluster_location) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 8); - msgpack_pack_str_body(&mp_pck, "location", 8); - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->cluster_location)); - msgpack_pack_str_body(&mp_pck, - ctx->cluster_location, - flb_sds_len(ctx->cluster_location)); - } - - if (ctx->cluster_name) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 12); - msgpack_pack_str_body(&mp_pck, "cluster_name", 12); - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->cluster_name)); - msgpack_pack_str_body(&mp_pck, - ctx->cluster_name, flb_sds_len(ctx->cluster_name)); - } - - if (ctx->namespace_name) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 14); - msgpack_pack_str_body(&mp_pck, "namespace_name", 14); - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->namespace_name)); - msgpack_pack_str_body(&mp_pck, - ctx->namespace_name, - flb_sds_len(ctx->namespace_name)); - } - - if (ctx->pod_name) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 8); - msgpack_pack_str_body(&mp_pck, "pod_name", 8); - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->pod_name)); - msgpack_pack_str_body(&mp_pck, - ctx->pod_name, flb_sds_len(ctx->pod_name)); - } - - if (ctx->container_name) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 14); - msgpack_pack_str_body(&mp_pck, "container_name", 14); - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->container_name)); - msgpack_pack_str_body(&mp_pck, - ctx->container_name, - flb_sds_len(ctx->container_name)); - } - - flb_mp_map_header_end(&mh); - } - else if (strcmp(ctx->resource, K8S_NODE) == 0) { - /* k8s_node resource has fields project_id, location, cluster_name, node_name - * - * The local_resource_id for k8s_node is in format: - * k8s_node. - */ - - ret = process_local_resource_id(ctx, tag, tag_len, K8S_NODE); - if (ret == -1) { - flb_plg_error(ctx->ins, "fail to process local_resource_id from " - "log entry for k8s_node"); - msgpack_sbuffer_destroy(&mp_sbuf); - return NULL; - } - - flb_mp_map_header_init(&mh, &mp_pck); - - if (ctx->project_id) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 10); - msgpack_pack_str_body(&mp_pck, "project_id", 10); - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->project_id)); - msgpack_pack_str_body(&mp_pck, - ctx->project_id, flb_sds_len(ctx->project_id)); - } - - if (ctx->cluster_location) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 8); - msgpack_pack_str_body(&mp_pck, "location", 8); - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->cluster_location)); - msgpack_pack_str_body(&mp_pck, - ctx->cluster_location, - flb_sds_len(ctx->cluster_location)); - } - - if (ctx->cluster_name) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 12); - msgpack_pack_str_body(&mp_pck, "cluster_name", 12); - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->cluster_name)); - msgpack_pack_str_body(&mp_pck, - ctx->cluster_name, flb_sds_len(ctx->cluster_name)); - } - - if (ctx->node_name) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 9); - msgpack_pack_str_body(&mp_pck, "node_name", 9); - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->node_name)); - msgpack_pack_str_body(&mp_pck, - ctx->node_name, flb_sds_len(ctx->node_name)); - } - - flb_mp_map_header_end(&mh); - } - else if (strcmp(ctx->resource, K8S_POD) == 0) { - /* k8s_pod resource has fields project_id, location, cluster_name, - * namespace_name, pod_name. - * - * The local_resource_id for k8s_pod is in format: - * k8s_pod.. - */ - - ret = process_local_resource_id(ctx, tag, tag_len, K8S_POD); - if (ret != 0) { - flb_plg_error(ctx->ins, "fail to process local_resource_id from " - "log entry for k8s_pod"); - msgpack_sbuffer_destroy(&mp_sbuf); - return NULL; - } - - flb_mp_map_header_init(&mh, &mp_pck); - - if (ctx->project_id) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 10); - msgpack_pack_str_body(&mp_pck, "project_id", 10); - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->project_id)); - msgpack_pack_str_body(&mp_pck, - ctx->project_id, flb_sds_len(ctx->project_id)); - } - - if (ctx->cluster_location) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 8); - msgpack_pack_str_body(&mp_pck, "location", 8); - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->cluster_location)); - msgpack_pack_str_body(&mp_pck, - ctx->cluster_location, - flb_sds_len(ctx->cluster_location)); - } - - if (ctx->cluster_name) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 12); - msgpack_pack_str_body(&mp_pck, "cluster_name", 12); - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->cluster_name)); - msgpack_pack_str_body(&mp_pck, - ctx->cluster_name, flb_sds_len(ctx->cluster_name)); - } - - if (ctx->namespace_name) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 14); - msgpack_pack_str_body(&mp_pck, "namespace_name", 14); - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->namespace_name)); - msgpack_pack_str_body(&mp_pck, - ctx->namespace_name, - flb_sds_len(ctx->namespace_name)); - } - - if (ctx->pod_name) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(&mp_pck, 8); - msgpack_pack_str_body(&mp_pck, "pod_name", 8); - msgpack_pack_str(&mp_pck, flb_sds_len(ctx->pod_name)); - msgpack_pack_str_body(&mp_pck, - ctx->pod_name, flb_sds_len(ctx->pod_name)); - } - - flb_mp_map_header_end(&mh); - } - else { - flb_plg_error(ctx->ins, "unsupported resource type '%s'", - ctx->resource); - msgpack_sbuffer_destroy(&mp_sbuf); - return NULL; - } - } - } - msgpack_pack_str(&mp_pck, 7); - msgpack_pack_str_body(&mp_pck, "entries", 7); - - /* Append entries */ - msgpack_pack_array(&mp_pck, array_size); - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - msgpack_sbuffer_destroy(&mp_sbuf); - - return NULL; - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - obj = log_event.body; - tms_status = extract_timestamp(obj, &log_event.timestamp); - - /* - * Pack entry - * - * { - * "severity": "...", - * "labels": "...", - * "logName": "...", - * "jsonPayload": {...}, - * "timestamp": "...", - * "spanId": "...", - * "traceSampled": , - * "trace": "..." - * } - */ - entry_size = 3; - - /* Extract severity */ - severity_extracted = FLB_FALSE; - if (ctx->severity_key - && get_severity_level(&severity, obj, ctx->severity_key) == 0) { - severity_extracted = FLB_TRUE; - entry_size += 1; - } - - /* Extract trace */ - trace_extracted = FLB_FALSE; - if (ctx->trace_key - && get_string(&trace, obj, ctx->trace_key) == 0) { - trace_extracted = FLB_TRUE; - entry_size += 1; - } - - /* Extract span id */ - span_id_extracted = FLB_FALSE; - if (ctx->span_id_key - && get_string(&span_id, obj, ctx->span_id_key) == 0) { - span_id_extracted = FLB_TRUE; - entry_size += 1; - } - - /* Extract trace sampled */ - trace_sampled_extracted = FLB_FALSE; - if (ctx->trace_sampled_key - && get_trace_sampled(&trace_sampled, obj, ctx->trace_sampled_key) == 0) { - trace_sampled_extracted = FLB_TRUE; - entry_size += 1; - } - - /* Extract log name */ - log_name_extracted = FLB_FALSE; - if (ctx->log_name_key - && get_string(&log_name, obj, ctx->log_name_key) == 0) { - log_name_extracted = FLB_TRUE; - } - - /* Extract insertId */ - in_status = validate_insert_id(&insert_id_obj, obj); - if (in_status == INSERTID_VALID) { - insert_id_extracted = FLB_TRUE; - entry_size += 1; - } - else if (in_status == INSERTID_NOT_PRESENT) { - insert_id_extracted = FLB_FALSE; - } - else { - if (log_name_extracted == FLB_TRUE) { - flb_sds_destroy(log_name); - } - continue; - } - - /* Extract operation */ - operation_id = flb_sds_create(""); - operation_producer = flb_sds_create(""); - operation_first = FLB_FALSE; - operation_last = FLB_FALSE; - operation_extra_size = 0; - operation_extracted = extract_operation(&operation_id, &operation_producer, - &operation_first, &operation_last, obj, - &operation_extra_size); - - if (operation_extracted == FLB_TRUE) { - entry_size += 1; - } - - /* Extract sourceLocation */ - source_location_file = flb_sds_create(""); - source_location_line = 0; - source_location_function = flb_sds_create(""); - source_location_extra_size = 0; - source_location_extracted = extract_source_location(&source_location_file, - &source_location_line, - &source_location_function, - obj, - &source_location_extra_size); - - if (source_location_extracted == FLB_TRUE) { - entry_size += 1; - } - - /* Extract httpRequest */ - init_http_request(&http_request); - http_request_extra_size = 0; - http_request_extracted = extract_http_request(&http_request, - ctx->http_request_key, - ctx->http_request_key_size, - obj, &http_request_extra_size); - if (http_request_extracted == FLB_TRUE) { - entry_size += 1; - } - - /* Extract payload labels */ - payload_labels_ptr = get_payload_labels(ctx, obj); - if (payload_labels_ptr != NULL && - payload_labels_ptr->type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "the type of payload labels should be map"); - flb_sds_destroy(operation_id); - flb_sds_destroy(operation_producer); - flb_log_event_decoder_destroy(&log_decoder); - msgpack_sbuffer_destroy(&mp_sbuf); - return NULL; - } - - /* Number of parsed labels */ - labels_size = mk_list_size(&ctx->config_labels); - if (payload_labels_ptr != NULL && - payload_labels_ptr->type == MSGPACK_OBJECT_MAP) { - labels_size += payload_labels_ptr->via.map.size; - } - - if (labels_size > 0) { - entry_size += 1; - } - - msgpack_pack_map(&mp_pck, entry_size); - - /* Add severity into the log entry */ - if (severity_extracted == FLB_TRUE) { - msgpack_pack_str(&mp_pck, 8); - msgpack_pack_str_body(&mp_pck, "severity", 8); - msgpack_pack_int(&mp_pck, severity); - } - - /* Add trace into the log entry */ - if (trace_extracted == FLB_TRUE) { - msgpack_pack_str(&mp_pck, 5); - msgpack_pack_str_body(&mp_pck, "trace", 5); - - if (ctx->autoformat_stackdriver_trace) { - len = snprintf(stackdriver_trace, sizeof(stackdriver_trace) - 1, - "projects/%s/traces/%s", ctx->project_id, trace); - new_trace = stackdriver_trace; - } - else { - len = flb_sds_len(trace); - new_trace = trace; - } - - msgpack_pack_str(&mp_pck, len); - msgpack_pack_str_body(&mp_pck, new_trace, len); - flb_sds_destroy(trace); - } - - /* Add spanId field into the log entry */ - if (span_id_extracted == FLB_TRUE) { - msgpack_pack_str_with_body(&mp_pck, "spanId", 6); - len = flb_sds_len(span_id); - msgpack_pack_str_with_body(&mp_pck, span_id, len); - flb_sds_destroy(span_id); - } - - /* Add traceSampled field into the log entry */ - if (trace_sampled_extracted == FLB_TRUE) { - msgpack_pack_str_with_body(&mp_pck, "traceSampled", 12); - - if (trace_sampled == FLB_TRUE) { - msgpack_pack_true(&mp_pck); - } else { - msgpack_pack_false(&mp_pck); - } - - } - - /* Add insertId field into the log entry */ - if (insert_id_extracted == FLB_TRUE) { - msgpack_pack_str(&mp_pck, 8); - msgpack_pack_str_body(&mp_pck, "insertId", 8); - msgpack_pack_object(&mp_pck, insert_id_obj); - } - - /* Add operation field into the log entry */ - if (operation_extracted == FLB_TRUE) { - add_operation_field(&operation_id, &operation_producer, - &operation_first, &operation_last, &mp_pck); - } - - /* Add sourceLocation field into the log entry */ - if (source_location_extracted == FLB_TRUE) { - add_source_location_field(&source_location_file, source_location_line, - &source_location_function, &mp_pck); - } - - /* Add httpRequest field into the log entry */ - if (http_request_extracted == FLB_TRUE) { - add_http_request_field(&http_request, &mp_pck); - } - - /* labels */ - if (labels_size > 0) { - msgpack_pack_str(&mp_pck, 6); - msgpack_pack_str_body(&mp_pck, "labels", 6); - pack_labels(ctx, &mp_pck, payload_labels_ptr); - } - - /* Clean up id and producer if operation extracted */ - flb_sds_destroy(operation_id); - flb_sds_destroy(operation_producer); - flb_sds_destroy(source_location_file); - flb_sds_destroy(source_location_function); - destroy_http_request(&http_request); - - /* jsonPayload */ - msgpack_pack_str(&mp_pck, 11); - msgpack_pack_str_body(&mp_pck, "jsonPayload", 11); - pack_json_payload(insert_id_extracted, - operation_extracted, operation_extra_size, - source_location_extracted, - source_location_extra_size, - http_request_extracted, - http_request_extra_size, - tms_status, - &mp_pck, obj, ctx); - - /* avoid modifying the original tag */ - newtag = tag; - stream_key = flb_sds_create("stream"); - if (ctx->resource_type == RESOURCE_TYPE_K8S - && get_string(&stream, obj, stream_key) == 0) { - if (flb_sds_cmp(stream, STDOUT, flb_sds_len(stream)) == 0) { - newtag = "stdout"; - } - else if (flb_sds_cmp(stream, STDERR, flb_sds_len(stream)) == 0) { - newtag = "stderr"; - } - } - - if (log_name_extracted == FLB_FALSE) { - new_log_name = newtag; - } - else { - new_log_name = log_name; - } - - /* logName */ - len = snprintf(path, sizeof(path) - 1, - "projects/%s/logs/%s", ctx->export_to_project_id, new_log_name); - - if (log_name_extracted == FLB_TRUE) { - flb_sds_destroy(log_name); - } - - msgpack_pack_str(&mp_pck, 7); - msgpack_pack_str_body(&mp_pck, "logName", 7); - msgpack_pack_str(&mp_pck, len); - msgpack_pack_str_body(&mp_pck, path, len); - flb_sds_destroy(stream_key); - flb_sds_destroy(stream); - - /* timestamp */ - msgpack_pack_str(&mp_pck, 9); - msgpack_pack_str_body(&mp_pck, "timestamp", 9); - - /* Format the time */ - /* - * If format is timestamp_object or timestamp_duo_fields, - * tms has been updated. - * - * If timestamp is not presen, - * use the default tms(current time). - */ - - gmtime_r(&log_event.timestamp.tm.tv_sec, &tm); - s = strftime(time_formatted, sizeof(time_formatted) - 1, - FLB_STD_TIME_FMT, &tm); - len = snprintf(time_formatted + s, sizeof(time_formatted) - 1 - s, - ".%09" PRIu64 "Z", - (uint64_t) log_event.timestamp.tm.tv_nsec); - s += len; - - msgpack_pack_str(&mp_pck, s); - msgpack_pack_str_body(&mp_pck, time_formatted, s); - } - - flb_log_event_decoder_destroy(&log_decoder); - - /* Convert from msgpack to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); - msgpack_sbuffer_destroy(&mp_sbuf); - - if (!out_buf) { - flb_plg_error(ctx->ins, "error formatting JSON payload"); - return NULL; - } - - return out_buf; -} - -static int stackdriver_format_test(struct flb_config *config, - struct flb_input_instance *ins, - void *plugin_context, - void *flush_ctx, - int event_type, - const char *tag, int tag_len, - const void *data, size_t bytes, - void **out_data, size_t *out_size) -{ - int total_records; - flb_sds_t payload = NULL; - struct flb_stackdriver *ctx = plugin_context; - - /* Count number of records */ - total_records = flb_mp_count(data, bytes); - - payload = stackdriver_format(ctx, total_records, - (char *) tag, tag_len, data, bytes); - if (payload == NULL) { - return -1; - } - - *out_data = payload; - *out_size = flb_sds_len(payload); - - return 0; - -} - -#ifdef FLB_HAVE_METRICS -static void update_http_metrics(struct flb_stackdriver *ctx, - struct flb_event_chunk *event_chunk, - uint64_t ts, - int http_status) -{ - char tmp[32]; - - /* convert status to string format */ - snprintf(tmp, sizeof(tmp) - 1, "%i", http_status); - char *name = (char *) flb_output_name(ctx->ins); - - /* processed records total */ - cmt_counter_add(ctx->cmt_proc_records_total, ts, event_chunk->total_events, - 2, (char *[]) {tmp, name}); - - /* HTTP status */ - if (http_status != STACKDRIVER_NET_ERROR) { - cmt_counter_inc(ctx->cmt_requests_total, ts, 2, (char *[]) {tmp, name}); - } -} - -static void update_retry_metric(struct flb_stackdriver *ctx, - struct flb_event_chunk *event_chunk, - uint64_t ts, - int http_status, int ret_code) -{ - char tmp[32]; - char *name = (char *) flb_output_name(ctx->ins); - - if (ret_code != FLB_RETRY) { - return; - } - - /* convert status to string format */ - snprintf(tmp, sizeof(tmp) - 1, "%i", http_status); - cmt_counter_add(ctx->cmt_retried_records_total, - ts, event_chunk->total_events, 2, (char *[]) {tmp, name}); - -} -#endif - -static void cb_stackdriver_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - (void) i_ins; - (void) config; - int ret; - int ret_code = FLB_RETRY; - size_t b_sent; - flb_sds_t token; - flb_sds_t payload_buf; - void *compressed_payload_buffer = NULL; - size_t compressed_payload_size; - struct flb_stackdriver *ctx = out_context; - struct flb_connection *u_conn; - struct flb_http_client *c; - int compressed = FLB_FALSE; -#ifdef FLB_HAVE_METRICS - char *name = (char *) flb_output_name(ctx->ins); - uint64_t ts = cfl_time_now(); -#endif - - /* Get upstream connection */ - u_conn = flb_upstream_conn_get(ctx->u); - if (!u_conn) { -#ifdef FLB_HAVE_METRICS - cmt_counter_inc(ctx->cmt_failed_requests, - ts, 1, (char *[]) {name}); - - /* OLD api */ - flb_metrics_sum(FLB_STACKDRIVER_FAILED_REQUESTS, 1, ctx->ins->metrics); - - update_http_metrics(ctx, event_chunk, ts, STACKDRIVER_NET_ERROR); - update_retry_metric(ctx, event_chunk, ts, STACKDRIVER_NET_ERROR, FLB_RETRY); -#endif - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Reformat msgpack to stackdriver JSON payload */ - payload_buf = stackdriver_format(ctx, - event_chunk->total_events, - event_chunk->tag, flb_sds_len(event_chunk->tag), - event_chunk->data, event_chunk->size); - if (!payload_buf) { -#ifdef FLB_HAVE_METRICS - cmt_counter_inc(ctx->cmt_failed_requests, - ts, 1, (char *[]) {name}); - - /* OLD api */ - flb_metrics_sum(FLB_STACKDRIVER_FAILED_REQUESTS, 1, ctx->ins->metrics); -#endif - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Get or renew Token */ - token = get_google_token(ctx); - if (!token) { - flb_plg_error(ctx->ins, "cannot retrieve oauth2 token"); - flb_upstream_conn_release(u_conn); - flb_sds_destroy(payload_buf); -#ifdef FLB_HAVE_METRICS - cmt_counter_inc(ctx->cmt_failed_requests, - ts, 1, (char *[]) {name}); - - /* OLD api */ - flb_metrics_sum(FLB_STACKDRIVER_FAILED_REQUESTS, 1, ctx->ins->metrics); -#endif - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - compressed_payload_buffer = payload_buf; - compressed_payload_size = flb_sds_len(payload_buf); - if (ctx->compress_gzip == FLB_TRUE) { - ret = flb_gzip_compress((void *) payload_buf, flb_sds_len(payload_buf), - &compressed_payload_buffer, &compressed_payload_size); - if (ret == -1) { - flb_plg_error(ctx->ins, "cannot gzip payload, disabling compression"); - } else { - compressed = FLB_TRUE; - flb_sds_destroy(payload_buf); - } - } - - /* Compose HTTP Client request */ - c = flb_http_client(u_conn, FLB_HTTP_POST, FLB_STD_WRITE_URI, - compressed_payload_buffer, compressed_payload_size, NULL, 0, NULL, 0); - - flb_http_buffer_size(c, 4192); - - if (ctx->stackdriver_agent) { - flb_http_add_header(c, "User-Agent", 10, - ctx->stackdriver_agent, - flb_sds_len(ctx->stackdriver_agent)); - } - else { - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - } - - flb_http_add_header(c, "Content-Type", 12, "application/json", 16); - flb_http_add_header(c, "Authorization", 13, token, flb_sds_len(token)); - /* Content Encoding: gzip */ - if (compressed == FLB_TRUE) { - flb_http_set_content_encoding_gzip(c); - } - - /* Send HTTP request */ - ret = flb_http_do(c, &b_sent); - - /* validate response */ - if (ret != 0) { - flb_plg_warn(ctx->ins, "http_do=%i", ret); - ret_code = FLB_RETRY; -#ifdef FLB_HAVE_METRICS - update_http_metrics(ctx, event_chunk, ts, STACKDRIVER_NET_ERROR); -#endif - } - else { - /* The request was issued successfully, validate the 'error' field */ - flb_plg_debug(ctx->ins, "HTTP Status=%i", c->resp.status); - if (c->resp.status == 200) { - ret_code = FLB_OK; - } - else if (c->resp.status >= 400 && c->resp.status < 500) { - ret_code = FLB_ERROR; - flb_plg_warn(ctx->ins, "error\n%s", - c->resp.payload); - } - else { - if (c->resp.payload_size > 0) { - /* we got an error */ - flb_plg_warn(ctx->ins, "error\n%s", - c->resp.payload); - } - else { - flb_plg_debug(ctx->ins, "response\n%s", - c->resp.payload); - } - ret_code = FLB_RETRY; - } - } - - /* Update specific stackdriver metrics */ -#ifdef FLB_HAVE_METRICS - if (ret_code == FLB_OK) { - cmt_counter_inc(ctx->cmt_successful_requests, - ts, 1, (char *[]) {name}); - - /* OLD api */ - flb_metrics_sum(FLB_STACKDRIVER_SUCCESSFUL_REQUESTS, 1, ctx->ins->metrics); - } - else { - cmt_counter_inc(ctx->cmt_failed_requests, - ts, 1, (char *[]) {name}); - - /* OLD api */ - flb_metrics_sum(FLB_STACKDRIVER_FAILED_REQUESTS, 1, ctx->ins->metrics); - } - - /* Update metrics counter by using labels/http status code */ - if (ret == 0) { - update_http_metrics(ctx, event_chunk, ts, c->resp.status); - } - - /* Update retry count if necessary */ - update_retry_metric(ctx, event_chunk, ts, c->resp.status, ret_code); -#endif - - - /* Cleanup */ - if (compressed == FLB_TRUE) { - flb_free(compressed_payload_buffer); - } - else { - flb_sds_destroy(payload_buf); - } - flb_sds_destroy(token); - flb_http_client_destroy(c); - flb_upstream_conn_release(u_conn); - - /* Done */ - FLB_OUTPUT_RETURN(ret_code); -} - -static int cb_stackdriver_exit(void *data, struct flb_config *config) -{ - struct flb_stackdriver *ctx = data; - - if (!ctx) { - return -1; - } - - flb_stackdriver_conf_destroy(ctx); - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "google_service_credentials", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_stackdriver, credentials_file), - "Set the path for the google service credentials file" - }, - { - FLB_CONFIG_MAP_STR, "metadata_server", (char *)NULL, - 0, FLB_FALSE, 0, - "Set the metadata server" - }, - { - FLB_CONFIG_MAP_STR, "service_account_email", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_stackdriver, client_email), - "Set the service account email" - }, - // set in flb_bigquery_oauth_credentials - { - FLB_CONFIG_MAP_STR, "service_account_secret", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_stackdriver, private_key), - "Set the service account secret" - }, - { - FLB_CONFIG_MAP_STR, "export_to_project_id", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_stackdriver, export_to_project_id), - "Export to project id" - }, - { - FLB_CONFIG_MAP_STR, "resource", FLB_SDS_RESOURCE_TYPE, - 0, FLB_TRUE, offsetof(struct flb_stackdriver, resource), - "Set the resource" - }, - { - FLB_CONFIG_MAP_STR, "severity_key", DEFAULT_SEVERITY_KEY, - 0, FLB_TRUE, offsetof(struct flb_stackdriver, severity_key), - "Set the severity key" - }, - { - FLB_CONFIG_MAP_BOOL, "autoformat_stackdriver_trace", "false", - 0, FLB_TRUE, offsetof(struct flb_stackdriver, autoformat_stackdriver_trace), - "Autoformat the stackdriver trace" - }, - { - FLB_CONFIG_MAP_STR, "trace_key", DEFAULT_TRACE_KEY, - 0, FLB_TRUE, offsetof(struct flb_stackdriver, trace_key), - "Set the trace key" - }, - { - FLB_CONFIG_MAP_STR, "span_id_key", DEFAULT_SPAN_ID_KEY, - 0, FLB_TRUE, offsetof(struct flb_stackdriver, span_id_key), - "Set the span id key" - }, - { - FLB_CONFIG_MAP_STR, "trace_sampled_key", DEFAULT_TRACE_SAMPLED_KEY, - 0, FLB_TRUE, offsetof(struct flb_stackdriver, trace_sampled_key), - "Set the trace sampled key" - }, - { - FLB_CONFIG_MAP_STR, "log_name_key", DEFAULT_LOG_NAME_KEY, - 0, FLB_TRUE, offsetof(struct flb_stackdriver, log_name_key), - "Set the logname key" - }, - { - FLB_CONFIG_MAP_STR, "http_request_key", HTTPREQUEST_FIELD_IN_JSON, - 0, FLB_TRUE, offsetof(struct flb_stackdriver, http_request_key), - "Set the http request key" - }, - { - FLB_CONFIG_MAP_STR, "k8s_cluster_name", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_stackdriver, cluster_name), - "Set the kubernetes cluster name" - }, - { - FLB_CONFIG_MAP_STR, "k8s_cluster_location", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_stackdriver, cluster_location), - "Set the kubernetes cluster location" - }, - { - FLB_CONFIG_MAP_STR, "location", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_stackdriver, location), - "Set the resource location" - }, - { - FLB_CONFIG_MAP_STR, "namespace", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_stackdriver, namespace_id), - "Set the resource namespace" - }, - { - FLB_CONFIG_MAP_STR, "node_id", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_stackdriver, node_id), - "Set the resource node id" - }, - { - FLB_CONFIG_MAP_STR, "job", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_stackdriver, job), - "Set the resource job" - }, - { - FLB_CONFIG_MAP_STR, "task_id", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_stackdriver, task_id), - "Set the resource task id" - }, - { - FLB_CONFIG_MAP_STR, "compress", NULL, - 0, FLB_FALSE, 0, - "Set log payload compression method. Option available is 'gzip'" - }, - { - FLB_CONFIG_MAP_CLIST, "labels", NULL, - 0, FLB_TRUE, offsetof(struct flb_stackdriver, labels), - "Set the labels" - }, - { - FLB_CONFIG_MAP_STR, "labels_key", DEFAULT_LABELS_KEY, - 0, FLB_TRUE, offsetof(struct flb_stackdriver, labels_key), - "Set the labels key" - }, - { - FLB_CONFIG_MAP_STR, "tag_prefix", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_stackdriver, tag_prefix), - "Set the tag prefix" - }, - { - FLB_CONFIG_MAP_STR, "stackdriver_agent", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_stackdriver, stackdriver_agent), - "Set the stackdriver agent" - }, - /* Custom Regex */ - { - FLB_CONFIG_MAP_STR, "custom_k8s_regex", DEFAULT_TAG_REGEX, - 0, FLB_TRUE, offsetof(struct flb_stackdriver, custom_k8s_regex), - "Set a custom kubernetes regex filter" - }, - { - FLB_CONFIG_MAP_CLIST, "resource_labels", NULL, - 0, FLB_TRUE, offsetof(struct flb_stackdriver, resource_labels), - "Set the resource labels" - }, - /* EOF */ - {0} -}; - -struct flb_output_plugin out_stackdriver_plugin = { - .name = "stackdriver", - .description = "Send events to Google Stackdriver Logging", - .cb_init = cb_stackdriver_init, - .cb_flush = cb_stackdriver_flush, - .cb_exit = cb_stackdriver_exit, - .workers = 1, - .config_map = config_map, - - /* Test */ - .test_formatter.callback = stackdriver_format_test, - - /* Plugin flags */ - .flags = FLB_OUTPUT_NET | FLB_IO_TLS, -}; diff --git a/fluent-bit/plugins/out_stackdriver/stackdriver.h b/fluent-bit/plugins/out_stackdriver/stackdriver.h deleted file mode 100644 index 239a3ee31..000000000 --- a/fluent-bit/plugins/out_stackdriver/stackdriver.h +++ /dev/null @@ -1,241 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_STACKDRIVER_H -#define FLB_OUT_STACKDRIVER_H - -#include -#include -#include -#include -#include -#include - -/* refresh token every 50 minutes */ -#define FLB_STD_TOKEN_REFRESH 3000 - -/* Stackdriver Logging write scope */ -#define FLB_STD_SCOPE "https://www.googleapis.com/auth/logging.write" - -/* Stackdriver authorization URL */ -#define FLB_STD_AUTH_URL "https://oauth2.googleapis.com/token" - -/* Stackdriver Logging 'write' end-point */ -#define FLB_STD_WRITE_URI "/v2/entries:write" -#define FLB_STD_WRITE_URL "https://logging.googleapis.com" FLB_STD_WRITE_URI - -/* Timestamp format */ -#define FLB_STD_TIME_FMT "%Y-%m-%dT%H:%M:%S" - -/* Default Resource type */ -#define FLB_SDS_RESOURCE_TYPE "global" -#define OPERATION_FIELD_IN_JSON "logging.googleapis.com/operation" -#define MONITORED_RESOURCE_KEY "logging.googleapis.com/monitored_resource" -#define LOCAL_RESOURCE_ID_KEY "logging.googleapis.com/local_resource_id" -#define DEFAULT_LABELS_KEY "logging.googleapis.com/labels" -#define DEFAULT_SEVERITY_KEY "logging.googleapis.com/severity" -#define DEFAULT_TRACE_KEY "logging.googleapis.com/trace" -#define DEFAULT_SPAN_ID_KEY "logging.googleapis.com/spanId" -#define DEFAULT_TRACE_SAMPLED_KEY "logging.googleapis.com/traceSampled" -#define DEFAULT_LOG_NAME_KEY "logging.googleapis.com/logName" -#define DEFAULT_INSERT_ID_KEY "logging.googleapis.com/insertId" -#define SOURCELOCATION_FIELD_IN_JSON "logging.googleapis.com/sourceLocation" -#define HTTPREQUEST_FIELD_IN_JSON "logging.googleapis.com/http_request" -#define INSERT_ID_SIZE 31 -#define LEN_LOCAL_RESOURCE_ID_KEY 40 -#define OPERATION_KEY_SIZE 32 -#define SOURCE_LOCATION_SIZE 37 -#define HTTP_REQUEST_KEY_SIZE 35 - -/* - * Stackdriver implements a specific HTTP status code that is used internally in clients to keep - * track of networking errors that could happen before a successful HTTP request/response. For - * metrics counting purposes, every failed networking connection will use a 502 HTTP status code - * that can be used later to query the metrics by using labels with that value. - */ -#define STACKDRIVER_NET_ERROR 502 - -#define K8S_CONTAINER "k8s_container" -#define K8S_NODE "k8s_node" -#define K8S_POD "k8s_pod" - -#define STDOUT "stdout" -#define STDERR "stderr" - -#define DEFAULT_TAG_REGEX "(?[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?[^_]+)_(?.+)-(?[a-z0-9]{64})\\.log$" - -/* Metrics */ -#ifdef FLB_HAVE_METRICS -#define FLB_STACKDRIVER_SUCCESSFUL_REQUESTS 1000 /* successful requests */ -#define FLB_STACKDRIVER_FAILED_REQUESTS 1001 /* failed requests */ -#endif - -struct flb_stackdriver_oauth_credentials { - /* parsed credentials file */ - flb_sds_t type; - flb_sds_t private_key_id; - flb_sds_t private_key; - flb_sds_t client_email; - flb_sds_t client_id; - flb_sds_t auth_uri; - flb_sds_t token_uri; -}; - -struct flb_stackdriver_env { - flb_sds_t creds_file; - flb_sds_t metadata_server; -}; - -struct flb_stackdriver { - /* credentials */ - flb_sds_t credentials_file; - - /* parsed credentials file */ - flb_sds_t type; - flb_sds_t project_id; - flb_sds_t private_key_id; - flb_sds_t private_key; - flb_sds_t client_email; - flb_sds_t client_id; - flb_sds_t auth_uri; - flb_sds_t token_uri; - bool metadata_server_auth; - - /* metadata server (GCP specific, WIP) */ - flb_sds_t metadata_server; - flb_sds_t zone; - flb_sds_t instance_id; - flb_sds_t instance_name; - - /* kubernetes specific */ - flb_sds_t cluster_name; - flb_sds_t cluster_location; - flb_sds_t namespace_name; - flb_sds_t pod_name; - flb_sds_t container_name; - flb_sds_t node_name; - - flb_sds_t local_resource_id; - flb_sds_t tag_prefix; - /* shadow tag_prefix for safe deallocation */ - flb_sds_t tag_prefix_k8s; - - /* labels */ - flb_sds_t labels_key; - struct mk_list *labels; - struct mk_list config_labels; - - /* resource type flag */ - int resource_type; - - /* resource labels api */ - struct mk_list *resource_labels; - struct mk_list resource_labels_kvs; - int should_skip_resource_labels_api; - - /* generic resources */ - flb_sds_t location; - flb_sds_t namespace_id; - - /* generic_node specific */ - flb_sds_t node_id; - - /* generic_task specific */ - flb_sds_t job; - flb_sds_t task_id; - - /* Internal variable to reduce string comparisons */ - int compress_gzip; - - /* other */ - flb_sds_t export_to_project_id; - flb_sds_t resource; - flb_sds_t severity_key; - flb_sds_t trace_key; - flb_sds_t span_id_key; - flb_sds_t trace_sampled_key; - flb_sds_t log_name_key; - flb_sds_t http_request_key; - int http_request_key_size; - bool autoformat_stackdriver_trace; - - flb_sds_t stackdriver_agent; - - /* Regex context to parse tags */ - flb_sds_t custom_k8s_regex; - struct flb_regex *regex; - - /* oauth2 context */ - struct flb_oauth2 *o; - - /* parsed oauth2 credentials */ - struct flb_stackdriver_oauth_credentials *creds; - - /* environment variable settings */ - struct flb_stackdriver_env *env; - - /* mutex for acquiring oauth tokens */ - pthread_mutex_t token_mutex; - - /* upstream context for stackdriver write end-point */ - struct flb_upstream *u; - - /* upstream context for metadata end-point */ - struct flb_upstream *metadata_u; - -#ifdef FLB_HAVE_METRICS - /* metrics */ - struct cmt_counter *cmt_successful_requests; - struct cmt_counter *cmt_failed_requests; - struct cmt_counter *cmt_requests_total; - struct cmt_counter *cmt_proc_records_total; - struct cmt_counter *cmt_retried_records_total; -#endif - - /* plugin instance */ - struct flb_output_instance *ins; - - /* Fluent Bit context */ - struct flb_config *config; -}; - -typedef enum { - FLB_STD_EMERGENCY = 800, - FLB_STD_ALERT = 700, - FLB_STD_CRITICAL = 600, - FLB_STD_ERROR = 500, - FLB_STD_WARNING = 400, - FLB_STD_NOTICE = 300, - FLB_STD_INFO = 200, - FLB_STD_DEBUG = 100, - FLB_STD_DEFAULT = 0 -} severity_t; - -struct local_resource_id_list { - flb_sds_t val; - struct mk_list _head; -}; - -typedef enum { - INSERTID_VALID = 0, - INSERTID_INVALID = 1, - INSERTID_NOT_PRESENT = 2 -} insert_id_status; - -#endif diff --git a/fluent-bit/plugins/out_stackdriver/stackdriver_conf.c b/fluent-bit/plugins/out_stackdriver/stackdriver_conf.c deleted file mode 100644 index 9f3f28a35..000000000 --- a/fluent-bit/plugins/out_stackdriver/stackdriver_conf.c +++ /dev/null @@ -1,667 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "gce_metadata.h" -#include "stackdriver.h" -#include "stackdriver_conf.h" -#include "stackdriver_resource_types.h" - -static inline int key_cmp(const char *str, int len, const char *cmp) { - - if (strlen(cmp) != len) { - return -1; - } - - return strncasecmp(str, cmp, len); -} - -static int read_credentials_file(const char *cred_file, struct flb_stackdriver *ctx) -{ - int i; - int ret; - int key_len; - int val_len; - int tok_size = 32; - char *buf; - char *key; - char *val; - flb_sds_t tmp; - struct stat st; - jsmn_parser parser; - jsmntok_t *t; - jsmntok_t *tokens; - - /* Validate credentials path */ - ret = stat(cred_file, &st); - if (ret == -1) { - flb_errno(); - flb_plg_error(ctx->ins, "cannot open credentials file: %s", - cred_file); - return -1; - } - - if (!S_ISREG(st.st_mode) && !S_ISLNK(st.st_mode)) { - flb_plg_error(ctx->ins, "credentials file " - "is not a valid file: %s", cred_file); - return -1; - } - - /* Read file content */ - buf = mk_file_to_buffer(cred_file); - if (!buf) { - flb_plg_error(ctx->ins, "error reading credentials file: %s", - cred_file); - return -1; - } - - /* Parse content */ - jsmn_init(&parser); - tokens = flb_calloc(1, sizeof(jsmntok_t) * tok_size); - if (!tokens) { - flb_errno(); - flb_free(buf); - return -1; - } - - ret = jsmn_parse(&parser, buf, st.st_size, tokens, tok_size); - if (ret <= 0) { - flb_plg_error(ctx->ins, "invalid JSON credentials file: %s", - cred_file); - flb_free(buf); - flb_free(tokens); - return -1; - } - - t = &tokens[0]; - if (t->type != JSMN_OBJECT) { - flb_plg_error(ctx->ins, "invalid JSON map on file: %s", - cred_file); - flb_free(buf); - flb_free(tokens); - return -1; - } - - /* Parse JSON tokens */ - for (i = 1; i < ret; i++) { - t = &tokens[i]; - if (t->type != JSMN_STRING) { - continue; - } - - if (t->start == -1 || t->end == -1 || (t->start == 0 && t->end == 0)){ - break; - } - - /* Key */ - key = buf + t->start; - key_len = (t->end - t->start); - - /* Value */ - i++; - t = &tokens[i]; - val = buf + t->start; - val_len = (t->end - t->start); - - if (key_cmp(key, key_len, "type") == 0) { - ctx->creds->type = flb_sds_create_len(val, val_len); - } - else if (key_cmp(key, key_len, "project_id") == 0) { - ctx->project_id = flb_sds_create_len(val, val_len); - } - else if (key_cmp(key, key_len, "private_key_id") == 0) { - ctx->creds->private_key_id = flb_sds_create_len(val, val_len); - } - else if (key_cmp(key, key_len, "private_key") == 0) { - tmp = flb_sds_create_len(val, val_len); - if (tmp) { - /* Unescape private key */ - ctx->creds->private_key = flb_sds_create_size(val_len); - flb_unescape_string(tmp, flb_sds_len(tmp), - &ctx->creds->private_key); - flb_sds_destroy(tmp); - } - } - else if (key_cmp(key, key_len, "client_email") == 0) { - ctx->creds->client_email = flb_sds_create_len(val, val_len); - } - else if (key_cmp(key, key_len, "client_id") == 0) { - ctx->creds->client_id = flb_sds_create_len(val, val_len); - } - else if (key_cmp(key, key_len, "auth_uri") == 0) { - ctx->creds->auth_uri = flb_sds_create_len(val, val_len); - } - else if (key_cmp(key, key_len, "token_uri") == 0) { - ctx->creds->token_uri = flb_sds_create_len(val, val_len); - } - } - - flb_free(buf); - flb_free(tokens); - - return 0; -} - -/* - * parse_key_value_list(): - * - Parses an origin list of comma seperated string specifying key=value. - * - Appends the parsed key value pairs into the destination list. - * - Returns the length of the destination list. - */ -static int parse_key_value_list(struct flb_stackdriver *ctx, - struct mk_list *origin, - struct mk_list *dest, - int shouldTrim) -{ - char *p; - flb_sds_t key; - flb_sds_t val; - struct flb_kv *kv; - struct mk_list *head; - struct flb_slist_entry *entry; - - if (origin) { - mk_list_foreach(head, origin) { - entry = mk_list_entry(head, struct flb_slist_entry, _head); - - p = strchr(entry->str, '='); - if (!p) { - flb_plg_error(ctx->ins, "invalid key value pair on '%s'", - entry->str); - return -1; - } - - key = flb_sds_create_size((p - entry->str) + 1); - flb_sds_cat(key, entry->str, p - entry->str); - val = flb_sds_create(p + 1); - if (shouldTrim) { - flb_sds_trim(key); - flb_sds_trim(val); - } - if (!key || flb_sds_len(key) == 0) { - flb_plg_error(ctx->ins, - "invalid key value pair on '%s'", - entry->str); - return -1; - } - if (!val || flb_sds_len(val) == 0) { - flb_plg_error(ctx->ins, - "invalid key value pair on '%s'", - entry->str); - flb_sds_destroy(key); - return -1; - } - - kv = flb_kv_item_create(dest, key, val); - flb_sds_destroy(key); - flb_sds_destroy(val); - - if (!kv) { - return -1; - } - } - } - - return mk_list_size(dest); -} - -/* - * parse_configuration_labels - * - Parse labels set in configuration - * - Returns the number of configuration labels - */ -static int parse_configuration_labels(struct flb_stackdriver *ctx) -{ - return parse_key_value_list(ctx, ctx->labels, - &ctx->config_labels, FLB_FALSE); -} - -/* - * parse_resource_labels(): - * - Parses resource labels set in configuration. - * - Returns the number of resource label mappings. - */ -static int parse_resource_labels(struct flb_stackdriver *ctx) -{ - return parse_key_value_list(ctx, ctx->resource_labels, - &ctx->resource_labels_kvs, FLB_TRUE); -} - -struct flb_stackdriver *flb_stackdriver_conf_create(struct flb_output_instance *ins, - struct flb_config *config) -{ - int ret; - const char *tmp; - const char *backwards_compatible_env_var; - struct flb_stackdriver *ctx; - size_t http_request_key_size; - - /* Allocate config context */ - ctx = flb_calloc(1, sizeof(struct flb_stackdriver)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - ctx->config = config; - - ret = flb_output_config_map_set(ins, (void *)ctx); - if (ret == -1) { - flb_plg_error(ins, "unable to load configuration"); - flb_free(ctx); - return NULL; - } - - /* Compress (gzip) */ - tmp = flb_output_get_property("compress", ins); - ctx->compress_gzip = FLB_FALSE; - if (tmp && strcasecmp(tmp, "gzip") == 0) { - ctx->compress_gzip = FLB_TRUE; - } - - /* labels */ - flb_kv_init(&ctx->config_labels); - ret = parse_configuration_labels((void *)ctx); - if (ret == -1) { - flb_plg_error(ins, "unable to parse configuration labels"); - flb_kv_release(&ctx->config_labels); - flb_free(ctx); - return NULL; - } - - /* resource labels */ - flb_kv_init(&ctx->resource_labels_kvs); - ret = parse_resource_labels((void *)ctx); - if (ret == -1) { - flb_plg_error(ins, "unable to parse resource label list"); - flb_kv_release(&ctx->resource_labels_kvs); - flb_free(ctx); - return NULL; - } - - /* Lookup metadata server URL */ - ctx->metadata_server = NULL; - tmp = flb_output_get_property("metadata_server", ins); - if (tmp == NULL) { - tmp = getenv("METADATA_SERVER"); - if(tmp) { - if (ctx->env == NULL) { - ctx->env = flb_calloc(1, sizeof(struct flb_stackdriver_env)); - if (ctx->env == NULL) { - flb_plg_error(ins, "unable to allocate env variables"); - flb_free(ctx); - return NULL; - } - } - ctx->env->metadata_server = flb_sds_create(tmp); - ctx->metadata_server = ctx->env->metadata_server; - } - else { - ctx->metadata_server = flb_sds_create(FLB_STD_METADATA_SERVER); - } - } - else { - ctx->metadata_server = flb_sds_create(tmp); - } - flb_plg_info(ctx->ins, "metadata_server set to %s", ctx->metadata_server); - - /* Lookup credentials file */ - if (ctx->credentials_file == NULL) { - /* - * Use GOOGLE_APPLICATION_CREDENTIALS to fetch the credentials. - * GOOGLE_SERVICE_CREDENTIALS is checked for backwards compatibility. - */ - tmp = getenv("GOOGLE_APPLICATION_CREDENTIALS"); - backwards_compatible_env_var = getenv("GOOGLE_SERVICE_CREDENTIALS"); - if (tmp && backwards_compatible_env_var) { - flb_plg_warn(ctx->ins, "GOOGLE_APPLICATION_CREDENTIALS and " - "GOOGLE_SERVICE_CREDENTIALS are both defined. " - "Defaulting to GOOGLE_APPLICATION_CREDENTIALS"); - } - if ((tmp || backwards_compatible_env_var) && (ctx->env == NULL)) { - ctx->env = flb_calloc(1, sizeof(struct flb_stackdriver_env)); - if (ctx->env == NULL) { - flb_plg_error(ins, "unable to allocate env variables"); - flb_free(ctx); - return NULL; - } - } - if (tmp) { - ctx->env->creds_file = flb_sds_create(tmp); - ctx->credentials_file = ctx->env->creds_file; - } - else if (backwards_compatible_env_var) { - ctx->env->creds_file = flb_sds_create(backwards_compatible_env_var); - ctx->credentials_file = ctx->env->creds_file; - } - } - - if (ctx->credentials_file) { - ctx->creds = flb_calloc(1, sizeof(struct flb_stackdriver_oauth_credentials)); - if (ctx->creds == NULL) { - flb_plg_error(ctx->ins, "unable to allocate credentials"); - flb_stackdriver_conf_destroy(ctx); - return NULL; - } - ret = read_credentials_file(ctx->credentials_file, ctx); - if (ret != 0) { - flb_stackdriver_conf_destroy(ctx); - return NULL; - } - ctx->type = ctx->creds->type; - ctx->private_key_id = ctx->creds->private_key_id; - ctx->private_key = ctx->creds->private_key; - ctx->client_email = ctx->creds->client_email; - ctx->client_id = ctx->creds->client_id; - ctx->auth_uri = ctx->creds->auth_uri; - ctx->token_uri = ctx->creds->token_uri; - } - else { - /* - * If no credentials file has been defined, do manual lookup of the - * client email and the private key - */ - ctx->creds = flb_calloc(1, sizeof(struct flb_stackdriver_oauth_credentials)); - if (ctx->creds == NULL) { - flb_plg_error(ctx->ins, "unable to allocate credentials"); - flb_stackdriver_conf_destroy(ctx); - return NULL; - } - - /* Service Account Email */ - if (ctx->client_email == NULL) { - tmp = getenv("SERVICE_ACCOUNT_EMAIL"); - if (tmp) { - ctx->creds->client_email = flb_sds_create(tmp); - } - } - - /* Service Account Secret */ - if (ctx->private_key == NULL) { - tmp = getenv("SERVICE_ACCOUNT_SECRET"); - if (tmp) { - ctx->creds->private_key = flb_sds_create(tmp); - } - } - - ctx->private_key = ctx->creds->private_key; - ctx->client_email = ctx->creds->client_email; - } - - /* - * If only client email has been provided, fetch token from - * the GCE metadata server. - * - * If no credentials have been provided, fetch token from the GCE - * metadata server for default account. - */ - if (!ctx->client_email && ctx->private_key) { - flb_plg_error(ctx->ins, "client_email is not defined"); - flb_stackdriver_conf_destroy(ctx); - return NULL; - } - - if (!ctx->client_email) { - flb_plg_warn(ctx->ins, "client_email is not defined, using " - "a default one"); - if (ctx->creds == NULL) { - ctx->creds = flb_calloc(1, sizeof(struct flb_stackdriver_oauth_credentials)); - if (ctx->creds == NULL) { - flb_plg_error(ctx->ins, "unable to allocate credentials"); - flb_stackdriver_conf_destroy(ctx); - return NULL; - } - } - ctx->creds->client_email = flb_sds_create("default"); - ctx->client_email = ctx->creds->client_email; - } - if (!ctx->private_key) { - flb_plg_warn(ctx->ins, "private_key is not defined, fetching " - "it from metadata server"); - ctx->metadata_server_auth = true; - } - - if (ctx->http_request_key) { - http_request_key_size = flb_sds_len(ctx->http_request_key); - if (http_request_key_size >= INT_MAX) { - flb_plg_error(ctx->ins, "http_request_key is too long"); - flb_sds_destroy(ctx->http_request_key); - ctx->http_request_key = NULL; - ctx->http_request_key_size = 0; - } else { - ctx->http_request_key_size = http_request_key_size; - } - } - - set_resource_type(ctx); - - if (resource_api_has_required_labels(ctx) == FLB_FALSE) { - - if (ctx->resource_type == RESOURCE_TYPE_K8S) { - if (!ctx->cluster_name || !ctx->cluster_location) { - flb_plg_error(ctx->ins, "missing k8s_cluster_name " - "or k8s_cluster_location in configuration"); - flb_stackdriver_conf_destroy(ctx); - return NULL; - } - } - - else if (ctx->resource_type == RESOURCE_TYPE_GENERIC_NODE - || ctx->resource_type == RESOURCE_TYPE_GENERIC_TASK) { - - if (ctx->location == NULL) { - flb_plg_error(ctx->ins, "missing generic resource's location"); - } - - if (ctx->namespace_id == NULL) { - flb_plg_error(ctx->ins, "missing generic resource's namespace"); - } - - if (ctx->resource_type == RESOURCE_TYPE_GENERIC_NODE) { - if (ctx->node_id == NULL) { - flb_plg_error(ctx->ins, "missing generic_node's node_id"); - flb_stackdriver_conf_destroy(ctx); - return NULL; - } - } - else { - if (ctx->job == NULL) { - flb_plg_error(ctx->ins, "missing generic_task's job"); - } - - if (ctx->task_id == NULL) { - flb_plg_error(ctx->ins, "missing generic_task's task_id"); - } - - if (!ctx->job || !ctx->task_id) { - flb_stackdriver_conf_destroy(ctx); - return NULL; - } - } - - if (!ctx->location || !ctx->namespace_id) { - flb_stackdriver_conf_destroy(ctx); - return NULL; - } - } - } - - - if (ctx->tag_prefix == NULL && ctx->resource_type == RESOURCE_TYPE_K8S) { - /* allocate the flb_sds_t to tag_prefix_k8s so we can safely deallocate it */ - ctx->tag_prefix_k8s = flb_sds_create(ctx->resource); - ctx->tag_prefix_k8s = flb_sds_cat(ctx->tag_prefix_k8s, ".", 1); - ctx->tag_prefix = ctx->tag_prefix_k8s; - } - - /* Register metrics */ -#ifdef FLB_HAVE_METRICS - ctx->cmt_successful_requests = cmt_counter_create(ins->cmt, - "fluentbit", - "stackdriver", - "successful_requests", - "Total number of successful " - "requests.", - 1, (char *[]) {"name"}); - - ctx->cmt_failed_requests = cmt_counter_create(ins->cmt, - "fluentbit", - "stackdriver", - "failed_requests", - "Total number of failed " - "requests.", - 1, (char *[]) {"name"}); - - ctx->cmt_requests_total = cmt_counter_create(ins->cmt, - "fluentbit", - "stackdriver", - "requests_total", - "Total number of requests.", - 2, (char *[]) {"status", "name"}); - - ctx->cmt_proc_records_total = cmt_counter_create(ins->cmt, - "fluentbit", - "stackdriver", - "proc_records_total", - "Total number of processed records.", - 2, (char *[]) {"status", "name"}); - - ctx->cmt_retried_records_total = cmt_counter_create(ins->cmt, - "fluentbit", - "stackdriver", - "retried_records_total", - "Total number of retried records.", - 2, (char *[]) {"status", "name"}); - - /* OLD api */ - flb_metrics_add(FLB_STACKDRIVER_SUCCESSFUL_REQUESTS, - "stackdriver_successful_requests", ctx->ins->metrics); - flb_metrics_add(FLB_STACKDRIVER_FAILED_REQUESTS, - "stackdriver_failed_requests", ctx->ins->metrics); -#endif - - return ctx; -} - -int flb_stackdriver_conf_destroy(struct flb_stackdriver *ctx) -{ - if (!ctx) { - return -1; - } - - if (ctx->creds) { - if (ctx->creds->type) { - flb_sds_destroy(ctx->creds->type); - } - if (ctx->creds->private_key_id) { - flb_sds_destroy(ctx->creds->private_key_id); - } - if (ctx->creds->private_key) { - flb_sds_destroy(ctx->creds->private_key); - } - if (ctx->creds->client_email) { - flb_sds_destroy(ctx->creds->client_email); - } - if (ctx->creds->client_id) { - flb_sds_destroy(ctx->creds->client_id); - } - if (ctx->creds->auth_uri) { - flb_sds_destroy(ctx->creds->auth_uri); - } - if (ctx->creds->token_uri) { - flb_sds_destroy(ctx->creds->token_uri); - } - flb_free(ctx->creds); - } - - if (ctx->env) { - if (ctx->env->creds_file) { - flb_sds_destroy(ctx->env->creds_file); - } - if (ctx->env->metadata_server) { - flb_sds_destroy(ctx->env->metadata_server); - /* - * If ctx->env is not NULL, - * ctx->metadata_server points ctx->env->metadata_server. - * - * We set ctx->metadata_server to NULL to prevent double free. - */ - ctx->metadata_server = NULL; - } - flb_free(ctx->env); - } - - if (ctx->metadata_server) { - flb_sds_destroy(ctx->metadata_server); - } - - if (ctx->resource_type == RESOURCE_TYPE_K8S){ - flb_sds_destroy(ctx->namespace_name); - flb_sds_destroy(ctx->pod_name); - flb_sds_destroy(ctx->container_name); - flb_sds_destroy(ctx->node_name); - flb_sds_destroy(ctx->local_resource_id); - } - - if (ctx->metadata_server_auth) { - flb_sds_destroy(ctx->zone); - flb_sds_destroy(ctx->instance_id); - } - - if (ctx->metadata_u) { - flb_upstream_destroy(ctx->metadata_u); - } - - if (ctx->u) { - flb_upstream_destroy(ctx->u); - } - - if (ctx->o) { - flb_oauth2_destroy(ctx->o); - } - - if (ctx->regex) { - flb_regex_destroy(ctx->regex); - } - - if (ctx->project_id) { - flb_sds_destroy(ctx->project_id); - } - - if (ctx->tag_prefix_k8s) { - flb_sds_destroy(ctx->tag_prefix_k8s); - } - - flb_kv_release(&ctx->config_labels); - flb_kv_release(&ctx->resource_labels_kvs); - flb_free(ctx); - - return 0; -} diff --git a/fluent-bit/plugins/out_stackdriver/stackdriver_conf.h b/fluent-bit/plugins/out_stackdriver/stackdriver_conf.h deleted file mode 100644 index 6244e6851..000000000 --- a/fluent-bit/plugins/out_stackdriver/stackdriver_conf.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_STACKDRIVER_CONF_H -#define FLB_OUT_STACKDRIVER_CONF_H - -#include "stackdriver.h" - -struct flb_stackdriver *flb_stackdriver_conf_create(struct flb_output_instance *ins, - struct flb_config *config); -int flb_stackdriver_conf_destroy(struct flb_stackdriver *ctx); - -#endif diff --git a/fluent-bit/plugins/out_stackdriver/stackdriver_helper.c b/fluent-bit/plugins/out_stackdriver/stackdriver_helper.c deleted file mode 100644 index e5b94c481..000000000 --- a/fluent-bit/plugins/out_stackdriver/stackdriver_helper.c +++ /dev/null @@ -1,63 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#include "stackdriver.h" - -int equal_obj_str(msgpack_object obj, const char *str, const int size) { - if (obj.type != MSGPACK_OBJECT_STR) { - return FLB_FALSE; - } - if (size != obj.via.str.size - || strncmp(str, obj.via.str.ptr, obj.via.str.size) != 0) { - return FLB_FALSE; - } - return FLB_TRUE; -} - -int validate_key(msgpack_object obj, const char *str, const int size) { - return equal_obj_str(obj, str, size); -} - -void try_assign_subfield_str(msgpack_object obj, flb_sds_t *subfield) { - if (obj.type == MSGPACK_OBJECT_STR) { - *subfield = flb_sds_copy(*subfield, obj.via.str.ptr, - obj.via.str.size); - } -} - -void try_assign_subfield_bool(msgpack_object obj, int *subfield) { - if (obj.type == MSGPACK_OBJECT_BOOLEAN) { - if (obj.via.boolean) { - *subfield = FLB_TRUE; - } - else { - *subfield = FLB_FALSE; - } - } -} - -void try_assign_subfield_int(msgpack_object obj, int64_t *subfield) { - if (obj.type == MSGPACK_OBJECT_STR) { - *subfield = atoll(obj.via.str.ptr); - } - else if (obj.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - *subfield = obj.via.i64; - } -} diff --git a/fluent-bit/plugins/out_stackdriver/stackdriver_helper.h b/fluent-bit/plugins/out_stackdriver/stackdriver_helper.h deleted file mode 100644 index ab8ac30a8..000000000 --- a/fluent-bit/plugins/out_stackdriver/stackdriver_helper.h +++ /dev/null @@ -1,51 +0,0 @@ -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#ifndef FLB_STD_HELPER_H -#define FLB_STD_HELPER_H - -#include "stackdriver.h" - -/* - * Compare obj->via.str and str. - * Return FLB_TRUE if they are equal. - * Return FLB_FALSE if obj->type is not string or they are not equal - */ -int equal_obj_str(msgpack_object obj, const char *str, const int size); - -int validate_key(msgpack_object obj, const char *str, const int size); - -/* - * if obj->type is string, assign obj->val to subfield - * Otherwise leave the subfield untouched - */ -void try_assign_subfield_str(msgpack_object obj, flb_sds_t *subfield); - -/* - * if obj->type is boolean, assign obj->val to subfield - * Otherwise leave the subfield untouched - */ -void try_assign_subfield_bool(msgpack_object obj, int *subfield); - -/* - * if obj->type is valid, assign obj->val to subfield - * Otherwise leave the subfield untouched - */ -void try_assign_subfield_int(msgpack_object obj, int64_t *subfield); - -#endif diff --git a/fluent-bit/plugins/out_stackdriver/stackdriver_http_request.c b/fluent-bit/plugins/out_stackdriver/stackdriver_http_request.c deleted file mode 100644 index 9a1c814c0..000000000 --- a/fluent-bit/plugins/out_stackdriver/stackdriver_http_request.c +++ /dev/null @@ -1,393 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include "stackdriver.h" -#include "stackdriver_helper.h" -#include "stackdriver_http_request.h" - -#include - -typedef enum { - NO_HTTPREQUEST = 1, - HTTPREQUEST_EXISTS = 2 -} http_request_status; - -void init_http_request(struct http_request_field *http_request) -{ - http_request->latency = flb_sds_create(""); - http_request->protocol = flb_sds_create(""); - http_request->referer = flb_sds_create(""); - http_request->remoteIp = flb_sds_create(""); - http_request->requestMethod = flb_sds_create(""); - http_request->requestUrl = flb_sds_create(""); - http_request->serverIp = flb_sds_create(""); - http_request->userAgent = flb_sds_create(""); - - http_request->cacheFillBytes = 0; - http_request->requestSize = 0; - http_request->responseSize = 0; - http_request->status = 0; - - http_request->cacheHit = FLB_FALSE; - http_request->cacheLookup = FLB_FALSE; - http_request->cacheValidatedWithOriginServer = FLB_FALSE; -} - -void destroy_http_request(struct http_request_field *http_request) -{ - flb_sds_destroy(http_request->latency); - flb_sds_destroy(http_request->protocol); - flb_sds_destroy(http_request->referer); - flb_sds_destroy(http_request->remoteIp); - flb_sds_destroy(http_request->requestMethod); - flb_sds_destroy(http_request->requestUrl); - flb_sds_destroy(http_request->serverIp); - flb_sds_destroy(http_request->userAgent); -} - -void add_http_request_field(struct http_request_field *http_request, - msgpack_packer *mp_pck) -{ - msgpack_pack_str(mp_pck, 11); - msgpack_pack_str_body(mp_pck, "httpRequest", 11); - - if (flb_sds_is_empty(http_request->latency) == FLB_TRUE) { - msgpack_pack_map(mp_pck, 14); - } - else { - msgpack_pack_map(mp_pck, 15); - - msgpack_pack_str(mp_pck, HTTP_REQUEST_LATENCY_SIZE); - msgpack_pack_str_body(mp_pck, HTTP_REQUEST_LATENCY, - HTTP_REQUEST_LATENCY_SIZE); - msgpack_pack_str(mp_pck, flb_sds_len(http_request->latency)); - msgpack_pack_str_body(mp_pck, http_request->latency, - flb_sds_len(http_request->latency)); - } - - /* String sub-fields */ - msgpack_pack_str(mp_pck, HTTP_REQUEST_REQUEST_METHOD_SIZE); - msgpack_pack_str_body(mp_pck, HTTP_REQUEST_REQUEST_METHOD, - HTTP_REQUEST_REQUEST_METHOD_SIZE); - msgpack_pack_str(mp_pck, flb_sds_len(http_request->requestMethod)); - msgpack_pack_str_body(mp_pck, http_request->requestMethod, - flb_sds_len(http_request->requestMethod)); - - msgpack_pack_str(mp_pck, HTTP_REQUEST_REQUEST_URL_SIZE); - msgpack_pack_str_body(mp_pck, HTTP_REQUEST_REQUEST_URL, - HTTP_REQUEST_REQUEST_URL_SIZE); - msgpack_pack_str(mp_pck, flb_sds_len(http_request->requestUrl)); - msgpack_pack_str_body(mp_pck, http_request->requestUrl, - flb_sds_len(http_request->requestUrl)); - - msgpack_pack_str(mp_pck, HTTP_REQUEST_USER_AGENT_SIZE); - msgpack_pack_str_body(mp_pck, HTTP_REQUEST_USER_AGENT, - HTTP_REQUEST_USER_AGENT_SIZE); - msgpack_pack_str(mp_pck, flb_sds_len(http_request->userAgent)); - msgpack_pack_str_body(mp_pck, http_request->userAgent, - flb_sds_len(http_request->userAgent)); - - msgpack_pack_str(mp_pck, HTTP_REQUEST_REMOTE_IP_SIZE); - msgpack_pack_str_body(mp_pck, HTTP_REQUEST_REMOTE_IP, - HTTP_REQUEST_REMOTE_IP_SIZE); - msgpack_pack_str(mp_pck, flb_sds_len(http_request->remoteIp)); - msgpack_pack_str_body(mp_pck, http_request->remoteIp, - flb_sds_len(http_request->remoteIp)); - - msgpack_pack_str(mp_pck, HTTP_REQUEST_SERVER_IP_SIZE); - msgpack_pack_str_body(mp_pck, HTTP_REQUEST_SERVER_IP, - HTTP_REQUEST_SERVER_IP_SIZE); - msgpack_pack_str(mp_pck, flb_sds_len(http_request->serverIp)); - msgpack_pack_str_body(mp_pck, http_request->serverIp, - flb_sds_len(http_request->serverIp)); - - msgpack_pack_str(mp_pck, HTTP_REQUEST_REFERER_SIZE); - msgpack_pack_str_body(mp_pck, HTTP_REQUEST_REFERER, - HTTP_REQUEST_REFERER_SIZE); - msgpack_pack_str(mp_pck, flb_sds_len(http_request->referer)); - msgpack_pack_str_body(mp_pck, http_request->referer, - flb_sds_len(http_request->referer)); - - msgpack_pack_str(mp_pck, HTTP_REQUEST_PROTOCOL_SIZE); - msgpack_pack_str_body(mp_pck, HTTP_REQUEST_PROTOCOL, - HTTP_REQUEST_PROTOCOL_SIZE); - msgpack_pack_str(mp_pck, flb_sds_len(http_request->protocol)); - msgpack_pack_str_body(mp_pck, http_request->protocol, - flb_sds_len(http_request->protocol)); - - /* Integer sub-fields */ - msgpack_pack_str(mp_pck, HTTP_REQUEST_REQUESTSIZE_SIZE); - msgpack_pack_str_body(mp_pck, HTTP_REQUEST_REQUESTSIZE, - HTTP_REQUEST_REQUESTSIZE_SIZE); - msgpack_pack_int64(mp_pck, http_request->requestSize); - - msgpack_pack_str(mp_pck, HTTP_REQUEST_RESPONSESIZE_SIZE); - msgpack_pack_str_body(mp_pck, HTTP_REQUEST_RESPONSESIZE, - HTTP_REQUEST_RESPONSESIZE_SIZE); - msgpack_pack_int64(mp_pck, http_request->responseSize); - - msgpack_pack_str(mp_pck, HTTP_REQUEST_STATUS_SIZE); - msgpack_pack_str_body(mp_pck, HTTP_REQUEST_STATUS, HTTP_REQUEST_STATUS_SIZE); - msgpack_pack_int64(mp_pck, http_request->status); - - msgpack_pack_str(mp_pck, HTTP_REQUEST_CACHE_FILL_BYTES_SIZE); - msgpack_pack_str_body(mp_pck, HTTP_REQUEST_CACHE_FILL_BYTES, - HTTP_REQUEST_CACHE_FILL_BYTES_SIZE); - msgpack_pack_int64(mp_pck, http_request->cacheFillBytes); - - /* Boolean sub-fields */ - msgpack_pack_str(mp_pck, HTTP_REQUEST_CACHE_LOOKUP_SIZE); - msgpack_pack_str_body(mp_pck, HTTP_REQUEST_CACHE_LOOKUP, - HTTP_REQUEST_CACHE_LOOKUP_SIZE); - if (http_request->cacheLookup == FLB_TRUE) { - msgpack_pack_true(mp_pck); - } - else { - msgpack_pack_false(mp_pck); - } - - msgpack_pack_str(mp_pck, HTTP_REQUEST_CACHE_HIT_SIZE); - msgpack_pack_str_body(mp_pck, HTTP_REQUEST_CACHE_HIT, - HTTP_REQUEST_CACHE_HIT_SIZE); - if (http_request->cacheLookup == FLB_TRUE) { - msgpack_pack_true(mp_pck); - } - else { - msgpack_pack_false(mp_pck); - } - - msgpack_pack_str(mp_pck, HTTP_REQUEST_CACHE_VALIDATE_WITH_ORIGIN_SERVER_SIZE); - msgpack_pack_str_body(mp_pck, HTTP_REQUEST_CACHE_VALIDATE_WITH_ORIGIN_SERVER, - HTTP_REQUEST_CACHE_VALIDATE_WITH_ORIGIN_SERVER_SIZE); - if (http_request->cacheValidatedWithOriginServer == FLB_TRUE) { - msgpack_pack_true(mp_pck); - } - else { - msgpack_pack_false(mp_pck); - } -} - -/* latency should be in the format: - * whitespace (opt.) + integer + point & decimal (opt.) - * + whitespace (opt.) + "s" + whitespace (opt.) - * - * latency is Duration, so the maximum value is "315576000000.999999999s". - * (23 characters in length) - */ -static void validate_latency(msgpack_object_str latency_in_payload, - struct http_request_field *http_request) { - int i = 0; - int j = 0; - int status = 0; - char extract_latency[32]; - flb_sds_t pattern; - struct flb_regex *regex; - - pattern = flb_sds_create("^\\s*\\d+(.\\d+)?\\s*s\\s*$"); - if (!pattern) { - return; - } - - if (latency_in_payload.size > sizeof(extract_latency)) { - flb_sds_destroy(pattern); - return; - } - - regex = flb_regex_create(pattern); - status = flb_regex_match(regex, - (unsigned char *) latency_in_payload.ptr, - latency_in_payload.size); - flb_regex_destroy(regex); - flb_sds_destroy(pattern); - - if (status == 1) { - for (; i < latency_in_payload.size; ++ i) { - if (latency_in_payload.ptr[i] == '.' || latency_in_payload.ptr[i] == 's' - || isdigit(latency_in_payload.ptr[i])) { - extract_latency[j] = latency_in_payload.ptr[i]; - ++ j; - } - } - http_request->latency = flb_sds_copy(http_request->latency, extract_latency, j); - } -} - -/* Return true if httpRequest extracted */ -int extract_http_request(struct http_request_field *http_request, - flb_sds_t http_request_key, - int http_request_key_size, - msgpack_object *obj, int *extra_subfields) -{ - http_request_status op_status = NO_HTTPREQUEST; - msgpack_object_kv *p; - msgpack_object_kv *pend; - msgpack_object_kv *tmp_p; - msgpack_object_kv *tmp_pend; - - if (obj->via.map.size == 0) { - return FLB_FALSE; - } - - p = obj->via.map.ptr; - pend = obj->via.map.ptr + obj->via.map.size; - - for (; p < pend && op_status == NO_HTTPREQUEST; ++p) { - - if (p->val.type != MSGPACK_OBJECT_MAP - || !validate_key(p->key, http_request_key, - http_request_key_size)) { - - continue; - } - - op_status = HTTPREQUEST_EXISTS; - msgpack_object sub_field = p->val; - - tmp_p = sub_field.via.map.ptr; - tmp_pend = sub_field.via.map.ptr + sub_field.via.map.size; - - /* Validate the subfields of httpRequest */ - for (; tmp_p < tmp_pend; ++tmp_p) { - if (tmp_p->key.type != MSGPACK_OBJECT_STR) { - continue; - } - - if (validate_key(tmp_p->key, HTTP_REQUEST_LATENCY, - HTTP_REQUEST_LATENCY_SIZE)) { - if (tmp_p->val.type != MSGPACK_OBJECT_STR) { - continue; - } - validate_latency(tmp_p->val.via.str, http_request); - } - else if (validate_key(tmp_p->key, HTTP_REQUEST_PROTOCOL, - HTTP_REQUEST_PROTOCOL_SIZE)) { - try_assign_subfield_str(tmp_p->val, &http_request->protocol); - } - else if (validate_key(tmp_p->key, HTTP_REQUEST_REFERER, - HTTP_REQUEST_REFERER_SIZE)) { - try_assign_subfield_str(tmp_p->val, &http_request->referer); - } - else if (validate_key(tmp_p->key, HTTP_REQUEST_REMOTE_IP, - HTTP_REQUEST_REMOTE_IP_SIZE)) { - try_assign_subfield_str(tmp_p->val, &http_request->remoteIp); - } - else if (validate_key(tmp_p->key, HTTP_REQUEST_REQUEST_METHOD, - HTTP_REQUEST_REQUEST_METHOD_SIZE)) { - try_assign_subfield_str(tmp_p->val, &http_request->requestMethod); - } - else if (validate_key(tmp_p->key, HTTP_REQUEST_REQUEST_URL, - HTTP_REQUEST_REQUEST_URL_SIZE)) { - try_assign_subfield_str(tmp_p->val, &http_request->requestUrl); - } - else if (validate_key(tmp_p->key, HTTP_REQUEST_SERVER_IP, - HTTP_REQUEST_SERVER_IP_SIZE)) { - try_assign_subfield_str(tmp_p->val, &http_request->serverIp); - } - else if (validate_key(tmp_p->key, HTTP_REQUEST_USER_AGENT, - HTTP_REQUEST_USER_AGENT_SIZE)) { - try_assign_subfield_str(tmp_p->val, &http_request->userAgent); - } - - else if (validate_key(tmp_p->key, HTTP_REQUEST_CACHE_FILL_BYTES, - HTTP_REQUEST_CACHE_FILL_BYTES_SIZE)) { - try_assign_subfield_int(tmp_p->val, &http_request->cacheFillBytes); - } - else if (validate_key(tmp_p->key, HTTP_REQUEST_REQUESTSIZE, - HTTP_REQUEST_REQUESTSIZE_SIZE)) { - try_assign_subfield_int(tmp_p->val, &http_request->requestSize); - } - else if (validate_key(tmp_p->key, HTTP_REQUEST_RESPONSESIZE, - HTTP_REQUEST_RESPONSESIZE_SIZE)) { - try_assign_subfield_int(tmp_p->val, &http_request->responseSize); - } - else if (validate_key(tmp_p->key, HTTP_REQUEST_STATUS, - HTTP_REQUEST_STATUS_SIZE)) { - try_assign_subfield_int(tmp_p->val, &http_request->status); - } - - else if (validate_key(tmp_p->key, HTTP_REQUEST_CACHE_HIT, - HTTP_REQUEST_CACHE_HIT_SIZE)) { - try_assign_subfield_bool(tmp_p->val, &http_request->cacheHit); - } - else if (validate_key(tmp_p->key, HTTP_REQUEST_CACHE_LOOKUP, - HTTP_REQUEST_CACHE_LOOKUP_SIZE)) { - try_assign_subfield_bool(tmp_p->val, &http_request->cacheLookup); - } - else if (validate_key(tmp_p->key, HTTP_REQUEST_CACHE_VALIDATE_WITH_ORIGIN_SERVER, - HTTP_REQUEST_CACHE_VALIDATE_WITH_ORIGIN_SERVER_SIZE)) { - try_assign_subfield_bool(tmp_p->val, - &http_request->cacheValidatedWithOriginServer); - } - - else { - *extra_subfields += 1; - } - } - } - - return op_status == HTTPREQUEST_EXISTS; -} - -void pack_extra_http_request_subfields(msgpack_packer *mp_pck, - msgpack_object *http_request, - int extra_subfields) { - msgpack_object_kv *p = http_request->via.map.ptr; - msgpack_object_kv *const pend = http_request->via.map.ptr + http_request->via.map.size; - - msgpack_pack_map(mp_pck, extra_subfields); - - for (; p < pend; ++p) { - if (validate_key(p->key, HTTP_REQUEST_LATENCY, - HTTP_REQUEST_LATENCY_SIZE) - || validate_key(p->key, HTTP_REQUEST_PROTOCOL, - HTTP_REQUEST_PROTOCOL_SIZE) - || validate_key(p->key, HTTP_REQUEST_REFERER, - HTTP_REQUEST_REFERER_SIZE) - || validate_key(p->key, HTTP_REQUEST_REMOTE_IP, - HTTP_REQUEST_REMOTE_IP_SIZE) - || validate_key(p->key, HTTP_REQUEST_REQUEST_METHOD, - HTTP_REQUEST_REQUEST_METHOD_SIZE) - || validate_key(p->key, HTTP_REQUEST_REQUEST_URL, - HTTP_REQUEST_REQUEST_URL_SIZE) - || validate_key(p->key, HTTP_REQUEST_SERVER_IP, - HTTP_REQUEST_SERVER_IP_SIZE) - || validate_key(p->key, HTTP_REQUEST_USER_AGENT, - HTTP_REQUEST_USER_AGENT_SIZE) - || validate_key(p->key, HTTP_REQUEST_CACHE_FILL_BYTES, - HTTP_REQUEST_CACHE_FILL_BYTES_SIZE) - || validate_key(p->key, HTTP_REQUEST_REQUESTSIZE, - HTTP_REQUEST_REQUESTSIZE_SIZE) - || validate_key(p->key, HTTP_REQUEST_RESPONSESIZE, - HTTP_REQUEST_RESPONSESIZE_SIZE) - || validate_key(p->key, HTTP_REQUEST_STATUS, - HTTP_REQUEST_STATUS_SIZE) - || validate_key(p->key, HTTP_REQUEST_CACHE_HIT, - HTTP_REQUEST_CACHE_HIT_SIZE) - || validate_key(p->key, HTTP_REQUEST_CACHE_LOOKUP, - HTTP_REQUEST_CACHE_LOOKUP_SIZE) - || validate_key(p->key, HTTP_REQUEST_CACHE_VALIDATE_WITH_ORIGIN_SERVER, - HTTP_REQUEST_CACHE_VALIDATE_WITH_ORIGIN_SERVER_SIZE)) { - - continue; - } - - msgpack_pack_object(mp_pck, p->key); - msgpack_pack_object(mp_pck, p->val); - } -} diff --git a/fluent-bit/plugins/out_stackdriver/stackdriver_http_request.h b/fluent-bit/plugins/out_stackdriver/stackdriver_http_request.h deleted file mode 100644 index 8b935c3f7..000000000 --- a/fluent-bit/plugins/out_stackdriver/stackdriver_http_request.h +++ /dev/null @@ -1,120 +0,0 @@ -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#ifndef FLB_STD_HTTPREQUEST_H -#define FLB_STD_HTTPREQUEST_H - -#include "stackdriver.h" - -/* subfield name and size */ -#define HTTP_REQUEST_LATENCY "latency" -#define HTTP_REQUEST_PROTOCOL "protocol" -#define HTTP_REQUEST_REFERER "referer" -#define HTTP_REQUEST_REMOTE_IP "remoteIp" -#define HTTP_REQUEST_REQUEST_METHOD "requestMethod" -#define HTTP_REQUEST_REQUEST_URL "requestUrl" -#define HTTP_REQUEST_SERVER_IP "serverIp" -#define HTTP_REQUEST_USER_AGENT "userAgent" -#define HTTP_REQUEST_CACHE_FILL_BYTES "cacheFillBytes" -#define HTTP_REQUEST_REQUESTSIZE "requestSize" -#define HTTP_REQUEST_RESPONSESIZE "responseSize" -#define HTTP_REQUEST_STATUS "status" -#define HTTP_REQUEST_CACHE_HIT "cacheHit" -#define HTTP_REQUEST_CACHE_LOOKUP "cacheLookup" -#define HTTP_REQUEST_CACHE_VALIDATE_WITH_ORIGIN_SERVER "cacheValidatedWithOriginServer" - -#define HTTP_REQUEST_LATENCY_SIZE 7 -#define HTTP_REQUEST_PROTOCOL_SIZE 8 -#define HTTP_REQUEST_REFERER_SIZE 7 -#define HTTP_REQUEST_REMOTE_IP_SIZE 8 -#define HTTP_REQUEST_REQUEST_METHOD_SIZE 13 -#define HTTP_REQUEST_REQUEST_URL_SIZE 10 -#define HTTP_REQUEST_SERVER_IP_SIZE 8 -#define HTTP_REQUEST_USER_AGENT_SIZE 9 -#define HTTP_REQUEST_CACHE_FILL_BYTES_SIZE 14 -#define HTTP_REQUEST_REQUESTSIZE_SIZE 11 -#define HTTP_REQUEST_RESPONSESIZE_SIZE 12 -#define HTTP_REQUEST_STATUS_SIZE 6 -#define HTTP_REQUEST_CACHE_HIT_SIZE 8 -#define HTTP_REQUEST_CACHE_LOOKUP_SIZE 11 -#define HTTP_REQUEST_CACHE_VALIDATE_WITH_ORIGIN_SERVER_SIZE 30 - - -struct http_request_field { - flb_sds_t latency; - flb_sds_t protocol; - flb_sds_t referer; - flb_sds_t remoteIp; - flb_sds_t requestMethod; - flb_sds_t requestUrl; - flb_sds_t serverIp; - flb_sds_t userAgent; - - int64_t cacheFillBytes; - int64_t requestSize; - int64_t responseSize; - int64_t status; - - int cacheHit; - int cacheLookup; - int cacheValidatedWithOriginServer; -}; - -void init_http_request(struct http_request_field *http_request); -void destroy_http_request(struct http_request_field *http_request); - -/* - * Add httpRequest field to the entries. - * The structure of httpRequest is as shown in struct http_request_field - */ -void add_http_request_field(struct http_request_field *http_request, - msgpack_packer *mp_pck); - -/* - * Extract the httpRequest field from the jsonPayload. - * If the httpRequest field exists, return TRUE and store the subfields. - * If there are extra subfields, count the number. - */ -int extract_http_request(struct http_request_field *http_request, - flb_sds_t http_request_key, - int http_request_key_size, - msgpack_object *obj, int *extra_subfields); - -/* - * When there are extra subfields, we will preserve the extra subfields inside jsonPayload - * For example, if the jsonPayload is as followed： - * jsonPayload { - * "logging.googleapis.com/http_request": { - * "requestMethod": "GET", - * "latency": "1s", - * "cacheLookup": true, - * "extra": "some string" #extra subfield - * } - * } - * We will preserve the extra subfields. The jsonPayload after extracting is: - * jsonPayload { - * "logging.googleapis.com/http_request": { - * "extra": "some string" - * } - * } - */ -void pack_extra_http_request_subfields(msgpack_packer *mp_pck, - msgpack_object *http_request, - int extra_subfields); - -#endif diff --git a/fluent-bit/plugins/out_stackdriver/stackdriver_operation.c b/fluent-bit/plugins/out_stackdriver/stackdriver_operation.c deleted file mode 100644 index 548e8b473..000000000 --- a/fluent-bit/plugins/out_stackdriver/stackdriver_operation.c +++ /dev/null @@ -1,147 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include "stackdriver.h" -#include "stackdriver_helper.h" -#include "stackdriver_operation.h" - -typedef enum { - NO_OPERATION = 1, - OPERATION_EXISTED = 2 -} operation_status; - -void add_operation_field(flb_sds_t *operation_id, flb_sds_t *operation_producer, - int *operation_first, int *operation_last, - msgpack_packer *mp_pck) -{ - msgpack_pack_str(mp_pck, 9); - msgpack_pack_str_body(mp_pck, "operation", 9); - - msgpack_pack_map(mp_pck, 4); - - msgpack_pack_str(mp_pck, OPERATION_ID_SIZE); - msgpack_pack_str_body(mp_pck, OPERATION_ID, OPERATION_ID_SIZE); - msgpack_pack_str(mp_pck, flb_sds_len(*operation_id)); - msgpack_pack_str_body(mp_pck, *operation_id, flb_sds_len(*operation_id)); - - msgpack_pack_str(mp_pck, OPERATION_PRODUCER_SIZE); - msgpack_pack_str_body(mp_pck, OPERATION_PRODUCER, OPERATION_PRODUCER_SIZE); - msgpack_pack_str(mp_pck, flb_sds_len(*operation_producer)); - msgpack_pack_str_body(mp_pck, *operation_producer, - flb_sds_len(*operation_producer)); - - msgpack_pack_str(mp_pck, OPERATION_FIRST_SIZE); - msgpack_pack_str_body(mp_pck, OPERATION_FIRST, OPERATION_FIRST_SIZE); - if (*operation_first == FLB_TRUE) { - msgpack_pack_true(mp_pck); - } - else { - msgpack_pack_false(mp_pck); - } - - msgpack_pack_str(mp_pck, OPERATION_LAST_SIZE); - msgpack_pack_str_body(mp_pck, OPERATION_LAST, OPERATION_LAST_SIZE); - if (*operation_last == FLB_TRUE) { - msgpack_pack_true(mp_pck); - } - else { - msgpack_pack_false(mp_pck); - } -} - -/* Return true if operation extracted */ -int extract_operation(flb_sds_t *operation_id, flb_sds_t *operation_producer, - int *operation_first, int *operation_last, - msgpack_object *obj, int *extra_subfields) -{ - operation_status op_status = NO_OPERATION; - msgpack_object_kv *p; - msgpack_object_kv *pend; - msgpack_object_kv *tmp_p; - msgpack_object_kv *tmp_pend; - - if (obj->via.map.size == 0) { - return FLB_FALSE; - } - p = obj->via.map.ptr; - pend = obj->via.map.ptr + obj->via.map.size; - - for (; p < pend && op_status == NO_OPERATION; ++p) { - - if (p->val.type != MSGPACK_OBJECT_MAP - || !validate_key(p->key, OPERATION_FIELD_IN_JSON, - OPERATION_KEY_SIZE)) { - continue; - } - - op_status = OPERATION_EXISTED; - msgpack_object sub_field = p->val; - - tmp_p = sub_field.via.map.ptr; - tmp_pend = sub_field.via.map.ptr + sub_field.via.map.size; - - /* Validate the subfields of operation */ - for (; tmp_p < tmp_pend; ++tmp_p) { - if (tmp_p->key.type != MSGPACK_OBJECT_STR) { - continue; - } - - if (validate_key(tmp_p->key, OPERATION_ID, OPERATION_ID_SIZE)) { - try_assign_subfield_str(tmp_p->val, operation_id); - } - else if (validate_key(tmp_p->key, OPERATION_PRODUCER, - OPERATION_PRODUCER_SIZE)) { - try_assign_subfield_str(tmp_p->val, operation_producer); - } - else if (validate_key(tmp_p->key, OPERATION_FIRST, OPERATION_FIRST_SIZE)) { - try_assign_subfield_bool(tmp_p->val, operation_first); - } - else if (validate_key(tmp_p->key, OPERATION_LAST, OPERATION_LAST_SIZE)) { - try_assign_subfield_bool(tmp_p->val, operation_last); - } - else { - *extra_subfields += 1; - } - } - } - - return op_status == OPERATION_EXISTED; -} - -void pack_extra_operation_subfields(msgpack_packer *mp_pck, - msgpack_object *operation, int extra_subfields) { - msgpack_object_kv *p = operation->via.map.ptr; - msgpack_object_kv *const pend = operation->via.map.ptr + operation->via.map.size; - - msgpack_pack_map(mp_pck, extra_subfields); - - for (; p < pend; ++p) { - if (validate_key(p->key, OPERATION_ID, OPERATION_ID_SIZE) - || validate_key(p->key, OPERATION_PRODUCER, OPERATION_PRODUCER_SIZE) - || validate_key(p->key, OPERATION_FIRST, OPERATION_FIRST_SIZE) - || validate_key(p->key, OPERATION_LAST, OPERATION_LAST_SIZE)) { - continue; - } - - msgpack_pack_object(mp_pck, p->key); - msgpack_pack_object(mp_pck, p->val); - } -} diff --git a/fluent-bit/plugins/out_stackdriver/stackdriver_operation.h b/fluent-bit/plugins/out_stackdriver/stackdriver_operation.h deleted file mode 100644 index ded886c3b..000000000 --- a/fluent-bit/plugins/out_stackdriver/stackdriver_operation.h +++ /dev/null @@ -1,82 +0,0 @@ -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#ifndef FLB_STD_OPERATION_H -#define FLB_STD_OPERATION_H - -#include "stackdriver.h" - -/* subfield name and size */ -#define OPERATION_ID "id" -#define OPERATION_PRODUCER "producer" -#define OPERATION_FIRST "first" -#define OPERATION_LAST "last" - -#define OPERATION_ID_SIZE 2 -#define OPERATION_PRODUCER_SIZE 8 -#define OPERATION_FIRST_SIZE 5 -#define OPERATION_LAST_SIZE 4 - -/* - * Add operation field to the entries. - * The structure of operation is: - * { - * "id": string, - * "producer": string, - * "first": boolean, - * "last": boolean - * } - * - */ -void add_operation_field(flb_sds_t *operation_id, flb_sds_t *operation_producer, - int *operation_first, int *operation_last, - msgpack_packer *mp_pck); - -/* - * Extract the operation field from the jsonPayload. - * If the operation field exists, return TRUE and store the subfields. - * If there are extra subfields, count the number. - */ -int extract_operation(flb_sds_t *operation_id, flb_sds_t *operation_producer, - int *operation_first, int *operation_last, - msgpack_object *obj, int *extra_subfields); - -/* - * When there are extra subfields, we will preserve the extra subfields inside jsonPayload - * For example, if the jsonPayload is as followed： - * jsonPayload { - * "logging.googleapis.com/operation": { - * "id": "id1", - * "producer": "id2", - * "first": true, - * "last": true, - * "extra": "some string" #extra subfield - * } - * } - * We will preserve the extra subfields. The jsonPayload after extracting is: - * jsonPayload { - * "logging.googleapis.com/operation": { - * "extra": "some string" - * } - * } - */ -void pack_extra_operation_subfields(msgpack_packer *mp_pck, msgpack_object *operation, - int extra_subfields); - - -#endif diff --git a/fluent-bit/plugins/out_stackdriver/stackdriver_resource_types.c b/fluent-bit/plugins/out_stackdriver/stackdriver_resource_types.c deleted file mode 100644 index b114e4922..000000000 --- a/fluent-bit/plugins/out_stackdriver/stackdriver_resource_types.c +++ /dev/null @@ -1,143 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include "stackdriver.h" -#include "stackdriver_resource_types.h" - -static const struct resource_type resource_types[] = { - { - .id = RESOURCE_TYPE_K8S, - .resources = {"k8s_container", "k8s_node", "k8s_pod"}, - .required_labels = {"cluster_name", "location"} - }, - { - .id = RESOURCE_TYPE_GENERIC_NODE, - .resources = {"generic_node"}, - .required_labels = {"location", "namespace", "node_id"} - }, - { - .id = RESOURCE_TYPE_GENERIC_TASK, - .resources = {"generic_task"}, - .required_labels = {"location", "namespace", "job", "task_id"} - } -}; - -static char **get_required_labels(int resource_type) -{ - int i; - int len; - - len = sizeof(resource_types) / sizeof(resource_types[0]); - for(i = 0; i < len; i++) { - if (resource_types[i].id == resource_type) { - return (char **) resource_types[i].required_labels; - } - } - return NULL; -} - -/* - * set_resource_type(): - * - Iterates through resource_types that are set up for validation and sets the - * resource_type if it matches one of them. - * - A resource may not be in the resource types list but still be accepted - * and processed (e.g. global) if it does not require / is not set up for validation. - */ -void set_resource_type(struct flb_stackdriver *ctx) -{ - int i; - int j; - int len; - char *resource; - struct resource_type resource_type; - - len = sizeof(resource_types) / sizeof(resource_types[0]); - for(i = 0; i < len; i++) { - resource_type = resource_types[i]; - for(j = 0; j < MAX_RESOURCE_ENTRIES; j++) { - if (resource_type.resources[j] != NULL) { - resource = resource_type.resources[j]; - if (flb_sds_cmp(ctx->resource, resource, strlen(resource)) == 0) { - ctx->resource_type = resource_type.id; - return; - } - } - } - } -} - -/* - * resource_api_has_required_labels(): - * - Determines if all required labels for the set resource type are present as - * keys on the resource labels key-value pairs. - */ -int resource_api_has_required_labels(struct flb_stackdriver *ctx) -{ - struct mk_list *head; - struct flb_hash_table *ht; - struct flb_kv *label_kv; - char** required_labels; - int i; - int found; - void *tmp_buf; - size_t tmp_size; - - if (mk_list_size(&ctx->resource_labels_kvs) == 0) { - return FLB_FALSE; - } - - required_labels = get_required_labels(ctx->resource_type); - if (required_labels == NULL) { - flb_plg_warn(ctx->ins, "no validation applied to resource_labels " - "for set resource type"); - return FLB_FALSE; - } - - ht = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, MAX_REQUIRED_LABEL_ENTRIES, 0); - mk_list_foreach(head, &ctx->resource_labels_kvs) { - label_kv = mk_list_entry(head, struct flb_kv, _head); - for (i = 0; i < MAX_REQUIRED_LABEL_ENTRIES; i++) { - if (required_labels[i] != NULL && flb_sds_cmp(label_kv->key, - required_labels[i], strlen(required_labels[i])) == 0) { - flb_hash_table_add(ht, required_labels[i], strlen(required_labels[i]), - NULL, 0); - } - } - } - - for (i = 0; i < MAX_REQUIRED_LABEL_ENTRIES; i++) { - if (required_labels[i] != NULL) { - found = flb_hash_table_get(ht, required_labels[i], strlen(required_labels[i]), - &tmp_buf, &tmp_size); - if (found == -1) { - flb_plg_warn(ctx->ins, "labels set in resource_labels will not be applied" - ", as the required resource label [%s] is missing", required_labels[i]); - ctx->should_skip_resource_labels_api = FLB_TRUE; - flb_hash_table_destroy(ht); - return FLB_FALSE; - } - } - } - flb_hash_table_destroy(ht); - return FLB_TRUE; -} diff --git a/fluent-bit/plugins/out_stackdriver/stackdriver_resource_types.h b/fluent-bit/plugins/out_stackdriver/stackdriver_resource_types.h deleted file mode 100644 index 5e45c8745..000000000 --- a/fluent-bit/plugins/out_stackdriver/stackdriver_resource_types.h +++ /dev/null @@ -1,41 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_STACKDRIVER_RESOURCE_TYPES_H -#define FLB_OUT_STACKDRIVER_RESOURCE_TYPES_H - -#include "stackdriver.h" - -#define MAX_RESOURCE_ENTRIES 10 -#define MAX_REQUIRED_LABEL_ENTRIES 10 - -#define RESOURCE_TYPE_K8S 1 -#define RESOURCE_TYPE_GENERIC_NODE 2 -#define RESOURCE_TYPE_GENERIC_TASK 3 - -struct resource_type { - int id; - char* resources[MAX_RESOURCE_ENTRIES]; - char* required_labels[MAX_REQUIRED_LABEL_ENTRIES]; -}; - -void set_resource_type(struct flb_stackdriver *ctx); -int resource_api_has_required_labels(struct flb_stackdriver *ctx); - -#endif diff --git a/fluent-bit/plugins/out_stackdriver/stackdriver_source_location.c b/fluent-bit/plugins/out_stackdriver/stackdriver_source_location.c deleted file mode 100644 index 58102c91e..000000000 --- a/fluent-bit/plugins/out_stackdriver/stackdriver_source_location.c +++ /dev/null @@ -1,139 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "stackdriver.h" -#include "stackdriver_helper.h" -#include "stackdriver_source_location.h" - -typedef enum { - NO_SOURCELOCATION = 1, - SOURCELOCATION_EXISTED = 2 -} source_location_status; - - -void add_source_location_field(flb_sds_t *source_location_file, - int64_t source_location_line, - flb_sds_t *source_location_function, - msgpack_packer *mp_pck) -{ - msgpack_pack_str(mp_pck, 14); - msgpack_pack_str_body(mp_pck, "sourceLocation", 14); - msgpack_pack_map(mp_pck, 3); - - msgpack_pack_str(mp_pck, SOURCE_LOCATION_FILE_SIZE); - msgpack_pack_str_body(mp_pck, SOURCE_LOCATION_FILE, SOURCE_LOCATION_FILE_SIZE); - msgpack_pack_str(mp_pck, flb_sds_len(*source_location_file)); - msgpack_pack_str_body(mp_pck, *source_location_file, - flb_sds_len(*source_location_file)); - - msgpack_pack_str(mp_pck, SOURCE_LOCATION_LINE_SIZE); - msgpack_pack_str_body(mp_pck, SOURCE_LOCATION_LINE, SOURCE_LOCATION_LINE_SIZE); - msgpack_pack_int64(mp_pck, source_location_line); - - msgpack_pack_str(mp_pck, SOURCE_LOCATION_FUNCTION_SIZE); - msgpack_pack_str_body(mp_pck, SOURCE_LOCATION_FUNCTION, - SOURCE_LOCATION_FUNCTION_SIZE); - msgpack_pack_str(mp_pck, flb_sds_len(*source_location_function)); - msgpack_pack_str_body(mp_pck, *source_location_function, - flb_sds_len(*source_location_function)); -} - -/* Return FLB_TRUE if sourceLocation extracted */ -int extract_source_location(flb_sds_t *source_location_file, - int64_t *source_location_line, - flb_sds_t *source_location_function, - msgpack_object *obj, int *extra_subfields) -{ - source_location_status op_status = NO_SOURCELOCATION; - msgpack_object_kv *p; - msgpack_object_kv *pend; - msgpack_object_kv *tmp_p; - msgpack_object_kv *tmp_pend; - - if (obj->via.map.size == 0) { - return FLB_FALSE; - } - p = obj->via.map.ptr; - pend = obj->via.map.ptr + obj->via.map.size; - - for (; p < pend && op_status == NO_SOURCELOCATION; ++p) { - - if (p->val.type != MSGPACK_OBJECT_MAP - || p->key.type != MSGPACK_OBJECT_STR - || !validate_key(p->key, SOURCELOCATION_FIELD_IN_JSON, - SOURCE_LOCATION_SIZE)) { - - continue; - } - - op_status = SOURCELOCATION_EXISTED; - msgpack_object sub_field = p->val; - - tmp_p = sub_field.via.map.ptr; - tmp_pend = sub_field.via.map.ptr + sub_field.via.map.size; - - /* Validate the subfields of sourceLocation */ - for (; tmp_p < tmp_pend; ++tmp_p) { - if (tmp_p->key.type != MSGPACK_OBJECT_STR) { - continue; - } - - if (validate_key(tmp_p->key, - SOURCE_LOCATION_FILE, - SOURCE_LOCATION_FILE_SIZE)) { - try_assign_subfield_str(tmp_p->val, source_location_file); - } - else if (validate_key(tmp_p->key, - SOURCE_LOCATION_FUNCTION, - SOURCE_LOCATION_FUNCTION_SIZE)) { - try_assign_subfield_str(tmp_p->val, source_location_function); - } - else if (validate_key(tmp_p->key, - SOURCE_LOCATION_LINE, - SOURCE_LOCATION_LINE_SIZE)) { - try_assign_subfield_int(tmp_p->val, source_location_line); - } - else { - *extra_subfields += 1; - } - } - } - - return op_status == SOURCELOCATION_EXISTED; -} - -void pack_extra_source_location_subfields(msgpack_packer *mp_pck, - msgpack_object *source_location, - int extra_subfields) { - msgpack_object_kv *p = source_location->via.map.ptr; - msgpack_object_kv *const pend = source_location->via.map.ptr + source_location->via.map.size; - - msgpack_pack_map(mp_pck, extra_subfields); - - for (; p < pend; ++p) { - if (validate_key(p->key, SOURCE_LOCATION_FILE, SOURCE_LOCATION_FILE_SIZE) - || validate_key(p->key, SOURCE_LOCATION_LINE, SOURCE_LOCATION_LINE_SIZE) - || validate_key(p->key, SOURCE_LOCATION_FUNCTION, - SOURCE_LOCATION_FUNCTION_SIZE)) { - continue; - } - - msgpack_pack_object(mp_pck, p->key); - msgpack_pack_object(mp_pck, p->val); - } -} diff --git a/fluent-bit/plugins/out_stackdriver/stackdriver_source_location.h b/fluent-bit/plugins/out_stackdriver/stackdriver_source_location.h deleted file mode 100644 index 4f703d330..000000000 --- a/fluent-bit/plugins/out_stackdriver/stackdriver_source_location.h +++ /dev/null @@ -1,80 +0,0 @@ -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#ifndef FLB_STD_SOURCELOCATION_H -#define FLB_STD_SOURCELOCATION_H - -#include "stackdriver.h" - -/* subfield name and size */ -#define SOURCE_LOCATION_FILE "file" -#define SOURCE_LOCATION_LINE "line" -#define SOURCE_LOCATION_FUNCTION "function" - -#define SOURCE_LOCATION_FILE_SIZE 4 -#define SOURCE_LOCATION_LINE_SIZE 4 -#define SOURCE_LOCATION_FUNCTION_SIZE 8 - -/* - * Add sourceLocation field to the entries. - * The structure of sourceLocation is: - * { - * "file": string, - * "line": int, - * "function": string - * } - */ -void add_source_location_field(flb_sds_t *source_location_file, - int64_t source_location_line, - flb_sds_t *source_location_function, - msgpack_packer *mp_pck); - -/* - * Extract the sourceLocation field from the jsonPayload. - * If the sourceLocation field exists, return TRUE and store the subfields. - * If there are extra subfields, count the number. - */ -int extract_source_location(flb_sds_t *source_location_file, - int64_t *source_location_line, - flb_sds_t *source_location_function, - msgpack_object *obj, int *extra_subfields); - -/* - * When there are extra subfields, we will preserve the extra subfields inside jsonPayload - * For example, if the jsonPayload is as followed： - * jsonPayload { - * "logging.googleapis.com/sourceLocation": { - * "file": "file1", - * "line": 1, - * "function": "func1", - * "extra": "some string" #extra subfield - * } - * } - * We will preserve the extra subfields. The jsonPayload after extracting is: - * jsonPayload { - * "logging.googleapis.com/sourceLocation": { - * "extra": "some string" - * } - * } - */ -void pack_extra_source_location_subfields(msgpack_packer *mp_pck, - msgpack_object *source_location, - int extra_subfields); - - -#endif diff --git a/fluent-bit/plugins/out_stackdriver/stackdriver_timestamp.c b/fluent-bit/plugins/out_stackdriver/stackdriver_timestamp.c deleted file mode 100644 index a9b350d22..000000000 --- a/fluent-bit/plugins/out_stackdriver/stackdriver_timestamp.c +++ /dev/null @@ -1,180 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include "stackdriver.h" -#include "stackdriver_helper.h" -#include "stackdriver_timestamp.h" -#include - -#include - -static int is_integer(char *str, int size) { - int i; - for (i = 0; i < size; ++ i) { - if (!isdigit(str[i])) { - return FLB_FALSE; - } - } - return FLB_TRUE; -} - -static void try_assign_time(long long seconds, long long nanos, - struct flb_time *tms) -{ - if (seconds != 0) { - tms->tm.tv_sec = seconds; - tms->tm.tv_nsec = nanos; - } -} - -static long long get_integer(msgpack_object obj) -{ - char tmp[32]; - - if (obj.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - return obj.via.i64; - } - else if (obj.type == MSGPACK_OBJECT_STR - && is_integer((char *) obj.via.str.ptr, - obj.via.str.size)) { - - /* - * use an intermediary buffer to perform the conversion to avoid any - * overflow by atoll. LLONG_MAX value is +9,223,372,036,854,775,807, - * so using a 32 bytes buffer is enough. - */ - if (obj.via.str.size > sizeof(tmp) - 1) { - return 0; - } - - memcpy(tmp, obj.via.str.ptr, obj.via.str.size); - tmp[obj.via.str.size] = '\0'; - - return atoll(tmp); - } - - return 0; -} - -static int extract_format_timestamp_object(msgpack_object *obj, - struct flb_time *tms) -{ - int seconds_found = FLB_FALSE; - int nanos_found = FLB_FALSE; - long long seconds = 0; - long long nanos = 0; - - msgpack_object_kv *p; - msgpack_object_kv *pend; - msgpack_object_kv *tmp_p; - msgpack_object_kv *tmp_pend; - - if (obj->via.map.size == 0) { - return FLB_FALSE; - } - p = obj->via.map.ptr; - pend = obj->via.map.ptr + obj->via.map.size; - - for (; p < pend; ++p) { - if (!validate_key(p->key, "timestamp", 9) - || p->val.type != MSGPACK_OBJECT_MAP) { - continue; - } - - tmp_p = p->val.via.map.ptr; - tmp_pend = p->val.via.map.ptr + p->val.via.map.size; - - for (; tmp_p < tmp_pend; ++tmp_p) { - if (validate_key(tmp_p->key, "seconds", 7)) { - seconds_found = FLB_TRUE; - seconds = get_integer(tmp_p->val); - - if (nanos_found == FLB_TRUE) { - try_assign_time(seconds, nanos, tms); - return FLB_TRUE; - } - } - else if (validate_key(tmp_p->key, "nanos", 5)) { - nanos_found = FLB_TRUE; - nanos = get_integer(tmp_p->val); - - if (seconds_found == FLB_TRUE) { - try_assign_time(seconds, nanos, tms); - return FLB_TRUE; - } - } - } - } - return FLB_FALSE; -} - -static int extract_format_timestamp_duo_fields(msgpack_object *obj, - struct flb_time *tms) -{ - int seconds_found = FLB_FALSE; - int nanos_found = FLB_FALSE; - long long seconds = 0; - long long nanos = 0; - - msgpack_object_kv *p; - msgpack_object_kv *pend; - - if (obj->via.map.size == 0) { - return FLB_FALSE; - } - p = obj->via.map.ptr; - pend = obj->via.map.ptr + obj->via.map.size; - - for (; p < pend; ++p) { - if (validate_key(p->key, "timestampSeconds", 16)) { - seconds_found = FLB_TRUE; - seconds = get_integer(p->val); - - if (nanos_found == FLB_TRUE) { - try_assign_time(seconds, nanos, tms); - return FLB_TRUE; - } - } - else if (validate_key(p->key, "timestampNanos", 14)) { - nanos_found = FLB_TRUE; - nanos = get_integer(p->val); - - if (seconds_found == FLB_TRUE) { - try_assign_time(seconds, nanos, tms); - return FLB_TRUE; - } - } - } - - return FLB_FALSE; -} - -timestamp_status extract_timestamp(msgpack_object *obj, - struct flb_time *tms) -{ - if (extract_format_timestamp_object(obj, tms)) { - return FORMAT_TIMESTAMP_OBJECT; - } - if (extract_format_timestamp_duo_fields(obj, tms)) { - return FORMAT_TIMESTAMP_DUO_FIELDS; - } - return TIMESTAMP_NOT_PRESENT; -} diff --git a/fluent-bit/plugins/out_stackdriver/stackdriver_timestamp.h b/fluent-bit/plugins/out_stackdriver/stackdriver_timestamp.h deleted file mode 100644 index f3c025864..000000000 --- a/fluent-bit/plugins/out_stackdriver/stackdriver_timestamp.h +++ /dev/null @@ -1,47 +0,0 @@ -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#ifndef FLB_STD_TIMESTAMP_H -#define FLB_STD_TIMESTAMP_H - -#include "stackdriver.h" -#include - -typedef enum { - TIMESTAMP_NOT_PRESENT = 0, - FORMAT_TIMESTAMP_OBJECT = 1, - FORMAT_TIMESTAMP_DUO_FIELDS = 2 -} timestamp_status; - -/* - * Currently support two formats of time-related fields - * - "timestamp":{"seconds", "nanos"} - * - "timestampSeconds"/"timestampNanos" - * - * If timestamp field is not existed, return TIMESTAMP_NOT_PRESENT - * If timestamp format is "timestamp":{"seconds", "nanos"}, - * set the time and return FORMAT_TIMESTAMP - * - * If timestamp format is "timestampSeconds"/"timestampNanos", - * set the time and return FORMAT_TIMESTAMPSECONDS - */ -timestamp_status extract_timestamp(msgpack_object *obj, - struct flb_time *tms); - - -#endif diff --git a/fluent-bit/plugins/out_stdout/CMakeLists.txt b/fluent-bit/plugins/out_stdout/CMakeLists.txt deleted file mode 100644 index 2331680c1..000000000 --- a/fluent-bit/plugins/out_stdout/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - stdout.c) - -FLB_PLUGIN(out_stdout "${src}" "") diff --git a/fluent-bit/plugins/out_stdout/stdout.c b/fluent-bit/plugins/out_stdout/stdout.c deleted file mode 100644 index 2d6bff598..000000000 --- a/fluent-bit/plugins/out_stdout/stdout.c +++ /dev/null @@ -1,301 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include -#include "stdout.h" - - -static int cb_stdout_init(struct flb_output_instance *ins, - struct flb_config *config, void *data) -{ - int ret; - const char *tmp; - struct flb_stdout *ctx = NULL; - (void) ins; - (void) config; - (void) data; - - ctx = flb_calloc(1, sizeof(struct flb_stdout)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->ins = ins; - - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return -1; - } - - ctx->out_format = FLB_PACK_JSON_FORMAT_NONE; - tmp = flb_output_get_property("format", ins); - if (tmp) { - ret = flb_pack_to_json_format_type(tmp); - if (ret == -1) { - flb_plg_error(ctx->ins, "unrecognized 'format' option. " - "Using 'msgpack'"); - } - else { - ctx->out_format = ret; - } - } - - /* Date key */ - ctx->date_key = ctx->json_date_key; - tmp = flb_output_get_property("json_date_key", ins); - if (tmp) { - /* Just check if we have to disable it */ - if (flb_utils_bool(tmp) == FLB_FALSE) { - ctx->date_key = NULL; - } - } - - /* Date format for JSON output */ - ctx->json_date_format = FLB_PACK_JSON_DATE_DOUBLE; - tmp = flb_output_get_property("json_date_format", ins); - if (tmp) { - ret = flb_pack_to_json_date_type(tmp); - if (ret == -1) { - flb_plg_error(ctx->ins, "invalid json_date_format '%s'. " - "Using 'double' type", tmp); - } - else { - ctx->json_date_format = ret; - } - } - - /* Export context */ - flb_output_set_context(ins, ctx); - - return 0; -} - -#ifdef FLB_HAVE_METRICS -static void print_metrics_text(struct flb_output_instance *ins, - const void *data, size_t bytes) -{ - int ret; - size_t off = 0; - cfl_sds_t text; - struct cmt *cmt = NULL; - - /* get cmetrics context */ - ret = cmt_decode_msgpack_create(&cmt, (char *) data, bytes, &off); - if (ret != 0) { - flb_plg_error(ins, "could not process metrics payload"); - return; - } - - /* convert to text representation */ - text = cmt_encode_text_create(cmt); - - /* destroy cmt context */ - cmt_destroy(cmt); - - printf("%s", text); - fflush(stdout); - - cmt_encode_text_destroy(text); -} -#endif - -static void print_traces_text(struct flb_output_instance *ins, - const void *data, size_t bytes) -{ - int ret; - size_t off = 0; - cfl_sds_t text; - struct ctrace *ctr = NULL; - - /* get cmetrics context */ - ret = ctr_decode_msgpack_create(&ctr, (char *) data, bytes, &off); - if (ret != 0) { - flb_plg_error(ins, "could not process traces payload (ret=%i)", ret); - return; - } - - /* convert to text representation */ - text = ctr_encode_text_create(ctr); - - /* destroy cmt context */ - ctr_destroy(ctr); - - printf("%s", text); - fflush(stdout); - - ctr_encode_text_destroy(text); -} - -static void cb_stdout_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - int result; - flb_sds_t json; - struct flb_stdout *ctx; - size_t cnt; - - (void) config; - - result = FLB_EVENT_DECODER_SUCCESS; - ctx = (struct flb_stdout *) out_context; - cnt = 0; - -#ifdef FLB_HAVE_METRICS - /* Check if the event type is metrics, handle the payload differently */ - if (event_chunk->type == FLB_EVENT_TYPE_METRICS) { - print_metrics_text(ctx->ins, (char *) - event_chunk->data, - event_chunk->size); - FLB_OUTPUT_RETURN(FLB_OK); - } -#endif - - if (event_chunk->type == FLB_EVENT_TYPE_TRACES) { - print_traces_text(ctx->ins, (char *) - event_chunk->data, - event_chunk->size); - FLB_OUTPUT_RETURN(FLB_OK); - } - - /* Assuming data is a log entry...*/ - if (ctx->out_format != FLB_PACK_JSON_FORMAT_NONE) { - json = flb_pack_msgpack_to_json_format(event_chunk->data, - event_chunk->size, - ctx->out_format, - ctx->json_date_format, - ctx->date_key); - write(STDOUT_FILENO, json, flb_sds_len(json)); - flb_sds_destroy(json); - - /* - * If we are 'not' in json_lines mode, we need to add an extra - * breakline. - */ - if (ctx->out_format != FLB_PACK_JSON_FORMAT_LINES) { - printf("\n"); - } - fflush(stdout); - } - else { - result = flb_log_event_decoder_init(&log_decoder, - (char *) event_chunk->data, - event_chunk->size); - - if (result != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", result); - - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - while (flb_log_event_decoder_next(&log_decoder, - &log_event) == FLB_EVENT_DECODER_SUCCESS) { - printf("[%zd] %s: [[", cnt++, event_chunk->tag); - - printf("%"PRIu32".%09lu, ", - (uint32_t)log_event.timestamp.tm.tv_sec, - log_event.timestamp.tm.tv_nsec); - - msgpack_object_print(stdout, *log_event.metadata); - - printf("], "); - - msgpack_object_print(stdout, *log_event.body); - - printf("]\n"); - } - result = flb_log_event_decoder_get_last_result(&log_decoder); - - flb_log_event_decoder_destroy(&log_decoder); - } - - fflush(stdout); - - if (result != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, "Log event decoder error : %d", result); - FLB_OUTPUT_RETURN(FLB_ERROR); - } - - FLB_OUTPUT_RETURN(FLB_OK); -} - -static int cb_stdout_exit(void *data, struct flb_config *config) -{ - struct flb_stdout *ctx = data; - - if (!ctx) { - return 0; - } - - flb_free(ctx); - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "format", NULL, - 0, FLB_FALSE, 0, - "Specifies the data format to be printed. Supported formats are msgpack json, json_lines and json_stream." - }, - { - FLB_CONFIG_MAP_STR, "json_date_format", NULL, - 0, FLB_FALSE, 0, - FBL_PACK_JSON_DATE_FORMAT_DESCRIPTION - }, - { - FLB_CONFIG_MAP_STR, "json_date_key", "date", - 0, FLB_TRUE, offsetof(struct flb_stdout, json_date_key), - "Specifies the name of the date field in output." - }, - - /* EOF */ - {0} -}; - -/* Plugin registration */ -struct flb_output_plugin out_stdout_plugin = { - .name = "stdout", - .description = "Prints events to STDOUT", - .cb_init = cb_stdout_init, - .cb_flush = cb_stdout_flush, - .cb_exit = cb_stdout_exit, - .flags = 0, - .workers = 1, - .event_type = FLB_OUTPUT_LOGS | FLB_OUTPUT_METRICS | FLB_OUTPUT_TRACES, - .config_map = config_map -}; diff --git a/fluent-bit/plugins/out_stdout/stdout.h b/fluent-bit/plugins/out_stdout/stdout.h deleted file mode 100644 index 0db82f389..000000000 --- a/fluent-bit/plugins/out_stdout/stdout.h +++ /dev/null @@ -1,34 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_STDOUT -#define FLB_OUT_STDOUT - -#include -#include - -struct flb_stdout { - int out_format; - int json_date_format; - flb_sds_t json_date_key; - flb_sds_t date_key; - struct flb_output_instance *ins; -}; - -#endif diff --git a/fluent-bit/plugins/out_syslog/CMakeLists.txt b/fluent-bit/plugins/out_syslog/CMakeLists.txt deleted file mode 100644 index 556d8e1a4..000000000 --- a/fluent-bit/plugins/out_syslog/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -set(src - syslog.c - syslog_conf.c) - -FLB_PLUGIN(out_syslog "${src}" "") diff --git a/fluent-bit/plugins/out_syslog/syslog.c b/fluent-bit/plugins/out_syslog/syslog.c deleted file mode 100644 index a33351354..000000000 --- a/fluent-bit/plugins/out_syslog/syslog.c +++ /dev/null @@ -1,1170 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "syslog_conf.h" - -#ifndef MSG_DONTWAIT - #define MSG_DONTWAIT 0 -#endif - -#ifndef MSG_NOSIGNAL - #define MSG_NOSIGNAL 0 -#endif - -#define RFC5424_MAXSIZE 2048 -#define RFC3164_MAXSIZE 1024 - -struct syslog_msg { - int severity; - int facility; - flb_sds_t hostname; - flb_sds_t appname; - flb_sds_t procid; - flb_sds_t msgid; - flb_sds_t sd; - flb_sds_t message; -}; - -static const char *rfc3164_mon[] = {"Jan", "Feb", "Mar", "Apr", - "May", "Jun", "Jul", "Aug", - "Sep", "Oct", "Nov", "Dec"}; - -static struct { - char *name; - int len; - int value; -} syslog_severity[] = { - { "emerg", 5, 0 }, - { "alert", 5, 1 }, - { "crit", 4, 2 }, - { "err", 3, 3 }, - { "warning", 7, 4 }, - { "notice", 6, 5 }, - { "info", 4, 6 }, - { "debug", 5, 7 }, - { NULL, 0,-1 } -}; - -static struct { - char *name; - int len; - int value; -} syslog_facility[] = { - { "kern", 4, 0 }, - { "user", 4, 1 }, - { "mail", 4, 2 }, - { "daemon", 6, 3 }, - { "auth", 4, 4 }, - { "syslog", 6, 5 }, - { "lpr", 3, 6 }, - { "news", 4, 7 }, - { "uucp", 4, 8 }, - { "cron", 4, 9 }, - { "authpriv", 8, 10 }, - { "ftp", 3, 11 }, - { "ntp", 3, 12 }, - { "security", 8, 13 }, - { "console", 7, 14 }, - { "local0", 6, 16 }, - { "local1", 6, 17 }, - { "local2", 6, 18 }, - { "local3", 6, 19 }, - { "local4", 6, 20 }, - { "local5", 6, 21 }, - { "local6", 6, 22 }, - { "local7", 6, 23 }, - { NULL, 0,-1 }, -}; - -/* '"', '\' ']' */ -static char rfc5424_sp_value[256] = { - 0, 0, 0 , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 , 0 , 0, 0, - 0, 0, 0 , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 , 0 , 0, 0, - 0, 0,'"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 , 0 , 0, 0, - 0, 0, 0 , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 , 0 , 0, 0, - 0, 0, 0 , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 , 0 , 0, 0, - 0, 0, 0 , 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\',']', 0, 0, - 0, 0, 0 , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 , 0 , 0, 0, - 0, 0, 0 , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 , 0 , 0, 0, - 0, 0, 0 , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 , 0 , 0, 0, - 0, 0, 0 , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 , 0 , 0, 0, - 0, 0, 0 , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 , 0 , 0, 0, - 0, 0, 0 , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 , 0 , 0, 0, - 0, 0, 0 , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 , 0 , 0, 0, - 0, 0, 0 , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 , 0 , 0, 0, - 0, 0, 0 , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 , 0 , 0, 0, - 0, 0, 0 , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 , 0 , 0, 0 -}; - -/* '=', ' ', ']', '"' */ -static char rfc5424_sp_name[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 -}; - -static flb_sds_t syslog_rfc5424(flb_sds_t *s, struct flb_time *tms, - struct syslog_msg *msg) -{ - struct tm tm; - flb_sds_t tmp; - uint8_t prival; - - prival = (msg->facility << 3) + msg->severity; - - if (gmtime_r(&(tms->tm.tv_sec), &tm) == NULL) { - return NULL; - } - - tmp = flb_sds_printf(s, "<%i>%i %d-%02d-%02dT%02d:%02d:%02d.%06"PRIu64"Z ", - prival, 1, tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, - tm.tm_hour, tm.tm_min, tm.tm_sec, - (uint64_t) tms->tm.tv_nsec/1000); - if (!tmp) { - return NULL; - } - *s = tmp; - - if (msg->hostname) { - int len = flb_sds_len(msg->hostname); - tmp = flb_sds_cat(*s, msg->hostname, len > 255 ? 255 : len); - if (!tmp) { - return NULL; - } - *s = tmp; - } - else { - tmp = flb_sds_cat(*s, "-" , 1); - if (!tmp) { - return NULL; - } - *s = tmp; - } - - tmp = flb_sds_cat(*s, " ", 1); - if (!tmp) { - return NULL; - } - *s = tmp; - - if (msg->appname) { - int len = flb_sds_len(msg->appname); - tmp = flb_sds_cat(*s, msg->appname, len > 48 ? 48 : len); - if (!tmp) { - return NULL; - } - *s = tmp; - } - else { - tmp = flb_sds_cat(*s, "-" , 1); - if (!tmp) { - return NULL; - } - *s = tmp; - } - - tmp = flb_sds_cat(*s, " ", 1); - if (!tmp) { - return NULL; - } - *s = tmp; - - if (msg->procid) { - int len = flb_sds_len(msg->procid); - tmp = flb_sds_cat(*s, msg->procid, len > 128 ? 128 : len); - if (!tmp) { - return NULL; - } - *s = tmp; - } - else { - tmp = flb_sds_cat(*s, "-" , 1); - if (!tmp) { - return NULL; - } - *s = tmp; - } - - tmp = flb_sds_cat(*s, " ", 1); - if (!tmp) { - return NULL; - } - *s = tmp; - - if (msg->msgid) { - int len = flb_sds_len(msg->msgid); - tmp = flb_sds_cat(*s, msg->msgid, len > 32 ? 32 : len); - if (!tmp) { - return NULL; - } - *s = tmp; - } - else { - tmp = flb_sds_cat(*s, "-" , 1); - if (!tmp) { - return NULL; - } - *s = tmp; - } - - tmp = flb_sds_cat(*s, " ", 1); - if (!tmp) { - return NULL; - } - *s = tmp; - - if (msg->sd) { - tmp = flb_sds_cat(*s, msg->sd, flb_sds_len(msg->sd)); - if (!tmp) { - return NULL; - } - *s = tmp; - } - else { - tmp = flb_sds_cat(*s, "-" , 1); - if (!tmp) { - return NULL; - } - *s = tmp; - } - - if (msg->message) { - int len = flb_sds_len(msg->message); - tmp = flb_sds_cat(*s, " \xef\xbb\xbf", 4); - if (!tmp) { - return NULL; - } - *s = tmp; - tmp = flb_sds_cat(*s, msg->message, len); - if (!tmp) { - return NULL; - } - *s = tmp; - } - - return *s; -} - -static flb_sds_t syslog_rfc3164 (flb_sds_t *s, struct flb_time *tms, - struct syslog_msg *msg) -{ - struct tm tm; - flb_sds_t tmp; - uint8_t prival; - - prival = (msg->facility << 3) + msg->severity; - - if (gmtime_r(&(tms->tm.tv_sec), &tm) == NULL) { - return NULL; - } - - tmp = flb_sds_printf(s, "<%i>%s %2d %02d:%02d:%02d ", prival, - rfc3164_mon[tm.tm_mon], tm.tm_mday, - tm.tm_hour, tm.tm_min, tm.tm_sec); - if (!tmp) { - return NULL; - } - *s = tmp; - - if (msg->hostname) { - tmp = flb_sds_cat(*s, msg->hostname, flb_sds_len(msg->hostname)); - if (!tmp) { - return NULL; - } - *s = tmp; - tmp = flb_sds_cat(*s, " ", 1); - if (!tmp) { - return NULL; - } - *s = tmp; - } - - if (msg->appname) { - tmp = flb_sds_cat(*s, msg->appname, flb_sds_len(msg->appname)); - if (!tmp) { - return NULL; - } - *s = tmp; - if (msg->procid) { - tmp = flb_sds_cat(*s, "[" , 1); - if (!tmp) { - return NULL; - } - *s = tmp; - tmp = flb_sds_cat(*s, msg->procid, flb_sds_len(msg->procid)); - if (!tmp) { - return NULL; - } - *s = tmp; - tmp = flb_sds_cat(*s, "]" , 1); - if (!tmp) { - return NULL; - } - *s = tmp; - } - tmp = flb_sds_cat(*s, ": " , 2); - if (!tmp) { - return NULL; - } - *s = tmp; - } - - if (msg->message) { - tmp = flb_sds_cat(*s, msg->message, flb_sds_len(msg->message)); - if (!tmp) { - return NULL; - } - *s = tmp; - } - - return *s; -} - -static flb_sds_t msgpack_to_sd(struct flb_syslog *ctx, - flb_sds_t *s, const char *sd, int sd_len, - msgpack_object *o) -{ - flb_sds_t tmp; - int i; - int loop; - int n, start_len, end_len; - - if (*s == NULL) { - *s = flb_sds_create_size(512); - if (*s == NULL) { - return NULL; - } - } - - tmp = flb_sds_cat(*s, "[" , 1); - if (!tmp) { - return NULL; - } - *s = tmp; - - start_len = flb_sds_len(*s); - if (ctx->allow_longer_sd_id != FLB_TRUE && sd_len > 32) { - /* - * RFC5424 defines - * SD-NAME = 1*32PRINTUSASCII - * ; except '=', SP, ']', %d34 (") - * - * https://www.rfc-editor.org/rfc/rfc5424#section-6 - */ - sd_len = 32; - } - tmp = flb_sds_cat(*s, sd, sd_len); - if (!tmp) { - return NULL; - } - *s = tmp; - - end_len = flb_sds_len(*s); - for(n=start_len; n < end_len; n++) { - if (!rfc5424_sp_name[(unsigned char)(*s)[n]]) { - (*s)[n] = '_'; - } - } - - loop = o->via.map.size; - if (loop != 0) { - msgpack_object_kv *p = o->via.map.ptr; - for (i = 0; i < loop; i++) { - char temp[48] = {0}; - const char *key = NULL; - int key_len = 0; - const char *val = NULL; - int val_len = 0; - - msgpack_object *k = &p[i].key; - msgpack_object *v = &p[i].val; - - if (k->type != MSGPACK_OBJECT_BIN && k->type != MSGPACK_OBJECT_STR) { - continue; - } - - if (k->type == MSGPACK_OBJECT_STR) { - key = k->via.str.ptr; - key_len = k->via.str.size; - } - else { - key = k->via.bin.ptr; - key_len = k->via.bin.size; - } - - if (v->type == MSGPACK_OBJECT_BOOLEAN) { - val = v->via.boolean ? "true" : "false"; - val_len = v->via.boolean ? 4 : 5; - } - else if (v->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - val = temp; - val_len = snprintf(temp, sizeof(temp) - 1, - "%" PRIu64, v->via.u64); - } - else if (v->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { - val = temp; - val_len = snprintf(temp, sizeof(temp) - 1, - "%" PRId64, v->via.i64); - } - else if (v->type == MSGPACK_OBJECT_FLOAT) { - val = temp; - val_len = snprintf(temp, sizeof(temp) - 1, - "%f", v->via.f64); - } - else if (v->type == MSGPACK_OBJECT_STR) { - /* String value */ - val = v->via.str.ptr; - val_len = v->via.str.size; - } - else if (v->type == MSGPACK_OBJECT_BIN) { - /* Bin value */ - val = v->via.bin.ptr; - val_len = v->via.bin.size; - } - - if (!val || !key) { - continue; - } - - tmp = flb_sds_cat(*s, " " , 1); - if (!tmp) { - return NULL; - } - *s = tmp; - - start_len = flb_sds_len(*s); - if (ctx->allow_longer_sd_id != FLB_TRUE && key_len > 32 ) { - /* - * RFC5424 defines - * PARAM-NAME = SD-NAME - * SD-NAME = 1*32PRINTUSASCII - * ; except '=', SP, ']', %d34 (") - * - * https://www.rfc-editor.org/rfc/rfc5424#section-6 - */ - key_len = 32; - } - tmp = flb_sds_cat(*s, key, key_len); - if (!tmp) { - return NULL; - } - *s = tmp; - - end_len = flb_sds_len(*s); - for(n=start_len; n < end_len; n++) { - if (!rfc5424_sp_name[(unsigned char)(*s)[n]]) { - (*s)[n] = '_'; - } - } - - tmp = flb_sds_cat(*s, "=\"" , 2); - if (!tmp) { - return NULL; - } - *s = tmp; - - tmp = flb_sds_cat_esc(*s, val , val_len, - rfc5424_sp_value, sizeof(rfc5424_sp_value)); - if (!tmp) { - return NULL; - } - *s = tmp; - - tmp = flb_sds_cat(*s, "\"" , 1); - if (!tmp) { - return NULL; - } - *s = tmp; - } - } - - tmp = flb_sds_cat(*s, "]" , 1); - if (!tmp) return NULL; - *s = tmp; - - return *s; -} - -static int msgpack_to_syslog(struct flb_syslog *ctx, msgpack_object *o, - struct syslog_msg *msg) -{ - int i; - int loop; - struct mk_list *head; - struct flb_config_map_val *mv; - - if (o == NULL) { - return -1; - } - - loop = o->via.map.size; - if (loop != 0) { - msgpack_object_kv *p = o->via.map.ptr; - - for (i = 0; i < loop; i++) { - char temp[48] = {0}; - const char *key = NULL; - int key_len = 0; - const char *val = NULL; - int val_len = 0; - - msgpack_object *k = &p[i].key; - msgpack_object *v = &p[i].val; - - if (k->type != MSGPACK_OBJECT_BIN && k->type != MSGPACK_OBJECT_STR){ - continue; - } - - if (k->type == MSGPACK_OBJECT_STR) { - key = k->via.str.ptr; - key_len = k->via.str.size; - } - else { - key = k->via.bin.ptr; - key_len = k->via.bin.size; - } - - if (v->type == MSGPACK_OBJECT_MAP) { - if (ctx->sd_keys) { - flb_config_map_foreach(head, mv, ctx->sd_keys) { - if ((key_len == flb_sds_len(mv->val.str)) && - strncmp(key, mv->val.str, flb_sds_len(mv->val.str)) == 0) { - msgpack_to_sd(ctx, &(msg->sd), key, key_len, v); - break; - } - } - } - continue; - } - - if (v->type == MSGPACK_OBJECT_BOOLEAN) { - val = v->via.boolean ? "true" : "false"; - val_len = v->via.boolean ? 4 : 5; - } - else if (v->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - val = temp; - val_len = snprintf(temp, sizeof(temp) - 1, - "%" PRIu64, v->via.u64); - } - else if (v->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { - val = temp; - val_len = snprintf(temp, sizeof(temp) - 1, - "%" PRId64, v->via.i64); - } - else if (v->type == MSGPACK_OBJECT_FLOAT) { - val = temp; - val_len = snprintf(temp, sizeof(temp) - 1, - "%f", v->via.f64); - } - else if (v->type == MSGPACK_OBJECT_STR) { - /* String value */ - val = v->via.str.ptr; - val_len = v->via.str.size; - } - else if (v->type == MSGPACK_OBJECT_BIN) { - /* Bin value */ - val = v->via.bin.ptr; - val_len = v->via.bin.size; - } - - if (!val || !key) { - continue; - } - - if ((ctx->severity_key != NULL) && - flb_sds_cmp(ctx->severity_key, key, key_len) == 0) { - if (msg->severity == -1) { - if ((val_len == 1) && (val[0] >= '0' && val[0] <= '7')) { - msg->severity = val[0]-'0'; - } - else { - int i; - for (i=0; syslog_severity[i].name != NULL; i++) { - if ((syslog_severity[i].len == val_len) && - (!strncasecmp(syslog_severity[i].name, val, val_len))) { - msg->severity = syslog_severity[i].value; - } - } - if (!syslog_severity[i].name) { - flb_plg_warn(ctx->ins, "invalid severity: '%.*s'", - val_len, val); - } - } - } - } - else if ((ctx->facility_key != NULL) && - flb_sds_cmp(ctx->facility_key, key, key_len) == 0) { - if (msg->facility == -1) { - if ((val_len == 1) && (val[0] >= '0' && val[0] <= '9')) { - msg->facility = val[0]-'0'; - } - else if ((val_len == 2) && - (val[0] >= '0' && val[0] <= '2') && - (val[1] >= '0' && val[1] <= '9')) { - msg->facility = (val[0]-'0')*10; - msg->facility += (val[1]-'0'); - if (!((msg->facility >= 0) && (msg->facility <=23))) { - flb_plg_warn(ctx->ins, "invalid facility: '%.*s'", - val_len, val); - msg->facility= -1; - } - } - else { - int i; - for (i=0; syslog_facility[i].name != NULL; i++) { - if ((syslog_facility[i].len == val_len) && - (!strncasecmp(syslog_facility[i].name, val, val_len))) { - msg->facility = syslog_facility[i].value; - } - } - if (!syslog_facility[i].name) { - flb_plg_warn(ctx->ins, "invalid facility: '%.*s'", - val_len, val); - } - } - } - } - else if ((ctx->hostname_key != NULL) && - flb_sds_cmp(ctx->hostname_key, key, key_len) == 0) { - if (!msg->hostname) { - msg->hostname = flb_sds_create_len(val, val_len); - } - } - else if ((ctx->appname_key != NULL) && - flb_sds_cmp(ctx->appname_key, key, key_len) == 0) { - if (!msg->appname) { - msg->appname = flb_sds_create_len(val, val_len); - } - } - else if ((ctx->procid_key != NULL) && - flb_sds_cmp(ctx->procid_key, key, key_len) == 0) { - if (!msg->procid) { - msg->procid = flb_sds_create_len(val, val_len); - } - } - else if ((ctx->msgid_key != NULL) && - flb_sds_cmp(ctx->msgid_key, key, key_len) == 0) { - if (!msg->msgid) { - msg->msgid = flb_sds_create_len(val, val_len); - } - } - else if ((ctx->message_key != NULL) && - flb_sds_cmp(ctx->message_key, key, key_len) == 0) { - if (!msg->message) { - msg->message = flb_sds_create_len(val, val_len); - } - } - } - } - - return 0; -} - -static flb_sds_t syslog_format(struct flb_syslog *ctx, msgpack_object *o, - flb_sds_t *s, struct flb_time *tm) -{ - struct syslog_msg msg; - flb_sds_t tmp; - flb_sds_t ret_sds; - int ret; - - msg.severity = -1; - msg.facility = -1; - msg.hostname = NULL; - msg.appname = NULL; - msg.procid = NULL; - msg.msgid = NULL; - msg.sd = NULL; - msg.message = NULL; - - ret = msgpack_to_syslog(ctx, o, &msg); - if (!ret) { - if (msg.severity < 0) { - msg.severity = ctx->severity_preset; - } - if (msg.facility < 0) { - msg.facility = ctx->facility_preset; - } - if (msg.hostname == NULL && ctx->hostname_preset) { - msg.hostname = flb_sds_create(ctx->hostname_preset); - } - if (msg.appname == NULL && ctx->appname_preset) { - msg.appname = flb_sds_create(ctx->appname_preset); - } - if (msg.procid == NULL && ctx->procid_preset) { - msg.procid = flb_sds_create(ctx->procid_preset); - } - if (msg.msgid == NULL && ctx->msgid_preset) { - msg.msgid = flb_sds_create(ctx->msgid_preset); - } - - if (ctx->parsed_format == FLB_SYSLOG_RFC3164) { - tmp = syslog_rfc3164(s, tm, &msg); - } - else { - tmp = syslog_rfc5424(s, tm, &msg); - } - - if (!tmp) { - ret_sds = NULL; - goto clean; - } - *s = tmp; - - if (flb_sds_len(*s) > ctx->maxsize) { - flb_sds_len_set(*s, ctx->maxsize); - } - - if (ctx->parsed_mode != FLB_SYSLOG_UDP) { - tmp = flb_sds_cat(*s, "\n", 1); - if (!tmp) { - ret_sds = NULL; - goto clean; - } - *s = tmp; - } - } - else { - ret_sds = NULL; - goto clean; - } - - ret_sds = *s; -clean: - flb_sds_destroy(msg.hostname); - flb_sds_destroy(msg.appname); - flb_sds_destroy(msg.procid); - flb_sds_destroy(msg.msgid); - flb_sds_destroy(msg.sd); - flb_sds_destroy(msg.message); - - return ret_sds; -} - -static void cb_syslog_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - struct flb_syslog *ctx = out_context; - flb_sds_t s; - flb_sds_t tmp; - size_t bytes_sent; - msgpack_object map; - struct flb_connection *u_conn = NULL; - int ret; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - if (ctx->parsed_mode != FLB_SYSLOG_UDP) { - u_conn = flb_upstream_conn_get(ctx->u); - if (!u_conn) { - flb_plg_error(ctx->ins, "no upstream connections available"); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - } - - s = flb_sds_create_size(ctx->maxsize); - if (s == NULL) { - FLB_OUTPUT_RETURN(FLB_ERROR); - } - - ret = flb_log_event_decoder_init(&log_decoder, - (char *) event_chunk->data, - event_chunk->size); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - flb_sds_destroy(s); - - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - map = *log_event.body; - - flb_sds_len_set(s, 0); - - tmp = syslog_format(ctx, &map, &s, &log_event.timestamp); - if (tmp != NULL) { - s = tmp; - if (ctx->parsed_mode == FLB_SYSLOG_UDP) { - ret = send(ctx->fd, s, flb_sds_len(s), MSG_DONTWAIT | MSG_NOSIGNAL); - if (ret == -1) { - flb_log_event_decoder_destroy(&log_decoder); - flb_sds_destroy(s); - - FLB_OUTPUT_RETURN(FLB_RETRY); - } - } - else { - ret = flb_io_net_write(u_conn, - s, flb_sds_len(s), &bytes_sent); - if (ret == -1) { - flb_errno(); - flb_log_event_decoder_destroy(&log_decoder); - flb_upstream_conn_release(u_conn); - flb_sds_destroy(s); - - FLB_OUTPUT_RETURN(FLB_RETRY); - } - } - } - else { - flb_plg_error(ctx->ins, "error formating message"); - } - } - - flb_sds_destroy(s); - flb_log_event_decoder_destroy(&log_decoder); - - if (ctx->parsed_mode != FLB_SYSLOG_UDP) { - flb_upstream_conn_release(u_conn); - } - - FLB_OUTPUT_RETURN(FLB_OK); -} - -static int cb_syslog_init(struct flb_output_instance *ins, struct flb_config *config, - void *data) -{ - int io_flags; - struct flb_syslog *ctx = NULL; - - /* Set default network configuration */ - flb_output_net_default("127.0.0.1", 514, ins); - - /* Create config context */ - ctx = flb_syslog_config_create(ins, config); - if (ctx == NULL) { - flb_plg_error(ins, "error configuring plugin"); - return -1; - } - - if (ctx->maxsize < 0) { - if (ctx->parsed_format == FLB_SYSLOG_RFC3164) { - ctx->maxsize = RFC3164_MAXSIZE; - } - else { - ctx->maxsize = RFC5424_MAXSIZE; - } - } - - ctx->fd = -1; - if (ctx->parsed_mode == FLB_SYSLOG_UDP) { - ctx->fd = flb_net_udp_connect(ins->host.name, ins->host.port, - ins->net_setup.source_address); - if (ctx->fd < 0) { - flb_syslog_config_destroy(ctx); - return -1; - } - } - else { - - /* use TLS ? */ - if (ins->use_tls == FLB_TRUE) { - io_flags = FLB_IO_TLS; - } - else { - io_flags = FLB_IO_TCP; - } - - if (ins->host.ipv6 == FLB_TRUE) { - io_flags |= FLB_IO_IPV6; - } - - ctx->u = flb_upstream_create(config, ins->host.name, ins->host.port, - io_flags, ins->tls); - if (!(ctx->u)) { - flb_syslog_config_destroy(ctx); - return -1; - } - flb_output_upstream_set(ctx->u, ins); - } - - /* Set the plugin context */ - flb_output_set_context(ins, ctx); - - flb_plg_info(ctx->ins, "setup done for %s:%i (TLS=%s)", - ins->host.name, ins->host.port, - ins->use_tls ? "on" : "off"); - return 0; -} - -static int cb_syslog_exit(void *data, struct flb_config *config) -{ - struct flb_syslog *ctx = data; - - if (ctx == NULL) { - return 0; - } - - if (ctx->u) { - flb_upstream_destroy(ctx->u); - } - - if (ctx->fd > 0) { - close(ctx->fd); - } - - flb_syslog_config_destroy(ctx); - - return 0; -} - - -/* for testing */ -static int cb_syslog_format_test(struct flb_config *config, - struct flb_input_instance *ins, - void *plugin_context, - void *flush_ctx, - int event_type, - const char *tag, int tag_len, - const void *data, size_t bytes, - void **out_data, size_t *out_size) -{ - struct flb_syslog *ctx = plugin_context; - flb_sds_t tmp; - flb_sds_t s; - msgpack_object map; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - int ret; - - s = flb_sds_create_size(ctx->maxsize); - if (s == NULL) { - flb_error("flb_sds_create_size failed"); - return -1; - } - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - flb_sds_destroy(s); - - return -1; - } - - flb_log_event_decoder_next(&log_decoder, &log_event); - ret = flb_log_event_decoder_get_last_result(&log_decoder); - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_error("msgpack_unpack_next failed"); - - flb_log_event_decoder_destroy(&log_decoder); - - return -1; - } - - map = *log_event.body; - flb_sds_len_set(s, 0); - tmp = syslog_format(ctx, &map, &s, &log_event.timestamp); - - flb_log_event_decoder_destroy(&log_decoder); - - if (tmp == NULL) { - flb_error("syslog_fromat returns NULL"); - return -1; - } - - *out_data = tmp; - *out_size = flb_sds_len(tmp); - - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "mode", "udp", - 0, FLB_TRUE, offsetof(struct flb_syslog, mode), - "Set the desired transport type, the available options are tcp and udp. If you need to " - "use a TLS secure channel, choose 'tcp' mode here and enable the 'tls' option separately." - }, - - { - FLB_CONFIG_MAP_STR, "syslog_format", "rfc5424", - 0, FLB_TRUE, offsetof(struct flb_syslog, format), - "Specify the Syslog protocol format to use, the available options are rfc3164 " - "and rfc5424." - }, - - { - FLB_CONFIG_MAP_SIZE, "syslog_maxsize", "0", - 0, FLB_TRUE, offsetof(struct flb_syslog, maxsize), - "Set the maximum size allowed per message. The value must be only integers " - "representing the number of bytes allowed. If no value is provided, the " - "default size is set depending of the protocol version specified by " - "syslog_format , rfc3164 sets max size to 1024 bytes, while rfc5424 sets " - "the size to 2048 bytes." - }, - - { - FLB_CONFIG_MAP_STR, "syslog_severity_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_syslog, severity_key), - "Specify the name of the key from the original record that contains the Syslog " - "severity number. This configuration is optional." - }, - - { - FLB_CONFIG_MAP_INT, "syslog_severity_preset", "6", - 0, FLB_TRUE, offsetof(struct flb_syslog, severity_preset), - "Specify the preset severity number. It must be 0-7. " - " This configuration is optional." - }, - - { - FLB_CONFIG_MAP_STR, "syslog_facility_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_syslog, facility_key), - "Specify the name of the key from the original record that contains the Syslog " - "facility number. This configuration is optional." - }, - - { - FLB_CONFIG_MAP_INT, "syslog_facility_preset", "1", - 0, FLB_TRUE, offsetof(struct flb_syslog, facility_preset), - "Specify the preset facility number. It must be 0-23. " - " This configuration is optional." - }, - - { - FLB_CONFIG_MAP_STR, "syslog_hostname_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_syslog, hostname_key), - "Specify the key name from the original record that contains the hostname that " - "generated the message. This configuration is optional." - }, - - { - FLB_CONFIG_MAP_STR, "syslog_hostname_preset", NULL, - 0, FLB_TRUE, offsetof(struct flb_syslog, hostname_preset), - "Specify the preset hostname. This configuration is optional." - }, - - { - FLB_CONFIG_MAP_STR, "syslog_appname_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_syslog, appname_key), - "Specify the key name from the original record that contains the application " - "name that generated the message. This configuration is optional." - }, - - { - FLB_CONFIG_MAP_STR, "syslog_appname_preset", NULL, - 0, FLB_TRUE, offsetof(struct flb_syslog, appname_preset), - "Specify the preset appname. This configuration is optional." - }, - - { - FLB_CONFIG_MAP_STR, "syslog_procid_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_syslog, procid_key), - "Specify the key name from the original record that contains the Process ID " - "that generated the message. This configuration is optional." - }, - - { - FLB_CONFIG_MAP_STR, "syslog_procid_preset", NULL, - 0, FLB_TRUE, offsetof(struct flb_syslog, procid_preset), - "Specify the preset procid. This configuration is optional." - }, - - { - FLB_CONFIG_MAP_STR, "syslog_msgid_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_syslog, msgid_key), - "Specify the key name from the original record that contains the Message ID " - "associated to the message. This configuration is optional." - }, - - { - FLB_CONFIG_MAP_STR, "syslog_msgid_preset", NULL, - 0, FLB_TRUE, offsetof(struct flb_syslog, msgid_preset), - "Specify the preset msgid. This configuration is optional." - }, - - { - FLB_CONFIG_MAP_STR, "syslog_sd_key", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct flb_syslog, sd_keys), - "Specify the key name from the original record that contains the " - "Structured Data (SD) content. If set, the value of the key must be a map." - "This option can be set multiple times." - }, - - { - FLB_CONFIG_MAP_STR, "syslog_message_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_syslog, message_key), - "Specify the key name that contains the message to deliver. Note that if " - "this property is mandatory, otherwise the message will be empty." - }, - - { - FLB_CONFIG_MAP_BOOL, "allow_longer_sd_id", "false", - 0, FLB_TRUE, offsetof(struct flb_syslog, allow_longer_sd_id), - "If true, Fluent-bit allows SD-ID that is longer than 32 characters. " - "Such long SD-ID violates RFC 5424." - }, - - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_output_plugin out_syslog_plugin = { - .name = "syslog", - .description = "Syslog", - .cb_init = cb_syslog_init, - .cb_pre_run = NULL, - .cb_flush = cb_syslog_flush, - .cb_exit = cb_syslog_exit, - - /* Configuration */ - .config_map = config_map, - - /* for testing */ - .test_formatter.callback = cb_syslog_format_test, - - /* Plugin flags */ - .flags = FLB_OUTPUT_NET | FLB_IO_OPT_TLS, -}; diff --git a/fluent-bit/plugins/out_syslog/syslog_conf.c b/fluent-bit/plugins/out_syslog/syslog_conf.c deleted file mode 100644 index ea16fce26..000000000 --- a/fluent-bit/plugins/out_syslog/syslog_conf.c +++ /dev/null @@ -1,162 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include "syslog_conf.h" - -static int is_valid_severity(struct flb_output_instance *ins, int val, int format) -{ - if (format != FLB_SYSLOG_RFC5424 && format != FLB_SYSLOG_RFC3164) { - flb_plg_error(ins, "[%s] unknown syslog format.", __FUNCTION__); - return -1; - } - - if (ins == NULL) { - flb_plg_error(ins, "[%s] arg is null. ins=%p", __FUNCTION__, ins); - return -1; - } - if (val < 0 || val > 7) { - flb_plg_error(ins, "[%s] invalid severity level %d. It should be 0-7.", __FUNCTION__, val); - return -1; - } - - return 0; -} - -static int is_valid_facility(struct flb_output_instance *ins, int val, int format) -{ - if (format != FLB_SYSLOG_RFC5424 && format != FLB_SYSLOG_RFC3164) { - flb_plg_error(ins, "[%s] unknown syslog format.", __FUNCTION__); - return -1; - } - - if (ins == NULL) { - flb_plg_error(ins, "[%s] arg is null. ins=%p", __FUNCTION__, ins); - return -1; - } - - if (val < 0 || val > 23) { - flb_plg_error(ins, "[%s] invalid facility level %d. It should be 0-23.", __FUNCTION__, val); - return -1; - } - return 0; -} - - -struct flb_syslog *flb_syslog_config_create(struct flb_output_instance *ins, - struct flb_config *config) -{ - int ret; - const char *tmp; - struct flb_syslog *ctx = NULL; - - /* Allocate plugin context */ - ctx = flb_calloc(1, sizeof(struct flb_syslog)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - ctx->parsed_mode = FLB_SYSLOG_UDP; - ctx->parsed_format = FLB_SYSLOG_RFC5424; - ctx->maxsize = -1; - - /* Populate context with config map defaults and incoming properties */ - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_plg_error(ctx->ins, "configuration error"); - flb_syslog_config_destroy(ctx); - return NULL; - } - - /* Set context */ - flb_output_set_context(ins, ctx); - - /* Config Mode */ - tmp = flb_output_get_property("mode", ins); - if (tmp) { - if (!strcasecmp(tmp, "tcp")) { - ctx->parsed_mode = FLB_SYSLOG_TCP; - } - else if (!strcasecmp(tmp, "tls")) { - ctx->parsed_mode = FLB_SYSLOG_TLS; - } - else if (!strcasecmp(tmp, "udp")) { - ctx->parsed_mode = FLB_SYSLOG_UDP; - } - else { - flb_plg_error(ctx->ins, "unknown syslog mode %s", tmp); - flb_syslog_config_destroy(ctx); - return NULL; - } - } - - /* syslog_format */ - tmp = flb_output_get_property("syslog_format", ins); - if (tmp) { - if (strcasecmp(tmp, "rfc3164") == 0) { - ctx->parsed_format = FLB_SYSLOG_RFC3164; - } - else if (strcasecmp(tmp, "rfc5424") == 0) { - ctx->parsed_format = FLB_SYSLOG_RFC5424; - } - else { - flb_plg_error(ctx->ins, "unknown syslog format %s", tmp); - flb_syslog_config_destroy(ctx); - return NULL; - } - } - - if (ctx->parsed_format == FLB_SYSLOG_RFC5424 && ctx->allow_longer_sd_id == FLB_TRUE) { - flb_plg_warn(ctx->ins, "Allow longer SD-ID. It may violate RFC5424."); - } - - /* validate preset values */ - ret = is_valid_severity(ctx->ins, ctx->severity_preset, ctx->parsed_format); - if (ret != 0) { - flb_syslog_config_destroy(ctx); - return NULL; - } - - ret = is_valid_facility(ctx->ins, ctx->facility_preset, ctx->parsed_format); - if (ret != 0) { - flb_syslog_config_destroy(ctx); - return NULL; - } - - - /* syslog maxsize */ - if (ctx->maxsize <= 0) { - if (ctx->parsed_format == FLB_SYSLOG_RFC3164) { - ctx->maxsize = 1024; - } - else if (ctx->parsed_format == FLB_SYSLOG_RFC5424) { - ctx->maxsize = 2048; - } - } - - return ctx; -} - -void flb_syslog_config_destroy(struct flb_syslog *ctx) -{ - flb_free(ctx); -} diff --git a/fluent-bit/plugins/out_syslog/syslog_conf.h b/fluent-bit/plugins/out_syslog/syslog_conf.h deleted file mode 100644 index 8bdb027a5..000000000 --- a/fluent-bit/plugins/out_syslog/syslog_conf.h +++ /dev/null @@ -1,70 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_SYSLOG_CONF_H -#define FLB_OUT_SYSLOG_CONF_H - -#include -#include -#include - - -#define FLB_SYSLOG_UDP 0 -#define FLB_SYSLOG_TCP 1 -#define FLB_SYSLOG_TLS 2 - -#define FLB_SYSLOG_RFC3164 0 -#define FLB_SYSLOG_RFC5424 1 - -struct flb_syslog { - flb_sockfd_t fd; - struct flb_upstream *u; - flb_sds_t mode; - flb_sds_t format; - size_t maxsize; - flb_sds_t severity_key; - flb_sds_t facility_key; - flb_sds_t timestamp_key; - flb_sds_t hostname_key; - flb_sds_t appname_key; - flb_sds_t procid_key; - flb_sds_t msgid_key; - struct mk_list *sd_keys; - int allow_longer_sd_id; - flb_sds_t message_key; - - /* Preset */ - int severity_preset; - int facility_preset; - flb_sds_t hostname_preset; - flb_sds_t appname_preset; - flb_sds_t procid_preset; - flb_sds_t msgid_preset; - - /* Internal */ - int parsed_mode; - int parsed_format; - struct flb_output_instance *ins; -}; - -struct flb_syslog *flb_syslog_config_create(struct flb_output_instance *ins, - struct flb_config *config); -void flb_syslog_config_destroy(struct flb_syslog *ctx); - -#endif diff --git a/fluent-bit/plugins/out_tcp/CMakeLists.txt b/fluent-bit/plugins/out_tcp/CMakeLists.txt deleted file mode 100644 index 6dd000fdf..000000000 --- a/fluent-bit/plugins/out_tcp/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(src - tcp.c - tcp_conf.c - ) - -FLB_PLUGIN(out_tcp "${src}" "mk_core") diff --git a/fluent-bit/plugins/out_tcp/tcp.c b/fluent-bit/plugins/out_tcp/tcp.c deleted file mode 100644 index f74730b70..000000000 --- a/fluent-bit/plugins/out_tcp/tcp.c +++ /dev/null @@ -1,269 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "tcp.h" -#include "tcp_conf.h" - -static int cb_tcp_init(struct flb_output_instance *ins, - struct flb_config *config, void *data) -{ - struct flb_out_tcp *ctx = NULL; - (void) data; - - ctx = flb_tcp_conf_create(ins, config); - if (!ctx) { - return -1; - } - - /* Set the plugin context */ - flb_output_set_context(ins, ctx); - - return 0; -} - -static int compose_payload(struct flb_out_tcp *ctx, - const char *tag, int tag_len, - const void *in_data, size_t in_size, - void **out_payload, size_t *out_size) -{ - int ret; - flb_sds_t buf = NULL; - flb_sds_t json = NULL; - flb_sds_t str; - msgpack_object map; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - /* raw message key by using a record accessor */ - if (ctx->ra_raw_message_key) { - ret = flb_log_event_decoder_init(&log_decoder, (char *) in_data, in_size); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return -1; - } - - buf = flb_sds_create_size(in_size); - if (!buf) { - flb_log_event_decoder_destroy(&log_decoder); - return FLB_ERROR; - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - - map = *log_event.body; - - str = flb_ra_translate(ctx->ra_raw_message_key, (char *) tag, tag_len, map, NULL); - if (!str) { - continue; - } - - ret = flb_sds_cat_safe(&buf, str, flb_sds_len(str)); - if (ret != 0) { - flb_plg_error(ctx->ins, "failed to compose payload from '%s'", str); - } - flb_sds_destroy(str); - - /* append a new line */ - flb_sds_cat_safe(&buf, "\n", 1); - } - - flb_log_event_decoder_destroy(&log_decoder); - - if (flb_sds_len(buf) == 0) { - flb_sds_destroy(buf); - return FLB_ERROR; - } - - *out_payload = buf; - *out_size = flb_sds_len(buf); - return FLB_OK; - } - - if (ctx->out_format == FLB_PACK_JSON_FORMAT_NONE) { - /* nothing to do */ - *out_payload = (void*)in_data; - *out_size = in_size; - return FLB_OK; - } - - json = flb_pack_msgpack_to_json_format(in_data, - in_size, - ctx->out_format, - ctx->json_date_format, - ctx->date_key); - if (!json) { - flb_plg_error(ctx->ins, "error formatting JSON payload"); - return FLB_ERROR; - } - *out_payload = (void*)json; - *out_size = flb_sds_len(json); - - return FLB_OK; -} - -static void cb_tcp_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - int ret = FLB_ERROR; - size_t bytes_sent; - struct flb_upstream *u; - struct flb_connection *u_conn; - struct flb_out_tcp *ctx = out_context; - void *out_payload = NULL; - size_t out_size = 0; - (void) i_ins; - - /* Get upstream context and connection */ - u = ctx->u; - u_conn = flb_upstream_conn_get(u); - if (!u_conn) { - flb_plg_error(ctx->ins, "no upstream connections available to %s:%i", - u->tcp_host, u->tcp_port); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - ret = compose_payload(ctx, - event_chunk->tag, flb_sds_len(event_chunk->tag), - event_chunk->data, event_chunk->size, - &out_payload, &out_size); - if (ret != FLB_OK) { - flb_upstream_conn_release(u_conn); - return FLB_OUTPUT_RETURN(ret); - } - - if (ctx->ra_raw_message_key) { - ret = flb_io_net_write(u_conn, out_payload, out_size, &bytes_sent); - flb_sds_destroy(out_payload); - } - else if (ctx->out_format == FLB_PACK_JSON_FORMAT_NONE) { - ret = flb_io_net_write(u_conn, - event_chunk->data, event_chunk->size, - &bytes_sent); - } - else { - ret = flb_io_net_write(u_conn, out_payload, out_size, &bytes_sent); - flb_sds_destroy(out_payload); - } - if (ret == -1) { - flb_errno(); - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_OK); -} - -static int cb_tcp_exit(void *data, struct flb_config *config) -{ - struct flb_out_tcp *ctx = data; - - flb_tcp_conf_destroy(ctx); - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "format", "msgpack", - 0, FLB_FALSE, 0, - "Specify the payload format, supported formats: msgpack, json, " - "json_lines or json_stream." - }, - - { - FLB_CONFIG_MAP_STR, "json_date_format", "double", - 0, FLB_FALSE, 0, - FBL_PACK_JSON_DATE_FORMAT_DESCRIPTION - }, - - { - FLB_CONFIG_MAP_STR, "json_date_key", "date", - 0, FLB_TRUE, offsetof(struct flb_out_tcp, json_date_key), - "Specify the name of the date field in output." - }, - - { - FLB_CONFIG_MAP_STR, "raw_message_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_out_tcp, raw_message_key), - "use a raw message key for the message." - }, - - /* EOF */ - {0} -}; - -static int cb_tcp_format_test(struct flb_config *config, - struct flb_input_instance *ins, - void *plugin_context, - void *flush_ctx, - int event_type, - const char *tag, int tag_len, - const void *data, size_t bytes, - void **out_data, size_t *out_size) -{ - struct flb_out_tcp *ctx = plugin_context; - int ret; - - ret = compose_payload(ctx, tag, tag_len, data, bytes, out_data, out_size); - if (ret != FLB_OK) { - flb_error("ret=%d", ret); - return -1; - } - - return 0; -} - -/* Plugin reference */ -struct flb_output_plugin out_tcp_plugin = { - .name = "tcp", - .description = "TCP Output", - .cb_init = cb_tcp_init, - .cb_flush = cb_tcp_flush, - .cb_exit = cb_tcp_exit, - .config_map = config_map, - /* for testing */ - .test_formatter.callback = cb_tcp_format_test, - - .workers = 2, - .flags = FLB_OUTPUT_NET | FLB_IO_OPT_TLS, -}; diff --git a/fluent-bit/plugins/out_tcp/tcp.h b/fluent-bit/plugins/out_tcp/tcp.h deleted file mode 100644 index a133bb3f8..000000000 --- a/fluent-bit/plugins/out_tcp/tcp.h +++ /dev/null @@ -1,46 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_TCP_H -#define FLB_OUT_TCP_H - -#include -#include - -struct flb_out_tcp { - /* Output format */ - int out_format; - flb_sds_t raw_message_key; - struct flb_record_accessor *ra_raw_message_key; - - char *host; - int port; - - /* Timestamp format */ - int json_date_format; - flb_sds_t json_date_key; - flb_sds_t date_key; - - /* Upstream connection to the backend server */ - struct flb_upstream *u; - - struct flb_output_instance *ins; -}; - -#endif diff --git a/fluent-bit/plugins/out_tcp/tcp_conf.c b/fluent-bit/plugins/out_tcp/tcp_conf.c deleted file mode 100644 index 7d6e4b0b3..000000000 --- a/fluent-bit/plugins/out_tcp/tcp_conf.c +++ /dev/null @@ -1,154 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include "tcp.h" -#include "tcp_conf.h" - -struct flb_out_tcp *flb_tcp_conf_create(struct flb_output_instance *ins, - struct flb_config *config) -{ - int ret; - int io_flags = 0; - const char *tmp; - struct flb_upstream *upstream; - struct flb_out_tcp *ctx = NULL; - - /* Allocate plugin context */ - ctx = flb_calloc(1, sizeof(struct flb_out_tcp)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return NULL; - } - - /* Set default network configuration if not set */ - flb_output_net_default("127.0.0.1", 5170, ins); - - /* Check if SSL/TLS is enabled */ -#ifdef FLB_HAVE_TLS - if (ins->use_tls == FLB_TRUE) { - io_flags = FLB_IO_TLS; - } - else { - io_flags = FLB_IO_TCP; - } -#else - io_flags = FLB_IO_TCP; -#endif - - if (ins->host.ipv6 == FLB_TRUE) { - io_flags |= FLB_IO_IPV6; - } - - /* raw message key mode */ - if (ctx->raw_message_key) { - ctx->ra_raw_message_key = flb_ra_create(ctx->raw_message_key, FLB_TRUE); - if (!ctx->ra_raw_message_key) { - flb_plg_error(ctx->ins, "could not create record accessor for raw_message_key"); - flb_free(ctx); - return NULL; - } - } - - /* Upstream context */ - upstream = flb_upstream_create(config, - ins->host.name, - ins->host.port, - io_flags, ins->tls); - if (!upstream) { - flb_plg_error(ctx->ins, "could not create upstream context"); - flb_free(ctx); - return NULL; - } - - /* Output format */ - ctx->out_format = FLB_PACK_JSON_FORMAT_NONE; - tmp = flb_output_get_property("format", ins); - if (tmp) { - ret = flb_pack_to_json_format_type(tmp); - if (ret == -1) { - flb_plg_error(ctx->ins, "unrecognized 'format' option '%s'. " - "Using 'msgpack'", tmp); - } - else { - ctx->out_format = ret; - } - } - - /* Date key */ - ctx->date_key = ctx->json_date_key; - tmp = flb_output_get_property("json_date_key", ins); - if (tmp) { - /* Just check if we have to disable it */ - if (flb_utils_bool(tmp) == FLB_FALSE) { - ctx->date_key = NULL; - } - } - - /* Date format for JSON output */ - ctx->json_date_format = FLB_PACK_JSON_DATE_DOUBLE; - tmp = flb_output_get_property("json_date_format", ins); - if (tmp) { - ret = flb_pack_to_json_date_type(tmp); - if (ret == -1) { - flb_plg_error(ctx->ins, "unrecognized 'json_date_format' option '%s'. " - "Using 'double'", tmp); - } - else { - ctx->json_date_format = ret; - } - } - - ctx->u = upstream; - flb_output_upstream_set(ctx->u, ins); - - ctx->host = ins->host.name; - ctx->port = ins->host.port; - - return ctx; -} - -void flb_tcp_conf_destroy(struct flb_out_tcp *ctx) -{ - if (!ctx) { - return; - } - - if (ctx->ra_raw_message_key) { - flb_ra_destroy(ctx->ra_raw_message_key); - } - - if (ctx->u) { - flb_upstream_destroy(ctx->u); - } - - flb_free(ctx); - ctx = NULL; -} diff --git a/fluent-bit/plugins/out_tcp/tcp_conf.h b/fluent-bit/plugins/out_tcp/tcp_conf.h deleted file mode 100644 index c91def5ca..000000000 --- a/fluent-bit/plugins/out_tcp/tcp_conf.h +++ /dev/null @@ -1,32 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_TCP_CONF_H -#define FLB_OUT_TCP_CONF_H - -#include -#include - -#include "tcp.h" - -struct flb_out_tcp *flb_tcp_conf_create(struct flb_output_instance *ins, - struct flb_config *config); -void flb_tcp_conf_destroy(struct flb_out_tcp *ctx); - -#endif diff --git a/fluent-bit/plugins/out_td/CMakeLists.txt b/fluent-bit/plugins/out_td/CMakeLists.txt deleted file mode 100644 index 3641be389..000000000 --- a/fluent-bit/plugins/out_td/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -set(src - td_http.c - td_config.c - td.c) - -FLB_PLUGIN(out_td "${src}" "mk_core") -target_link_libraries(flb-plugin-out_td) diff --git a/fluent-bit/plugins/out_td/td.c b/fluent-bit/plugins/out_td/td.c deleted file mode 100644 index 7836c384a..000000000 --- a/fluent-bit/plugins/out_td/td.c +++ /dev/null @@ -1,271 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "td.h" -#include "td_http.h" -#include "td_config.h" - -#include -#include -#include -#include - -/* - * Convert the internal Fluent Bit data representation to the required - * one by Treasure Data cloud service. - * - * This function returns a new msgpack buffer and store the bytes length - * in the out_size variable. - */ -static char *td_format(struct flb_td *ctx, const void *data, size_t bytes, int *out_size) -{ - int i; - int ret; - int n_size; - time_t atime; - char *buf; - struct msgpack_sbuffer mp_sbuf; - struct msgpack_packer mp_pck; - msgpack_object map; - msgpack_sbuffer *sbuf; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - /* Initialize contexts for new output */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return NULL; - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - atime = log_event.timestamp.tm.tv_sec; - map = *log_event.body; - - n_size = map.via.map.size + 1; - msgpack_pack_map(&mp_pck, n_size); - msgpack_pack_str(&mp_pck, 4); - msgpack_pack_str_body(&mp_pck, "time", 4); - msgpack_pack_int32(&mp_pck, atime); - - for (i = 0; i < n_size - 1; i++) { - msgpack_pack_object(&mp_pck, map.via.map.ptr[i].key); - msgpack_pack_object(&mp_pck, map.via.map.ptr[i].val); - } - } - - flb_log_event_decoder_destroy(&log_decoder); - - /* Create new buffer */ - sbuf = &mp_sbuf; - *out_size = sbuf->size; - buf = flb_malloc(sbuf->size); - if (!buf) { - flb_errno(); - return NULL; - } - - /* set a new buffer and re-initialize our MessagePack context */ - memcpy(buf, sbuf->data, sbuf->size); - msgpack_sbuffer_destroy(&mp_sbuf); - - return buf; -} - -static int cb_td_init(struct flb_output_instance *ins, struct flb_config *config, - void *data) -{ - struct flb_td *ctx; - struct flb_upstream *upstream; - (void) data; - - ctx = td_config_init(ins); - if (!ctx) { - flb_plg_warn(ins, "Error reading configuration"); - return -1; - } - - if (ctx->region == FLB_TD_REGION_US) { - flb_output_net_default("api.treasuredata.com", 443, ins); - } - else if (ctx->region == FLB_TD_REGION_JP) { - flb_output_net_default("api.treasuredata.co.jp", 443, ins); - } - - upstream = flb_upstream_create(config, - ins->host.name, - ins->host.port, - FLB_IO_TLS, ins->tls); - if (!upstream) { - flb_free(ctx); - return -1; - } - ctx->u = upstream; - flb_output_upstream_set(ctx->u, ins); - - flb_output_set_context(ins, ctx); - return 0; -} - -static void cb_td_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - int ret; - int bytes_out; - char *pack; - size_t b_sent; - char *body = NULL; - struct flb_td *ctx = out_context; - struct flb_connection *u_conn; - struct flb_http_client *c; - (void) i_ins; - - /* Convert format */ - pack = td_format(ctx, event_chunk->data, event_chunk->size, &bytes_out); - if (!pack) { - FLB_OUTPUT_RETURN(FLB_ERROR); - } - - /* Lookup an available connection context */ - u_conn = flb_upstream_conn_get(ctx->u); - if (!u_conn) { - flb_plg_error(ctx->ins, "no upstream connections available"); - flb_free(pack); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Compose request */ - c = td_http_client(u_conn, pack, bytes_out, &body, ctx, config); - if (!c) { - flb_free(pack); - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Issue HTTP request */ - ret = flb_http_do(c, &b_sent); - - /* Release Resources */ - flb_free(pack); - flb_free(body); - - /* Validate HTTP status */ - if (ret == 0) { - /* We expect a HTTP 200 OK */ - if (c->resp.status != 200) { - if (c->resp.payload_size > 0) { - flb_plg_warn(ctx->ins, "HTTP status %i\n%s", - c->resp.status, c->resp.payload); - } - else { - flb_plg_warn(ctx->ins, "HTTP status %i", c->resp.status); - } - goto retry; - } - else { - flb_plg_info(ctx->ins, "HTTP status 200 OK"); - } - } - else { - flb_plg_error(ctx->ins, "http_do=%i", ret); - goto retry; - } - - /* release */ - flb_upstream_conn_release(u_conn); - flb_http_client_destroy(c); - - FLB_OUTPUT_RETURN(FLB_OK); - - retry: - flb_upstream_conn_release(u_conn); - flb_http_client_destroy(c); - - FLB_OUTPUT_RETURN(FLB_RETRY); -} - -static int cb_td_exit(void *data, struct flb_config *config) -{ - struct flb_td *ctx = data; - - if (!ctx) { - return 0; - } - - flb_upstream_destroy(ctx->u); - flb_free(ctx); - - return 0; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "API", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_td, api), - "Set the API key" - }, - { - FLB_CONFIG_MAP_STR, "Database", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_td, db_name), - "Set the Database file" - }, - { - FLB_CONFIG_MAP_STR, "Table", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_td, db_table), - "Set the Database Table" - }, - { - FLB_CONFIG_MAP_STR, "Region", (char *)NULL, - 0, FLB_TRUE, offsetof(struct flb_td, region_str), - "Set the Region: us or jp" - }, - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_output_plugin out_td_plugin = { - .name = "td", - .description = "Treasure Data", - .cb_init = cb_td_init, - .cb_pre_run = NULL, - .cb_flush = cb_td_flush, - .cb_exit = cb_td_exit, - .config_map = config_map, - .flags = FLB_IO_TLS, -}; diff --git a/fluent-bit/plugins/out_td/td.h b/fluent-bit/plugins/out_td/td.h deleted file mode 100644 index 1050289ce..000000000 --- a/fluent-bit/plugins/out_td/td.h +++ /dev/null @@ -1,24 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_TD_H -#define FLB_OUT_TD_H - - -#endif diff --git a/fluent-bit/plugins/out_td/td_config.c b/fluent-bit/plugins/out_td/td_config.c deleted file mode 100644 index ac5be6693..000000000 --- a/fluent-bit/plugins/out_td/td_config.c +++ /dev/null @@ -1,86 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include "td_config.h" -#include - -struct flb_td *td_config_init(struct flb_output_instance *ins) -{ - int ret; - struct flb_td *ctx; - - - /* Allocate context */ - ctx = flb_calloc(1, sizeof(struct flb_td)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - ctx->fd = -1; - - ret = flb_output_config_map_set(ins, (void *)ctx); - if (ret == -1) { - flb_plg_error(ins, "unable to load configuration"); - flb_free(ctx); - return NULL; - } - - if (ctx->api == NULL) { - flb_plg_error(ins, "error reading API key value"); - flb_free(ctx); - return NULL; - } - - if (ctx->db_name == NULL) { - flb_plg_error(ins, "error reading Database name"); - flb_free(ctx); - return NULL; - } - - if (ctx->db_table == NULL) { - flb_plg_error(ins, "error reading Table name"); - flb_free(ctx); - return NULL; - } - - /* Lookup desired region */ - if (ctx->region_str) { - if (strcasecmp(ctx->region_str, "us") == 0) { - ctx->region = FLB_TD_REGION_US; - } - else if (strcasecmp(ctx->region_str, "jp") == 0) { - ctx->region = FLB_TD_REGION_JP; - } - else { - flb_plg_error(ctx->ins, "invalid region in configuration"); - flb_free(ctx); - return NULL; - } - } - else { - ctx->region = FLB_TD_REGION_US; - } - - flb_plg_info(ctx->ins, "Treasure Data / database='%s' table='%s'", - ctx->db_name, ctx->db_table); - - return ctx; -} diff --git a/fluent-bit/plugins/out_td/td_config.h b/fluent-bit/plugins/out_td/td_config.h deleted file mode 100644 index f2412fddb..000000000 --- a/fluent-bit/plugins/out_td/td_config.h +++ /dev/null @@ -1,41 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_TD_CONFIG_H -#define FLB_TD_CONFIG_H - -#include - -#define FLB_TD_REGION_US 0 -#define FLB_TD_REGION_JP 1 - -struct flb_td { - int fd; /* Socket to destination/backend */ - int region; /* TD Region end-point */ - flb_sds_t region_str; - const char *api; - const char *db_name; - const char *db_table; - struct flb_upstream *u; - struct flb_output_instance *ins; -}; - -struct flb_td *td_config_init(struct flb_output_instance *ins); - -#endif diff --git a/fluent-bit/plugins/out_td/td_http.c b/fluent-bit/plugins/out_td/td_http.c deleted file mode 100644 index 539408044..000000000 --- a/fluent-bit/plugins/out_td/td_http.c +++ /dev/null @@ -1,94 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include "td_config.h" - -#include -#include -#include - - -#define TD_HTTP_HEADER_SIZE 512 - -struct flb_http_client *td_http_client(struct flb_connection *u_conn, - void *data, size_t len, - char **body, - struct flb_td *ctx, - struct flb_config *config) -{ - int ret; - int pos = 0; - int api_len; - size_t gz_size; - void *gz_data; - char *tmp; - struct flb_http_client *c; - - /* Compress data */ - ret = flb_gzip_compress(data, len, &gz_data, &gz_size); - if (ret == -1) { - flb_plg_error(ctx->ins, "error compressing data"); - return NULL; - } - - /* Compose URI */ - tmp = flb_malloc(512); - if (!tmp) { - flb_free(gz_data); - return NULL; - } - snprintf(tmp, 256, - "/v3/table/import/%s/%s/msgpack.gz", - ctx->db_name, ctx->db_table); - - /* Create client */ - c = flb_http_client(u_conn, FLB_HTTP_PUT, tmp, - gz_data, gz_size, NULL, 0, NULL, 0); - if (!c) { - flb_free(tmp); - flb_free(gz_data); - return NULL; - } - - /* Add custom headers */ - tmp[pos++] = 'T'; - tmp[pos++] = 'D'; - tmp[pos++] = '1'; - tmp[pos++] = ' '; - - api_len = strlen(ctx->api); - memcpy(tmp + pos, ctx->api, api_len); - pos += api_len; - - flb_http_add_header(c, - "Authorization", 13, - tmp, pos); - flb_http_add_header(c, - "Content-Type", 12, - "application/gzip", 16); - flb_free(tmp); - *body = gz_data; - - return c; -} diff --git a/fluent-bit/plugins/out_td/td_http.h b/fluent-bit/plugins/out_td/td_http.h deleted file mode 100644 index 30d398fe1..000000000 --- a/fluent-bit/plugins/out_td/td_http.h +++ /dev/null @@ -1,35 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_TD_HTTP_H -#define FLB_OUT_TD_HTTP_H - -#include -#include "td_config.h" - -char *td_http_request(void *data, size_t len, - size_t *out_len, - struct flb_td *ctx, struct flb_config *config); - -struct flb_http_client *td_http_client(struct flb_connection *u_conn, - void *data, size_t len, - char **body, - struct flb_td *ctx, - struct flb_config *config); -#endif diff --git a/fluent-bit/plugins/out_udp/CMakeLists.txt b/fluent-bit/plugins/out_udp/CMakeLists.txt deleted file mode 100644 index ad4f7e411..000000000 --- a/fluent-bit/plugins/out_udp/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(src - udp.c - udp_conf.c - ) - -FLB_PLUGIN(out_udp "${src}" "mk_core") diff --git a/fluent-bit/plugins/out_udp/udp.c b/fluent-bit/plugins/out_udp/udp.c deleted file mode 100644 index 8e0044bfe..000000000 --- a/fluent-bit/plugins/out_udp/udp.c +++ /dev/null @@ -1,351 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "udp.h" -#include "udp_conf.h" - -static int cb_udp_init(struct flb_output_instance *ins, - struct flb_config *config, void *data) -{ - struct flb_out_udp *ctx = NULL; - (void) data; - - ctx = flb_udp_conf_create(ins, config); - if (!ctx) { - return -1; - } - - /* Set the plugin context */ - flb_output_set_context(ins, ctx); - - return 0; -} - -static int deliver_chunks_raw(struct flb_out_udp *ctx, - const char *tag, int tag_len, - const void *in_data, size_t in_size) -{ - int ret; - flb_sds_t buf = NULL; - flb_sds_t str; - msgpack_object map; - ssize_t send_result; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - buf = flb_sds_create_size(in_size); - if (!buf) { - return FLB_ERROR; - } - - ret = flb_log_event_decoder_init(&log_decoder, (char *) in_data, in_size); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - flb_sds_destroy(buf); - - return -1; - } - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - map = *log_event.body; - - str = flb_ra_translate(ctx->ra_raw_message_key, (char *) tag, tag_len, map, NULL); - if (!str) { - continue; - } - - ret = flb_sds_cat_safe(&buf, str, flb_sds_len(str)); - if (ret != 0) { - flb_plg_error(ctx->ins, "failed to compose payload from '%s'", str); - } - flb_sds_destroy(str); - - /* append a new line */ - flb_sds_cat_safe(&buf, "\n", 1); - - if (flb_sds_len(buf) > 65535) { - flb_plg_debug(ctx->ins, "record size exceeds maximum datagram size : %zu", flb_sds_len(buf)); - } - - send_result = send(ctx->endpoint_descriptor, - buf, - flb_sds_len(buf), - 0); - - if (send_result == -1) { - flb_log_event_decoder_destroy(&log_decoder); - flb_sds_destroy(buf); - - return FLB_RETRY; - } - - flb_sds_len_set(buf, 0); - buf[0] = '\0'; - } - - flb_log_event_decoder_destroy(&log_decoder); - flb_sds_destroy(buf); - - return FLB_OK; -} - -static int deliver_chunks_json(struct flb_out_udp *ctx, - const char *tag, int tag_len, - const void *in_data, size_t in_size) -{ - int ret; - size_t off = 0; - flb_sds_t json = NULL; - ssize_t send_result; - size_t previous_offset; - int append_new_line; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) in_data, in_size); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return FLB_ERROR; - } - - previous_offset = 0; - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - off = log_decoder.offset; - - json = flb_pack_msgpack_to_json_format(&((char *) in_data)[previous_offset], - off - previous_offset, - ctx->out_format, - ctx->json_date_format, - ctx->date_key); - if (!json) { - flb_plg_error(ctx->ins, "error formatting JSON payload"); - - flb_log_event_decoder_destroy(&log_decoder); - - return FLB_ERROR; - } - - previous_offset = off; - append_new_line = FLB_FALSE; - - if (flb_sds_len(json) > 0) { - if (json[flb_sds_len(json) - 1] != '\n') { - append_new_line = FLB_TRUE; - } - - if (append_new_line) { - ret = flb_sds_cat_safe(&json, "\n", 1); - - if (ret != 0) { - flb_log_event_decoder_destroy(&log_decoder); - flb_sds_destroy(json); - - return FLB_RETRY; - } - } - - if (flb_sds_len(json) > 65535) { - flb_plg_debug(ctx->ins, "record size exceeds maximum datagram size : %zu", flb_sds_len(json)); - } - - send_result = send(ctx->endpoint_descriptor, - json, - flb_sds_len(json), - 0); - - if (send_result == -1) { - flb_log_event_decoder_destroy(&log_decoder); - flb_sds_destroy(json); - - return FLB_RETRY; - } - } - - flb_sds_destroy(json); - } - - flb_log_event_decoder_destroy(&log_decoder); - - return FLB_OK; -} - -static int deliver_chunks_msgpack(struct flb_out_udp *ctx, - const char *tag, int tag_len, - const void *in_data, size_t in_size) -{ - size_t off = 0; - ssize_t send_result; - size_t previous_offset; - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - int ret; - - ret = flb_log_event_decoder_init(&log_decoder, (char *) in_data, in_size); - - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_plg_error(ctx->ins, - "Log event decoder initialization error : %d", ret); - - return FLB_RETRY; - } - - previous_offset = 0; - - while ((ret = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - off = log_decoder.offset; - - if ((off - previous_offset) > 65535) { - flb_plg_debug(ctx->ins, "record size exceeds maximum datagram size : %zu", (off - previous_offset)); - } - - send_result = send(ctx->endpoint_descriptor, - &((char *) in_data)[previous_offset], - off - previous_offset, - 0); - - if (send_result == -1) { - flb_log_event_decoder_destroy(&log_decoder); - - return FLB_RETRY; - } - - previous_offset = off; - } - - flb_log_event_decoder_destroy(&log_decoder); - - return FLB_OK; -} - -static void cb_udp_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - int ret = FLB_ERROR; - struct flb_out_udp *ctx = out_context; - - (void) i_ins; - - if (ctx->ra_raw_message_key != NULL) { - ret = deliver_chunks_raw(ctx, - event_chunk->tag, - flb_sds_len(event_chunk->tag), - event_chunk->data, - event_chunk->size); - } - else if (ctx->out_format == FLB_PACK_JSON_FORMAT_NONE) { - ret = deliver_chunks_msgpack(ctx, - event_chunk->tag, - flb_sds_len(event_chunk->tag), - event_chunk->data, - event_chunk->size); - } - else { - ret = deliver_chunks_json(ctx, - event_chunk->tag, - flb_sds_len(event_chunk->tag), - event_chunk->data, - event_chunk->size); - } - - return FLB_OUTPUT_RETURN(ret); -} - -static int cb_udp_exit(void *data, struct flb_config *config) -{ - struct flb_out_udp *ctx = data; - - flb_udp_conf_destroy(ctx); - - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "format", "json_lines", - 0, FLB_FALSE, 0, - "Specify the payload format, supported formats: msgpack, json, " - "json_lines or json_stream." - }, - - { - FLB_CONFIG_MAP_STR, "json_date_format", "double", - 0, FLB_FALSE, 0, - FBL_PACK_JSON_DATE_FORMAT_DESCRIPTION - }, - - { - FLB_CONFIG_MAP_STR, "json_date_key", "date", - 0, FLB_TRUE, offsetof(struct flb_out_udp, json_date_key), - "Specify the name of the date field in output." - }, - - { - FLB_CONFIG_MAP_STR, "raw_message_key", NULL, - 0, FLB_TRUE, offsetof(struct flb_out_udp, raw_message_key), - "use a raw message key for the message." - }, - - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_output_plugin out_udp_plugin = { - .name = "udp", - .description = "UDP Output", - .cb_init = cb_udp_init, - .cb_flush = cb_udp_flush, - .cb_exit = cb_udp_exit, - .config_map = config_map, - - .workers = 2, - .flags = FLB_OUTPUT_NET, -}; diff --git a/fluent-bit/plugins/out_udp/udp.h b/fluent-bit/plugins/out_udp/udp.h deleted file mode 100644 index 522367ee0..000000000 --- a/fluent-bit/plugins/out_udp/udp.h +++ /dev/null @@ -1,47 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_UDP_H -#define FLB_OUT_UDP_H - -#include -#include -#include -#include - -struct flb_out_udp { - /* Output format */ - int out_format; - flb_sds_t raw_message_key; - struct flb_record_accessor *ra_raw_message_key; - - char *host; - int port; - - flb_sockfd_t endpoint_descriptor; - - /* Timestamp format */ - int json_date_format; - flb_sds_t json_date_key; - flb_sds_t date_key; - - struct flb_output_instance *ins; -}; - -#endif diff --git a/fluent-bit/plugins/out_udp/udp_conf.c b/fluent-bit/plugins/out_udp/udp_conf.c deleted file mode 100644 index 2f5509d3a..000000000 --- a/fluent-bit/plugins/out_udp/udp_conf.c +++ /dev/null @@ -1,135 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include "udp.h" -#include "udp_conf.h" - -struct flb_out_udp *flb_udp_conf_create(struct flb_output_instance *ins, - struct flb_config *config) -{ - int ret; - const char *tmp; - struct flb_out_udp *ctx = NULL; - - /* Allocate plugin context */ - ctx = flb_calloc(1, sizeof(struct flb_out_udp)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return NULL; - } - - /* Set default network configuration if not set */ - flb_output_net_default("127.0.0.1", 5170, ins); - - /* raw message key mode */ - if (ctx->raw_message_key) { - ctx->ra_raw_message_key = flb_ra_create(ctx->raw_message_key, FLB_TRUE); - if (!ctx->ra_raw_message_key) { - flb_plg_error(ctx->ins, "could not create record accessor for raw_message_key"); - flb_free(ctx); - return NULL; - } - } - - /* Output format */ - ctx->out_format = FLB_PACK_JSON_FORMAT_NONE; - tmp = flb_output_get_property("format", ins); - if (tmp) { - ret = flb_pack_to_json_format_type(tmp); - if (ret == -1) { - flb_plg_error(ctx->ins, "unrecognized 'format' option '%s'. " - "Using 'msgpack'", tmp); - } - else { - ctx->out_format = ret; - } - } - - /* Date key */ - ctx->date_key = ctx->json_date_key; - tmp = flb_output_get_property("json_date_key", ins); - if (tmp) { - /* Just check if we have to disable it */ - if (flb_utils_bool(tmp) == FLB_FALSE) { - ctx->date_key = NULL; - } - } - - /* Date format for JSON output */ - ctx->json_date_format = FLB_PACK_JSON_DATE_DOUBLE; - tmp = flb_output_get_property("json_date_format", ins); - if (tmp) { - ret = flb_pack_to_json_date_type(tmp); - if (ret == -1) { - flb_plg_error(ctx->ins, "unrecognized 'json_date_format' option '%s'. " - "Using 'double'", tmp); - } - else { - ctx->json_date_format = ret; - } - } - - ctx->host = ins->host.name; - ctx->port = ins->host.port; - - ctx->endpoint_descriptor = flb_net_udp_connect(ins->host.name, - ins->host.port, - ins->net_setup.source_address); - - if (ctx->endpoint_descriptor < 0) { - flb_udp_conf_destroy(ctx); - - flb_plg_error(ctx->ins, "Error creating upstream socket"); - - ctx = NULL; - } - - return ctx; -} - -void flb_udp_conf_destroy(struct flb_out_udp *ctx) -{ - if (!ctx) { - return; - } - - if (ctx->ra_raw_message_key) { - flb_ra_destroy(ctx->ra_raw_message_key); - } - - if (ctx->endpoint_descriptor >= 0) { - flb_socket_close(ctx->endpoint_descriptor); - } - - flb_free(ctx); - - ctx = NULL; -} diff --git a/fluent-bit/plugins/out_udp/udp_conf.h b/fluent-bit/plugins/out_udp/udp_conf.h deleted file mode 100644 index 58cba4564..000000000 --- a/fluent-bit/plugins/out_udp/udp_conf.h +++ /dev/null @@ -1,32 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_UDP_CONF_H -#define FLB_OUT_UDP_CONF_H - -#include -#include - -#include "udp.h" - -struct flb_out_udp *flb_udp_conf_create(struct flb_output_instance *ins, - struct flb_config *config); -void flb_udp_conf_destroy(struct flb_out_udp *ctx); - -#endif diff --git a/fluent-bit/plugins/out_vivo_exporter/CMakeLists.txt b/fluent-bit/plugins/out_vivo_exporter/CMakeLists.txt deleted file mode 100644 index e458b1ff4..000000000 --- a/fluent-bit/plugins/out_vivo_exporter/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -if(NOT FLB_HTTP_SERVER) - message( - FATAL_ERROR - "Vivo Exporter output plugin requires built-in HTTP Server be enabled: - Use -DFLB_HTTP_SERVER=On option to enable it" - ) -endif() - -set(src - vivo_http.c - vivo_stream.c - vivo.c - ) - -FLB_PLUGIN(out_vivo_exporter "${src}" "") diff --git a/fluent-bit/plugins/out_vivo_exporter/vivo.c b/fluent-bit/plugins/out_vivo_exporter/vivo.c deleted file mode 100644 index 85e1e0159..000000000 --- a/fluent-bit/plugins/out_vivo_exporter/vivo.c +++ /dev/null @@ -1,343 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "vivo.h" -#include "vivo_http.h" -#include "vivo_stream.h" - -static flb_sds_t format_logs(struct flb_event_chunk *event_chunk) -{ - struct flb_log_event_decoder log_decoder; - struct flb_log_event log_event; - int result; - int i; - flb_sds_t out_js; - flb_sds_t out_buf = NULL; - msgpack_sbuffer tmp_sbuf; - msgpack_packer tmp_pck; - - result = flb_log_event_decoder_init(&log_decoder, - (char *) event_chunk->data, - event_chunk->size); - - if (result != FLB_EVENT_DECODER_SUCCESS) { - return NULL; - } - - out_buf = flb_sds_create_size((event_chunk->size * 2) / 4); - if (!out_buf) { - flb_errno(); - return NULL; - } - - /* Create temporary msgpack buffer */ - msgpack_sbuffer_init(&tmp_sbuf); - msgpack_packer_init(&tmp_pck, &tmp_sbuf, msgpack_sbuffer_write); - - while ((result = flb_log_event_decoder_next( - &log_decoder, - &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - /* - * If the caller specified FLB_PACK_JSON_DATE_FLUENT, we format the data - * by using the following structure: - * - * [[TIMESTAMP, {"_tag": "...", ...MORE_METADATA}], {RECORD CONTENT}] - */ - msgpack_pack_array(&tmp_pck, 2); - msgpack_pack_array(&tmp_pck, 2); - msgpack_pack_uint64(&tmp_pck, flb_time_to_nanosec(&log_event.timestamp)); - - /* add tag only */ - msgpack_pack_map(&tmp_pck, 1 + log_event.metadata->via.map.size); - - msgpack_pack_str(&tmp_pck, 4); - msgpack_pack_str_body(&tmp_pck, "_tag", 4); - - msgpack_pack_str(&tmp_pck, flb_sds_len(event_chunk->tag)); - msgpack_pack_str_body(&tmp_pck, event_chunk->tag, flb_sds_len(event_chunk->tag)); - - /* Append remaining keys/values */ - for (i = 0; - i < log_event.metadata->via.map.size; - i++) { - msgpack_pack_object(&tmp_pck, - log_event.metadata->via.map.ptr[i].key); - msgpack_pack_object(&tmp_pck, - log_event.metadata->via.map.ptr[i].val); - } - - /* pack the remaining content */ - msgpack_pack_map(&tmp_pck, log_event.body->via.map.size); - - /* Append remaining keys/values */ - for (i = 0; - i < log_event.body->via.map.size; - i++) { - msgpack_pack_object(&tmp_pck, - log_event.body->via.map.ptr[i].key); - msgpack_pack_object(&tmp_pck, - log_event.body->via.map.ptr[i].val); - } - - /* Concatenate by using break lines */ - out_js = flb_msgpack_raw_to_json_sds(tmp_sbuf.data, tmp_sbuf.size); - if (!out_js) { - flb_sds_destroy(out_buf); - msgpack_sbuffer_destroy(&tmp_sbuf); - flb_log_event_decoder_destroy(&log_decoder); - return NULL; - } - - /* - * One map record has been converted, now append it to the - * outgoing out_buf sds variable. - */ - flb_sds_cat_safe(&out_buf, out_js, flb_sds_len(out_js)); - flb_sds_cat_safe(&out_buf, "\n", 1); - - flb_sds_destroy(out_js); - msgpack_sbuffer_clear(&tmp_sbuf); - } - - /* Release the unpacker */ - flb_log_event_decoder_destroy(&log_decoder); - - msgpack_sbuffer_destroy(&tmp_sbuf); - - return out_buf; -} - -static int logs_event_chunk_append(struct vivo_exporter *ctx, - struct flb_event_chunk *event_chunk) -{ - size_t len; - flb_sds_t json; - struct vivo_stream_entry *entry; - - - json = format_logs(event_chunk); - if (!json) { - flb_plg_error(ctx->ins, "cannot convert logs chunk to JSON"); - return -1; - } - - /* append content to the stream */ - len = flb_sds_len(json); - entry = vivo_stream_append(ctx->stream_logs, json, len); - - flb_sds_destroy(json); - - if (!entry) { - flb_plg_error(ctx->ins, "cannot append JSON log to stream"); - return -1; - } - - return 0; -} - -static int metrics_traces_event_chunk_append(struct vivo_exporter *ctx, - struct vivo_stream *vs, - struct flb_event_chunk *event_chunk) -{ - size_t len; - flb_sds_t json; - struct vivo_stream_entry *entry; - - /* Convert msgpack to readable JSON format */ - json = flb_msgpack_raw_to_json_sds(event_chunk->data, event_chunk->size); - if (!json) { - flb_plg_error(ctx->ins, "cannot convert metrics chunk to JSON"); - return -1; - } - - flb_sds_cat_safe(&json, "\n", 1); - - /* append content to the stream */ - len = flb_sds_len(json); - entry = vivo_stream_append(vs, json, len); - - flb_sds_destroy(json); - - if (!entry) { - flb_plg_error(ctx->ins, "cannot append JSON log to stream"); - return -1; - } - - return 0; -} - -static int cb_vivo_init(struct flb_output_instance *ins, - struct flb_config *config, - void *data) -{ - int ret; - struct vivo_exporter *ctx; - - flb_output_net_default("0.0.0.0", 2025 , ins); - - ctx = flb_calloc(1, sizeof(struct vivo_exporter)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->ins = ins; - - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return -1; - } - - flb_output_set_context(ins, ctx); - - /* Load config map */ - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - return -1; - } - - /* Create Streams */ - ctx->stream_logs = vivo_stream_create(ctx); - if (!ctx->stream_logs) { - return -1; - } - - ctx->stream_metrics = vivo_stream_create(ctx); - if (!ctx->stream_metrics) { - return -1; - } - - ctx->stream_traces = vivo_stream_create(ctx); - if (!ctx->stream_traces) { - return -1; - } - - /* HTTP Server context */ - ctx->http = vivo_http_server_create(ctx, - ins->host.name, ins->host.port, config); - if (!ctx->http) { - flb_plg_error(ctx->ins, "could not initialize HTTP server, aborting"); - return -1; - } - - /* Start HTTP Server */ - ret = vivo_http_server_start(ctx->http); - if (ret == -1) { - return -1; - } - - flb_plg_info(ctx->ins, "listening iface=%s tcp_port=%d", - ins->host.name, ins->host.port); - - return 0; -} - -static void cb_vivo_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *ins, void *out_context, - struct flb_config *config) -{ - int ret = -1; - struct vivo_exporter *ctx = out_context; - -#ifdef FLB_HAVE_METRICS - if (event_chunk->type == FLB_EVENT_TYPE_METRICS) { - ret = metrics_traces_event_chunk_append(ctx, ctx->stream_metrics, event_chunk); - } -#endif - if (event_chunk->type == FLB_EVENT_TYPE_LOGS) { - ret = logs_event_chunk_append(ctx, event_chunk); - } - else if (event_chunk->type == FLB_EVENT_TYPE_TRACES) { - ret = metrics_traces_event_chunk_append(ctx, ctx->stream_traces, event_chunk); - } - - if (ret == 0) { - FLB_OUTPUT_RETURN(FLB_OK); - } - - FLB_OUTPUT_RETURN(FLB_ERROR); -} - -static int cb_vivo_exit(void *data, struct flb_config *config) -{ - struct vivo_exporter *ctx = data; - - if (!ctx) { - return 0; - } - - if (ctx->http) { - vivo_http_server_stop(ctx->http); - vivo_http_server_destroy(ctx->http); - } - - vivo_stream_destroy(ctx->stream_logs); - vivo_stream_destroy(ctx->stream_metrics); - vivo_stream_destroy(ctx->stream_traces); - - flb_free(ctx); - - return 0; -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_BOOL, "empty_stream_on_read", "off", - 0, FLB_TRUE, offsetof(struct vivo_exporter, empty_stream_on_read), - "If enabled, when an HTTP client consumes the data from a stream, the queue " - "content will be removed" - }, - - { - FLB_CONFIG_MAP_SIZE, "stream_queue_size", "20M", - 0, FLB_TRUE, offsetof(struct vivo_exporter, stream_queue_size), - "Specify the maximum queue size per stream. Each specific stream for logs, metrics " - "and traces can hold up to 'stream_queue_size' bytes." - }, - - { - FLB_CONFIG_MAP_STR, "http_cors_allow_origin", NULL, - 0, FLB_TRUE, offsetof(struct vivo_exporter, http_cors_allow_origin), - "Specify the value for the HTTP Access-Control-Allow-Origin header (CORS)" - }, - - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_output_plugin out_vivo_exporter_plugin = { - .name = "vivo_exporter", - .description = "Vivo Exporter", - .cb_init = cb_vivo_init, - .cb_flush = cb_vivo_flush, - .cb_exit = cb_vivo_exit, - .flags = FLB_OUTPUT_NET, - .event_type = FLB_OUTPUT_LOGS | FLB_OUTPUT_METRICS | FLB_OUTPUT_TRACES, - .config_map = config_map, - .workers = 1, -}; diff --git a/fluent-bit/plugins/out_vivo_exporter/vivo.h b/fluent-bit/plugins/out_vivo_exporter/vivo.h deleted file mode 100644 index 943c40364..000000000 --- a/fluent-bit/plugins/out_vivo_exporter/vivo.h +++ /dev/null @@ -1,45 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_VIVO_EXPORTER_H -#define FLB_VIVO_EXPORTER_H - -#include -#include - -#define VIVO_RING_BUFFER_SIZE 10 - -/* Plugin context */ -struct vivo_exporter { - void *http; - - void *stream_logs; - void *stream_metrics; - void *stream_traces; - - /* options */ - int empty_stream_on_read; - size_t stream_queue_size; - flb_sds_t http_cors_allow_origin; - - /* instance context */ - struct flb_output_instance *ins; -}; - -#endif diff --git a/fluent-bit/plugins/out_vivo_exporter/vivo_http.c b/fluent-bit/plugins/out_vivo_exporter/vivo_http.c deleted file mode 100644 index efd39dcc8..000000000 --- a/fluent-bit/plugins/out_vivo_exporter/vivo_http.c +++ /dev/null @@ -1,266 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include "vivo.h" -#include "vivo_http.h" -#include "vivo_stream.h" - -#define VIVO_CONTENT_TYPE "Content-Type" -#define VIVO_CONTENT_TYPE_JSON "application/json" -#define VIVO_STREAM_START_ID "Vivo-Stream-Start-ID" -#define VIVO_STREAM_END_ID "Vivo-Stream-End-ID" - -static int stream_get_uri_properties(mk_request_t *request, - int64_t *from, int64_t *to, int64_t *limit) -{ - char *ptr; - flb_sds_t buf; - - *from = -1; - *to = -1; - *limit = -1; - - buf = flb_sds_create_len(request->query_string.data, request->query_string.len); - if (!buf) { - return -1; - } - - ptr = strstr(buf, "from="); - if (ptr) { - *from = atol(ptr + 5); - } - - ptr = strstr(buf, "to="); - if (ptr) { - *to = atol(ptr + 3); - } - - ptr = strstr(buf, "limit="); - if (ptr) { - *limit = atol(ptr + 6); - } - - flb_sds_destroy(buf); - - return 0; -} - -static void headers_set(mk_request_t *request, struct vivo_stream *vs) -{ - struct vivo_exporter *ctx; - - - /* parent context */ - ctx = vs->parent; - - /* content type */ - mk_http_header(request, - VIVO_CONTENT_TYPE, sizeof(VIVO_CONTENT_TYPE) - 1, - VIVO_CONTENT_TYPE_JSON, sizeof(VIVO_CONTENT_TYPE_JSON) - 1); - - /* CORS */ - if (ctx->http_cors_allow_origin) { - mk_http_header(request, - "Access-Control-Allow-Origin", - sizeof("Access-Control-Allow-Origin") - 1, - ctx->http_cors_allow_origin, - flb_sds_len(ctx->http_cors_allow_origin)); - - mk_http_header(request, - "Access-Control-Allow-Headers", - sizeof("Access-Control-Allow-Headers") - 1, - "Origin, X-Requested-With, Content-Type, Accept", - sizeof("Origin, X-Requested-With, Content-Type, Accept") - 1); - - mk_http_header(request, - "Access-Control-Expose-Headers", - sizeof("Access-Control-Expose-Headers") - 1, - "vivo-stream-start-id, vivo-stream-end-id", - sizeof("vivo-stream-start-id, vivo-stream-end-id") - 1); - - } -} - -static void serve_content(mk_request_t *request, struct vivo_stream *vs) -{ - int64_t from = -1; - int64_t to = -1; - int64_t limit = -1; - int64_t stream_start_id = -1; - int64_t stream_end_id = -1; - flb_sds_t payload; - flb_sds_t str_start; - flb_sds_t str_end; - - - if (request->query_string.len > 0) { - stream_get_uri_properties(request, &from, &to, &limit); - } - - payload = vivo_stream_get_content(vs, from, to, limit, - &stream_start_id, &stream_end_id); - if (!payload) { - mk_http_status(request, 500); - return; - } - - if (flb_sds_len(payload) == 0) { - mk_http_status(request, 200); - headers_set(request, vs); - flb_sds_destroy(payload); - return; - } - - mk_http_status(request, 200); - - /* set response headers */ - headers_set(request, vs); - - /* stream ids served: compose buffer and set headers */ - str_start = flb_sds_create_size(32); - flb_sds_printf(&str_start, "%" PRId64, stream_start_id); - - str_end = flb_sds_create_size(32); - flb_sds_printf(&str_end, "%" PRId64, stream_end_id); - - mk_http_header(request, - VIVO_STREAM_START_ID, sizeof(VIVO_STREAM_START_ID) - 1, - str_start, flb_sds_len(str_start)); - - mk_http_header(request, - VIVO_STREAM_END_ID, sizeof(VIVO_STREAM_END_ID) - 1, - str_end, flb_sds_len(str_end)); - - /* send payload */ - mk_http_send(request, payload, flb_sds_len(payload), NULL); - - /* release */ - flb_sds_destroy(payload); - flb_sds_destroy(str_start); - flb_sds_destroy(str_end); -} - -/* HTTP endpoint: /logs */ -static void cb_logs(mk_request_t *request, void *data) -{ - struct vivo_exporter *ctx; - - ctx = (struct vivo_exporter *) data; - - serve_content(request, ctx->stream_logs); - mk_http_done(request); -} - -/* HTTP endpoint: /metrics */ -static void cb_metrics(mk_request_t *request, void *data) -{ - struct vivo_exporter *ctx; - - ctx = (struct vivo_exporter *) data; - - serve_content(request, ctx->stream_metrics); - mk_http_done(request); -} - -static void cb_traces(mk_request_t *request, void *data) -{ - struct vivo_exporter *ctx; - - ctx = (struct vivo_exporter *) data; - - serve_content(request, ctx->stream_traces); - mk_http_done(request); -} - -/* HTTP endpoint: / (root) */ -static void cb_root(mk_request_t *request, void *data) -{ - (void) data; - - mk_http_status(request, 200); - mk_http_send(request, "Fluent Bit Vivo Exporter\n", 24, NULL); - mk_http_done(request); -} - -struct vivo_http *vivo_http_server_create(struct vivo_exporter *ctx, - const char *listen, - int tcp_port, - struct flb_config *config) -{ - int vid; - char tmp[32]; - struct vivo_http *ph; - - ph = flb_malloc(sizeof(struct vivo_http)); - if (!ph) { - flb_errno(); - return NULL; - } - ph->config = config; - - /* HTTP Server context */ - ph->ctx = mk_create(); - if (!ph->ctx) { - flb_free(ph); - return NULL; - } - - /* Compose listen address */ - snprintf(tmp, sizeof(tmp) -1, "%s:%d", listen, tcp_port); - mk_config_set(ph->ctx, - "Listen", tmp, - "Workers", "1", - NULL); - - /* Virtual host */ - vid = mk_vhost_create(ph->ctx, NULL); - ph->vid = vid; - - /* Set HTTP URI callbacks */ - mk_vhost_handler(ph->ctx, vid, "/logs", cb_logs, ctx); - mk_vhost_handler(ph->ctx, vid, "/metrics", cb_metrics, ctx); - mk_vhost_handler(ph->ctx, vid, "/traces", cb_traces, ctx); - mk_vhost_handler(ph->ctx, vid, "/", cb_root, NULL); - - return ph; -} - -void vivo_http_server_destroy(struct vivo_http *ph) -{ - if (ph) { - /* TODO: release mk_vhost */ - if (ph->ctx) { - mk_destroy(ph->ctx); - } - flb_free(ph); - } -} - -int vivo_http_server_start(struct vivo_http *ph) -{ - return mk_start(ph->ctx); -} - -int vivo_http_server_stop(struct vivo_http *ph) -{ - return mk_stop(ph->ctx); -} diff --git a/fluent-bit/plugins/out_vivo_exporter/vivo_http.h b/fluent-bit/plugins/out_vivo_exporter/vivo_http.h deleted file mode 100644 index 77453d289..000000000 --- a/fluent-bit/plugins/out_vivo_exporter/vivo_http.h +++ /dev/null @@ -1,56 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_VIVO_EXPORTER_HTTP_H -#define FLB_VIVO_EXPORTER_HTTP_H - -#include -#include - -#include "vivo.h" - -/* HTTP response payload received through a Message Queue */ -struct vivo_http_buf { - int users; - char *buf_data; - size_t buf_size; - struct mk_list _head; -}; - -/* Vivo HTTP Server context */ -struct vivo_http { - mk_ctx_t *ctx; /* Monkey HTTP Context */ - int vid; /* Virtual host ID */ - int qid_metrics; /* Queue ID for Metrics buffer */ - struct flb_config *config; /* Fluent Bit context */ -}; - -struct vivo_http *vivo_http_server_create(struct vivo_exporter *ctx, - const char *listen, - int tcp_port, - struct flb_config *config); -void vivo_http_server_destroy(struct vivo_http *ph); - -int vivo_http_server_start(struct vivo_http *ph); -int vivo_http_server_stop(struct vivo_http *ph); - -int vivo_http_server_mq_push_metrics(struct vivo_http *ph, - void *data, size_t size); - -#endif diff --git a/fluent-bit/plugins/out_vivo_exporter/vivo_stream.c b/fluent-bit/plugins/out_vivo_exporter/vivo_stream.c deleted file mode 100644 index 9c8edb9ea..000000000 --- a/fluent-bit/plugins/out_vivo_exporter/vivo_stream.c +++ /dev/null @@ -1,239 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include "vivo.h" -#include "vivo_stream.h" - -static inline void stream_lock(struct vivo_stream *vs) -{ - pthread_mutex_lock(&vs->stream_mutex); -} - -static inline void stream_unlock(struct vivo_stream *vs) -{ - pthread_mutex_unlock(&vs->stream_mutex); -} - -struct vivo_stream *vivo_stream_create(struct vivo_exporter *ctx) -{ - struct vivo_stream *vs; - - vs = flb_calloc(1, sizeof(struct vivo_stream)); - if (!vs) { - flb_errno(); - return NULL; - } - vs->parent = ctx; - vs->entries_added = 0; - pthread_mutex_init(&vs->stream_mutex, NULL); - mk_list_init(&vs->entries); - mk_list_init(&vs->purge); - - return vs; -} - -static uint64_t vivo_stream_get_new_id(struct vivo_stream *vs) -{ - uint64_t id = 0; - - stream_lock(vs); - - /* to get the next id, we simply use the value of the counter 'entries' added */ - id = vs->entries_added; - - stream_unlock(vs); - - return id; -} - - -struct vivo_stream_entry *vivo_stream_entry_create(struct vivo_stream *vs, - void *data, size_t size) -{ - struct vivo_stream_entry *e; - - if (size == 0) { - return NULL; - } - - e = flb_calloc(1, sizeof(struct vivo_stream_entry)); - if (!e) { - flb_errno(); - return NULL; - } - e->id = vivo_stream_get_new_id(vs); - - e->data = flb_sds_create_len(data, size); - if (!e->data) { - flb_free(e); - return NULL; - } - - return e; -} - -/* - * NOTE: this function must always invoked under the stream_mutex in a locked state, we don't do the lock - * inside the function since the caller might be itering the parent list - */ -static void vivo_stream_entry_destroy(struct vivo_stream *vs, struct vivo_stream_entry *e) -{ - mk_list_del(&e->_head); - vs->current_bytes_size -= flb_sds_len(e->data); - flb_sds_destroy(e->data); - flb_free(e); -} - -/* NOTE: this function must run inside a stream_lock()/stream_unlock() protection */ -static void vivo_stream_cleanup(struct vivo_stream *vs) -{ - struct mk_list *tmp; - struct mk_list *head; - struct vivo_stream_entry *e; - - mk_list_foreach_safe(head, tmp, &vs->entries) { - e = mk_list_entry(head, struct vivo_stream_entry, _head); - vivo_stream_entry_destroy(vs, e); - } -} - -void vivo_stream_destroy(struct vivo_stream *vs) -{ - struct mk_list *tmp; - struct mk_list *head; - struct vivo_stream_entry *e; - - stream_lock(vs); - mk_list_foreach_safe(head, tmp, &vs->entries) { - e = mk_list_entry(head, struct vivo_stream_entry, _head); - vivo_stream_entry_destroy(vs, e); - } - stream_unlock(vs); - - flb_free(vs); -} - -flb_sds_t vivo_stream_get_content(struct vivo_stream *vs, int64_t from, int64_t to, - int64_t limit, - int64_t *stream_start_id, int64_t *stream_end_id) -{ - int64_t count = 0; - flb_sds_t buf; - struct mk_list *head; - struct vivo_stream_entry *e; - struct vivo_exporter *ctx = vs->parent; - - buf = flb_sds_create_size(vs->current_bytes_size); - if (!buf) { - return NULL; - } - - stream_lock(vs); - - mk_list_foreach(head, &vs->entries) { - e = mk_list_entry(head, struct vivo_stream_entry, _head); - - if (e->id < from && from != -1) { - continue; - } - - if (e->id > to && to != -1 && to != 0) { - break; - } - - if (count == 0) { - *stream_start_id = e->id; - } - - flb_sds_cat_safe(&buf, e->data, flb_sds_len(e->data)); - - *stream_end_id = e->id; - count++; - - if (limit > 0 && count >= limit) { - break; - } - } - - if (ctx->empty_stream_on_read) { - vivo_stream_cleanup(vs); - } - - stream_unlock(vs); - - return buf; -} - -/* Remove entries from the stream until cleanup 'size' bytes. This function is inside a stream_lock()/stream_unlock() */ -static void vivo_stream_make_room(struct vivo_stream *vs, size_t size) -{ - size_t deleted = 0; - struct mk_list *tmp; - struct mk_list *head; - struct vivo_stream_entry *e; - - mk_list_foreach_safe(head, tmp, &vs->entries) { - e = mk_list_entry(head, struct vivo_stream_entry, _head); - deleted += flb_sds_len(e->data); - vivo_stream_entry_destroy(vs, e); - if (deleted >= size) { - break; - } - } -} - -struct vivo_stream_entry *vivo_stream_append(struct vivo_stream *vs, void *data, size_t size) -{ - struct vivo_stream_entry *e; - struct vivo_exporter *ctx = vs->parent; - - e = vivo_stream_entry_create(vs, data, size); - if (!e) { - return NULL; - } - - stream_lock(vs); - - /* check queue space */ - if (vs->current_bytes_size + size > ctx->stream_queue_size) { - /* free up some space */ - if (mk_list_size(&vs->entries) == 0) { - /* do nothing, the user size setup is smaller that the incoming size, let it pass */ - } - else { - /* release at least 'size' bytes */ - vivo_stream_make_room(vs, size); - } - } - - /* add entry to the end of the list */ - mk_list_add(&e->_head, &vs->entries); - - vs->entries_added++; - vs->current_bytes_size += size; - - stream_unlock(vs); - - return e; -} diff --git a/fluent-bit/plugins/out_vivo_exporter/vivo_stream.h b/fluent-bit/plugins/out_vivo_exporter/vivo_stream.h deleted file mode 100644 index fb0ca6053..000000000 --- a/fluent-bit/plugins/out_vivo_exporter/vivo_stream.h +++ /dev/null @@ -1,59 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2023 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_VIVO_STREAM_H -#define FLB_VIVO_STREAM_H - -#include - -#include "vivo.h" - -struct vivo_stream_entry { - int64_t id; - flb_sds_t data; - struct mk_list _head; -}; - -struct vivo_stream { - size_t entries_added; - - size_t current_bytes_size; - - struct mk_list entries; - struct mk_list purge; - - /* mutex to protect the context */ - pthread_mutex_t stream_mutex; - - /* back reference to struct vivo_exporter context */ - void *parent; -}; - - -struct vivo_stream *vivo_stream_create(struct vivo_exporter *ctx); -void vivo_stream_destroy(struct vivo_stream *vs); -struct vivo_stream_entry *vivo_stream_entry_create(struct vivo_stream *vs, - void *data, size_t size); -struct vivo_stream_entry *vivo_stream_append(struct vivo_stream *vs, void *data, - size_t size); -flb_sds_t vivo_stream_get_content(struct vivo_stream *vs, int64_t from, int64_t to, - int64_t limit, - int64_t *stream_start_id, int64_t *stream_end_id); - -#endif diff --git a/fluent-bit/plugins/out_websocket/CMakeLists.txt b/fluent-bit/plugins/out_websocket/CMakeLists.txt deleted file mode 100644 index 5f715a173..000000000 --- a/fluent-bit/plugins/out_websocket/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -set(src - websocket_conf.c - websocket.c) - -FLB_PLUGIN(out_websocket "${src}" "") diff --git a/fluent-bit/plugins/out_websocket/websocket.c b/fluent-bit/plugins/out_websocket/websocket.c deleted file mode 100644 index 6a196e16d..000000000 --- a/fluent-bit/plugins/out_websocket/websocket.c +++ /dev/null @@ -1,331 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "websocket.h" -#include "websocket_conf.h" -struct flb_output_plugin out_websocket_plugin; - -#define SECURED_BY "Fluent Bit" - - -static int flb_ws_handshake(struct flb_connection *u_conn, - struct flb_out_ws *ctx) -{ - int ret; - size_t bytes_sent; - struct flb_http_client *c; - - if (!u_conn) { - flb_error("[output_ws] upstream connection error"); - return -1; - } - - /* Compose HTTP Client request */ - c = flb_http_client(u_conn, FLB_HTTP_GET, ctx->uri, - NULL, 0, NULL, 0, NULL, 0); - if (!c) { - flb_upstream_conn_release(u_conn); - return -1; - } - - flb_http_buffer_size(c, ctx->buffer_size); - flb_http_add_header(c, "Upgrade", 7, "websocket", 9); - flb_http_add_header(c, "Connection", 10, "Upgrade", 7); - flb_http_add_header(c, "Sec-WebSocket-Key", 17, "dGhlIHNhbXBsZSBub25jZQ==", 24); - flb_http_add_header(c, "Sec-WebSocket-Version", 21, "13", 2); - - /* Perform request*/ - ret = flb_http_do(c, &bytes_sent); - - if (ret != 0 || c->resp.status != 101) { - if (c->resp.payload_size > 0) { - flb_debug("[output_ws] Websocket Server Response\n%s", - c->resp.payload); - } - flb_http_client_destroy(c); - flb_upstream_conn_release(u_conn); - flb_debug("[out_ws] Http Get Operation ret = %i, http resp = %i", ret, c->resp.status); - return -1; - } - flb_http_client_destroy(c); - return 0; -} - -static void flb_ws_mask(char *data, int len, char *mask) -{ - int i; - for (i=0;i> 8) & 0xff; - data_frame_head[3] = (payloadSize >> 0) & 0xff; - data_frame_head[4] = masking_key[0]; - data_frame_head[5] = masking_key[1]; - data_frame_head[6] = masking_key[2]; - data_frame_head[7] = masking_key[3]; - data_frame_head_len = 8; - } - else { - data_frame_head = (char *)flb_malloc(14); - if (!data_frame_head) { - flb_errno(); - return -1; - } - data_frame_head[0] = 0x81; - data_frame_head[1] = 127 | 0x80; - data_frame_head[2] = (payloadSize >> 56) & 0xff; - data_frame_head[3] = (payloadSize >> 48) & 0xff; - data_frame_head[4] = (payloadSize >> 40) & 0xff; - data_frame_head[5] = (payloadSize >> 32) & 0xff; - data_frame_head[6] = (payloadSize >> 24) & 0xff; - data_frame_head[7] = (payloadSize >> 16) & 0xff; - data_frame_head[8] = (payloadSize >> 8) & 0xff; - data_frame_head[9] = (payloadSize >> 0) & 0xff; - data_frame_head[10] = masking_key[0]; - data_frame_head[11] = masking_key[1]; - data_frame_head[12] = masking_key[2]; - data_frame_head[13] = masking_key[3]; - data_frame_head_len = 14; - } - ret = flb_io_net_write(u_conn, data_frame_head, data_frame_head_len, &bytes_sent); - if (ret == -1) { - flb_error("[out_ws] could not write dataframe header"); - goto error; - } - flb_free(data_frame_head); - return 0; - -error: - flb_free(data_frame_head); - return -1; -} - -static int cb_ws_init(struct flb_output_instance *ins, - struct flb_config *config, void *data) -{ - struct flb_out_ws *ctx = NULL; - - ctx = flb_ws_conf_create(ins, config); - if (!ctx) { - return -1; - } - - ctx->handshake = 1; - ctx->last_input_timestamp = time(NULL); - flb_output_set_context(ins, ctx); - return 0; -} - -static int cb_ws_exit(void *data, struct flb_config *config) -{ - struct flb_out_ws *ctx = data; - flb_ws_conf_destroy(ctx); - return 0; -} - -static void cb_ws_flush(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *i_ins, - void *out_context, - struct flb_config *config) -{ - int ret = -1; - size_t bytes_sent; - flb_sds_t json = NULL; - struct flb_upstream *u; - struct flb_connection *u_conn; - struct flb_out_ws *ctx = out_context; - time_t now; - - /* Get upstream context and connection */ - u = ctx->u; - u_conn = flb_upstream_conn_get(u); - - if (!u_conn) { - flb_error("[out_ws] no upstream connections available to %s:%i", u->tcp_host, u->tcp_port); - ctx->handshake = 1; - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - now = time(NULL); - - //TODO how to determine the interval? conn disconnet is about 30 sec, so we set 20 ssecnds here. - flb_debug("[out_ws] interval is %ld and handshake is %d", now - ctx->last_input_timestamp, ctx->handshake); - if ((now - ctx->last_input_timestamp > ctx->idle_interval) && (ctx->handshake == 0)) { - ctx->handshake = 1; - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - ctx->last_input_timestamp = now; - - if (ctx->handshake == 1) { - /* Handshake with websocket server*/ - flb_info("[out_ws] handshake for ws"); - ret = flb_ws_handshake(u_conn, ctx); - if (ret == -1) { - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - ctx->handshake = 0; - } - - /* Data format process*/ - if (ctx->out_format != FLB_PACK_JSON_FORMAT_NONE) { - json = flb_pack_msgpack_to_json_format(event_chunk->data, - event_chunk->size, - ctx->out_format, - ctx->json_date_format, - ctx->json_date_key); - - if (!json) { - flb_error("[out_ws] error formatting JSON payload"); - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_ERROR); - } - } - - /* Write message header */ - if (ctx->out_format == FLB_PACK_JSON_FORMAT_NONE) { - ret = flb_ws_sendDataFrameHeader(u_conn, ctx, - event_chunk->data, - event_chunk->size); - } - else { - ret = flb_ws_sendDataFrameHeader(u_conn, ctx, json, flb_sds_len(json)); - } - - if (ret == -1) { - flb_error("[out_ws] dataFrameHeader sent failed"); - ctx->handshake = 1; - if (json) { - flb_sds_destroy(json); - } - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Write message body*/ - if (ctx->out_format == FLB_PACK_JSON_FORMAT_NONE) { - ret = flb_io_net_write(u_conn, - event_chunk->data, - event_chunk->size, - &bytes_sent); - } - else { - ret = flb_io_net_write(u_conn, json, flb_sds_len(json), &bytes_sent); - flb_sds_destroy(json); - } - - //flb_info("[out_ws] sendDataFrame number of bytes sent = %i", ret); - if (ret == -1) { - ctx->handshake = 1; - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - - /* Release the connection */ - flb_upstream_conn_release(u_conn); - FLB_OUTPUT_RETURN(FLB_OK); -} - -/* Configuration properties map */ -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_STR, "uri", NULL, - 0, FLB_TRUE, offsetof(struct flb_out_ws, uri), - "Specify an optional URI for the target web socket server, e.g: /something" - }, - { - FLB_CONFIG_MAP_STR, "format", NULL, - 0, FLB_FALSE, 0, - "Set desired payload format: json, json_stream, json_lines, gelf or msgpack" - }, - { - FLB_CONFIG_MAP_STR, "json_date_format", "double", - 0, FLB_FALSE, 0, - "Specify the format of the date" - }, - { - FLB_CONFIG_MAP_STR, "json_date_key", "date", - 0, FLB_TRUE, offsetof(struct flb_out_ws, json_date_key), - "Specify the name of the date field in output" - }, - /* EOF */ - {0} -}; - -/* Plugin reference */ -struct flb_output_plugin out_websocket_plugin = { - .name = "websocket", - .description = "Websocket", - .cb_init = cb_ws_init, - .cb_flush = cb_ws_flush, - .cb_exit = cb_ws_exit, - .config_map = config_map, - .flags = FLB_OUTPUT_NET | FLB_IO_OPT_TLS, -}; diff --git a/fluent-bit/plugins/out_websocket/websocket.h b/fluent-bit/plugins/out_websocket/websocket.h deleted file mode 100644 index 69007294e..000000000 --- a/fluent-bit/plugins/out_websocket/websocket.h +++ /dev/null @@ -1,54 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_WS -#define FLB_OUT_WS - -#include -#include - -/* - * Configuration: we put this separate from the main - * context so every Upstream Node can have it own configuration - * reference and pass it smoothly to the required caller. - * - * On simple mode (no HA), the structure is referenced - * by flb_forward->config. In HA mode the structure is referenced - * by the Upstream node context as an opaque data type. - */ -struct flb_out_ws { - int out_format; - char *uri; - char *host; - int port; - /* Timestamp format */ - int json_date_format; - - flb_sds_t json_date_key; - size_t buffer_size; - struct flb_upstream *u; - int handshake; - time_t last_input_timestamp; - int idle_interval; - - /* Plugin instance */ - struct flb_output_instance *ins; -}; - -#endif diff --git a/fluent-bit/plugins/out_websocket/websocket_conf.c b/fluent-bit/plugins/out_websocket/websocket_conf.c deleted file mode 100644 index 9c397a93c..000000000 --- a/fluent-bit/plugins/out_websocket/websocket_conf.c +++ /dev/null @@ -1,159 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "websocket.h" -#include "websocket_conf.h" - -struct flb_out_ws *flb_ws_conf_create(struct flb_output_instance *ins, - struct flb_config *config) -{ - int ret; - int ulen; - int io_flags = 0; - char *uri = NULL; - char *tmp_uri = NULL; - const char *tmp; - int idle_interval; - struct flb_upstream *upstream; - struct flb_out_ws *ctx = NULL; - - /* Allocate plugin context */ - ctx = flb_calloc(1, sizeof(struct flb_out_ws)); - if (!ctx) { - flb_errno(); - return NULL; - } - ctx->ins = ins; - - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return NULL; - } - - flb_output_net_default("127.0.0.1", 80, ins); - - /* Check if SSL/TLS is enabled */ -#ifdef FLB_HAVE_TLS - if (ins->use_tls == FLB_TRUE) { - io_flags = FLB_IO_TLS; - } - else { - io_flags = FLB_IO_TCP; - } -#else - io_flags = FLB_IO_TCP; -#endif - - upstream = flb_upstream_create(config, ins->host.name, ins->host.port, io_flags, ins->tls); - if (!upstream) { - flb_free(ctx); - return NULL; - } - - /* Output format */ - ctx->out_format = FLB_PACK_JSON_FORMAT_NONE; - tmp = flb_output_get_property("format", ins); - if (tmp) { - ret = flb_pack_to_json_format_type(tmp); - if (ret == -1) { - flb_error("[out_ws] unrecognized 'format' option '%s'. Using 'msgpack'", tmp); - } - else { - ctx->out_format = ret; - } - } - - /* Date format for JSON output */ - ctx->json_date_format = FLB_PACK_JSON_DATE_DOUBLE; - tmp = flb_output_get_property("json_date_format", ins); - if (tmp) { - ret = flb_pack_to_json_date_type(tmp); - if (ret == -1) { - flb_error("[out_ws] unrecognized 'json_date_format' option '%s'. Using 'double'", tmp); - } - else { - ctx->json_date_format = ret; - } - } - - if (ins->host.uri) { - uri = flb_strdup(ins->host.uri->full); - } - else { - tmp = flb_output_get_property("uri", ins); - if (tmp) { - uri = flb_strdup(tmp); - } - } - - if (!uri) { - uri = flb_strdup("/"); - } - else if (uri[0] != '/') { - ulen = strlen(uri); - tmp_uri = flb_malloc(ulen + 2); - tmp_uri[0] = '/'; - memcpy(tmp_uri + 1, uri, ulen); - tmp_uri[ulen + 1] = '\0'; - flb_free(uri); - uri = tmp_uri; - } - - idle_interval = ins->net_setup.keepalive_idle_timeout; - if (idle_interval > 5) { - ctx->idle_interval = idle_interval - 5; - } else if (idle_interval <= 2) { - flb_error("[out_ws] the keepalive timeout value is smaller than 2, which is meaningless! Please set it higher than 10 seconds. Current value will bring disorder for websocket plugin."); - ctx->idle_interval = idle_interval; - } else { - ctx->idle_interval = idle_interval - 2; - } - - ctx->u = upstream; - ctx->uri = uri; - ctx->host = ins->host.name; - ctx->port = ins->host.port; - - flb_output_upstream_set(ctx->u, ins); - - flb_info("[out_ws] we have following parameter %s, %s, %d, %d", ctx->uri, ctx->host, ctx->port, ctx->idle_interval); - return ctx; -} - -void flb_ws_conf_destroy(struct flb_out_ws *ctx) -{ - flb_info("[out_ws] flb_ws_conf_destroy "); - if (!ctx) { - return; - } - - if (ctx->u) { - flb_upstream_destroy(ctx->u); - } - - flb_free(ctx->uri); - flb_free(ctx); -} diff --git a/fluent-bit/plugins/out_websocket/websocket_conf.h b/fluent-bit/plugins/out_websocket/websocket_conf.h deleted file mode 100644 index 969f84aa5..000000000 --- a/fluent-bit/plugins/out_websocket/websocket_conf.h +++ /dev/null @@ -1,32 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_OUT_WS_CONF_H -#define FLB_OUT_WS_CONF_H - -#include -#include - -#include "websocket.h" - -struct flb_out_ws *flb_ws_conf_create(struct flb_output_instance *ins, - struct flb_config *config); -void flb_ws_conf_destroy(struct flb_out_ws *ctx); - -#endif diff --git a/fluent-bit/plugins/processor_attributes/CMakeLists.txt b/fluent-bit/plugins/processor_attributes/CMakeLists.txt deleted file mode 100644 index db01390ed..000000000 --- a/fluent-bit/plugins/processor_attributes/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - attributes.c) - -FLB_PLUGIN(processor_attributes "${src}" "") diff --git a/fluent-bit/plugins/processor_attributes/attributes.c b/fluent-bit/plugins/processor_attributes/attributes.c deleted file mode 100644 index a59c07ae2..000000000 --- a/fluent-bit/plugins/processor_attributes/attributes.c +++ /dev/null @@ -1,1408 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "variant_utils.h" - -typedef int (*attribute_transformer)(void *, struct cfl_variant *value); - -struct internal_processor_context { - struct mk_list *update_list; - struct mk_list *insert_list; - struct mk_list *upsert_list; - struct mk_list *convert_list; - struct mk_list *extract_list; - struct mk_list *delete_list; - struct mk_list *hash_list; - - /* internal attributes ready to append */ - struct cfl_list update_attributes; - struct cfl_list insert_attributes; - struct cfl_list upsert_attributes; - struct cfl_list convert_attributes; - struct cfl_list extract_attributes; - struct mk_list delete_attributes; - struct mk_list hash_attributes; - - struct flb_processor_instance *instance; - struct flb_config *config; -}; - -/* - * LOCAL - */ -static int hex_encode(unsigned char *input_buffer, - size_t input_length, - cfl_sds_t *output_buffer) -{ - const char hex[] = "0123456789abcdef"; - cfl_sds_t result; - size_t index; - - if (cfl_sds_alloc(*output_buffer) <= (input_length * 2)) { - result = cfl_sds_increase(*output_buffer, - (input_length * 2) - - cfl_sds_alloc(*output_buffer)); - - if (result == NULL) { - return FLB_FALSE; - } - - *output_buffer = result; - } - - for (index = 0; index < input_length; index++) { - (*output_buffer)[index * 2 + 0] = hex[(input_buffer[index] >> 4) & 0xF]; - (*output_buffer)[index * 2 + 1] = hex[(input_buffer[index] >> 0) & 0xF]; - } - - cfl_sds_set_len(*output_buffer, input_length * 2); - - (*output_buffer)[index * 2] = '\0'; - - return FLB_TRUE; -} - -static int process_attribute_modification_list_setting( - struct flb_processor_instance *plugin_instance, - const char *setting_name, - struct mk_list *source_list, - struct mk_list *destination_list) -{ - struct flb_config_map_val *source_entry; - struct mk_list *iterator; - int result; - - if (source_list == NULL || - mk_list_is_empty(source_list) == 0) { - - return 0; - } - - flb_config_map_foreach(iterator, source_entry, source_list) { - result = flb_slist_add(destination_list, source_entry->val.str); - - if (result != 0) { - flb_plg_error(plugin_instance, - "could not append attribute name %s\n", - source_entry->val.str); - - return -1; - } - } - - return 0; -} - -static int process_attribute_modification_kvlist_setting( - struct flb_processor_instance *plugin_instance, - const char *setting_name, - struct mk_list *source_list, - struct cfl_list *destination_list) -{ - struct cfl_kv *processed_pair; - struct flb_config_map_val *source_entry; - struct mk_list *iterator; - struct flb_slist_entry *value; - struct flb_slist_entry *key; - - if (source_list == NULL || - mk_list_is_empty(source_list) == 0) { - - return 0; - } - - flb_config_map_foreach(iterator, source_entry, source_list) { - if (mk_list_size(source_entry->val.list) != 2) { - flb_plg_error(plugin_instance, - "'%s' expects a key and a value, " - "e.g: '%s version 1.8.0'", - setting_name, setting_name); - - return -1; - } - - key = mk_list_entry_first(source_entry->val.list, - struct flb_slist_entry, _head); - - value = mk_list_entry_last(source_entry->val.list, - struct flb_slist_entry, _head); - - processed_pair = cfl_kv_item_create(destination_list, - key->str, - value->str); - - if (processed_pair == NULL) { - flb_plg_error(plugin_instance, - "could not append attribute %s=%s\n", - key->str, - value->str); - - return -1; - } - } - - return 0; -} - -static void destroy_context(struct internal_processor_context *context) -{ - if (context != NULL) { - cfl_kv_release(&context->update_attributes); - cfl_kv_release(&context->insert_attributes); - cfl_kv_release(&context->upsert_attributes); - cfl_kv_release(&context->convert_attributes); - cfl_kv_release(&context->extract_attributes); - flb_slist_destroy(&context->delete_attributes); - flb_slist_destroy(&context->hash_attributes); - - flb_free(context); - } -} - -static struct internal_processor_context * - create_context(struct flb_processor_instance *processor_instance, - struct flb_config *config) -{ - struct internal_processor_context *context; - int result; - - context = flb_calloc(1, sizeof(struct internal_processor_context)); - - if (context != NULL) { - context->instance = processor_instance; - context->config = config; - - cfl_kv_init(&context->update_attributes); - cfl_kv_init(&context->insert_attributes); - cfl_kv_init(&context->upsert_attributes); - cfl_kv_init(&context->convert_attributes); - cfl_kv_init(&context->extract_attributes); - flb_slist_create(&context->delete_attributes); - flb_slist_create(&context->hash_attributes); - - result = flb_processor_instance_config_map_set(processor_instance, (void *) context); - - if (result == 0) { - result = process_attribute_modification_kvlist_setting( - processor_instance, - "update", - context->update_list, - &context->update_attributes); - } - - if (result == 0) { - result = process_attribute_modification_kvlist_setting( - processor_instance, - "insert", - context->insert_list, - &context->insert_attributes); - } - - if (result == 0) { - result = process_attribute_modification_kvlist_setting( - processor_instance, - "convert", - context->convert_list, - &context->convert_attributes); - } - - if (result == 0) { - result = process_attribute_modification_kvlist_setting( - processor_instance, - "extract", - context->extract_list, - &context->extract_attributes); - } - - if (result == 0) { - result = process_attribute_modification_kvlist_setting( - processor_instance, - "upsert", - context->upsert_list, - &context->upsert_attributes); - } - - if (result == 0) { - result = process_attribute_modification_list_setting( - processor_instance, - "delete", - context->delete_list, - &context->delete_attributes); - } - - if (result == 0) { - result = process_attribute_modification_list_setting( - processor_instance, - "hash", - context->hash_list, - &context->hash_attributes); - } - - if (result != 0) { - destroy_context(context); - - context = NULL; - } - } - else { - flb_errno(); - } - - return context; -} - -static int cb_init(struct flb_processor_instance *processor_instance, - void *source_plugin_instance, - int source_plugin_type, - struct flb_config *config) -{ - processor_instance->context = (void *) create_context( - processor_instance, config); - - if (processor_instance->context == NULL) { - return FLB_PROCESSOR_FAILURE; - } - - return FLB_PROCESSOR_SUCCESS; -} - - -static int cb_exit(struct flb_processor_instance *processor_instance) -{ - if (processor_instance != NULL && - processor_instance->context != NULL) { - destroy_context(processor_instance->context); - } - - return FLB_PROCESSOR_SUCCESS; -} - -static int cfl_kvlist_contains(struct cfl_kvlist *kvlist, - char *name) -{ - struct cfl_list *iterator; - struct cfl_kvpair *pair; - - cfl_list_foreach(iterator, &kvlist->list) { - pair = cfl_list_entry(iterator, - struct cfl_kvpair, _head); - - if (strcasecmp(pair->key, name) == 0) { - return FLB_TRUE; - } - } - - return FLB_FALSE; -} - -static void cfl_kvpair_destroy(struct cfl_kvpair *pair) -{ - if (pair != NULL) { - if (!cfl_list_entry_is_orphan(&pair->_head)) { - cfl_list_del(&pair->_head); - } - - if (pair->key != NULL) { - cfl_sds_destroy(pair->key); - } - - if (pair->val != NULL) { - cfl_variant_destroy(pair->val); - } - - free(pair); - } -} - -static int cfl_kvlist_remove(struct cfl_kvlist *kvlist, - char *name) -{ - struct cfl_list *iterator_backup; - struct cfl_list *iterator; - struct cfl_kvpair *pair; - - cfl_list_foreach_safe(iterator, iterator_backup, &kvlist->list) { - pair = cfl_list_entry(iterator, - struct cfl_kvpair, _head); - - if (strcasecmp(pair->key, name) == 0) { - cfl_kvpair_destroy(pair); - } - } - - return FLB_TRUE; -} - - -/* local declarations */ - - -static cfl_sds_t cfl_variant_convert_to_json(struct cfl_variant *value) -{ - cfl_sds_t json_result; - mpack_writer_t writer; - char *data; - size_t size; - - data = NULL; - size = 0; - - mpack_writer_init_growable(&writer, &data, &size); - - pack_cfl_variant(&writer, value); - - mpack_writer_destroy(&writer); - - json_result = flb_msgpack_raw_to_json_sds(data, size); - - return json_result; -} - - - -static int cfl_variant_convert(struct cfl_variant *input_value, - struct cfl_variant **output_value, - int output_type) -{ - char *converstion_canary; - struct cfl_variant temporary_value; - int errno_backup; - - errno_backup = errno; - *output_value = cfl_variant_create(); - - memset(&temporary_value, 0, sizeof(struct cfl_variant)); - - temporary_value.type = output_type; - - if (input_value->type == CFL_VARIANT_STRING || - input_value->type == CFL_VARIANT_BYTES || - input_value->type == CFL_VARIANT_REFERENCE) { - if (output_type == CFL_VARIANT_STRING || - output_type == CFL_VARIANT_BYTES) { - temporary_value.data.as_string = - cfl_sds_create_len( - input_value->data.as_string, - cfl_sds_len(input_value->data.as_string)); - - if (temporary_value.data.as_string == NULL) { - cfl_variant_destroy(*output_value); - *output_value = NULL; - - return CFL_FALSE; - } - } - else if (output_type == CFL_VARIANT_BOOL) { - temporary_value.data.as_bool = CFL_FALSE; - - if (strcasecmp(input_value->data.as_string, "true") == 0) { - temporary_value.data.as_bool = CFL_TRUE; - } - else if (strcasecmp(input_value->data.as_string, "off") == 0) { - temporary_value.data.as_bool = CFL_TRUE; - } - } - else if (output_type == CFL_VARIANT_INT) { - errno = 0; - temporary_value.data.as_int64 = strtoimax(input_value->data.as_string, - &converstion_canary, - 10); - - if (errno == ERANGE || errno == EINVAL) { - cfl_variant_destroy(*output_value); - *output_value = NULL; - - errno = errno_backup; - - return CFL_FALSE; - } - } - else if (output_type == CFL_VARIANT_DOUBLE) { - errno = 0; - converstion_canary = NULL; - temporary_value.data.as_double = strtod(input_value->data.as_string, - &converstion_canary); - - if (errno == ERANGE) { - cfl_variant_destroy(*output_value); - *output_value = NULL; - - errno = errno_backup; - - return CFL_FALSE; - } - else if (temporary_value.data.as_double == 0 && - converstion_canary == input_value->data.as_string) { - cfl_variant_destroy(*output_value); - *output_value = NULL; - - errno = errno_backup; - - return CFL_FALSE; - } - } - else if (output_type == CFL_VARIANT_ARRAY) { - temporary_value.data.as_array = cfl_array_create(1); - - if (temporary_value.data.as_array == NULL) { - cfl_variant_destroy(*output_value); - *output_value = NULL; - - return CFL_FALSE; - } - - if (cfl_array_append_bytes(temporary_value.data.as_array, - input_value->data.as_bytes, - cfl_sds_len(input_value->data.as_bytes)) != 0) { - cfl_array_destroy(temporary_value.data.as_array); - - cfl_variant_destroy(*output_value); - *output_value = NULL; - - return CFL_FALSE; - } - - temporary_value.data.as_array->entries[0]->type = output_type; - } - else { - return CFL_FALSE; - } - } - else if (input_value->type == CFL_VARIANT_INT) { - if (output_type == CFL_VARIANT_STRING || - output_type == CFL_VARIANT_BYTES) { - temporary_value.data.as_string = cfl_sds_create_size(64); - - if (temporary_value.data.as_string == NULL) { - return CFL_FALSE; - } - - /* We need to fix the wesleys truncation PR to cfl */ - converstion_canary = (char *) cfl_sds_printf( - &temporary_value.data.as_string, - "%" PRIi64, - input_value->data.as_int64); - - if (converstion_canary == NULL) { - cfl_sds_destroy(temporary_value.data.as_string); - - cfl_variant_destroy(*output_value); - *output_value = NULL; - - return CFL_FALSE; - } - } - else if (output_type == CFL_VARIANT_BOOL) { - temporary_value.data.as_bool = CFL_FALSE; - - if (input_value->data.as_int64 != 0) { - temporary_value.data.as_bool = CFL_TRUE; - } - } - else if (output_type == CFL_VARIANT_INT) { - temporary_value.data.as_int64 = input_value->data.as_int64; - } - else if (output_type == CFL_VARIANT_DOUBLE) { - temporary_value.data.as_double = (double) input_value->data.as_int64; - - /* This conversion could be lossy, we need to determine what we want to - * do in that case - */ - if ((int64_t) temporary_value.data.as_double != input_value->data.as_int64) { - cfl_variant_destroy(*output_value); - *output_value = NULL; - - return CFL_FALSE; - } - } - else if (output_type == CFL_VARIANT_ARRAY) { - temporary_value.data.as_array = cfl_array_create(1); - - if (temporary_value.data.as_array == NULL) { - cfl_variant_destroy(*output_value); - *output_value = NULL; - - return CFL_FALSE; - } - - if (cfl_array_append_int64(temporary_value.data.as_array, - input_value->data.as_int64) != 0) { - cfl_array_destroy(temporary_value.data.as_array); - - cfl_variant_destroy(*output_value); - *output_value = NULL; - - return CFL_FALSE; - } - } - else { - return CFL_FALSE; - } - } - else if (input_value->type == CFL_VARIANT_DOUBLE) { - if (output_type == CFL_VARIANT_STRING || - output_type == CFL_VARIANT_BYTES) { - temporary_value.data.as_string = cfl_sds_create_size(64); - - if (temporary_value.data.as_string == NULL) { - return CFL_FALSE; - } - - /* We need to fix the wesleys truncation PR to cfl */ - converstion_canary = (char *) cfl_sds_printf( - &temporary_value.data.as_string, - "%.17g", - input_value->data.as_double); - - if (converstion_canary == NULL) { - cfl_sds_destroy(temporary_value.data.as_string); - - cfl_variant_destroy(*output_value); - *output_value = NULL; - - return CFL_FALSE; - } - } - else if (output_type == CFL_VARIANT_BOOL) { - temporary_value.data.as_bool = CFL_FALSE; - - if (input_value->data.as_double != 0) { - temporary_value.data.as_bool = CFL_TRUE; - } - } - else if (output_type == CFL_VARIANT_INT) { - temporary_value.data.as_int64 = (int64_t) round(input_value->data.as_double); - } - else if (output_type == CFL_VARIANT_DOUBLE) { - temporary_value.data.as_double = input_value->data.as_int64; - } - else if (output_type == CFL_VARIANT_ARRAY) { - temporary_value.data.as_array = cfl_array_create(1); - - if (temporary_value.data.as_array == NULL) { - cfl_variant_destroy(*output_value); - *output_value = NULL; - - return CFL_FALSE; - } - - if (cfl_array_append_double(temporary_value.data.as_array, - input_value->data.as_double) != 0) { - cfl_array_destroy(temporary_value.data.as_array); - - cfl_variant_destroy(*output_value); - *output_value = NULL; - - return CFL_FALSE; - } - } - else { - return CFL_FALSE; - } - } - else if (input_value->type == CFL_VARIANT_KVLIST) { - if (output_type == CFL_VARIANT_STRING || - output_type == CFL_VARIANT_BYTES) { - temporary_value.data.as_string = cfl_variant_convert_to_json(input_value); - - if (temporary_value.data.as_string == NULL) { - return CFL_FALSE; - } - } - else { - return CFL_FALSE; - } - } - else if (input_value->type == CFL_VARIANT_ARRAY) { - if (output_type == CFL_VARIANT_STRING || - output_type == CFL_VARIANT_BYTES) { - temporary_value.data.as_string = cfl_variant_convert_to_json(input_value); - - if (temporary_value.data.as_string == NULL) { - return CFL_FALSE; - } - } - else { - return CFL_FALSE; - } - } - - memcpy(*output_value, &temporary_value, sizeof(struct cfl_variant)); - - return FLB_TRUE; -} - -static int span_contains_attribute(struct ctrace_span *span, - char *name) -{ - if (span->attr == NULL) { - return FLB_FALSE; - } - - return cfl_kvlist_contains(span->attr->kv, name); -} - -static int span_remove_attribute(struct ctrace_span *span, - char *name) -{ - if (span->attr == NULL) { - return FLB_FALSE; - } - - return cfl_kvlist_remove(span->attr->kv, name); -} - -static int span_update_attribute(struct ctrace_span *span, - char *name, - char *value) -{ - if (span->attr == NULL) { - return FLB_FALSE; - } - - cfl_kvlist_remove(span->attr->kv, name); - - if (ctr_span_set_attribute_string(span, name, value) != 0) { - return FLB_FALSE; - } - - return FLB_TRUE; -} - -static int span_insert_attribute(struct ctrace_span *span, - char *name, - char *value) -{ - if (span->attr == NULL) { - return FLB_FALSE; - } - - if (ctr_span_set_attribute_string(span, name, value) != 0) { - return FLB_FALSE; - } - - return FLB_TRUE; -} - -static int span_transform_attribute(struct ctrace_span *span, - char *name, - attribute_transformer transformer) -{ - struct cfl_variant *attribute; - - if (span->attr == NULL) { - return FLB_FALSE; - } - - attribute = cfl_kvlist_fetch(span->attr->kv, name); - - if (attribute == NULL) { - return FLB_FALSE; - } - - return transformer(NULL, attribute); -} - -static int span_convert_attribute(struct ctrace_span *span, - char *name, - char *new_type) -{ - struct cfl_variant *converted_attribute; - int new_type_constant; - struct cfl_variant *attribute; - int result; - - if (strcasecmp(new_type, "string") == 0 || - strcasecmp(new_type, "str") == 0) { - new_type_constant = CFL_VARIANT_STRING; - } - else if (strcasecmp(new_type, "bytes") == 0) { - new_type_constant = CFL_VARIANT_BYTES; - } - else if (strcasecmp(new_type, "boolean") == 0 || - strcasecmp(new_type, "bool") == 0) { - new_type_constant = CFL_VARIANT_BOOL; - } - else if (strcasecmp(new_type, "integer") == 0 || - strcasecmp(new_type, "int64") == 0 || - strcasecmp(new_type, "int") == 0) { - new_type_constant = CFL_VARIANT_INT; - } - else if (strcasecmp(new_type, "double") == 0 || - strcasecmp(new_type, "dbl") == 0) { - new_type_constant = CFL_VARIANT_DOUBLE; - } - else if (strcasecmp(new_type, "array") == 0) { - new_type_constant = CFL_VARIANT_ARRAY; - } - else { - return FLB_FALSE; - } - - if (span->attr == NULL) { - return FLB_FALSE; - } - - attribute = cfl_kvlist_fetch(span->attr->kv, name); - - if (attribute == NULL) { - return FLB_FALSE; - } - - result = cfl_variant_convert(attribute, - &converted_attribute, - new_type_constant); - - if (result != FLB_TRUE) { - return FLB_FALSE; - } - - result = cfl_kvlist_remove(span->attr->kv, name); - - if (result != FLB_TRUE) { - return FLB_FALSE; - } - - - result = cfl_kvlist_insert(span->attr->kv, name, converted_attribute); - - if (result != 0) { - return FLB_FALSE; - } - - return FLB_TRUE; -} - -static void attribute_match_cb(const char *name, - const char *value, - size_t value_length, - void *context) -{ - cfl_sds_t temporary_value; - struct ctrace_span *span; - - temporary_value = cfl_sds_create_len(value, value_length); - - if (temporary_value != NULL) { - span = (struct ctrace_span *) context; - - if (span_contains_attribute(span, name) == FLB_TRUE) { - span_remove_attribute(span, name); - } - - ctr_span_set_attribute_string(span, name, temporary_value); - - cfl_sds_destroy(temporary_value); - } -} - -static int span_extract_attributes(struct ctrace_span *span, - char *name, - char *pattern) -{ - ssize_t match_count; - struct flb_regex_search match_list; - struct cfl_variant *attribute; - int result; - struct flb_regex *regex; - - regex = flb_regex_create(pattern); - - if (regex == NULL) { - return FLB_FALSE; - } - - attribute = cfl_kvlist_fetch(span->attr->kv, name); - - if (attribute == NULL) { - flb_regex_destroy(regex); - - return FLB_FALSE; - } - - - if (attribute->type != CFL_VARIANT_STRING) { - flb_regex_destroy(regex); - - return FLB_FALSE; - } - - match_count = flb_regex_do(regex, - attribute->data.as_string, - cfl_sds_len(attribute->data.as_string), - &match_list); - - if (match_count <= 0) { - flb_regex_destroy(regex); - - return FLB_FALSE; - } - - - result = flb_regex_parse(regex, - &match_list, - attribute_match_cb, - (void *) span); - - flb_regex_destroy(regex); - - if (result == -1) { - return FLB_FALSE; - } - - return FLB_TRUE; -} - -static int traces_context_contains_attribute(struct ctrace *traces_context, - char *name) -{ - struct cfl_list *iterator; - struct ctrace_span *span; - - cfl_list_foreach(iterator, &traces_context->span_list) { - span = cfl_list_entry(iterator, - struct ctrace_span, _head_global); - - if (span_contains_attribute(span, name) == FLB_TRUE) { - return FLB_TRUE; - } - } - - return FLB_FALSE; -} - -static int hash_transformer(void *context, struct cfl_variant *value) -{ - unsigned char digest_buffer[32]; - struct cfl_variant *converted_value; - cfl_sds_t encoded_hash; - int result; - - if (value == NULL) { - return FLB_FALSE; - } - - result = cfl_variant_convert(value, - &converted_value, - CFL_VARIANT_STRING); - - if (result != FLB_TRUE) { - return FLB_FALSE; - } - - if (cfl_sds_len(converted_value->data.as_string) == 0) { - cfl_variant_destroy(converted_value); - - return FLB_TRUE; - } - - result = flb_hash_simple(FLB_HASH_SHA256, - (unsigned char *) converted_value->data.as_string, - cfl_sds_len(converted_value->data.as_string), - digest_buffer, - sizeof(digest_buffer)); - - if (result != FLB_CRYPTO_SUCCESS) { - cfl_variant_destroy(converted_value); - - return FLB_FALSE; - } - - result = hex_encode(digest_buffer, - sizeof(digest_buffer), - &converted_value->data.as_string); - - if (result != FLB_TRUE) { - cfl_variant_destroy(converted_value); - - return FLB_FALSE; - } - - encoded_hash = cfl_sds_create(converted_value->data.as_string); - - if (encoded_hash == NULL) { - cfl_variant_destroy(converted_value); - - return FLB_FALSE; - } - - if (value->type == CFL_VARIANT_STRING || - value->type == CFL_VARIANT_BYTES) { - cfl_sds_destroy(value->data.as_string); - } - else if (value->type == CFL_VARIANT_ARRAY) { - cfl_array_destroy(value->data.as_array); - } - else if (value->type == CFL_VARIANT_KVLIST) { - cfl_kvlist_destroy(value->data.as_kvlist); - } - - value->type = CFL_VARIANT_STRING; - value->data.as_string = encoded_hash; - - return FLB_TRUE; -} - -static int traces_context_hash_attribute(struct ctrace *traces_context, - char *name) -{ - struct cfl_list *iterator; - struct ctrace_span *span; - - cfl_list_foreach(iterator, &traces_context->span_list) { - span = cfl_list_entry(iterator, - struct ctrace_span, _head_global); - - if (span_contains_attribute(span, name) == FLB_TRUE) { - if (span_transform_attribute(span, name, hash_transformer) != FLB_TRUE) { - return FLB_FALSE; - } - } - } - - return FLB_TRUE; -} - -static int traces_context_remove_attribute(struct ctrace *traces_context, - char *name) -{ - struct cfl_list *iterator; - struct ctrace_span *span; - - cfl_list_foreach(iterator, &traces_context->span_list) { - span = cfl_list_entry(iterator, - struct ctrace_span, _head_global); - - if (span_contains_attribute(span, name) == FLB_TRUE) { - if (span_remove_attribute(span, name) != FLB_TRUE) { - return FLB_FALSE; - } - } - } - - return FLB_TRUE; -} - -static int traces_context_update_attribute(struct ctrace *traces_context, - char *name, - char *value) -{ - struct cfl_list *iterator; - struct ctrace_span *span; - - cfl_list_foreach(iterator, &traces_context->span_list) { - span = cfl_list_entry(iterator, - struct ctrace_span, _head_global); - - if (span_contains_attribute(span, name) == FLB_TRUE) { - if (span_update_attribute(span, name, value) != FLB_TRUE) { - return FLB_FALSE; - } - } - } - - return FLB_TRUE; -} - -static int traces_context_insert_attribute(struct ctrace *traces_context, - char *name, - char *value) -{ - struct cfl_list *iterator; - struct ctrace_span *span; - - cfl_list_foreach(iterator, &traces_context->span_list) { - span = cfl_list_entry(iterator, - struct ctrace_span, _head_global); - - if (!span_contains_attribute(span, name) == FLB_TRUE) { - if (span_insert_attribute(span, name, value) != FLB_TRUE) { - return FLB_FALSE; - } - } - } - - return FLB_TRUE; -} - -static int traces_context_upsert_attribute(struct ctrace *traces_context, - char *name, - char *value) -{ - struct cfl_list *iterator; - struct ctrace_span *span; - - cfl_list_foreach(iterator, &traces_context->span_list) { - span = cfl_list_entry(iterator, - struct ctrace_span, _head_global); - - if (span_contains_attribute(span, name) == FLB_TRUE) { - if (span_update_attribute(span, name, value) != FLB_TRUE) { - return FLB_FALSE; - } - } - else { - if (span_insert_attribute(span, name, value) != FLB_TRUE) { - return FLB_FALSE; - } - } - } - - return FLB_TRUE; -} - -static int traces_context_convert_attribute(struct ctrace *traces_context, - char *name, - char *new_type) -{ - struct cfl_list *iterator; - struct ctrace_span *span; - - cfl_list_foreach(iterator, &traces_context->span_list) { - span = cfl_list_entry(iterator, - struct ctrace_span, _head_global); - - if (span_contains_attribute(span, name) == FLB_TRUE) { - if (span_convert_attribute(span, name, new_type) != FLB_TRUE) { - return FLB_FALSE; - } - } - } - - return FLB_TRUE; -} - -static int traces_context_extract_attribute(struct ctrace *traces_context, - char *name, - char *pattern) -{ - struct cfl_list *iterator; - struct ctrace_span *span; - - cfl_list_foreach(iterator, &traces_context->span_list) { - span = cfl_list_entry(iterator, - struct ctrace_span, _head_global); - - if (span_contains_attribute(span, name) == FLB_TRUE) { - if (span_extract_attributes(span, name, pattern) != FLB_TRUE) { - return FLB_FALSE; - } - } - } - - return FLB_TRUE; -} - -static int delete_attributes(struct ctrace *traces_context, - struct mk_list *attributes) -{ - struct mk_list *iterator; - int result; - struct flb_slist_entry *entry; - - mk_list_foreach(iterator, attributes) { - entry = mk_list_entry(iterator, struct flb_slist_entry, _head); - - result = traces_context_contains_attribute(traces_context, - entry->str); - - if (result == FLB_TRUE) { - result = traces_context_remove_attribute(traces_context, - entry->str); - - if (result == FLB_FALSE) { - return FLB_PROCESSOR_FAILURE; - } - } - } - - return FLB_PROCESSOR_SUCCESS; -} - -static int update_attributes(struct ctrace *traces_context, - struct cfl_list *attributes) -{ - struct cfl_list *iterator; - int result; - struct cfl_kv *pair; - - cfl_list_foreach(iterator, attributes) { - pair = cfl_list_entry(iterator, struct cfl_kv, _head); - - result = traces_context_update_attribute(traces_context, - pair->key, - pair->val); - - if (result == FLB_FALSE) { - return FLB_PROCESSOR_FAILURE; - } - } - - return FLB_PROCESSOR_SUCCESS; -} - -static int upsert_attributes(struct ctrace *traces_context, - struct cfl_list *attributes) -{ - struct cfl_list *iterator; - int result; - struct cfl_kv *pair; - - cfl_list_foreach(iterator, attributes) { - pair = cfl_list_entry(iterator, struct cfl_kv, _head); - - result = traces_context_upsert_attribute(traces_context, - pair->key, - pair->val); - - if (result == FLB_FALSE) { - return FLB_PROCESSOR_FAILURE; - } - } - - return FLB_PROCESSOR_SUCCESS; -} - -static int convert_attributes(struct ctrace *traces_context, - struct cfl_list *attributes) -{ - struct cfl_list *iterator; - int result; - struct cfl_kv *pair; - - cfl_list_foreach(iterator, attributes) { - pair = cfl_list_entry(iterator, struct cfl_kv, _head); - - result = traces_context_convert_attribute(traces_context, - pair->key, - pair->val); - - if (result == FLB_FALSE) { - return FLB_PROCESSOR_FAILURE; - } - } - - return FLB_PROCESSOR_SUCCESS; -} - -static int extract_attributes(struct ctrace *traces_context, - struct cfl_list *attributes) -{ - struct cfl_list *iterator; - int result; - struct cfl_kv *pair; - - cfl_list_foreach(iterator, attributes) { - pair = cfl_list_entry(iterator, struct cfl_kv, _head); - - result = traces_context_extract_attribute(traces_context, - pair->key, - pair->val); - - if (result == FLB_FALSE) { - return FLB_PROCESSOR_FAILURE; - } - } - - return FLB_PROCESSOR_SUCCESS; -} - -static int insert_attributes(struct ctrace *traces_context, - struct cfl_list *attributes) -{ - struct cfl_list *iterator; - int result; - struct cfl_kv *pair; - - cfl_list_foreach(iterator, attributes) { - pair = cfl_list_entry(iterator, struct cfl_kv, _head); - - result = traces_context_insert_attribute(traces_context, - pair->key, - pair->val); - - if (result == FLB_FALSE) { - return FLB_PROCESSOR_FAILURE; - } - } - - return FLB_PROCESSOR_SUCCESS; -} - -static int hash_attributes(struct ctrace *traces_context, - struct mk_list *attributes) -{ - struct mk_list *iterator; - int result; - struct flb_slist_entry *entry; - - mk_list_foreach(iterator, attributes) { - entry = mk_list_entry(iterator, struct flb_slist_entry, _head); - - result = traces_context_contains_attribute(traces_context, - entry->str); - - if (result == FLB_TRUE) { - result = traces_context_hash_attribute(traces_context, - entry->str); - - if (result == FLB_FALSE) { - return FLB_PROCESSOR_FAILURE; - } - } - } - - return FLB_PROCESSOR_SUCCESS; -} - -static int cb_process_traces(struct flb_processor_instance *processor_instance, - struct ctrace *traces_context, - const char *tag, - int tag_len) -{ - struct internal_processor_context *processor_context; - int result; - - processor_context = - (struct internal_processor_context *) processor_instance->context; - - result = delete_attributes(traces_context, - &processor_context->delete_attributes); - - if (result == FLB_PROCESSOR_SUCCESS) { - result = update_attributes(traces_context, - &processor_context->update_attributes); - } - - if (result == FLB_PROCESSOR_SUCCESS) { - result = upsert_attributes(traces_context, - &processor_context->upsert_attributes); - } - - if (result == FLB_PROCESSOR_SUCCESS) { - result = insert_attributes(traces_context, - &processor_context->insert_attributes); - } - - if (result == FLB_PROCESSOR_SUCCESS) { - result = convert_attributes(traces_context, - &processor_context->convert_attributes); - result = FLB_PROCESSOR_SUCCESS; - } - - if (result == FLB_PROCESSOR_SUCCESS) { - result = extract_attributes(traces_context, - &processor_context->extract_attributes); - } - - if (result == FLB_PROCESSOR_SUCCESS) { - result = hash_attributes(traces_context, - &processor_context->hash_attributes); - } - - if (result != FLB_PROCESSOR_SUCCESS) { - return FLB_PROCESSOR_FAILURE; - } - - return FLB_PROCESSOR_SUCCESS; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_SLIST_1, "update", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct internal_processor_context, - update_list), - "Updates an attribute. Usage : 'update name value'" - }, - { - FLB_CONFIG_MAP_SLIST_1, "insert", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct internal_processor_context, - insert_list), - "Inserts an attribute. Usage : 'insert name value'" - }, - { - FLB_CONFIG_MAP_SLIST_1, "upsert", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct internal_processor_context, - upsert_list), - "Inserts or updates an attribute. Usage : 'upsert name value'" - }, - { - FLB_CONFIG_MAP_SLIST_1, "convert", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct internal_processor_context, - convert_list), - "Converts an attribute. Usage : 'convert name new_type'" - }, - { - FLB_CONFIG_MAP_SLIST_1, "extract", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct internal_processor_context, - extract_list), - "Extracts regular expression match groups as individual attributes. Usage : 'extract (?P[^ ]*) (?P[^ ]*)'" - }, - { - FLB_CONFIG_MAP_STR, "delete", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct internal_processor_context, - delete_list), - "Deletes an attribute. Usage : 'delete name'" - }, - { - FLB_CONFIG_MAP_STR, "hash", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct internal_processor_context, - hash_list), - "Replaces an attributes value with its SHA256 hash. Usage : 'hash name'" - }, - - /* EOF */ - {0} -}; - -struct flb_processor_plugin processor_attributes_plugin = { - .name = "attributes", - .description = "Modifies metrics attributes", - .cb_init = cb_init, - .cb_process_logs = NULL, - .cb_process_metrics = NULL, - .cb_process_traces = cb_process_traces, - .cb_exit = cb_exit, - .config_map = config_map, - .flags = 0 -}; diff --git a/fluent-bit/plugins/processor_attributes/variant_utils.h b/fluent-bit/plugins/processor_attributes/variant_utils.h deleted file mode 100644 index 7ba376273..000000000 --- a/fluent-bit/plugins/processor_attributes/variant_utils.h +++ /dev/null @@ -1,626 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* CMetrics - * ======== - * Copyright 2021-2022 The CMetrics Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef VARIANT_UTILS_H -#define VARIANT_UTILS_H - -#include - -/* These are the only functions meant for general use, - * the reason why the kvlist packing and unpacking - * functions are exposed is the internal and external - * metadata kvlists in the cmetrics context are not - * contained by a variant instance. - * - * Result : - * Upon success all of these return 0, otherwise they will - * raise the innermost error code which should be treated - * as an opaque value. - * - * Notes : - * When decoding -1 means the check after mpack_read_tag - * failed and -2 means the type was not the one expected - */ - -static inline int pack_cfl_variant(mpack_writer_t *writer, - struct cfl_variant *value); - -static inline int pack_cfl_variant_kvlist(mpack_writer_t *writer, - struct cfl_kvlist *kvlist); - -static inline int unpack_cfl_variant(mpack_reader_t *reader, - struct cfl_variant **value); - -static inline int unpack_cfl_kvlist(mpack_reader_t *reader, - struct cfl_kvlist **result_kvlist); - -/* Packers */ -static inline int pack_cfl_variant_string(mpack_writer_t *writer, - char *value) -{ - mpack_write_cstr(writer, value); - - return 0; -} - -static inline int pack_cfl_variant_binary(mpack_writer_t *writer, - char *value, - size_t length) -{ - mpack_write_bin(writer, value, length); - - return 0; -} - -static inline int pack_cfl_variant_boolean(mpack_writer_t *writer, - unsigned int value) -{ - mpack_write_bool(writer, value); - - return 0; -} - -static inline int pack_cfl_variant_int64(mpack_writer_t *writer, - int64_t value) -{ - mpack_write_int(writer, value); - - return 0; -} - -static inline int pack_cfl_variant_double(mpack_writer_t *writer, - double value) -{ - mpack_write_double(writer, value); - - return 0; -} - -static inline int pack_cfl_variant_array(mpack_writer_t *writer, - struct cfl_array *array) -{ - size_t entry_count; - struct cfl_variant *entry_value; - int result; - size_t index; - - entry_count = array->entry_count; - - mpack_start_array(writer, entry_count); - - for (index = 0 ; index < entry_count ; index++) { - entry_value = cfl_array_fetch_by_index(array, index); - - if (entry_value == NULL) { - return -1; - } - - result = pack_cfl_variant(writer, entry_value); - - if (result != 0) { - return result; - } - } - - mpack_finish_array(writer); - - return 0; -} - -static inline int pack_cfl_variant_kvlist(mpack_writer_t *writer, - struct cfl_kvlist *kvlist) { - size_t entry_count; - struct cfl_list *iterator; - struct cfl_kvpair *kvpair; - int result; - - entry_count = cfl_kvlist_count(kvlist); - - mpack_start_map(writer, entry_count); - - cfl_list_foreach(iterator, &kvlist->list) { - kvpair = cfl_list_entry(iterator, struct cfl_kvpair, _head); - - mpack_write_cstr(writer, kvpair->key); - - result = pack_cfl_variant(writer, kvpair->val); - - if (result != 0) { - return result; - } - } - - mpack_finish_map(writer); - - return 0; -} - -static inline int pack_cfl_variant(mpack_writer_t *writer, - struct cfl_variant *value) -{ - int result; - - if (value->type == CFL_VARIANT_STRING) { - result = pack_cfl_variant_string(writer, value->data.as_string); - } - else if (value->type == CFL_VARIANT_BOOL) { - result = pack_cfl_variant_boolean(writer, value->data.as_bool); - } - else if (value->type == CFL_VARIANT_INT) { - result = pack_cfl_variant_int64(writer, value->data.as_int64); - } - else if (value->type == CFL_VARIANT_DOUBLE) { - result = pack_cfl_variant_double(writer, value->data.as_double); - } - else if (value->type == CFL_VARIANT_ARRAY) { - result = pack_cfl_variant_array(writer, value->data.as_array); - } - else if (value->type == CFL_VARIANT_KVLIST) { - result = pack_cfl_variant_kvlist(writer, value->data.as_kvlist); - } - else if (value->type == CFL_VARIANT_BYTES) { - result = pack_cfl_variant_binary(writer, - value->data.as_bytes, - cfl_sds_len(value->data.as_bytes)); - } - else if (value->type == CFL_VARIANT_REFERENCE) { - result = pack_cfl_variant_string(writer, value->data.as_string); - } - else { - result = -1; - } - - return result; -} - -/* Unpackers */ - -static inline int unpack_cfl_variant_read_tag(mpack_reader_t *reader, - mpack_tag_t *tag, - mpack_type_t expected_type) -{ - *tag = mpack_read_tag(reader); - - if (mpack_ok != mpack_reader_error(reader)) { - return -1; - } - - if (mpack_tag_type(tag) != expected_type) { - return -2; - } - - return 0; -} - -static inline int unpack_cfl_array(mpack_reader_t *reader, - struct cfl_array **result_array) -{ - struct cfl_array *internal_array; - size_t entry_count; - struct cfl_variant *entry_value; - int result; - size_t index; - mpack_tag_t tag; - - result = unpack_cfl_variant_read_tag(reader, &tag, mpack_type_array); - - if (result != 0) { - return result; - } - - entry_count = mpack_tag_array_count(&tag); - - internal_array = cfl_array_create(entry_count); - - if (internal_array == NULL) { - return -3; - } - - for (index = 0 ; index < entry_count ; index++) { - result = unpack_cfl_variant(reader, &entry_value); - - if (result != 0) { - cfl_array_destroy(internal_array); - - return -4; - } - - result = cfl_array_append(internal_array, entry_value); - - if (result != 0) { - cfl_array_destroy(internal_array); - - return -5; - } - } - - mpack_done_array(reader); - - if (mpack_reader_error(reader) != mpack_ok) { - cfl_array_destroy(internal_array); - - return -6; - } - - *result_array = internal_array; - - return 0; -} - -static inline int unpack_cfl_kvlist(mpack_reader_t *reader, - struct cfl_kvlist **result_kvlist) -{ - struct cfl_kvlist *internal_kvlist; - char key_name[256]; - size_t entry_count; - size_t key_length; - struct cfl_variant *key_value; - mpack_tag_t key_tag; - int result; - size_t index; - mpack_tag_t tag; - - result = unpack_cfl_variant_read_tag(reader, &tag, mpack_type_map); - - if (result != 0) { - return result; - } - - entry_count = mpack_tag_map_count(&tag); - - internal_kvlist = cfl_kvlist_create(); - - if (internal_kvlist == NULL) { - return -3; - } - - result = 0; - key_value = NULL; - - for (index = 0 ; index < entry_count ; index++) { - result = unpack_cfl_variant_read_tag(reader, &key_tag, mpack_type_str); - - if (result != 0) { - result = -4; - - break; - } - - key_length = mpack_tag_str_length(&key_tag); - - if (key_length >= sizeof(key_name)) { - result = -5; - - break; - } - - mpack_read_cstr(reader, key_name, sizeof(key_name), key_length); - - key_name[key_length] = '\0'; - - mpack_done_str(reader); - - if (mpack_ok != mpack_reader_error(reader)) { - result = -6; - - break; - } - - result = unpack_cfl_variant(reader, &key_value); - - if (result != 0) { - result = -7; - - break; - } - - result = cfl_kvlist_insert(internal_kvlist, key_name, key_value); - - if (result != 0) { - result = -8; - - break; - } - - key_value = NULL; - } - - mpack_done_map(reader); - - if (mpack_reader_error(reader) != mpack_ok) { - result = -9; - } - - if (result != 0) { - cfl_kvlist_destroy(internal_kvlist); - - if (key_value != NULL) { - cfl_variant_destroy(key_value); - } - } - else { - *result_kvlist = internal_kvlist; - } - - return result; -} - -static inline int unpack_cfl_variant_string(mpack_reader_t *reader, - struct cfl_variant **value) -{ - size_t value_length; - char *value_data; - int result; - mpack_tag_t tag; - - result = unpack_cfl_variant_read_tag(reader, &tag, mpack_type_str); - - if (result != 0) { - return result; - } - - value_length = mpack_tag_str_length(&tag); - - value_data = cfl_sds_create_size(value_length + 1); - - if (value_data == NULL) { - return -3; - } - - cfl_sds_set_len(value_data, value_length); - - mpack_read_cstr(reader, value_data, value_length + 1, value_length); - - mpack_done_str(reader); - - if (mpack_reader_error(reader) != mpack_ok) { - cfl_sds_destroy(value_data); - - return -4; - } - - *value = cfl_variant_create_from_reference(value_data); - - if (*value == NULL) { - return -5; - } - - (*value)->type = CFL_VARIANT_STRING; - - return 0; -} - -static inline int unpack_cfl_variant_binary(mpack_reader_t *reader, - struct cfl_variant **value) -{ - size_t value_length; - char *value_data; - int result; - mpack_tag_t tag; - - result = unpack_cfl_variant_read_tag(reader, &tag, mpack_type_bin); - - if (result != 0) { - return result; - } - - value_length = mpack_tag_bin_length(&tag); - - value_data = cfl_sds_create_size(value_length); - - if (value_data == NULL) { - return -3; - } - - cfl_sds_set_len(value_data, value_length); - - mpack_read_bytes(reader, value_data, value_length); - - mpack_done_bin(reader); - - if (mpack_reader_error(reader) != mpack_ok) { - cfl_sds_destroy(value_data); - - return -4; - } - - *value = cfl_variant_create_from_reference(value_data); - - if (*value == NULL) { - return -5; - } - - (*value)->type = CFL_VARIANT_BYTES; - - return 0; -} - -static inline int unpack_cfl_variant_boolean(mpack_reader_t *reader, - struct cfl_variant **value) -{ - int result; - mpack_tag_t tag; - - result = unpack_cfl_variant_read_tag(reader, &tag, mpack_type_bool); - - if (result != 0) { - return result; - } - - *value = cfl_variant_create_from_bool((unsigned int) mpack_tag_bool_value(&tag)); - - if (*value == NULL) { - return -3; - } - - return 0; -} - -static inline int unpack_cfl_variant_uint64(mpack_reader_t *reader, - struct cfl_variant **value) -{ - int result; - mpack_tag_t tag; - - result = unpack_cfl_variant_read_tag(reader, &tag, mpack_type_uint); - - if (result != 0) { - return result; - } - - *value = cfl_variant_create_from_int64((int64_t) mpack_tag_uint_value(&tag)); - - if (*value == NULL) { - return -3; - } - - return 0; -} - -static inline int unpack_cfl_variant_int64(mpack_reader_t *reader, - struct cfl_variant **value) -{ - int result; - mpack_tag_t tag; - - result = unpack_cfl_variant_read_tag(reader, &tag, mpack_type_int); - - if (result != 0) { - return result; - } - - *value = cfl_variant_create_from_int64((int64_t) mpack_tag_int_value(&tag)); - - if (*value == NULL) { - return -3; - } - - return 0; -} - -static inline int unpack_cfl_variant_double(mpack_reader_t *reader, - struct cfl_variant **value) -{ - int result; - mpack_tag_t tag; - - result = unpack_cfl_variant_read_tag(reader, &tag, mpack_type_double); - - if (result != 0) { - return result; - } - - *value = cfl_variant_create_from_double(mpack_tag_double_value(&tag)); - - if (*value == NULL) { - return -3; - } - - return 0; -} - -static inline int unpack_cfl_variant_array(mpack_reader_t *reader, - struct cfl_variant **value) -{ - struct cfl_array *unpacked_array; - int result; - - result = unpack_cfl_array(reader, &unpacked_array); - - if (result != 0) { - return result; - } - - *value = cfl_variant_create_from_array(unpacked_array); - - if (*value == NULL) { - return -3; - } - - return 0; -} - -static inline int unpack_cfl_variant_kvlist(mpack_reader_t *reader, - struct cfl_variant **value) -{ - struct cfl_kvlist *unpacked_kvlist; - int result; - - result = unpack_cfl_kvlist(reader, &unpacked_kvlist); - - if (result != 0) { - return result; - } - - *value = cfl_variant_create_from_kvlist(unpacked_kvlist); - - if (*value == NULL) { - return -3; - } - - return 0; -} - -static inline int unpack_cfl_variant(mpack_reader_t *reader, - struct cfl_variant **value) -{ - mpack_type_t value_type; - int result; - mpack_tag_t tag; - - tag = mpack_peek_tag(reader); - - if (mpack_ok != mpack_reader_error(reader)) { - return -1; - } - - value_type = mpack_tag_type(&tag); - - if (value_type == mpack_type_str) { - result = unpack_cfl_variant_string(reader, value); - } - else if (value_type == mpack_type_str) { - result = unpack_cfl_variant_boolean(reader, value); - } - else if (value_type == mpack_type_int) { - result = unpack_cfl_variant_int64(reader, value); - } - else if (value_type == mpack_type_uint) { - result = unpack_cfl_variant_uint64(reader, value); - } - else if (value_type == mpack_type_double) { - result = unpack_cfl_variant_double(reader, value); - } - else if (value_type == mpack_type_array) { - result = unpack_cfl_variant_array(reader, value); - } - else if (value_type == mpack_type_map) { - result = unpack_cfl_variant_kvlist(reader, value); - } - else if (value_type == mpack_type_bin) { - result = unpack_cfl_variant_binary(reader, value); - } - else { - result = -1; - } - - return result; -} - -#endif diff --git a/fluent-bit/plugins/processor_labels/CMakeLists.txt b/fluent-bit/plugins/processor_labels/CMakeLists.txt deleted file mode 100644 index 93adb1190..000000000 --- a/fluent-bit/plugins/processor_labels/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(src - labels.c) - -FLB_PLUGIN(processor_labels "${src}" "") diff --git a/fluent-bit/plugins/processor_labels/labels.c b/fluent-bit/plugins/processor_labels/labels.c deleted file mode 100644 index 2caaadc31..000000000 --- a/fluent-bit/plugins/processor_labels/labels.c +++ /dev/null @@ -1,1784 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include - -#define PROMOTE_STATIC_METRICS_ON_LABEL_INSERT - -typedef int (*label_transformer)(struct cmt_metric *, cfl_sds_t *value); - -struct internal_processor_context { - struct mk_list *update_list; - struct mk_list *insert_list; - struct mk_list *upsert_list; - struct mk_list *delete_list; - struct mk_list *hash_list; - - /* internal labels ready to append */ - struct cfl_list update_labels; - struct cfl_list insert_labels; - struct cfl_list upsert_labels; - struct mk_list delete_labels; - struct mk_list hash_labels; - - struct flb_processor_instance *instance; - struct flb_config *config; -}; - - -/* - * CMETRICS - */ - -static void cmt_label_destroy(struct cmt_label *label) -{ - if (label != NULL) { - if (!cfl_list_entry_is_orphan(&label->_head)) { - cfl_list_del(&label->_head); - } - - if (label->key != NULL) { - cfl_sds_destroy(label->key); - } - - if (label->val != NULL) { - cfl_sds_destroy(label->val); - } - - free(label); - } -} - -/* we can't use flb_* memory functions here because this will - * be released by cmetrics using the standard allocator. - */ - -static struct cmt_map_label *cmt_map_label_create(char *name) -{ - struct cmt_map_label *label; - - label = calloc(1, sizeof(struct cmt_map_label)); - - if (label != NULL) { - label->name = cfl_sds_create(name); - - if (label->name == NULL) { - free(label); - - label = NULL; - } - - } - - return label; -} - -static void cmt_map_label_destroy(struct cmt_map_label *label) -{ - if (label != NULL) { - if (!cfl_list_entry_is_orphan(&label->_head)) { - cfl_list_del(&label->_head); - } - - if (label->name != NULL) { - cfl_sds_destroy(label->name); - } - - free(label); - } -} - -static struct cmt_metric *map_metric_create(uint64_t hash, - int labels_count, char **labels_val) -{ - int i; - char *name; - struct cmt_metric *metric; - struct cmt_map_label *label; - - metric = calloc(1, sizeof(struct cmt_metric)); - if (!metric) { - cmt_errno(); - return NULL; - } - cfl_list_init(&metric->labels); - metric->val = 0.0; - metric->hash = hash; - - for (i = 0; i < labels_count; i++) { - label = malloc(sizeof(struct cmt_map_label)); - if (!label) { - cmt_errno(); - goto error; - } - - name = labels_val[i]; - label->name = cfl_sds_create(name); - if (!label->name) { - cmt_errno(); - free(label); - goto error; - } - cfl_list_add(&label->_head, &metric->labels); - } - - return metric; - - error: - free(metric); - return NULL; -} - -static void map_metric_destroy(struct cmt_metric *metric) -{ - struct cfl_list *tmp; - struct cfl_list *head; - struct cmt_map_label *label; - - cfl_list_foreach_safe(head, tmp, &metric->labels) { - label = cfl_list_entry(head, struct cmt_map_label, _head); - cfl_sds_destroy(label->name); - cfl_list_del(&label->_head); - free(label); - } - - if (metric->hist_buckets) { - free(metric->hist_buckets); - } - if (metric->sum_quantiles) { - free(metric->sum_quantiles); - } - - cfl_list_del(&metric->_head); - free(metric); -} - - -/* - * LOCAL - */ -static int hex_encode(unsigned char *input_buffer, - size_t input_length, - cfl_sds_t *output_buffer) -{ - const char hex[] = "0123456789abcdef"; - cfl_sds_t result; - size_t index; - - if (cfl_sds_alloc(*output_buffer) <= (input_length * 2)) { - result = cfl_sds_increase(*output_buffer, - (input_length * 2) - - cfl_sds_alloc(*output_buffer)); - - if (result == NULL) { - return FLB_FALSE; - } - - *output_buffer = result; - } - - for (index = 0; index < input_length; index++) { - (*output_buffer)[index * 2 + 0] = hex[(input_buffer[index] >> 4) & 0xF]; - (*output_buffer)[index * 2 + 1] = hex[(input_buffer[index] >> 0) & 0xF]; - } - - cfl_sds_set_len(*output_buffer, input_length * 2); - - (*output_buffer)[index * 2] = '\0'; - - return FLB_TRUE; -} - -static int process_label_modification_list_setting( - struct flb_processor_instance *plugin_instance, - const char *setting_name, - struct mk_list *source_list, - struct mk_list *destination_list) -{ - struct flb_config_map_val *source_entry; - struct mk_list *iterator; - int result; - - if (source_list == NULL || - mk_list_is_empty(source_list) == 0) { - - return 0; - } - - flb_config_map_foreach(iterator, source_entry, source_list) { - result = flb_slist_add(destination_list, source_entry->val.str); - - if (result != 0) { - flb_plg_error(plugin_instance, - "could not append label name %s\n", - source_entry->val.str); - - return -1; - } - } - - return 0; -} - -static int process_label_modification_kvlist_setting( - struct flb_processor_instance *plugin_instance, - const char *setting_name, - struct mk_list *source_list, - struct cfl_list *destination_list) -{ - struct cfl_kv *processed_pair; - struct flb_config_map_val *source_entry; - struct mk_list *iterator; - struct flb_slist_entry *value; - struct flb_slist_entry *key; - - if (source_list == NULL || - mk_list_is_empty(source_list) == 0) { - - return 0; - } - - flb_config_map_foreach(iterator, source_entry, source_list) { - if (mk_list_size(source_entry->val.list) != 2) { - flb_plg_error(plugin_instance, - "'%s' expects a key and a value, " - "e.g: '%s version 1.8.0'", - setting_name, setting_name); - - return -1; - } - - key = mk_list_entry_first(source_entry->val.list, - struct flb_slist_entry, _head); - - value = mk_list_entry_last(source_entry->val.list, - struct flb_slist_entry, _head); - - processed_pair = cfl_kv_item_create(destination_list, - key->str, - value->str); - - if (processed_pair == NULL) { - flb_plg_error(plugin_instance, - "could not append label %s=%s\n", - key->str, - value->str); - - return -1; - } - } - - return 0; -} - -static void destroy_context(struct internal_processor_context *context) -{ - if (context != NULL) { - cfl_kv_release(&context->update_labels); - cfl_kv_release(&context->insert_labels); - cfl_kv_release(&context->upsert_labels); - flb_slist_destroy(&context->delete_labels); - flb_slist_destroy(&context->hash_labels); - - flb_free(context); - } -} - -static struct internal_processor_context * - create_context(struct flb_processor_instance *processor_instance, - struct flb_config *config) -{ - struct internal_processor_context *context; - int result; - - context = flb_calloc(1, sizeof(struct internal_processor_context)); - - if (context != NULL) { - context->instance = processor_instance; - context->config = config; - - cfl_kv_init(&context->update_labels); - cfl_kv_init(&context->insert_labels); - cfl_kv_init(&context->upsert_labels); - flb_slist_create(&context->delete_labels); - flb_slist_create(&context->hash_labels); - - result = flb_processor_instance_config_map_set(processor_instance, (void *) context); - - if (result == 0) { - result = process_label_modification_kvlist_setting(processor_instance, - "update", - context->update_list, - &context->update_labels); - } - - if (result == 0) { - result = process_label_modification_kvlist_setting(processor_instance, - "insert", - context->insert_list, - &context->insert_labels); - } - - if (result == 0) { - result = process_label_modification_kvlist_setting(processor_instance, - "upsert", - context->upsert_list, - &context->upsert_labels); - } - - if (result == 0) { - result = process_label_modification_list_setting(processor_instance, - "delete", - context->delete_list, - &context->delete_labels); - } - - if (result == 0) { - result = process_label_modification_list_setting(processor_instance, - "hash", - context->hash_list, - &context->hash_labels); - } - - if (result != 0) { - destroy_context(context); - - context = NULL; - } - } - else { - flb_errno(); - } - - return context; -} - -static int cb_init(struct flb_processor_instance *processor_instance, - void *source_plugin_instance, - int source_plugin_type, - struct flb_config *config) -{ - processor_instance->context = (void *) create_context( - processor_instance, config); - - if (processor_instance->context == NULL) { - return FLB_PROCESSOR_FAILURE; - } - - return FLB_PROCESSOR_SUCCESS; -} - - -static int cb_exit(struct flb_processor_instance *processor_instance) -{ - if (processor_instance != NULL && - processor_instance->context != NULL) { - destroy_context(processor_instance->context); - } - - return FLB_PROCESSOR_SUCCESS; -} - -static int metrics_context_contains_static_label(struct cmt *metrics_context, - char *label_name) -{ - struct cfl_list *label_iterator; - struct cmt_label *label; - - cfl_list_foreach(label_iterator, &metrics_context->static_labels->list) { - label = cfl_list_entry(label_iterator, - struct cmt_label, _head); - - if (strcasecmp(label_name, label->key) == 0) { - return FLB_TRUE; - } - } - - return FLB_FALSE; -} - -static int metrics_context_insert_static_label(struct cmt *metrics_context, - char *label_name, - char *label_value) -{ - if (cmt_label_add(metrics_context, label_name, label_value) != 0) { - return FLB_FALSE; - } - - return FLB_TRUE; -} - -static int metrics_context_update_static_label(struct cmt *metrics_context, - char *label_name, - char *label_value) -{ - struct cfl_list *iterator; - cfl_sds_t result; - struct cmt_label *label; - - cfl_list_foreach(iterator, &metrics_context->static_labels->list) { - label = cfl_list_entry(iterator, - struct cmt_label, _head); - - if (strcasecmp(label_name, label->key) == 0) { - cfl_sds_set_len(label->val, 0); - - result = cfl_sds_cat(label->val, label_value, strlen(label_value)); - - if (result == NULL) { - return FLB_FALSE; - } - - label->val = result; - - return FLB_TRUE; - } - } - - return FLB_FALSE; -} - -static int metrics_context_transform_static_label(struct cmt *metrics_context, - char *label_name, - label_transformer transformer) -{ - struct cfl_list *iterator; - struct cmt_label *label; - - cfl_list_foreach(iterator, &metrics_context->static_labels->list) { - label = cfl_list_entry(iterator, - struct cmt_label, _head); - - if (strcasecmp(label_name, label->key) == 0) { - return transformer(NULL, &label->val); - } - } - - return FLB_FALSE; -} - -static int metrics_context_upsert_static_label(struct cmt *metrics_context, - char *label_name, - char *label_value) -{ - int result; - - result = metrics_context_contains_static_label(metrics_context, - label_name); - - if (result == FLB_TRUE) { - return metrics_context_update_static_label(metrics_context, - label_name, - label_value); - } - - return metrics_context_insert_static_label(metrics_context, - label_name, - label_value); -} - -static int metrics_context_remove_static_label(struct cmt *metrics_context, - char *label_name) -{ - struct cfl_list *iterator; - struct cmt_label *label; - - cfl_list_foreach(iterator, - &metrics_context->static_labels->list) { - label = cfl_list_entry(iterator, struct cmt_label, _head); - - if (strcasecmp(label_name, label->key) == 0) { - cmt_label_destroy(label); - - return FLB_TRUE; - } - } - - return FLB_FALSE; -} - -static ssize_t metrics_map_get_label_index(struct cmt_map *map, char *label_name) -{ - struct cfl_list *iterator; - struct cmt_map_label *label; - ssize_t index; - - index = 0; - - cfl_list_foreach(iterator, &map->label_keys) { - label = cfl_list_entry(iterator, struct cmt_map_label, _head); - - if (strcasecmp(label_name, label->name) == 0) { - return index; - } - - index++; - } - - return -1; -} - -static ssize_t metrics_map_insert_label_name(struct cmt_map *map, char *label_name) -{ - struct cmt_map_label *label; - ssize_t index; - - label = cmt_map_label_create(label_name); - - if (label == NULL) { - return -1; - } - - map->label_count++; - - cfl_list_add(&label->_head, &map->label_keys); - - index = (ssize_t) cfl_list_size(&map->label_keys); - index--; - - return index; -} - -static int metrics_map_contains_label(struct cmt_map *map, char *label_name) -{ - ssize_t result; - - result = metrics_map_get_label_index(map, label_name); - - if (result != -1) { - return FLB_TRUE; - } - - return FLB_FALSE; -} - -static int metrics_map_remove_label_name(struct cmt_map *map, - size_t label_index) -{ - struct cfl_list *iterator; - struct cmt_map_label *label; - size_t index; - - index = 0; - - cfl_list_foreach(iterator, &map->label_keys) { - label = cfl_list_entry(iterator, struct cmt_map_label, _head); - - if (label_index == index) { - cmt_map_label_destroy(label); - - return FLB_TRUE; - } - - index++; - } - - return FLB_FALSE; -} - -int metrics_data_point_remove_label_value(struct cmt_metric *metric, - size_t label_index) -{ - struct cfl_list *iterator; - struct cmt_map_label *label; - size_t index; - - index = 0; - - cfl_list_foreach(iterator, &metric->labels) { - label = cfl_list_entry(iterator, struct cmt_map_label, _head); - - if (label_index == index) { - cmt_map_label_destroy(label); - - return FLB_TRUE; - } - - index++; - } - - return FLB_FALSE; -} - -int metrics_data_point_transform_label_value(struct cmt_metric *metric, - size_t label_index, - label_transformer transformer) -{ - struct cfl_list *iterator; - struct cmt_map_label *label; - size_t index; - - index = 0; - - cfl_list_foreach(iterator, &metric->labels) { - label = cfl_list_entry(iterator, struct cmt_map_label, _head); - - if (label_index == index) { - return transformer(metric, &label->name); - } - - index++; - } - - return FLB_FALSE; -} - -int metrics_data_point_set_label_value(struct cmt_metric *metric, - size_t label_index, - char *label_value, - int overwrite, - int insert) -{ - struct cmt_map_label *new_label; - struct cfl_list *iterator; - cfl_sds_t result; - size_t index; - struct cmt_map_label *label; - - label = NULL; - index = 0; - - cfl_list_foreach(iterator, &metric->labels) { - label = cfl_list_entry(iterator, struct cmt_map_label, _head); - - if (label_index == index) { - break; - } - - index++; - } - - if (label_index != index) { - return FLB_FALSE; - } - - if (insert == FLB_TRUE) { - new_label = cmt_map_label_create(label_value); - - if (new_label == NULL) { - return FLB_FALSE; - } - - if (label != NULL) { - cfl_list_add_after(&new_label->_head, - &label->_head, - &metric->labels); - } - else { - cfl_list_append(&new_label->_head, - &metric->labels); - } - } - else { - if (label == NULL) { - return FLB_FALSE; - } - - if (label->name == NULL) { - label->name = cfl_sds_create(label_value); - - if (label->name == NULL) { - return FLB_FALSE; - } - } - else { - if (overwrite == FLB_TRUE || - cfl_sds_len(label->name) == 0) { - cfl_sds_set_len(label->name, 0); - - result = cfl_sds_cat(label->name, - label_value, - strlen(label_value)); - - if (result == NULL) { - return FLB_FALSE; - } - - label->name = result; - } - } - } - - return FLB_TRUE; -} - - -int metrics_map_convert_static_metric(struct cmt_map *map, - size_t label_index, - char *label_value) -{ - struct cmt_metric *metric; - int result; - size_t index; - cfl_hash_state_t state; - uint64_t hash; - - cfl_hash_64bits_reset(&state); - - cfl_hash_64bits_update(&state, - map->opts->fqname, - cfl_sds_len(map->opts->fqname)); - - for (index = 0 ; index < map->label_count ; index++) { - if (index != label_index) { - cfl_hash_64bits_update(&state, - "_NULL_", - 6); - } - else { - cfl_hash_64bits_update(&state, - label_value, - strlen(label_value)); - } - } - - hash = cfl_hash_64bits_digest(&state); - - metric = map_metric_create(hash, 0, NULL); - - if (metric == NULL) { - return FLB_FALSE; - } - - for (index = 0 ; index < map->label_count ; index++) { - if (index != label_index) { - result = metrics_data_point_set_label_value(metric, - index, - "", - FLB_TRUE, - FLB_TRUE); - } - else { - result = metrics_data_point_set_label_value(metric, - index, - label_value, - FLB_TRUE, - FLB_TRUE); - } - - if (result != FLB_TRUE) { - map_metric_destroy(metric); - - return FLB_FALSE; - } - } - - metric->val = map->metric.val; - - metric->hist_buckets = map->metric.hist_buckets; - metric->hist_count = map->metric.hist_count; - metric->hist_sum = map->metric.hist_sum; - - metric->sum_quantiles_set = map->metric.sum_quantiles_set; - metric->sum_quantiles = map->metric.sum_quantiles; - metric->sum_quantiles_count = map->metric.sum_quantiles_count; - metric->sum_count = map->metric.sum_count; - metric->sum_sum = map->metric.sum_sum; - - metric->timestamp = map->metric.timestamp; - - map->metric_static_set = 0; - - cfl_list_add(&metric->_head, &map->metrics); - - memset(&map->metric, 0, sizeof(struct cmt_metric)); - - return FLB_TRUE; -} - -int metrics_map_remove_label_value(struct cmt_map *map, - size_t label_index) -{ - struct cfl_list *iterator; - struct cmt_metric *metric; - int result; - - result = FLB_TRUE; - - cfl_list_foreach(iterator, &map->metrics) { - metric = cfl_list_entry(iterator, struct cmt_metric, _head); - - result = metrics_data_point_remove_label_value(metric, label_index); - - if (result == FLB_FALSE) { - break; - } - } - - return result; -} - -int metrics_map_set_label_value(struct cmt_map *map, - size_t label_index, - char *label_value, - int overwrite, - int insert) -{ - struct cfl_list *iterator; - struct cmt_metric *metric; - int result; - - result = FLB_TRUE; - - cfl_list_foreach(iterator, &map->metrics) { - metric = cfl_list_entry(iterator, struct cmt_metric, _head); - - result = metrics_data_point_set_label_value(metric, - label_index, - label_value, - overwrite, - insert); - - if (result == FLB_FALSE) { - break; - } - } - -#ifdef PROMOTE_STATIC_METRICS_ON_LABEL_INSERT - if (map->metric_static_set == 1) { - result = metrics_map_convert_static_metric(map, - label_index, - label_value); - - if(result == FLB_FALSE) { - return FLB_FALSE; - } - } -#endif - - return result; -} - -int metrics_map_transform_label_value(struct cmt_map *map, - size_t label_index, - label_transformer transformer) -{ - struct cfl_list *iterator; - struct cmt_metric *metric; - int result; - - result = FLB_TRUE; - - cfl_list_foreach(iterator, &map->metrics) { - metric = cfl_list_entry(iterator, struct cmt_metric, _head); - - result = metrics_data_point_transform_label_value(metric, - label_index, - transformer); - - if (result == FLB_FALSE) { - break; - } - } - - return result; -} - -int metrics_map_update_label(struct cmt_map *map, - char *label_name, - char *label_value) -{ - ssize_t label_index; - int result; - - label_index = metrics_map_get_label_index(map, label_name); - - if (label_index == -1) { - return FLB_TRUE; - } - - result = metrics_map_set_label_value(map, - label_index, - label_value, - FLB_TRUE, - FLB_FALSE); - - if(result == FLB_FALSE) { - return FLB_FALSE; - } - - return FLB_TRUE; -} - -int metrics_map_transform_label(struct cmt_map *map, - char *label_name, - label_transformer transformer) -{ - ssize_t label_index; - int result; - - label_index = metrics_map_get_label_index(map, label_name); - - if (label_index == -1) { - return FLB_TRUE; - } - - result = metrics_map_transform_label_value(map, - label_index, - transformer); - - if(result == FLB_FALSE) { - return FLB_FALSE; - } - - return FLB_TRUE; -} - -int metrics_map_insert_label(struct cmt_map *map, - char *label_name, - char *label_value) -{ - ssize_t label_index; - int label_added; - int result; - - label_added = FLB_FALSE; - label_index = metrics_map_get_label_index(map, label_name); - - if (label_index == -1) { - label_index = metrics_map_insert_label_name(map, label_name); - label_added = FLB_TRUE; - } - - if (label_index == -1) { - return FLB_FALSE; - } - - result = metrics_map_set_label_value(map, - label_index, - label_value, - FLB_FALSE, - label_added); - - if(result == FLB_FALSE) { - return FLB_FALSE; - } - - return FLB_TRUE; -} - -int metrics_map_upsert_label(struct cmt_map *map, - char *label_name, - char *label_value) -{ - ssize_t label_index; - int label_added; - int result; - - label_added = FLB_FALSE; - label_index = metrics_map_get_label_index(map, label_name); - - if (label_index == -1) { - label_index = metrics_map_insert_label_name(map, label_name); - label_added = FLB_TRUE; - } - - if (label_index == -1) { - return FLB_FALSE; - } - - result = metrics_map_set_label_value(map, - label_index, - label_value, - FLB_TRUE, - label_added); - - if(result == FLB_FALSE) { - return FLB_FALSE; - } - - return FLB_TRUE; -} - -int metrics_map_remove_label(struct cmt_map *map, - char *label_name) -{ - ssize_t label_index; - int result; - - label_index = metrics_map_get_label_index(map, label_name); - - if (label_index == -1) { - return FLB_TRUE; - } - - map->label_count--; - - result = metrics_map_remove_label_name(map, label_index); - - if(result == FLB_TRUE) { - result = metrics_map_remove_label_value(map, label_index); - } - - return result; -} - -static int metrics_context_contains_dynamic_label(struct cmt *metrics_context, - char *label_name) -{ - struct cfl_list *metric_iterator; - struct cmt_histogram *histogram; - struct cmt_summary *summary; - struct cmt_untyped *untyped; - struct cmt_counter *counter; - struct cmt_gauge *gauge; - - cfl_list_foreach(metric_iterator, &metrics_context->histograms) { - histogram = cfl_list_entry(metric_iterator, struct cmt_histogram, _head); - - if(metrics_map_contains_label(histogram->map, label_name) == FLB_TRUE) { - return FLB_TRUE; - } - } - - cfl_list_foreach(metric_iterator, &metrics_context->summaries) { - summary = cfl_list_entry(metric_iterator, struct cmt_summary, _head); - - if(metrics_map_contains_label(summary->map, label_name) == FLB_TRUE) { - return FLB_TRUE; - } - } - - cfl_list_foreach(metric_iterator, &metrics_context->untypeds) { - untyped = cfl_list_entry(metric_iterator, struct cmt_untyped, _head); - - if(metrics_map_contains_label(untyped->map, label_name) == FLB_TRUE) { - return FLB_TRUE; - } - } - - cfl_list_foreach(metric_iterator, &metrics_context->counters) { - counter = cfl_list_entry(metric_iterator, struct cmt_counter, _head); - - if(metrics_map_contains_label(counter->map, label_name) == FLB_TRUE) { - return FLB_TRUE; - } - } - - cfl_list_foreach(metric_iterator, &metrics_context->gauges) { - gauge = cfl_list_entry(metric_iterator, struct cmt_gauge, _head); - - if(metrics_map_contains_label(gauge->map, label_name) == FLB_TRUE) { - return FLB_TRUE; - } - } - - return FLB_FALSE; -} - -static int metrics_context_insert_dynamic_label(struct cmt *metrics_context, - char *label_name, - char *label_value) -{ - struct cfl_list *metric_iterator; - struct cmt_histogram *histogram; - struct cmt_summary *summary; - struct cmt_untyped *untyped; - struct cmt_counter *counter; - int result; - struct cmt_gauge *gauge; - - cfl_list_foreach(metric_iterator, &metrics_context->histograms) { - histogram = cfl_list_entry(metric_iterator, struct cmt_histogram, _head); - - result = metrics_map_insert_label(histogram->map, - label_name, - label_value); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - - cfl_list_foreach(metric_iterator, &metrics_context->summaries) { - summary = cfl_list_entry(metric_iterator, struct cmt_summary, _head); - - result = metrics_map_insert_label(summary->map, - label_name, - label_value); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - - cfl_list_foreach(metric_iterator, &metrics_context->untypeds) { - untyped = cfl_list_entry(metric_iterator, struct cmt_untyped, _head); - - result = metrics_map_insert_label(untyped->map, - label_name, - label_value); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - - cfl_list_foreach(metric_iterator, &metrics_context->counters) { - counter = cfl_list_entry(metric_iterator, struct cmt_counter, _head); - - result = metrics_map_insert_label(counter->map, - label_name, - label_value); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - - cfl_list_foreach(metric_iterator, &metrics_context->gauges) { - gauge = cfl_list_entry(metric_iterator, struct cmt_gauge, _head); - - result = metrics_map_insert_label(gauge->map, - label_name, - label_value); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - - return FLB_TRUE; -} - -static int metrics_context_update_dynamic_label(struct cmt *metrics_context, - char *label_name, - char *label_value) -{ - struct cfl_list *metric_iterator; - struct cmt_histogram *histogram; - struct cmt_summary *summary; - struct cmt_untyped *untyped; - struct cmt_counter *counter; - int result; - struct cmt_gauge *gauge; - - cfl_list_foreach(metric_iterator, &metrics_context->histograms) { - histogram = cfl_list_entry(metric_iterator, struct cmt_histogram, _head); - - result = metrics_map_update_label(histogram->map, - label_name, - label_value); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - - cfl_list_foreach(metric_iterator, &metrics_context->summaries) { - summary = cfl_list_entry(metric_iterator, struct cmt_summary, _head); - - result = metrics_map_update_label(summary->map, - label_name, - label_value); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - - cfl_list_foreach(metric_iterator, &metrics_context->untypeds) { - untyped = cfl_list_entry(metric_iterator, struct cmt_untyped, _head); - - result = metrics_map_update_label(untyped->map, - label_name, - label_value); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - - cfl_list_foreach(metric_iterator, &metrics_context->counters) { - counter = cfl_list_entry(metric_iterator, struct cmt_counter, _head); - - result = metrics_map_update_label(counter->map, - label_name, - label_value); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - - cfl_list_foreach(metric_iterator, &metrics_context->gauges) { - gauge = cfl_list_entry(metric_iterator, struct cmt_gauge, _head); - - result = metrics_map_update_label(gauge->map, - label_name, - label_value); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - - return FLB_TRUE; -} - -static int metrics_context_transform_dynamic_label(struct cmt *metrics_context, - char *label_name, - label_transformer transformer) -{ - struct cfl_list *metric_iterator; - struct cmt_histogram *histogram; - struct cmt_summary *summary; - struct cmt_untyped *untyped; - struct cmt_counter *counter; - int result; - struct cmt_gauge *gauge; - - cfl_list_foreach(metric_iterator, &metrics_context->histograms) { - histogram = cfl_list_entry(metric_iterator, struct cmt_histogram, _head); - - result = metrics_map_transform_label(histogram->map, - label_name, - transformer); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - - cfl_list_foreach(metric_iterator, &metrics_context->summaries) { - summary = cfl_list_entry(metric_iterator, struct cmt_summary, _head); - - result = metrics_map_transform_label(summary->map, - label_name, - transformer); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - - cfl_list_foreach(metric_iterator, &metrics_context->untypeds) { - untyped = cfl_list_entry(metric_iterator, struct cmt_untyped, _head); - - result = metrics_map_transform_label(untyped->map, - label_name, - transformer); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - - cfl_list_foreach(metric_iterator, &metrics_context->counters) { - counter = cfl_list_entry(metric_iterator, struct cmt_counter, _head); - - result = metrics_map_transform_label(counter->map, - label_name, - transformer); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - - cfl_list_foreach(metric_iterator, &metrics_context->gauges) { - gauge = cfl_list_entry(metric_iterator, struct cmt_gauge, _head); - - result = metrics_map_transform_label(gauge->map, - label_name, - transformer); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - - return FLB_TRUE; -} - -static int metrics_context_upsert_dynamic_label(struct cmt *metrics_context, - char *label_name, - char *label_value) -{ - struct cfl_list *metric_iterator; - struct cmt_histogram *histogram; - struct cmt_summary *summary; - struct cmt_untyped *untyped; - struct cmt_counter *counter; - int result; - struct cmt_gauge *gauge; - - cfl_list_foreach(metric_iterator, &metrics_context->histograms) { - histogram = cfl_list_entry(metric_iterator, struct cmt_histogram, _head); - - result = metrics_map_upsert_label(histogram->map, - label_name, - label_value); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - - cfl_list_foreach(metric_iterator, &metrics_context->summaries) { - summary = cfl_list_entry(metric_iterator, struct cmt_summary, _head); - - result = metrics_map_upsert_label(summary->map, - label_name, - label_value); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - - cfl_list_foreach(metric_iterator, &metrics_context->untypeds) { - untyped = cfl_list_entry(metric_iterator, struct cmt_untyped, _head); - - result = metrics_map_upsert_label(untyped->map, - label_name, - label_value); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - - cfl_list_foreach(metric_iterator, &metrics_context->counters) { - counter = cfl_list_entry(metric_iterator, struct cmt_counter, _head); - - result = metrics_map_upsert_label(counter->map, - label_name, - label_value); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - - cfl_list_foreach(metric_iterator, &metrics_context->gauges) { - gauge = cfl_list_entry(metric_iterator, struct cmt_gauge, _head); - - result = metrics_map_upsert_label(gauge->map, - label_name, - label_value); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - - return FLB_TRUE; -} - -static int metrics_context_remove_dynamic_label(struct cmt *metrics_context, - char *label_name) -{ - struct cfl_list *metric_iterator; - struct cmt_histogram *histogram; - struct cmt_summary *summary; - struct cmt_untyped *untyped; - struct cmt_counter *counter; - int result; - struct cmt_gauge *gauge; - - cfl_list_foreach(metric_iterator, &metrics_context->histograms) { - histogram = cfl_list_entry(metric_iterator, struct cmt_histogram, _head); - - result = metrics_map_remove_label(histogram->map, - label_name); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - - cfl_list_foreach(metric_iterator, &metrics_context->summaries) { - summary = cfl_list_entry(metric_iterator, struct cmt_summary, _head); - - result = metrics_map_remove_label(summary->map, - label_name); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - - cfl_list_foreach(metric_iterator, &metrics_context->untypeds) { - untyped = cfl_list_entry(metric_iterator, struct cmt_untyped, _head); - - result = metrics_map_remove_label(untyped->map, - label_name); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - - cfl_list_foreach(metric_iterator, &metrics_context->counters) { - counter = cfl_list_entry(metric_iterator, struct cmt_counter, _head); - - result = metrics_map_remove_label(counter->map, - label_name); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - - cfl_list_foreach(metric_iterator, &metrics_context->gauges) { - gauge = cfl_list_entry(metric_iterator, struct cmt_gauge, _head); - - result = metrics_map_remove_label(gauge->map, - label_name); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - - return FLB_TRUE; -} - -static int update_labels(struct cmt *metrics_context, - struct cfl_list *labels) -{ - struct cfl_list *iterator; - int result; - struct cfl_kv *pair; - - cfl_list_foreach(iterator, labels) { - pair = cfl_list_entry(iterator, struct cfl_kv, _head); - - result = metrics_context_contains_dynamic_label(metrics_context, - pair->key); - - if (result == FLB_TRUE) { - result = metrics_context_update_dynamic_label(metrics_context, - pair->key, - pair->val); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - - result = metrics_context_contains_static_label(metrics_context, - pair->key); - - if (result == FLB_TRUE) { - result = metrics_context_update_static_label(metrics_context, - pair->key, - pair->val); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - } - - return FLB_PROCESSOR_SUCCESS; -} - -static int insert_labels(struct cmt *metrics_context, - struct cfl_list *labels) -{ - struct cfl_list *iterator; - int result; - struct cfl_kv *pair; - - cfl_list_foreach(iterator, labels) { - pair = cfl_list_entry(iterator, struct cfl_kv, _head); - - result = metrics_context_contains_dynamic_label(metrics_context, - pair->key); - - if (result == FLB_TRUE) { - result = metrics_context_insert_dynamic_label(metrics_context, - pair->key, - pair->val); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - else { - result = metrics_context_contains_static_label(metrics_context, - pair->key); - - if (result == FLB_FALSE) { - result = metrics_context_insert_static_label(metrics_context, - pair->key, - pair->val); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - } - } - - return FLB_PROCESSOR_SUCCESS; -} - -static int upsert_labels(struct cmt *metrics_context, - struct cfl_list *labels) -{ - struct cfl_list *iterator; - int result; - struct cfl_kv *pair; - - cfl_list_foreach(iterator, labels) { - pair = cfl_list_entry(iterator, struct cfl_kv, _head); - - result = metrics_context_contains_dynamic_label(metrics_context, - pair->key); - - if (result == FLB_TRUE) { - result = metrics_context_upsert_dynamic_label(metrics_context, - pair->key, - pair->val); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - else { - result = metrics_context_upsert_static_label(metrics_context, - pair->key, - pair->val); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - } - - return FLB_PROCESSOR_SUCCESS; -} - -static int delete_labels(struct cmt *metrics_context, - struct mk_list *labels) -{ - struct mk_list *iterator; - int result; - struct flb_slist_entry *entry; - - mk_list_foreach(iterator, labels) { - entry = mk_list_entry(iterator, struct flb_slist_entry, _head); - - result = metrics_context_contains_dynamic_label(metrics_context, - entry->str); - - if (result == FLB_TRUE) { - result = metrics_context_remove_dynamic_label(metrics_context, - entry->str); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - else { - result = metrics_context_contains_static_label(metrics_context, - entry->str); - - if (result == FLB_TRUE) { - result = metrics_context_remove_static_label(metrics_context, - entry->str); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - } - } - - return FLB_PROCESSOR_SUCCESS; -} - -static int hash_transformer(struct cmt_metric *metric, cfl_sds_t *value) -{ - unsigned char digest_buffer[32]; - int result; - - if (value == NULL) { - return FLB_FALSE; - } - - if (cfl_sds_len(*value) == 0) { - return FLB_TRUE; - } - - result = flb_hash_simple(FLB_HASH_SHA256, - (unsigned char *) *value, - cfl_sds_len(*value), - digest_buffer, - sizeof(digest_buffer)); - - if (result != FLB_CRYPTO_SUCCESS) { - return FLB_FALSE; - } - - return hex_encode(digest_buffer, sizeof(digest_buffer), value); -} - -static int hash_labels(struct cmt *metrics_context, - struct mk_list *labels) -{ - struct mk_list *iterator; - int result; - struct flb_slist_entry *entry; - - mk_list_foreach(iterator, labels) { - entry = mk_list_entry(iterator, struct flb_slist_entry, _head); - - result = metrics_context_contains_dynamic_label(metrics_context, - entry->str); - - if (result == FLB_TRUE) { - result = metrics_context_transform_dynamic_label(metrics_context, - entry->str, - hash_transformer); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - else { - result = metrics_context_contains_static_label(metrics_context, - entry->str); - - if (result == FLB_TRUE) { - result = metrics_context_transform_static_label(metrics_context, - entry->str, - hash_transformer); - - if (result == FLB_FALSE) { - return FLB_FALSE; - } - } - } - } - - return FLB_PROCESSOR_SUCCESS; -} - -static int cb_process_metrics(struct flb_processor_instance *processor_instance, - struct cmt *metrics_context, - const char *tag, - int tag_len) -{ - struct internal_processor_context *processor_context; - int result; - - processor_context = - (struct internal_processor_context *) processor_instance->context; - - result = delete_labels(metrics_context, - &processor_context->delete_labels); - - if (result == FLB_PROCESSOR_SUCCESS) { - result = update_labels(metrics_context, - &processor_context->update_labels); - } - - if (result == FLB_PROCESSOR_SUCCESS) { - result = upsert_labels(metrics_context, - &processor_context->upsert_labels); - } - - if (result == FLB_PROCESSOR_SUCCESS) { - result = insert_labels(metrics_context, - &processor_context->insert_labels); - } - - if (result == FLB_PROCESSOR_SUCCESS) { - result = hash_labels(metrics_context, - &processor_context->hash_labels); - } - - if (result != FLB_PROCESSOR_SUCCESS) { - return FLB_PROCESSOR_FAILURE; - } - - return FLB_PROCESSOR_SUCCESS; -} - -static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_SLIST_1, "update", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct internal_processor_context, - update_list), - "Updates a label. Usage : 'update label_name value'" - }, - { - FLB_CONFIG_MAP_SLIST_1, "insert", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct internal_processor_context, - insert_list), - "Inserts a label. Usage : 'insert label_name value'" - }, - { - FLB_CONFIG_MAP_SLIST_1, "upsert", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct internal_processor_context, - upsert_list), - "Inserts or updates a label. Usage : 'upsert label_name value'" - }, - { - FLB_CONFIG_MAP_STR, "delete", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct internal_processor_context, - delete_list), - "Deletes a label. Usage : 'delete label_name'" - }, - { - FLB_CONFIG_MAP_STR, "hash", NULL, - FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct internal_processor_context, - hash_list), - "Replaces a labels value with its SHA1 hash. Usage : 'hash label_name'" - }, - - /* EOF */ - {0} -}; - -struct flb_processor_plugin processor_labels_plugin = { - .name = "labels", - .description = "Modifies metrics labels", - .cb_init = cb_init, - .cb_process_logs = NULL, - .cb_process_metrics = cb_process_metrics, - .cb_process_traces = NULL, - .cb_exit = cb_exit, - .config_map = config_map, - .flags = 0 -}; -- cgit v1.2.3