diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-05 11:19:16 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-07-24 09:53:24 +0000 |
commit | b5f8ee61a7f7e9bd291dd26b0585d03eb686c941 (patch) | |
tree | d4d31289c39fc00da064a825df13a0b98ce95b10 /fluent-bit/lib/librdkafka-2.1.0/examples | |
parent | Adding upstream version 1.44.3. (diff) | |
download | netdata-b5f8ee61a7f7e9bd291dd26b0585d03eb686c941.tar.xz netdata-b5f8ee61a7f7e9bd291dd26b0585d03eb686c941.zip |
Adding upstream version 1.46.3.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'fluent-bit/lib/librdkafka-2.1.0/examples')
26 files changed, 0 insertions, 10846 deletions
diff --git a/fluent-bit/lib/librdkafka-2.1.0/examples/.gitignore b/fluent-bit/lib/librdkafka-2.1.0/examples/.gitignore deleted file mode 100644 index 4190608c4..000000000 --- a/fluent-bit/lib/librdkafka-2.1.0/examples/.gitignore +++ /dev/null @@ -1,19 +0,0 @@ -consumer -delete_records -idempotent_producer -kafkatest_verifiable_client -misc -openssl_engine_example_cpp -producer -producer_cpp -rdkafka_complex_consumer_example -rdkafka_complex_consumer_example_cpp -rdkafka_consume_batch -rdkafka_example -rdkafka_example_cpp -rdkafka_performance -transactions -list_consumer_groups -describe_consumer_groups -list_consumer_group_offsets -alter_consumer_group_offsets diff --git a/fluent-bit/lib/librdkafka-2.1.0/examples/CMakeLists.txt b/fluent-bit/lib/librdkafka-2.1.0/examples/CMakeLists.txt deleted file mode 100644 index bbbb89ad9..000000000 --- a/fluent-bit/lib/librdkafka-2.1.0/examples/CMakeLists.txt +++ /dev/null @@ -1,40 +0,0 @@ -if(WIN32) - set(win32_sources ../win32/wingetopt.c ../win32/wingetopt.h) -endif(WIN32) - -add_executable(producer producer.c ${win32_sources}) -target_link_libraries(producer PUBLIC rdkafka) - -add_executable(producer_cpp producer.cpp ${win32_sources}) -target_link_libraries(producer_cpp PUBLIC rdkafka++) - -add_executable(consumer consumer.c ${win32_sources}) -target_link_libraries(consumer PUBLIC rdkafka) - -add_executable(rdkafka_performance rdkafka_performance.c ${win32_sources}) -target_link_libraries(rdkafka_performance PUBLIC rdkafka) - -add_executable(rdkafka_example_cpp rdkafka_example.cpp ${win32_sources}) -target_link_libraries(rdkafka_example_cpp PUBLIC rdkafka++) - -add_executable(rdkafka_complex_consumer_example_cpp rdkafka_complex_consumer_example.cpp ${win32_sources}) -target_link_libraries(rdkafka_complex_consumer_example_cpp PUBLIC rdkafka++) - -add_executable(openssl_engine_example_cpp openssl_engine_example.cpp ${win32_sources}) -target_link_libraries(openssl_engine_example_cpp PUBLIC rdkafka++) - -add_executable(misc misc.c ${win32_sources}) -target_link_libraries(misc PUBLIC rdkafka) - - -# The targets below has Unix include dirs and do not compile on Windows. -if(NOT WIN32) - add_executable(rdkafka_example rdkafka_example.c) - target_link_libraries(rdkafka_example PUBLIC rdkafka) - - add_executable(rdkafka_complex_consumer_example rdkafka_complex_consumer_example.c) - target_link_libraries(rdkafka_complex_consumer_example PUBLIC rdkafka) - - add_executable(kafkatest_verifiable_client kafkatest_verifiable_client.cpp) - target_link_libraries(kafkatest_verifiable_client PUBLIC rdkafka++) -endif(NOT WIN32) diff --git a/fluent-bit/lib/librdkafka-2.1.0/examples/Makefile b/fluent-bit/lib/librdkafka-2.1.0/examples/Makefile deleted file mode 100644 index 15fba3c2a..000000000 --- a/fluent-bit/lib/librdkafka-2.1.0/examples/Makefile +++ /dev/null @@ -1,137 +0,0 @@ -EXAMPLES ?= rdkafka_example rdkafka_performance rdkafka_example_cpp \ - rdkafka_complex_consumer_example rdkafka_complex_consumer_example_cpp \ - kafkatest_verifiable_client \ - producer consumer idempotent_producer transactions \ - delete_records \ - openssl_engine_example_cpp \ - list_consumer_groups \ - describe_consumer_groups \ - list_consumer_group_offsets \ - alter_consumer_group_offsets \ - misc - -all: $(EXAMPLES) - -include ../mklove/Makefile.base - -CFLAGS += -I../src -CXXFLAGS += -I../src-cpp - -# librdkafka must be compiled with -gstrict-dwarf, but rdkafka_example must not, -# due to some clang bug on OSX 10.9 -CPPFLAGS := $(subst strict-dwarf,,$(CPPFLAGS)) - -rdkafka_example: ../src/librdkafka.a rdkafka_example.c - $(CC) $(CPPFLAGS) $(CFLAGS) rdkafka_example.c -o $@ $(LDFLAGS) \ - ../src/librdkafka.a $(LIBS) - @echo "# $@ is ready" - @echo "#" - @echo "# Run producer (write messages on stdin)" - @echo "./$@ -P -t <topic> -p <partition>" - @echo "" - @echo "# or consumer" - @echo "./$@ -C -t <topic> -p <partition>" - @echo "" - @echo "#" - @echo "# More usage options:" - @echo "./$@ -h" - -producer: ../src/librdkafka.a producer.c - $(CC) $(CPPFLAGS) $(CFLAGS) $@.c -o $@ $(LDFLAGS) \ - ../src/librdkafka.a $(LIBS) - -producer_cpp: ../src-cpp/librdkafka++.a ../src/librdkafka.a producer.cpp - $(CXX) $(CPPFLAGS) $(CXXFLAGS) producer.cpp -o $@ $(LDFLAGS) \ - ../src-cpp/librdkafka++.a ../src/librdkafka.a $(LIBS) - -consumer: ../src/librdkafka.a consumer.c - $(CC) $(CPPFLAGS) $(CFLAGS) $@.c -o $@ $(LDFLAGS) \ - ../src/librdkafka.a $(LIBS) - -idempotent_producer: ../src/librdkafka.a idempotent_producer.c - $(CC) $(CPPFLAGS) $(CFLAGS) $@.c -o $@ $(LDFLAGS) \ - ../src/librdkafka.a $(LIBS) - -transactions: ../src/librdkafka.a transactions.c - $(CC) $(CPPFLAGS) $(CFLAGS) $@.c -o $@ $(LDFLAGS) \ - ../src/librdkafka.a $(LIBS) - -transactions-older-broker.c: ../src/librdkafka.a transactions-older-broker.c - $(CC) $(CPPFLAGS) $(CFLAGS) $@.c -o $@ $(LDFLAGS) \ - ../src/librdkafka.a $(LIBS) - -delete_records: ../src/librdkafka.a delete_records.c - $(CC) $(CPPFLAGS) $(CFLAGS) $@.c -o $@ $(LDFLAGS) \ - ../src/librdkafka.a $(LIBS) - -list_consumer_groups: ../src/librdkafka.a list_consumer_groups.c - $(CC) $(CPPFLAGS) $(CFLAGS) $@.c -o $@ $(LDFLAGS) \ - ../src/librdkafka.a $(LIBS) - -describe_consumer_groups: ../src/librdkafka.a describe_consumer_groups.c - $(CC) $(CPPFLAGS) $(CFLAGS) $@.c -o $@ $(LDFLAGS) \ - ../src/librdkafka.a $(LIBS) - -list_consumer_group_offsets: ../src/librdkafka.a list_consumer_group_offsets.c - $(CC) $(CPPFLAGS) $(CFLAGS) $@.c -o $@ $(LDFLAGS) \ - ../src/librdkafka.a $(LIBS) - -alter_consumer_group_offsets: ../src/librdkafka.a alter_consumer_group_offsets.c - $(CC) $(CPPFLAGS) $(CFLAGS) $@.c -o $@ $(LDFLAGS) \ - ../src/librdkafka.a $(LIBS) - -rdkafka_complex_consumer_example: ../src/librdkafka.a rdkafka_complex_consumer_example.c - $(CC) $(CPPFLAGS) $(CFLAGS) rdkafka_complex_consumer_example.c -o $@ $(LDFLAGS) \ - ../src/librdkafka.a $(LIBS) - @echo "# $@ is ready" - @echo "#" - @echo "./$@ <topic[:part]> <topic2[:part]> .." - @echo "" - @echo "#" - @echo "# More usage options:" - @echo "./$@ -h" - -rdkafka_performance: ../src/librdkafka.a rdkafka_performance.c - $(CC) $(CPPFLAGS) $(CFLAGS) rdkafka_performance.c -o $@ $(LDFLAGS) \ - ../src/librdkafka.a $(LIBS) - @echo "# $@ is ready" - @echo "#" - @echo "# Run producer" - @echo "./$@ -P -t <topic> -p <partition> -s <msgsize>" - @echo "" - @echo "# or consumer" - @echo "./$@ -C -t <topic> -p <partition>" - @echo "" - @echo "#" - @echo "# More usage options:" - @echo "./$@ -h" - - -rdkafka_example_cpp: ../src-cpp/librdkafka++.a ../src/librdkafka.a rdkafka_example.cpp - $(CXX) $(CPPFLAGS) $(CXXFLAGS) rdkafka_example.cpp -o $@ $(LDFLAGS) \ - ../src-cpp/librdkafka++.a ../src/librdkafka.a $(LIBS) - -kafkatest_verifiable_client: ../src-cpp/librdkafka++.a ../src/librdkafka.a kafkatest_verifiable_client.cpp - $(CXX) $(CPPFLAGS) $(CXXFLAGS) kafkatest_verifiable_client.cpp -o $@ $(LDFLAGS) \ - ../src-cpp/librdkafka++.a ../src/librdkafka.a $(LIBS) - - -rdkafka_complex_consumer_example_cpp: ../src-cpp/librdkafka++.a ../src/librdkafka.a rdkafka_complex_consumer_example.cpp - $(CXX) $(CPPFLAGS) $(CXXFLAGS) rdkafka_complex_consumer_example.cpp -o $@ $(LDFLAGS) \ - ../src-cpp/librdkafka++.a ../src/librdkafka.a $(LIBS) - -rdkafka_consume_batch: ../src-cpp/librdkafka++.a ../src/librdkafka.a rdkafka_consume_batch.cpp - $(CXX) $(CPPFLAGS) $(CXXFLAGS) rdkafka_consume_batch.cpp -o $@ $(LDFLAGS) \ - ../src-cpp/librdkafka++.a ../src/librdkafka.a $(LIBS) - -openssl_engine_example_cpp: ../src-cpp/librdkafka++.a ../src/librdkafka.a openssl_engine_example.cpp - $(CXX) $(CPPFLAGS) $(CXXFLAGS) openssl_engine_example.cpp -o $@ $(LDFLAGS) \ - ../src-cpp/librdkafka++.a ../src/librdkafka.a $(LIBS) - -misc: ../src/librdkafka.a misc.c - $(CC) $(CPPFLAGS) $(CFLAGS) $@.c -o $@ $(LDFLAGS) \ - ../src/librdkafka.a $(LIBS) - -clean: - rm -f $(EXAMPLES) - diff --git a/fluent-bit/lib/librdkafka-2.1.0/examples/README.md b/fluent-bit/lib/librdkafka-2.1.0/examples/README.md deleted file mode 100644 index 3caee3b86..000000000 --- a/fluent-bit/lib/librdkafka-2.1.0/examples/README.md +++ /dev/null @@ -1,38 +0,0 @@ -# librdkafka examples - -This directory contains example applications utilizing librdkafka. -The examples are built by running `make` and they will be be linked -statically or dynamically to librdkafka in the parent `../src` directory. - -Begin with the following examples: - - * [consumer.c](consumer.c) - a typical C high-level consumer application. - * [producer.c](producer.c) - a typical C producer application. - * [producer.cpp](producer.cpp) - a typical C++ producer application. - * [idempotent_producer.c](idempotent_producer.c) - Idempotent producer. - * [transactions.c](transactions.c) - Full exactly once semantics (EOS) - transactional consumer-producer exammple. - Requires Apache Kafka 2.5 or later. - * [transactions-older-broker.c](transactions-older-broker.c) - Same as - `transactions.c` but for Apache Kafka versions 2.4.x and older which - lack KIP-447 support. - * [misc.c](misc.c) - a collection of miscellaneous usage examples. - - -For more complex uses, see: - * [rdkafka_example.c](rdkafka_example.c) - simple consumer, producer, metadata listing, kitchen sink, etc. - * [rdkafka_example.cpp](rdkafka_example.cpp) - simple consumer, producer, metadata listing in C++. - * [rdkafka_complex_consumer_example.c](rdkafka_complex_consumer_example.c) - a more contrived high-level C consumer example. - * [rdkafka_complex_consumer_example.cpp](rdkafka_complex_consumer_example.cpp) - a more contrived high-level C++ consumer example. - * [rdkafka_consume_batch.cpp](rdkafka_consume_batch.cpp) - batching high-level C++ consumer example. - * [rdkafka_performance.c](rdkafka_performance.c) - performance, benchmark, latency producer and consumer tool. - * [kafkatest_verifiable_client.cpp](kafkatest_verifiable_client.cpp) - for use with the official Apache Kafka client system tests. - * [openssl_engine_example.cpp](openssl_engine_example.cpp) - metadata listing in C++ over SSL channel established using OpenSSL engine. - - - For Admin API examples see: - * [delete_records.c](delete_records.c) - Delete records. - * [list_consumer_groups.c](list_consumer_groups.c) - List consumer groups. - * [describe_consumer_groups.c](describe_consumer_groups.c) - Describe consumer groups. - * [list_consumer_group_offsets.c](list_consumer_group_offsets.c) - List offsets of a consumer group. - * [alter_consumer_group_offsets.c](alter_consumer_group_offsets.c) - Alter offsets of a consumer group. diff --git a/fluent-bit/lib/librdkafka-2.1.0/examples/alter_consumer_group_offsets.c b/fluent-bit/lib/librdkafka-2.1.0/examples/alter_consumer_group_offsets.c deleted file mode 100644 index 09a52fd7e..000000000 --- a/fluent-bit/lib/librdkafka-2.1.0/examples/alter_consumer_group_offsets.c +++ /dev/null @@ -1,338 +0,0 @@ -/* - * librdkafka - Apache Kafka C library - * - * Copyright (c) 2022, Magnus Edenhill - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/** - * AlterConsumerGroupOffsets usage example. - */ - -#include <stdio.h> -#include <signal.h> -#include <string.h> -#include <stdlib.h> -#include <stdarg.h> - -#ifdef _WIN32 -#include "../win32/wingetopt.h" -#else -#include <getopt.h> -#endif - - -/* Typical include path would be <librdkafka/rdkafka.h>, but this program - * is builtin from within the librdkafka source tree and thus differs. */ -#include "rdkafka.h" - - -const char *argv0; - -static rd_kafka_queue_t *queue; /** Admin result queue. - * This is a global so we can - * yield in stop() */ -static volatile sig_atomic_t run = 1; - -/** - * @brief Signal termination of program - */ -static void stop(int sig) { - if (!run) { - fprintf(stderr, "%% Forced termination\n"); - exit(2); - } - run = 0; - rd_kafka_queue_yield(queue); -} - - -static void usage(const char *reason, ...) { - - fprintf(stderr, - "Alter consumer group offsets usage examples\n" - "\n" - "Usage: %s <options> <group_id> <topic>\n" - " <partition1> <offset1>\n" - " <partition2> <offset2>\n" - " ...\n" - "\n" - "Options:\n" - " -b <brokers> Bootstrap server list to connect to.\n" - " -X <prop=val> Set librdkafka configuration property.\n" - " See CONFIGURATION.md for full list.\n" - " -d <dbg,..> Enable librdkafka debugging (%s).\n" - "\n", - argv0, rd_kafka_get_debug_contexts()); - - if (reason) { - va_list ap; - char reasonbuf[512]; - - va_start(ap, reason); - vsnprintf(reasonbuf, sizeof(reasonbuf), reason, ap); - va_end(ap); - - fprintf(stderr, "ERROR: %s\n", reasonbuf); - } - - exit(reason ? 1 : 0); -} - - -#define fatal(...) \ - do { \ - fprintf(stderr, "ERROR: "); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr, "\n"); \ - exit(2); \ - } while (0) - - -/** - * @brief Set config property. Exit on failure. - */ -static void conf_set(rd_kafka_conf_t *conf, const char *name, const char *val) { - char errstr[512]; - - if (rd_kafka_conf_set(conf, name, val, errstr, sizeof(errstr)) != - RD_KAFKA_CONF_OK) - fatal("Failed to set %s=%s: %s", name, val, errstr); -} - - -static void -print_partition_list(FILE *fp, - const rd_kafka_topic_partition_list_t *partitions, - int print_offset, - const char *prefix) { - int i; - - if (partitions->cnt == 0) { - fprintf(fp, "%sNo partition found", prefix); - } - for (i = 0; i < partitions->cnt; i++) { - char offset_string[512] = {}; - *offset_string = '\0'; - if (print_offset) { - snprintf(offset_string, sizeof(offset_string), - " offset %" PRId64, - partitions->elems[i].offset); - } - fprintf(fp, "%s%s %s [%" PRId32 "]%s error %s", - i > 0 ? "\n" : "", prefix, partitions->elems[i].topic, - partitions->elems[i].partition, offset_string, - rd_kafka_err2str(partitions->elems[i].err)); - } - fprintf(fp, "\n"); -} - -/** - * @brief Parse an integer or fail. - */ -int64_t parse_int(const char *what, const char *str) { - char *end; - unsigned long n = strtoull(str, &end, 0); - - if (end != str + strlen(str)) { - fprintf(stderr, "%% Invalid input for %s: %s: not an integer\n", - what, str); - exit(1); - } - - return (int64_t)n; -} - -static void -cmd_alter_consumer_group_offsets(rd_kafka_conf_t *conf, int argc, char **argv) { - char errstr[512]; /* librdkafka API error reporting buffer */ - rd_kafka_t *rk; /* Admin client instance */ - rd_kafka_AdminOptions_t *options; /* (Optional) Options for - * AlterConsumerGroupOffsets() */ - rd_kafka_event_t *event; /* AlterConsumerGroupOffsets result event */ - const int min_argc = 2; - int i, num_partitions = 0; - const char *group_id, *topic; - rd_kafka_AlterConsumerGroupOffsets_t *alter_consumer_group_offsets; - - /* - * Argument validation - */ - if (argc < min_argc || (argc - min_argc) % 2 != 0) { - usage("Wrong number of arguments"); - } - - num_partitions = (argc - min_argc) / 2; - group_id = argv[0]; - topic = argv[1]; - - /* - * Create an admin client, it can be created using any client type, - * so we choose producer since it requires no extra configuration - * and is more light-weight than the consumer. - * - * NOTE: rd_kafka_new() takes ownership of the conf object - * and the application must not reference it again after - * this call. - */ - rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr)); - if (!rk) { - fprintf(stderr, "%% Failed to create new producer: %s\n", - errstr); - exit(1); - } - - /* The Admin API is completely asynchronous, results are emitted - * on the result queue that is passed to AlterConsumerGroupOffsets() */ - queue = rd_kafka_queue_new(rk); - - /* Signal handler for clean shutdown */ - signal(SIGINT, stop); - - /* Set timeout (optional) */ - options = rd_kafka_AdminOptions_new( - rk, RD_KAFKA_ADMIN_OP_ALTERCONSUMERGROUPOFFSETS); - if (rd_kafka_AdminOptions_set_request_timeout( - options, 30 * 1000 /* 30s */, errstr, sizeof(errstr))) { - fprintf(stderr, "%% Failed to set timeout: %s\n", errstr); - exit(1); - } - - /* Read passed partition-offsets */ - rd_kafka_topic_partition_list_t *partitions = - rd_kafka_topic_partition_list_new(num_partitions); - for (i = 0; i < num_partitions; i++) { - rd_kafka_topic_partition_list_add( - partitions, topic, - parse_int("partition", argv[min_argc + i * 2])) - ->offset = parse_int("offset", argv[min_argc + 1 + i * 2]); - } - - /* Create argument */ - alter_consumer_group_offsets = - rd_kafka_AlterConsumerGroupOffsets_new(group_id, partitions); - /* Call AlterConsumerGroupOffsets */ - rd_kafka_AlterConsumerGroupOffsets(rk, &alter_consumer_group_offsets, 1, - options, queue); - - /* Clean up input arguments */ - rd_kafka_AlterConsumerGroupOffsets_destroy( - alter_consumer_group_offsets); - rd_kafka_AdminOptions_destroy(options); - rd_kafka_topic_partition_list_destroy(partitions); - - - /* Wait for results */ - event = rd_kafka_queue_poll(queue, -1 /* indefinitely but limited by - * the request timeout set - * above (30s) */); - - if (!event) { - /* User hit Ctrl-C, - * see yield call in stop() signal handler */ - fprintf(stderr, "%% Cancelled by user\n"); - - } else if (rd_kafka_event_error(event)) { - /* AlterConsumerGroupOffsets request failed */ - fprintf(stderr, "%% AlterConsumerGroupOffsets failed: %s\n", - rd_kafka_event_error_string(event)); - exit(1); - - } else { - /* AlterConsumerGroupOffsets request succeeded, but individual - * partitions may have errors. */ - const rd_kafka_AlterConsumerGroupOffsets_result_t *result; - const rd_kafka_group_result_t **groups; - size_t n_groups, i; - - result = rd_kafka_event_AlterConsumerGroupOffsets_result(event); - groups = rd_kafka_AlterConsumerGroupOffsets_result_groups( - result, &n_groups); - - printf("AlterConsumerGroupOffsets results:\n"); - for (i = 0; i < n_groups; i++) { - const rd_kafka_group_result_t *group = groups[i]; - const rd_kafka_topic_partition_list_t *partitions = - rd_kafka_group_result_partitions(group); - print_partition_list(stderr, partitions, 1, " "); - } - } - - /* Destroy event object when we're done with it. - * Note: rd_kafka_event_destroy() allows a NULL event. */ - rd_kafka_event_destroy(event); - - /* Destroy queue */ - rd_kafka_queue_destroy(queue); - - /* Destroy the producer instance */ - rd_kafka_destroy(rk); -} - -int main(int argc, char **argv) { - rd_kafka_conf_t *conf; /**< Client configuration object */ - int opt; - argv0 = argv[0]; - - /* - * Create Kafka client configuration place-holder - */ - conf = rd_kafka_conf_new(); - - - /* - * Parse common options - */ - while ((opt = getopt(argc, argv, "b:X:d:")) != -1) { - switch (opt) { - case 'b': - conf_set(conf, "bootstrap.servers", optarg); - break; - - case 'X': { - char *name = optarg, *val; - - if (!(val = strchr(name, '='))) - fatal("-X expects a name=value argument"); - - *val = '\0'; - val++; - - conf_set(conf, name, val); - break; - } - - case 'd': - conf_set(conf, "debug", optarg); - break; - - default: - usage("Unknown option %c", (char)opt); - } - } - - cmd_alter_consumer_group_offsets(conf, argc - optind, &argv[optind]); - - return 0; -} diff --git a/fluent-bit/lib/librdkafka-2.1.0/examples/consumer.c b/fluent-bit/lib/librdkafka-2.1.0/examples/consumer.c deleted file mode 100644 index 21b27ca78..000000000 --- a/fluent-bit/lib/librdkafka-2.1.0/examples/consumer.c +++ /dev/null @@ -1,260 +0,0 @@ -/* - * librdkafka - Apache Kafka C library - * - * Copyright (c) 2019, Magnus Edenhill - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/** - * Simple high-level balanced Apache Kafka consumer - * using the Kafka driver from librdkafka - * (https://github.com/edenhill/librdkafka) - */ - -#include <stdio.h> -#include <signal.h> -#include <string.h> -#include <ctype.h> - - -/* Typical include path would be <librdkafka/rdkafka.h>, but this program - * is builtin from within the librdkafka source tree and thus differs. */ -//#include <librdkafka/rdkafka.h> -#include "rdkafka.h" - - -static volatile sig_atomic_t run = 1; - -/** - * @brief Signal termination of program - */ -static void stop(int sig) { - run = 0; -} - - - -/** - * @returns 1 if all bytes are printable, else 0. - */ -static int is_printable(const char *buf, size_t size) { - size_t i; - - for (i = 0; i < size; i++) - if (!isprint((int)buf[i])) - return 0; - - return 1; -} - - -int main(int argc, char **argv) { - rd_kafka_t *rk; /* Consumer instance handle */ - rd_kafka_conf_t *conf; /* Temporary configuration object */ - rd_kafka_resp_err_t err; /* librdkafka API error code */ - char errstr[512]; /* librdkafka API error reporting buffer */ - const char *brokers; /* Argument: broker list */ - const char *groupid; /* Argument: Consumer group id */ - char **topics; /* Argument: list of topics to subscribe to */ - int topic_cnt; /* Number of topics to subscribe to */ - rd_kafka_topic_partition_list_t *subscription; /* Subscribed topics */ - int i; - - /* - * Argument validation - */ - if (argc < 4) { - fprintf(stderr, - "%% Usage: " - "%s <broker> <group.id> <topic1> <topic2>..\n", - argv[0]); - return 1; - } - - brokers = argv[1]; - groupid = argv[2]; - topics = &argv[3]; - topic_cnt = argc - 3; - - - /* - * Create Kafka client configuration place-holder - */ - conf = rd_kafka_conf_new(); - - /* Set bootstrap broker(s) as a comma-separated list of - * host or host:port (default port 9092). - * librdkafka will use the bootstrap brokers to acquire the full - * set of brokers from the cluster. */ - if (rd_kafka_conf_set(conf, "bootstrap.servers", brokers, errstr, - sizeof(errstr)) != RD_KAFKA_CONF_OK) { - fprintf(stderr, "%s\n", errstr); - rd_kafka_conf_destroy(conf); - return 1; - } - - /* Set the consumer group id. - * All consumers sharing the same group id will join the same - * group, and the subscribed topic' partitions will be assigned - * according to the partition.assignment.strategy - * (consumer config property) to the consumers in the group. */ - if (rd_kafka_conf_set(conf, "group.id", groupid, errstr, - sizeof(errstr)) != RD_KAFKA_CONF_OK) { - fprintf(stderr, "%s\n", errstr); - rd_kafka_conf_destroy(conf); - return 1; - } - - /* If there is no previously committed offset for a partition - * the auto.offset.reset strategy will be used to decide where - * in the partition to start fetching messages. - * By setting this to earliest the consumer will read all messages - * in the partition if there was no previously committed offset. */ - if (rd_kafka_conf_set(conf, "auto.offset.reset", "earliest", errstr, - sizeof(errstr)) != RD_KAFKA_CONF_OK) { - fprintf(stderr, "%s\n", errstr); - rd_kafka_conf_destroy(conf); - return 1; - } - - /* - * Create consumer instance. - * - * NOTE: rd_kafka_new() takes ownership of the conf object - * and the application must not reference it again after - * this call. - */ - rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, sizeof(errstr)); - if (!rk) { - fprintf(stderr, "%% Failed to create new consumer: %s\n", - errstr); - return 1; - } - - conf = NULL; /* Configuration object is now owned, and freed, - * by the rd_kafka_t instance. */ - - - /* Redirect all messages from per-partition queues to - * the main queue so that messages can be consumed with one - * call from all assigned partitions. - * - * The alternative is to poll the main queue (for events) - * and each partition queue separately, which requires setting - * up a rebalance callback and keeping track of the assignment: - * but that is more complex and typically not recommended. */ - rd_kafka_poll_set_consumer(rk); - - - /* Convert the list of topics to a format suitable for librdkafka */ - subscription = rd_kafka_topic_partition_list_new(topic_cnt); - for (i = 0; i < topic_cnt; i++) - rd_kafka_topic_partition_list_add(subscription, topics[i], - /* the partition is ignored - * by subscribe() */ - RD_KAFKA_PARTITION_UA); - - /* Subscribe to the list of topics */ - err = rd_kafka_subscribe(rk, subscription); - if (err) { - fprintf(stderr, "%% Failed to subscribe to %d topics: %s\n", - subscription->cnt, rd_kafka_err2str(err)); - rd_kafka_topic_partition_list_destroy(subscription); - rd_kafka_destroy(rk); - return 1; - } - - fprintf(stderr, - "%% Subscribed to %d topic(s), " - "waiting for rebalance and messages...\n", - subscription->cnt); - - rd_kafka_topic_partition_list_destroy(subscription); - - - /* Signal handler for clean shutdown */ - signal(SIGINT, stop); - - /* Subscribing to topics will trigger a group rebalance - * which may take some time to finish, but there is no need - * for the application to handle this idle period in a special way - * since a rebalance may happen at any time. - * Start polling for messages. */ - - while (run) { - rd_kafka_message_t *rkm; - - rkm = rd_kafka_consumer_poll(rk, 100); - if (!rkm) - continue; /* Timeout: no message within 100ms, - * try again. This short timeout allows - * checking for `run` at frequent intervals. - */ - - /* consumer_poll() will return either a proper message - * or a consumer error (rkm->err is set). */ - if (rkm->err) { - /* Consumer errors are generally to be considered - * informational as the consumer will automatically - * try to recover from all types of errors. */ - fprintf(stderr, "%% Consumer error: %s\n", - rd_kafka_message_errstr(rkm)); - rd_kafka_message_destroy(rkm); - continue; - } - - /* Proper message. */ - printf("Message on %s [%" PRId32 "] at offset %" PRId64 - " (leader epoch %" PRId32 "):\n", - rd_kafka_topic_name(rkm->rkt), rkm->partition, - rkm->offset, rd_kafka_message_leader_epoch(rkm)); - - /* Print the message key. */ - if (rkm->key && is_printable(rkm->key, rkm->key_len)) - printf(" Key: %.*s\n", (int)rkm->key_len, - (const char *)rkm->key); - else if (rkm->key) - printf(" Key: (%d bytes)\n", (int)rkm->key_len); - - /* Print the message value/payload. */ - if (rkm->payload && is_printable(rkm->payload, rkm->len)) - printf(" Value: %.*s\n", (int)rkm->len, - (const char *)rkm->payload); - else if (rkm->payload) - printf(" Value: (%d bytes)\n", (int)rkm->len); - - rd_kafka_message_destroy(rkm); - } - - - /* Close the consumer: commit final offsets and leave the group. */ - fprintf(stderr, "%% Closing consumer\n"); - rd_kafka_consumer_close(rk); - - - /* Destroy the consumer */ - rd_kafka_destroy(rk); - - return 0; -} diff --git a/fluent-bit/lib/librdkafka-2.1.0/examples/delete_records.c b/fluent-bit/lib/librdkafka-2.1.0/examples/delete_records.c deleted file mode 100644 index 2660996a5..000000000 --- a/fluent-bit/lib/librdkafka-2.1.0/examples/delete_records.c +++ /dev/null @@ -1,233 +0,0 @@ -/* - * librdkafka - Apache Kafka C library - * - * Copyright (c) 2020, Magnus Edenhill - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/** - * Example utility that shows how to use DeleteRecords (AdminAPI) - * do delete all messages/records up to (but not including) a specific offset - * from one or more topic partitions. - */ - -#include <stdio.h> -#include <signal.h> -#include <string.h> -#include <stdlib.h> - - -/* Typical include path would be <librdkafka/rdkafka.h>, but this program - * is builtin from within the librdkafka source tree and thus differs. */ -#include "rdkafka.h" - - -static rd_kafka_queue_t *queue; /** Admin result queue. - * This is a global so we can - * yield in stop() */ -static volatile sig_atomic_t run = 1; - -/** - * @brief Signal termination of program - */ -static void stop(int sig) { - if (!run) { - fprintf(stderr, "%% Forced termination\n"); - exit(2); - } - run = 0; - rd_kafka_queue_yield(queue); -} - - -/** - * @brief Parse an integer or fail. - */ -int64_t parse_int(const char *what, const char *str) { - char *end; - unsigned long n = strtoull(str, &end, 0); - - if (end != str + strlen(str)) { - fprintf(stderr, "%% Invalid input for %s: %s: not an integer\n", - what, str); - exit(1); - } - - return (int64_t)n; -} - - -int main(int argc, char **argv) { - rd_kafka_conf_t *conf; /* Temporary configuration object */ - char errstr[512]; /* librdkafka API error reporting buffer */ - const char *brokers; /* Argument: broker list */ - rd_kafka_t *rk; /* Admin client instance */ - rd_kafka_topic_partition_list_t *offsets_before; /* Delete messages up - * to but not - * including these - * offsets */ - rd_kafka_DeleteRecords_t *del_records; /* Container for offsets_before*/ - rd_kafka_AdminOptions_t *options; /* (Optional) Options for - * DeleteRecords() */ - rd_kafka_event_t *event; /* DeleteRecords result event */ - int exitcode = 0; - int i; - - /* - * Argument validation - */ - if (argc < 5 || (argc - 2) % 3 != 0) { - fprintf(stderr, - "%% Usage: %s <broker> " - "<topic> <partition> <offset_before> " - "<topic2> <partition2> <offset_before2> ...\n" - "\n" - "Delete all messages up to but not including the " - "specified offset(s).\n" - "\n", - argv[0]); - return 1; - } - - brokers = argv[1]; - - /* Parse topic partition offset tuples and add to offsets list */ - offsets_before = rd_kafka_topic_partition_list_new((argc - 2) / 3); - for (i = 2; i < argc; i += 3) { - const char *topic = argv[i]; - int partition = parse_int("partition", argv[i + 1]); - int64_t offset = parse_int("offset_before", argv[i + 2]); - - rd_kafka_topic_partition_list_add(offsets_before, topic, - partition) - ->offset = offset; - } - - /* - * Create Kafka client configuration place-holder - */ - conf = rd_kafka_conf_new(); - - /* Set bootstrap broker(s) as a comma-separated list of - * host or host:port (default port 9092). - * librdkafka will use the bootstrap brokers to acquire the full - * set of brokers from the cluster. */ - if (rd_kafka_conf_set(conf, "bootstrap.servers", brokers, errstr, - sizeof(errstr)) != RD_KAFKA_CONF_OK) { - fprintf(stderr, "%s\n", errstr); - return 1; - } - rd_kafka_conf_set(conf, "debug", "admin,topic,metadata", NULL, 0); - - /* - * Create an admin client, it can be created using any client type, - * so we choose producer since it requires no extra configuration - * and is more light-weight than the consumer. - * - * NOTE: rd_kafka_new() takes ownership of the conf object - * and the application must not reference it again after - * this call. - */ - rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr)); - if (!rk) { - fprintf(stderr, "%% Failed to create new producer: %s\n", - errstr); - return 1; - } - - /* The Admin API is completely asynchronous, results are emitted - * on the result queue that is passed to DeleteRecords() */ - queue = rd_kafka_queue_new(rk); - - /* Signal handler for clean shutdown */ - signal(SIGINT, stop); - - /* Set timeout (optional) */ - options = - rd_kafka_AdminOptions_new(rk, RD_KAFKA_ADMIN_OP_DELETERECORDS); - if (rd_kafka_AdminOptions_set_request_timeout( - options, 30 * 1000 /* 30s */, errstr, sizeof(errstr))) { - fprintf(stderr, "%% Failed to set timeout: %s\n", errstr); - return 1; - } - - /* Create argument */ - del_records = rd_kafka_DeleteRecords_new(offsets_before); - /* We're now done with offsets_before */ - rd_kafka_topic_partition_list_destroy(offsets_before); - - /* Call DeleteRecords */ - rd_kafka_DeleteRecords(rk, &del_records, 1, options, queue); - - /* Clean up input arguments */ - rd_kafka_DeleteRecords_destroy(del_records); - rd_kafka_AdminOptions_destroy(options); - - - /* Wait for results */ - event = rd_kafka_queue_poll(queue, -1 /*indefinitely*/); - - if (!event) { - /* User hit Ctrl-C */ - fprintf(stderr, "%% Cancelled by user\n"); - - } else if (rd_kafka_event_error(event)) { - /* DeleteRecords request failed */ - fprintf(stderr, "%% DeleteRecords failed: %s\n", - rd_kafka_event_error_string(event)); - exitcode = 2; - - } else { - /* DeleteRecords request succeeded, but individual - * partitions may have errors. */ - const rd_kafka_DeleteRecords_result_t *result; - const rd_kafka_topic_partition_list_t *offsets; - int i; - - result = rd_kafka_event_DeleteRecords_result(event); - offsets = rd_kafka_DeleteRecords_result_offsets(result); - - printf("DeleteRecords results:\n"); - for (i = 0; i < offsets->cnt; i++) - printf(" %s [%" PRId32 "] offset %" PRId64 ": %s\n", - offsets->elems[i].topic, - offsets->elems[i].partition, - offsets->elems[i].offset, - rd_kafka_err2str(offsets->elems[i].err)); - } - - /* Destroy event object when we're done with it. - * Note: rd_kafka_event_destroy() allows a NULL event. */ - rd_kafka_event_destroy(event); - - signal(SIGINT, SIG_DFL); - - /* Destroy queue */ - rd_kafka_queue_destroy(queue); - - /* Destroy the producer instance */ - rd_kafka_destroy(rk); - - return exitcode; -} diff --git a/fluent-bit/lib/librdkafka-2.1.0/examples/describe_consumer_groups.c b/fluent-bit/lib/librdkafka-2.1.0/examples/describe_consumer_groups.c deleted file mode 100644 index 45b6b8d0b..000000000 --- a/fluent-bit/lib/librdkafka-2.1.0/examples/describe_consumer_groups.c +++ /dev/null @@ -1,373 +0,0 @@ -/* - * librdkafka - Apache Kafka C library - * - * Copyright (c) 2022, Magnus Edenhill - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/** - * DescribeConsumerGroups usage example. - */ - -#include <stdio.h> -#include <signal.h> -#include <string.h> -#include <stdlib.h> -#include <stdarg.h> - -#ifdef _WIN32 -#include "../win32/wingetopt.h" -#else -#include <getopt.h> -#endif - - -/* Typical include path would be <librdkafka/rdkafka.h>, but this program - * is builtin from within the librdkafka source tree and thus differs. */ -#include "rdkafka.h" - - -const char *argv0; - -static rd_kafka_queue_t *queue; /** Admin result queue. - * This is a global so we can - * yield in stop() */ -static volatile sig_atomic_t run = 1; - -/** - * @brief Signal termination of program - */ -static void stop(int sig) { - if (!run) { - fprintf(stderr, "%% Forced termination\n"); - exit(2); - } - run = 0; - rd_kafka_queue_yield(queue); -} - - -static void usage(const char *reason, ...) { - - fprintf(stderr, - "Describe groups usage examples\n" - "\n" - "Usage: %s <options> <group1> <group2> ...\n" - "\n" - "Options:\n" - " -b <brokers> Bootstrap server list to connect to.\n" - " -X <prop=val> Set librdkafka configuration property.\n" - " See CONFIGURATION.md for full list.\n" - " -d <dbg,..> Enable librdkafka debugging (%s).\n" - "\n", - argv0, rd_kafka_get_debug_contexts()); - - if (reason) { - va_list ap; - char reasonbuf[512]; - - va_start(ap, reason); - vsnprintf(reasonbuf, sizeof(reasonbuf), reason, ap); - va_end(ap); - - fprintf(stderr, "ERROR: %s\n", reasonbuf); - } - - exit(reason ? 1 : 0); -} - - -#define fatal(...) \ - do { \ - fprintf(stderr, "ERROR: "); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr, "\n"); \ - exit(2); \ - } while (0) - - -/** - * @brief Set config property. Exit on failure. - */ -static void conf_set(rd_kafka_conf_t *conf, const char *name, const char *val) { - char errstr[512]; - - if (rd_kafka_conf_set(conf, name, val, errstr, sizeof(errstr)) != - RD_KAFKA_CONF_OK) - fatal("Failed to set %s=%s: %s", name, val, errstr); -} - - -static void -print_partition_list(FILE *fp, - const rd_kafka_topic_partition_list_t *partitions, - int print_offset, - const char *prefix) { - int i; - - if (partitions->cnt == 0) { - fprintf(fp, "%sNo partition found", prefix); - } - for (i = 0; i < partitions->cnt; i++) { - char offset_string[512] = {}; - *offset_string = '\0'; - if (print_offset) { - snprintf(offset_string, sizeof(offset_string), - " offset %" PRId64, - partitions->elems[i].offset); - } - fprintf(fp, "%s%s %s [%" PRId32 "]%s error %s", - i > 0 ? "\n" : "", prefix, partitions->elems[i].topic, - partitions->elems[i].partition, offset_string, - rd_kafka_err2str(partitions->elems[i].err)); - } - fprintf(fp, "\n"); -} - -/** - * @brief Print group information. - */ -static int -print_groups_info(const rd_kafka_DescribeConsumerGroups_result_t *grpdesc, - int groups_cnt) { - size_t i; - const rd_kafka_ConsumerGroupDescription_t **result_groups; - size_t result_groups_cnt; - result_groups = rd_kafka_DescribeConsumerGroups_result_groups( - grpdesc, &result_groups_cnt); - - if (result_groups_cnt == 0) { - if (groups_cnt > 0) { - fprintf(stderr, "No matching groups found\n"); - return 1; - } else { - fprintf(stderr, "No groups in cluster\n"); - } - } - - for (i = 0; i < result_groups_cnt; i++) { - int j, member_cnt; - const rd_kafka_error_t *error; - const rd_kafka_ConsumerGroupDescription_t *group = - result_groups[i]; - char coordinator_desc[512]; - const rd_kafka_Node_t *coordinator = NULL; - const char *group_id = - rd_kafka_ConsumerGroupDescription_group_id(group); - const char *partition_assignor = - rd_kafka_ConsumerGroupDescription_partition_assignor(group); - rd_kafka_consumer_group_state_t state = - rd_kafka_ConsumerGroupDescription_state(group); - member_cnt = - rd_kafka_ConsumerGroupDescription_member_count(group); - error = rd_kafka_ConsumerGroupDescription_error(group); - coordinator = - rd_kafka_ConsumerGroupDescription_coordinator(group); - *coordinator_desc = '\0'; - - if (coordinator != NULL) { - snprintf(coordinator_desc, sizeof(coordinator_desc), - ", coordinator [id: %" PRId32 - ", host: %s" - ", port: %" PRIu16 "]", - rd_kafka_Node_id(coordinator), - rd_kafka_Node_host(coordinator), - rd_kafka_Node_port(coordinator)); - } - printf( - "Group \"%s\", partition assignor \"%s\", " - "state %s%s, with %" PRId32 " member(s)", - group_id, partition_assignor, - rd_kafka_consumer_group_state_name(state), coordinator_desc, - member_cnt); - if (error) - printf(" error[%" PRId32 "]: %s", - rd_kafka_error_code(error), - rd_kafka_error_string(error)); - printf("\n"); - for (j = 0; j < member_cnt; j++) { - const rd_kafka_MemberDescription_t *member = - rd_kafka_ConsumerGroupDescription_member(group, j); - printf( - " Member \"%s\" with client-id %s," - " group instance id: %s, host %s\n", - rd_kafka_MemberDescription_consumer_id(member), - rd_kafka_MemberDescription_client_id(member), - rd_kafka_MemberDescription_group_instance_id( - member), - rd_kafka_MemberDescription_host(member)); - const rd_kafka_MemberAssignment_t *assignment = - rd_kafka_MemberDescription_assignment(member); - const rd_kafka_topic_partition_list_t - *topic_partitions = - rd_kafka_MemberAssignment_partitions( - assignment); - if (!topic_partitions) { - printf(" No assignment\n"); - } else if (topic_partitions->cnt == 0) { - printf(" Empty assignment\n"); - } else { - printf(" Assignment:\n"); - print_partition_list(stdout, topic_partitions, - 0, " "); - } - } - } - return 0; -} - -/** - * @brief Call rd_kafka_DescribeConsumerGroups() with a list of - * groups. - */ -static void -cmd_describe_consumer_groups(rd_kafka_conf_t *conf, int argc, char **argv) { - rd_kafka_t *rk; - const char **groups = NULL; - char errstr[512]; - rd_kafka_AdminOptions_t *options; - rd_kafka_event_t *event = NULL; - int retval = 0; - int groups_cnt = 0; - - if (argc >= 1) { - groups = (const char **)&argv[0]; - groups_cnt = argc; - } - - /* - * Create consumer instance - * NOTE: rd_kafka_new() takes ownership of the conf object - * and the application must not reference it again after - * this call. - */ - rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, sizeof(errstr)); - if (!rk) - fatal("Failed to create new consumer: %s", errstr); - - /* - * Describe consumer groups - */ - queue = rd_kafka_queue_new(rk); - - /* Signal handler for clean shutdown */ - signal(SIGINT, stop); - - options = rd_kafka_AdminOptions_new( - rk, RD_KAFKA_ADMIN_OP_DESCRIBECONSUMERGROUPS); - - if (rd_kafka_AdminOptions_set_request_timeout( - options, 10 * 1000 /* 10s */, errstr, sizeof(errstr))) { - fprintf(stderr, "%% Failed to set timeout: %s\n", errstr); - goto exit; - } - - rd_kafka_DescribeConsumerGroups(rk, groups, groups_cnt, options, queue); - - /* Wait for results */ - event = rd_kafka_queue_poll(queue, -1 /* indefinitely but limited by - * the request timeout set - * above (10s) */); - - if (!event) { - /* User hit Ctrl-C, - * see yield call in stop() signal handler */ - fprintf(stderr, "%% Cancelled by user\n"); - - } else if (rd_kafka_event_error(event)) { - rd_kafka_resp_err_t err = rd_kafka_event_error(event); - /* DescribeConsumerGroups request failed */ - fprintf(stderr, - "%% DescribeConsumerGroups failed[%" PRId32 "]: %s\n", - err, rd_kafka_event_error_string(event)); - goto exit; - - } else { - /* DescribeConsumerGroups request succeeded, but individual - * groups may have errors. */ - const rd_kafka_DescribeConsumerGroups_result_t *result; - - result = rd_kafka_event_DescribeConsumerGroups_result(event); - printf("DescribeConsumerGroups results:\n"); - retval = print_groups_info(result, groups_cnt); - } - - -exit: - if (event) - rd_kafka_event_destroy(event); - rd_kafka_AdminOptions_destroy(options); - rd_kafka_queue_destroy(queue); - /* Destroy the client instance */ - rd_kafka_destroy(rk); - - exit(retval); -} - -int main(int argc, char **argv) { - rd_kafka_conf_t *conf; /**< Client configuration object */ - int opt; - argv0 = argv[0]; - - /* - * Create Kafka client configuration place-holder - */ - conf = rd_kafka_conf_new(); - - - /* - * Parse common options - */ - while ((opt = getopt(argc, argv, "b:X:d:")) != -1) { - switch (opt) { - case 'b': - conf_set(conf, "bootstrap.servers", optarg); - break; - - case 'X': { - char *name = optarg, *val; - - if (!(val = strchr(name, '='))) - fatal("-X expects a name=value argument"); - - *val = '\0'; - val++; - - conf_set(conf, name, val); - break; - } - - case 'd': - conf_set(conf, "debug", optarg); - break; - - default: - usage("Unknown option %c", (char)opt); - } - } - - cmd_describe_consumer_groups(conf, argc - optind, &argv[optind]); - - return 0; -} diff --git a/fluent-bit/lib/librdkafka-2.1.0/examples/globals.json b/fluent-bit/lib/librdkafka-2.1.0/examples/globals.json deleted file mode 100644 index 527e1262b..000000000 --- a/fluent-bit/lib/librdkafka-2.1.0/examples/globals.json +++ /dev/null @@ -1,11 +0,0 @@ -{"VerifiableConsumer": - { - "class": "kafkatest.services.verifiable_client.VerifiableClientApp", - "exec_cmd": "/vagrant/tests/c/kafkatest_verifiable_client --consumer --debug cgrp,topic,protocol,broker" - }, - "VerifiableProducer": - { - "class": "kafkatest.services.verifiable_client.VerifiableClientApp", - "exec_cmd": "/vagrant/tests/c/kafkatest_verifiable_client --producer --debug topic,broker" - } -} diff --git a/fluent-bit/lib/librdkafka-2.1.0/examples/idempotent_producer.c b/fluent-bit/lib/librdkafka-2.1.0/examples/idempotent_producer.c deleted file mode 100644 index 91b42a4b9..000000000 --- a/fluent-bit/lib/librdkafka-2.1.0/examples/idempotent_producer.c +++ /dev/null @@ -1,344 +0,0 @@ -/* - * librdkafka - Apache Kafka C library - * - * Copyright (c) 2019, Magnus Edenhill - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/** - * Idempotent Producer example. - * - * The idempotent producer provides strict ordering and - * exactly-once producing guarantees. - * - * From the application developer's perspective, the only difference - * from a standard producer is the enabling of the feature by setting - * the `enable.idempotence` configuration property to `true`, and - * handling fatal (RD_KAFKA_RESP_ERR__FATAL) errors which are raised when - * the idempotent guarantees can't be satisfied. - */ - -#define _DEFAULT_SOURCE /* avoid glibc deprecation warning of _BSD_SOURCE */ -#define _BSD_SOURCE /* vsnprintf() */ -#include <stdio.h> -#include <signal.h> -#include <unistd.h> -#include <string.h> - - -/* Typical include path would be <librdkafka/rdkafka.h>, but this program - * is builtin from within the librdkafka source tree and thus differs. */ -#include "rdkafka.h" - - -static volatile sig_atomic_t run = 1; - -/** - * @brief Signal termination of program - */ -static void stop(int sig) { - run = 0; -} - - -static int deliveredcnt = 0; -static int msgerrcnt = 0; - -/** - * @brief Message delivery report callback. - * - * This callback is called exactly once per message, indicating if - * the message was succesfully delivered - * (rkmessage->err == RD_KAFKA_RESP_ERR_NO_ERROR) or permanently - * failed delivery (rkmessage->err != RD_KAFKA_RESP_ERR_NO_ERROR). - * - * The callback is triggered from rd_kafka_poll() or rd_kafka_flush() and - * executes on the application's thread. - */ -static void -dr_msg_cb(rd_kafka_t *rk, const rd_kafka_message_t *rkmessage, void *opaque) { - if (rkmessage->err) { - fprintf(stderr, "%% Message delivery failed: %s\n", - rd_kafka_err2str(rkmessage->err)); - msgerrcnt++; - } else { - fprintf(stderr, - "%% Message delivered (%zd bytes, topic %s, " - "partition %" PRId32 ", offset %" PRId64 ")\n", - rkmessage->len, rd_kafka_topic_name(rkmessage->rkt), - rkmessage->partition, rkmessage->offset); - deliveredcnt++; - } - - /* The rkmessage is destroyed automatically by librdkafka */ -} - - -/** - * @brief Generic error handling callback. - * - * This callback is triggered by rd_kafka_poll() or rd_kafka_flush() - * for client instance-level errors, such as broker connection failures, - * authentication issues, etc. - * - * These errors should generally be considered informational as - * the underlying client will automatically try to recover from - * any errors encountered, the application does not need to take - * action on them. - * - * But with idempotence truly fatal errors can be raised when - * the idempotence guarantees can't be satisfied, these errors - * are identified by a the `RD_KAFKA_RESP_ERR__FATAL` error code. - */ -static void -error_cb(rd_kafka_t *rk, int err, const char *reason, void *opaque) { - rd_kafka_resp_err_t orig_err; - char errstr[512]; - - fprintf(stderr, "%% Error: %s: %s\n", rd_kafka_err2name(err), reason); - - if (err != RD_KAFKA_RESP_ERR__FATAL) - return; - - /* Fatal error handling. - * - * When a fatal error is detected by the producer instance, - * it will trigger an error_cb with ERR__FATAL set. - * The application should use rd_kafka_fatal_error() to extract - * the actual underlying error code and description, propagate it - * to the user (for troubleshooting), and then terminate the - * producer since it will no longer accept any new messages to - * produce(). - * - * Note: - * After a fatal error has been raised, rd_kafka_produce*() will - * fail with the original error code. - * - * Note: - * As an alternative to an error_cb, the application may call - * rd_kafka_fatal_error() at any time to check if a fatal error - * has occurred, typically after a failing rd_kafka_produce*() call. - */ - - orig_err = rd_kafka_fatal_error(rk, errstr, sizeof(errstr)); - fprintf(stderr, "%% FATAL ERROR: %s: %s\n", rd_kafka_err2name(orig_err), - errstr); - - /* Clean termination to get delivery results (from rd_kafka_flush()) - * for all outstanding/in-transit/queued messages. */ - fprintf(stderr, "%% Terminating on fatal error\n"); - run = 0; -} - - -int main(int argc, char **argv) { - rd_kafka_t *rk; /* Producer instance handle */ - rd_kafka_conf_t *conf; /* Temporary configuration object */ - char errstr[512]; /* librdkafka API error reporting buffer */ - rd_kafka_resp_err_t err; /* librdkafka API error code */ - const char *brokers; /* Argument: broker list */ - const char *topic; /* Argument: topic to produce to */ - int msgcnt = 0; /* Number of messages produced */ - - /* - * Argument validation - */ - if (argc != 3) { - fprintf(stderr, "%% Usage: %s <broker> <topic>\n", argv[0]); - return 1; - } - - brokers = argv[1]; - topic = argv[2]; - - - /* - * Create Kafka client configuration place-holder - */ - conf = rd_kafka_conf_new(); - - /* Set bootstrap broker(s) as a comma-separated list of - * host or host:port (default port 9092). - * librdkafka will use the bootstrap brokers to acquire the full - * set of brokers from the cluster. */ - if (rd_kafka_conf_set(conf, "bootstrap.servers", brokers, errstr, - sizeof(errstr)) != RD_KAFKA_CONF_OK) { - fprintf(stderr, "%s\n", errstr); - rd_kafka_conf_destroy(conf); - return 1; - } - - /* Enable the idempotent producer */ - if (rd_kafka_conf_set(conf, "enable.idempotence", "true", errstr, - sizeof(errstr)) != RD_KAFKA_CONF_OK) { - fprintf(stderr, "%s\n", errstr); - rd_kafka_conf_destroy(conf); - return 1; - } - - /* Set the delivery report callback. - * This callback will be called once per message to inform - * the application if delivery succeeded or failed. - * See dr_msg_cb() above. */ - rd_kafka_conf_set_dr_msg_cb(conf, dr_msg_cb); - - /* Set an error handler callback to catch generic instance-level - * errors. - * - * See the `error_cb()` handler above for how to handle the - * fatal errors. - */ - rd_kafka_conf_set_error_cb(conf, error_cb); - - - /* - * Create producer instance. - * - * NOTE: rd_kafka_new() takes ownership of the conf object - * and the application must not reference it again after - * this call. - */ - rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr)); - if (!rk) { - fprintf(stderr, "%% Failed to create new producer: %s\n", - errstr); - return 1; - } - - /* Signal handler for clean shutdown */ - signal(SIGINT, stop); - - fprintf(stderr, "%% Running producer loop. Press Ctrl-C to exit\n"); - - while (run) { - char buf[64]; - - snprintf(buf, sizeof(buf), - "Idempotent Producer example message #%d", msgcnt); - - /* - * Produce message. - * This is an asynchronous call, on success it will only - * enqueue the message on the internal producer queue. - * The actual delivery attempts to the broker are handled - * by background threads. - * The previously registered delivery report callback - * (dr_msg_cb) is used to signal back to the application - * when the message has been delivered (or failed), - * and is triggered when the application calls - * rd_kafka_poll() or rd_kafka_flush(). - */ - retry: - err = rd_kafka_producev( - rk, RD_KAFKA_V_TOPIC(topic), - RD_KAFKA_V_VALUE(buf, strlen(buf)), - /* Copy the message payload so the `buf` can - * be reused for the next message. */ - RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY), RD_KAFKA_V_END); - - if (err) { - /** - * Failed to *enqueue* message for producing. - */ - fprintf(stderr, - "%% Failed to produce to topic %s: %s\n", topic, - rd_kafka_err2str(err)); - - if (err == RD_KAFKA_RESP_ERR__QUEUE_FULL) { - /* If the internal queue is full, wait for - * messages to be delivered and then retry. - * The internal queue represents both - * messages to be sent and messages that have - * been sent or failed, awaiting their - * delivery report callback to be called. - * - * The internal queue is limited by the - * configuration property - * queue.buffering.max.messages and - * queue.buffering.max.kbytes */ - rd_kafka_poll(rk, - 1000 /*block for max 1000ms*/); - goto retry; - } else { - /* Produce failed, most likely due to a - * fatal error (will be handled by error_cb()), - * bail out. */ - - /* Instead of using the error_cb(), an - * application may check for fatal errors here - * by calling rd_kafka_fatal_error(). */ - break; - } - } - - /* A producer application should continually serve - * the delivery report queue by calling rd_kafka_poll() - * at frequent intervals. - * Either put the poll call in your main loop, or in a - * dedicated thread, or call it after or before every - * rd_kafka_produce*() call. - * Just make sure that rd_kafka_poll() is still called - * during periods where you are not producing any messages - * to make sure previously produced messages have their - * delivery report callback served (and any other callbacks - * you register). */ - rd_kafka_poll(rk, 0 /*non-blocking*/); - - msgcnt++; - - /* Since fatal errors can't be triggered in practice, - * use the test API to trigger a fabricated error after - * some time. */ - if (msgcnt == 13) - rd_kafka_test_fatal_error( - rk, RD_KAFKA_RESP_ERR_OUT_OF_ORDER_SEQUENCE_NUMBER, - "This is a fabricated error to test the " - "fatal error handling"); - - /* Short sleep to rate-limit this example. - * A real application should not do this. */ - usleep(500 * 1000); /* 500ms */ - } - - - /* Wait for final messages to be delivered or fail. - * rd_kafka_flush() is an abstraction over rd_kafka_poll() which - * waits for all messages to be delivered. */ - fprintf(stderr, "%% Flushing outstanding messages..\n"); - rd_kafka_flush(rk, 10 * 1000 /* wait for max 10 seconds */); - fprintf(stderr, "%% %d message(s) produced, %d delivered, %d failed\n", - msgcnt, deliveredcnt, msgerrcnt); - - /* Save fatal error prior for using with exit status below. */ - err = rd_kafka_fatal_error(rk, NULL, 0); - - /* Destroy the producer instance */ - rd_kafka_destroy(rk); - - /* Exit application with an error (1) if there was a fatal error. */ - if (err) - return 1; - else - return 0; -} diff --git a/fluent-bit/lib/librdkafka-2.1.0/examples/kafkatest_verifiable_client.cpp b/fluent-bit/lib/librdkafka-2.1.0/examples/kafkatest_verifiable_client.cpp deleted file mode 100644 index bdb8607a3..000000000 --- a/fluent-bit/lib/librdkafka-2.1.0/examples/kafkatest_verifiable_client.cpp +++ /dev/null @@ -1,961 +0,0 @@ -/* - * Copyright (c) 2015, Confluent Inc - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/** - * librdkafka version of the Java VerifiableProducer and VerifiableConsumer - * for use with the official Kafka client tests. - */ - - -#include <iostream> -#include <fstream> -#include <sstream> -#include <map> -#include <string> -#include <algorithm> -#include <cstdlib> -#include <cstdio> -#include <csignal> -#include <cstring> -#include <unistd.h> -#include <sys/time.h> -#include <assert.h> -#include <ctype.h> -#include <strings.h> - -#ifdef _WIN32 -#include "../win32/wingetopt.h" -#elif _AIX -#include <unistd.h> -#else -#include <getopt.h> -#endif - -/* - * Typically include path in a real application would be - * #include <librdkafka/rdkafkacpp.h> - */ -#include "rdkafkacpp.h" - -static volatile sig_atomic_t run = 1; -static bool exit_eof = false; -static int verbosity = 1; -static std::string value_prefix; - -class Assignment { - public: - static std::string name(const std::string &t, int partition) { - std::stringstream stm; - stm << t << "." << partition; - return stm.str(); - } - - Assignment() : - topic(""), - partition(-1), - consumedMessages(0), - minOffset(-1), - maxOffset(0) { - printf("Created assignment\n"); - } - Assignment(const Assignment &a) { - topic = a.topic; - partition = a.partition; - consumedMessages = a.consumedMessages; - minOffset = a.minOffset; - maxOffset = a.maxOffset; - } - - Assignment &operator=(const Assignment &a) { - this->topic = a.topic; - this->partition = a.partition; - this->consumedMessages = a.consumedMessages; - this->minOffset = a.minOffset; - this->maxOffset = a.maxOffset; - return *this; - } - - int operator==(const Assignment &a) const { - return !(this->topic == a.topic && this->partition == a.partition); - } - - int operator<(const Assignment &a) const { - if (this->topic < a.topic) - return 1; - if (this->topic >= a.topic) - return 0; - return (this->partition < a.partition); - } - - void setup(std::string t, int32_t p) { - assert(!t.empty()); - assert(topic.empty() || topic == t); - assert(partition == -1 || partition == p); - topic = t; - partition = p; - } - - std::string topic; - int partition; - int consumedMessages; - int64_t minOffset; - int64_t maxOffset; -}; - - - -static struct { - int maxMessages; - - struct { - int numAcked; - int numSent; - int numErr; - } producer; - - struct { - int consumedMessages; - int consumedMessagesLastReported; - int consumedMessagesAtLastCommit; - bool useAutoCommit; - std::map<std::string, Assignment> assignments; - } consumer; -} state = { - /* .maxMessages = */ -1}; - - -static RdKafka::KafkaConsumer *consumer; - - -static std::string now() { - struct timeval tv; - gettimeofday(&tv, NULL); - time_t t = tv.tv_sec; - struct tm tm; - char buf[64]; - - localtime_r(&t, &tm); - strftime(buf, sizeof(buf), "%H:%M:%S", &tm); - snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf), ".%03d", - (int)(tv.tv_usec / 1000)); - - return buf; -} - - -static time_t watchdog_last_kick; -static const int watchdog_timeout = 20; /* Must be > socket.timeout.ms */ -static void sigwatchdog(int sig) { - time_t t = time(NULL); - if (watchdog_last_kick + watchdog_timeout <= t) { - std::cerr << now() << ": WATCHDOG TIMEOUT (" - << (int)(t - watchdog_last_kick) << "s): TERMINATING" - << std::endl; - int *i = NULL; - *i = 100; - abort(); - } -} - -static void watchdog_kick() { - watchdog_last_kick = time(NULL); - - /* Safe guard against hangs-on-exit */ - alarm(watchdog_timeout); -} - - - -static void errorString(const std::string &name, - const std::string &errmsg, - const std::string &topic, - const std::string *key, - const std::string &value) { - std::cout << "{ " - << "\"name\": \"" << name << "\", " - << "\"_time\": \"" << now() << "\", " - << "\"message\": \"" << errmsg << "\", " - << "\"topic\": \"" << topic << "\", " - << "\"key\": \"" << (key ? *key : "NULL") << "\", " - << "\"value\": \"" << value << "\" " - << "}" << std::endl; -} - - -static void successString(const std::string &name, - const std::string &topic, - int partition, - int64_t offset, - const std::string *key, - const std::string &value) { - std::cout << "{ " - << "\"name\": \"" << name << "\", " - << "\"_time\": \"" << now() << "\", " - << "\"topic\": \"" << topic << "\", " - << "\"partition\": " << partition << ", " - << "\"offset\": " << offset << ", " - << "\"key\": \"" << (key ? *key : "NULL") << "\", " - << "\"value\": \"" << value << "\" " - << "}" << std::endl; -} - - -#if FIXME -static void offsetStatus(bool success, - const std::string &topic, - int partition, - int64_t offset, - const std::string &errstr) { - std::cout << "{ " - "\"name\": \"offsets_committed\", " - << "\"success\": " << success << ", " - << "\"offsets\": [ " - << " { " - << " \"topic\": \"" << topic << "\", " - << " \"partition\": " << partition << ", " - << " \"offset\": " << (int)offset << ", " - << " \"error\": \"" << errstr << "\" " - << " } " - << "] }" << std::endl; -} -#endif - - -static void sigterm(int sig) { - std::cerr << now() << ": Terminating because of signal " << sig << std::endl; - - if (!run) { - std::cerr << now() << ": Forced termination" << std::endl; - exit(1); - } - run = 0; -} - - -class ExampleDeliveryReportCb : public RdKafka::DeliveryReportCb { - public: - void dr_cb(RdKafka::Message &message) { - if (message.err()) { - state.producer.numErr++; - errorString("producer_send_error", message.errstr(), message.topic_name(), - message.key(), - std::string(static_cast<const char *>(message.payload()), - message.len())); - } else { - successString("producer_send_success", message.topic_name(), - (int)message.partition(), message.offset(), message.key(), - std::string(static_cast<const char *>(message.payload()), - message.len())); - state.producer.numAcked++; - } - } -}; - - -class ExampleEventCb : public RdKafka::EventCb { - public: - void event_cb(RdKafka::Event &event) { - switch (event.type()) { - case RdKafka::Event::EVENT_ERROR: - std::cerr << now() << ": ERROR (" << RdKafka::err2str(event.err()) - << "): " << event.str() << std::endl; - break; - - case RdKafka::Event::EVENT_STATS: - std::cerr << now() << ": \"STATS\": " << event.str() << std::endl; - break; - - case RdKafka::Event::EVENT_LOG: - std::cerr << now() << ": LOG-" << event.severity() << "-" << event.fac() - << ": " << event.str() << std::endl; - break; - - default: - std::cerr << now() << ": EVENT " << event.type() << " (" - << RdKafka::err2str(event.err()) << "): " << event.str() - << std::endl; - break; - } - } -}; - - -/* Use of this partitioner is pretty pointless since no key is provided - * in the produce() call. */ -class MyHashPartitionerCb : public RdKafka::PartitionerCb { - public: - int32_t partitioner_cb(const RdKafka::Topic *topic, - const std::string *key, - int32_t partition_cnt, - void *msg_opaque) { - return djb_hash(key->c_str(), key->size()) % partition_cnt; - } - - private: - static inline unsigned int djb_hash(const char *str, size_t len) { - unsigned int hash = 5381; - for (size_t i = 0; i < len; i++) - hash = ((hash << 5) + hash) + str[i]; - return hash; - } -}; - - - -/** - * Print number of records consumed, every 100 messages or on timeout. - */ -static void report_records_consumed(int immediate) { - std::map<std::string, Assignment> *assignments = &state.consumer.assignments; - - if (state.consumer.consumedMessages <= - state.consumer.consumedMessagesLastReported + (immediate ? 0 : 999)) - return; - - std::cout << "{ " - "\"name\": \"records_consumed\", " - << "\"_totcount\": " << state.consumer.consumedMessages << ", " - << "\"count\": " - << (state.consumer.consumedMessages - - state.consumer.consumedMessagesLastReported) - << ", " - << "\"partitions\": [ "; - - for (std::map<std::string, Assignment>::iterator ii = assignments->begin(); - ii != assignments->end(); ii++) { - Assignment *a = &(*ii).second; - assert(!a->topic.empty()); - std::cout << (ii == assignments->begin() ? "" : ", ") << " { " - << " \"topic\": \"" << a->topic << "\", " - << " \"partition\": " << a->partition << ", " - << " \"minOffset\": " << a->minOffset << ", " - << " \"maxOffset\": " << a->maxOffset << " " - << " } "; - a->minOffset = -1; - } - - std::cout << "] }" << std::endl; - - state.consumer.consumedMessagesLastReported = state.consumer.consumedMessages; -} - - -class ExampleOffsetCommitCb : public RdKafka::OffsetCommitCb { - public: - void offset_commit_cb(RdKafka::ErrorCode err, - std::vector<RdKafka::TopicPartition *> &offsets) { - std::cerr << now() << ": Propagate offset for " << offsets.size() - << " partitions, error: " << RdKafka::err2str(err) << std::endl; - - /* No offsets to commit, dont report anything. */ - if (err == RdKafka::ERR__NO_OFFSET) - return; - - /* Send up-to-date records_consumed report to make sure consumed > committed - */ - report_records_consumed(1); - - std::cout << "{ " - << "\"name\": \"offsets_committed\", " - << "\"success\": " << (err ? "false" : "true") << ", " - << "\"error\": \"" << (err ? RdKafka::err2str(err) : "") << "\", " - << "\"_autocommit\": " - << (state.consumer.useAutoCommit ? "true" : "false") << ", " - << "\"offsets\": [ "; - assert(offsets.size() > 0); - for (unsigned int i = 0; i < offsets.size(); i++) { - std::cout << (i == 0 ? "" : ", ") << "{ " - << " \"topic\": \"" << offsets[i]->topic() << "\", " - << " \"partition\": " << offsets[i]->partition() << ", " - << " \"offset\": " << (int)offsets[i]->offset() << ", " - << " \"error\": \"" - << (offsets[i]->err() ? RdKafka::err2str(offsets[i]->err()) - : "") - << "\" " - << " }"; - } - std::cout << " ] }" << std::endl; - } -}; - -static ExampleOffsetCommitCb ex_offset_commit_cb; - - -/** - * Commit every 1000 messages or whenever there is a consume timeout. - */ -static void do_commit(RdKafka::KafkaConsumer *consumer, int immediate) { - if (!immediate && (state.consumer.useAutoCommit || - state.consumer.consumedMessagesAtLastCommit + 1000 > - state.consumer.consumedMessages)) - return; - - /* Make sure we report consumption before commit, - * otherwise tests may fail because of commit > consumed. */ - if (state.consumer.consumedMessagesLastReported < - state.consumer.consumedMessages) - report_records_consumed(1); - - std::cerr << now() << ": committing " - << (state.consumer.consumedMessages - - state.consumer.consumedMessagesAtLastCommit) - << " messages" << std::endl; - - RdKafka::ErrorCode err; - err = consumer->commitSync(&ex_offset_commit_cb); - - std::cerr << now() << ": " - << "sync commit returned " << RdKafka::err2str(err) << std::endl; - - state.consumer.consumedMessagesAtLastCommit = state.consumer.consumedMessages; -} - - -void msg_consume(RdKafka::KafkaConsumer *consumer, - RdKafka::Message *msg, - void *opaque) { - switch (msg->err()) { - case RdKafka::ERR__TIMED_OUT: - /* Try reporting consumed messages */ - report_records_consumed(1); - /* Commit one every consume() timeout instead of on every message. - * Also commit on every 1000 messages, whichever comes first. */ - do_commit(consumer, 1); - break; - - - case RdKafka::ERR_NO_ERROR: { - /* Real message */ - if (verbosity > 2) - std::cerr << now() << ": Read msg from " << msg->topic_name() << " [" - << (int)msg->partition() << "] at offset " << msg->offset() - << std::endl; - - if (state.maxMessages >= 0 && - state.consumer.consumedMessages >= state.maxMessages) - return; - - - Assignment *a = &state.consumer.assignments[Assignment::name( - msg->topic_name(), msg->partition())]; - a->setup(msg->topic_name(), msg->partition()); - - a->consumedMessages++; - if (a->minOffset == -1) - a->minOffset = msg->offset(); - if (a->maxOffset < msg->offset()) - a->maxOffset = msg->offset(); - - if (msg->key()) { - if (verbosity >= 3) - std::cerr << now() << ": Key: " << *msg->key() << std::endl; - } - - if (verbosity >= 3) - fprintf(stderr, "%.*s\n", static_cast<int>(msg->len()), - static_cast<const char *>(msg->payload())); - - state.consumer.consumedMessages++; - - report_records_consumed(0); - - do_commit(consumer, 0); - } break; - - case RdKafka::ERR__PARTITION_EOF: - /* Last message */ - if (exit_eof) { - std::cerr << now() << ": Terminate: exit on EOF" << std::endl; - run = 0; - } - break; - - case RdKafka::ERR__UNKNOWN_TOPIC: - case RdKafka::ERR__UNKNOWN_PARTITION: - std::cerr << now() << ": Consume failed: " << msg->errstr() << std::endl; - run = 0; - break; - - case RdKafka::ERR_GROUP_COORDINATOR_NOT_AVAILABLE: - std::cerr << now() << ": Warning: " << msg->errstr() << std::endl; - break; - - default: - /* Errors */ - std::cerr << now() << ": Consume failed: " << msg->errstr() << std::endl; - run = 0; - } -} - - - -class ExampleConsumeCb : public RdKafka::ConsumeCb { - public: - void consume_cb(RdKafka::Message &msg, void *opaque) { - msg_consume(consumer_, &msg, opaque); - } - RdKafka::KafkaConsumer *consumer_; -}; - -class ExampleRebalanceCb : public RdKafka::RebalanceCb { - private: - static std::string part_list_json( - const std::vector<RdKafka::TopicPartition *> &partitions) { - std::ostringstream out; - for (unsigned int i = 0; i < partitions.size(); i++) - out << (i == 0 ? "" : ", ") << "{ " - << " \"topic\": \"" << partitions[i]->topic() << "\", " - << " \"partition\": " << partitions[i]->partition() << " }"; - return out.str(); - } - - public: - void rebalance_cb(RdKafka::KafkaConsumer *consumer, - RdKafka::ErrorCode err, - std::vector<RdKafka::TopicPartition *> &partitions) { - std::cerr << now() << ": rebalance_cb " << RdKafka::err2str(err) << " for " - << partitions.size() << " partitions" << std::endl; - /* Send message report prior to rebalancing event to make sure they - * are accounted for on the "right side" of the rebalance. */ - report_records_consumed(1); - - if (err == RdKafka::ERR__ASSIGN_PARTITIONS) - consumer->assign(partitions); - else { - do_commit(consumer, 1); - consumer->unassign(); - } - - std::cout << "{ " - << "\"name\": \"partitions_" - << (err == RdKafka::ERR__ASSIGN_PARTITIONS ? "assigned" - : "revoked") - << "\", " - << "\"partitions\": [ " << part_list_json(partitions) << "] }" - << std::endl; - } -}; - - - -/** - * @brief Read (Java client) configuration file - */ -static void read_conf_file(RdKafka::Conf *conf, const std::string &conf_file) { - std::ifstream inf(conf_file.c_str()); - - if (!inf) { - std::cerr << now() << ": " << conf_file << ": could not open file" - << std::endl; - exit(1); - } - - std::cerr << now() << ": " << conf_file << ": read config file" << std::endl; - - std::string line; - int linenr = 0; - - while (std::getline(inf, line)) { - linenr++; - - // Ignore comments and empty lines - if (line[0] == '#' || line.length() == 0) - continue; - - // Match on key=value.. - size_t d = line.find("="); - if (d == 0 || d == std::string::npos) { - std::cerr << now() << ": " << conf_file << ":" << linenr << ": " << line - << ": ignoring invalid line (expect key=value): " - << ::std::endl; - continue; - } - - std::string key = line.substr(0, d); - std::string val = line.substr(d + 1); - - std::string errstr; - if (conf->set(key, val, errstr)) { - std::cerr << now() << ": " << conf_file << ":" << linenr << ": " << key - << "=" << val << ": " << errstr << ": ignoring error" - << std::endl; - } else { - std::cerr << now() << ": " << conf_file << ":" << linenr << ": " << key - << "=" << val << ": applied to configuration" << std::endl; - } - } - - inf.close(); -} - - - -int main(int argc, char **argv) { - std::string brokers = "localhost"; - std::string errstr; - std::vector<std::string> topics; - std::string mode = "P"; - int throughput = 0; - int32_t partition = RdKafka::Topic::PARTITION_UA; - MyHashPartitionerCb hash_partitioner; - int64_t create_time = -1; - - std::cerr << now() << ": librdkafka version " << RdKafka::version_str() - << " (" << RdKafka::version() << ")" << std::endl; - - /* - * Create configuration objects - */ - RdKafka::Conf *conf = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL); - - /* Java VerifiableProducer defaults to acks=all */ - if (conf->set("acks", "all", errstr)) { - std::cerr << now() << ": " << errstr << std::endl; - exit(1); - } - - /* Avoid slow shutdown on error */ - if (conf->set("message.timeout.ms", "60000", errstr)) { - std::cerr << now() << ": " << errstr << std::endl; - exit(1); - } - - { - char hostname[128]; - gethostname(hostname, sizeof(hostname) - 1); - conf->set("client.id", std::string("rdkafka@") + hostname, errstr); - } - - conf->set("log.thread.name", "true", errstr); - - /* auto commit is explicitly enabled with --enable-autocommit */ - conf->set("enable.auto.commit", "false", errstr); - - /* keep protocol request timeouts under the watchdog timeout - * to make sure things like commitSync() dont fall victim to the watchdog. */ - conf->set("socket.timeout.ms", "10000", errstr); - - conf->set("fetch.wait.max.ms", "500", errstr); - conf->set("fetch.min.bytes", "4096", errstr); - - conf->set("enable.partition.eof", "true", errstr); - - for (int i = 1; i < argc; i++) { - const char *name = argv[i]; - const char *val = i + 1 < argc ? argv[i + 1] : NULL; - - if (val && !strncmp(val, "-", 1)) - val = NULL; - - std::cout << now() << ": argument: " << name << " " << (val ? val : "") - << std::endl; - - if (val) { - if (!strcmp(name, "--topic")) - topics.push_back(val); - else if (!strcmp(name, "--broker-list")) - brokers = val; - else if (!strcmp(name, "--max-messages")) - state.maxMessages = atoi(val); - else if (!strcmp(name, "--throughput")) - throughput = atoi(val); - else if (!strcmp(name, "--producer.config") || - !strcmp(name, "--consumer.config")) - read_conf_file(conf, val); - else if (!strcmp(name, "--group-id")) - conf->set("group.id", val, errstr); - else if (!strcmp(name, "--session-timeout")) - conf->set("session.timeout.ms", val, errstr); - else if (!strcmp(name, "--reset-policy")) { - if (conf->set("auto.offset.reset", val, errstr)) { - std::cerr << now() << ": " << errstr << std::endl; - exit(1); - } - } else if (!strcmp(name, "--assignment-strategy")) { - /* The system tests pass the Java class name(s) rather than - * the configuration value. Fix it. - * "org.apache.kafka.clients.consumer.RangeAssignor,.." -> "range,.." - */ - std::string s = val; - size_t pos; - - while ((pos = s.find("org.apache.kafka.clients.consumer.")) != - std::string::npos) - s.erase(pos, strlen("org.apache.kafka.clients.consumer.")); - - while ((pos = s.find("Assignor")) != std::string::npos) - s.erase(pos, strlen("Assignor")); - - std::transform(s.begin(), s.end(), s.begin(), tolower); - - std::cerr << now() << ": converted " << name << " " << val << " to " - << s << std::endl; - - if (conf->set("partition.assignment.strategy", s.c_str(), errstr)) { - std::cerr << now() << ": " << errstr << std::endl; - exit(1); - } - } else if (!strcmp(name, "--value-prefix")) { - value_prefix = std::string(val) + "."; - } else if (!strcmp(name, "--acks")) { - if (conf->set("acks", val, errstr)) { - std::cerr << now() << ": " << errstr << std::endl; - exit(1); - } - } else if (!strcmp(name, "--message-create-time")) { - create_time = (int64_t)atoi(val); - } else if (!strcmp(name, "--debug")) { - conf->set("debug", val, errstr); - } else if (!strcmp(name, "-X")) { - char *s = strdup(val); - char *t = strchr(s, '='); - if (!t) - t = (char *)""; - else { - *t = '\0'; - t++; - } - if (conf->set(s, t, errstr)) { - std::cerr << now() << ": " << errstr << std::endl; - exit(1); - } - free(s); - } else { - std::cerr << now() << ": Unknown option " << name << std::endl; - exit(1); - } - - i++; - - } else { - if (!strcmp(name, "--consumer")) - mode = "C"; - else if (!strcmp(name, "--producer")) - mode = "P"; - else if (!strcmp(name, "--enable-autocommit")) { - state.consumer.useAutoCommit = true; - conf->set("enable.auto.commit", "true", errstr); - } else if (!strcmp(name, "-v")) - verbosity++; - else if (!strcmp(name, "-q")) - verbosity--; - else { - std::cerr << now() << ": Unknown option or missing argument to " << name - << std::endl; - exit(1); - } - } - } - - if (topics.empty() || brokers.empty()) { - std::cerr << now() << ": Missing --topic and --broker-list" << std::endl; - exit(1); - } - - - /* - * Set configuration properties - */ - conf->set("metadata.broker.list", brokers, errstr); - - ExampleEventCb ex_event_cb; - conf->set("event_cb", &ex_event_cb, errstr); - - signal(SIGINT, sigterm); - signal(SIGTERM, sigterm); - signal(SIGALRM, sigwatchdog); - - - if (mode == "P") { - /* - * Producer mode - */ - - ExampleDeliveryReportCb ex_dr_cb; - - /* Set delivery report callback */ - conf->set("dr_cb", &ex_dr_cb, errstr); - - /* - * Create producer using accumulated global configuration. - */ - RdKafka::Producer *producer = RdKafka::Producer::create(conf, errstr); - if (!producer) { - std::cerr << now() << ": Failed to create producer: " << errstr - << std::endl; - exit(1); - } - - std::cerr << now() << ": % Created producer " << producer->name() - << std::endl; - - /* - * Create topic handle. - */ - RdKafka::Topic *topic = - RdKafka::Topic::create(producer, topics[0], NULL, errstr); - if (!topic) { - std::cerr << now() << ": Failed to create topic: " << errstr << std::endl; - exit(1); - } - - static const int delay_us = throughput ? 1000000 / throughput : 10; - - if (state.maxMessages == -1) - state.maxMessages = 1000000; /* Avoid infinite produce */ - - for (int i = 0; run && i < state.maxMessages; i++) { - /* - * Produce message - */ - std::ostringstream msg; - msg << value_prefix << i; - while (true) { - RdKafka::ErrorCode resp; - if (create_time == -1) { - resp = producer->produce( - topic, partition, - RdKafka::Producer::RK_MSG_COPY /* Copy payload */, - const_cast<char *>(msg.str().c_str()), msg.str().size(), NULL, - NULL); - } else { - resp = producer->produce( - topics[0], partition, - RdKafka::Producer::RK_MSG_COPY /* Copy payload */, - const_cast<char *>(msg.str().c_str()), msg.str().size(), NULL, 0, - create_time, NULL); - } - - if (resp == RdKafka::ERR__QUEUE_FULL) { - producer->poll(100); - continue; - } else if (resp != RdKafka::ERR_NO_ERROR) { - errorString("producer_send_error", RdKafka::err2str(resp), - topic->name(), NULL, msg.str()); - state.producer.numErr++; - } else { - state.producer.numSent++; - } - break; - } - - producer->poll(delay_us / 1000); - usleep(1000); - watchdog_kick(); - } - run = 1; - - while (run && producer->outq_len() > 0) { - std::cerr << now() << ": Waiting for " << producer->outq_len() - << std::endl; - producer->poll(1000); - watchdog_kick(); - } - - std::cerr << now() << ": " << state.producer.numAcked << "/" - << state.producer.numSent << "/" << state.maxMessages - << " msgs acked/sent/max, " << state.producer.numErr << " errored" - << std::endl; - - delete topic; - delete producer; - - - } else if (mode == "C") { - /* - * Consumer mode - */ - - conf->set("auto.offset.reset", "smallest", errstr); - - ExampleRebalanceCb ex_rebalance_cb; - conf->set("rebalance_cb", &ex_rebalance_cb, errstr); - - conf->set("offset_commit_cb", &ex_offset_commit_cb, errstr); - - - /* - * Create consumer using accumulated global configuration. - */ - consumer = RdKafka::KafkaConsumer::create(conf, errstr); - if (!consumer) { - std::cerr << now() << ": Failed to create consumer: " << errstr - << std::endl; - exit(1); - } - - std::cerr << now() << ": % Created consumer " << consumer->name() - << std::endl; - - /* - * Subscribe to topic(s) - */ - RdKafka::ErrorCode resp = consumer->subscribe(topics); - if (resp != RdKafka::ERR_NO_ERROR) { - std::cerr << now() << ": Failed to subscribe to " << topics.size() - << " topics: " << RdKafka::err2str(resp) << std::endl; - exit(1); - } - - watchdog_kick(); - - /* - * Consume messages - */ - while (run) { - RdKafka::Message *msg = consumer->consume(500); - msg_consume(consumer, msg, NULL); - delete msg; - watchdog_kick(); - } - - std::cerr << now() << ": Final commit on termination" << std::endl; - - /* Final commit */ - do_commit(consumer, 1); - - /* - * Stop consumer - */ - consumer->close(); - - delete consumer; - } - - std::cout << "{ \"name\": \"shutdown_complete\" }" << std::endl; - - /* - * Wait for RdKafka to decommission. - * This is not strictly needed (when check outq_len() above), but - * allows RdKafka to clean up all its resources before the application - * exits so that memory profilers such as valgrind wont complain about - * memory leaks. - */ - RdKafka::wait_destroyed(5000); - - std::cerr << now() << ": EXITING WITH RETURN VALUE 0" << std::endl; - return 0; -} diff --git a/fluent-bit/lib/librdkafka-2.1.0/examples/list_consumer_group_offsets.c b/fluent-bit/lib/librdkafka-2.1.0/examples/list_consumer_group_offsets.c deleted file mode 100644 index 03e878ee1..000000000 --- a/fluent-bit/lib/librdkafka-2.1.0/examples/list_consumer_group_offsets.c +++ /dev/null @@ -1,359 +0,0 @@ -/* - * librdkafka - Apache Kafka C library - * - * Copyright (c) 2022, Magnus Edenhill - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/** - * ListConsumerGroupOffsets usage example. - */ - -#include <stdio.h> -#include <signal.h> -#include <string.h> -#include <stdlib.h> -#include <stdarg.h> - -#ifdef _WIN32 -#include "../win32/wingetopt.h" -#else -#include <getopt.h> -#endif - - -/* Typical include path would be <librdkafka/rdkafka.h>, but this program - * is builtin from within the librdkafka source tree and thus differs. */ -#include "rdkafka.h" - - -const char *argv0; - -static rd_kafka_queue_t *queue; /** Admin result queue. - * This is a global so we can - * yield in stop() */ -static volatile sig_atomic_t run = 1; - -/** - * @brief Signal termination of program - */ -static void stop(int sig) { - if (!run) { - fprintf(stderr, "%% Forced termination\n"); - exit(2); - } - run = 0; - rd_kafka_queue_yield(queue); -} - - -static void usage(const char *reason, ...) { - - fprintf(stderr, - "List consumer group offsets usage examples\n" - "\n" - "Usage: %s <options> <group_id> " - "<require_stable_offsets>\n" - " <topic1> <partition1>\n" - " <topic2> <partition2>\n" - " ...\n" - "\n" - "Options:\n" - " -b <brokers> Bootstrap server list to connect to.\n" - " -X <prop=val> Set librdkafka configuration property.\n" - " See CONFIGURATION.md for full list.\n" - " -d <dbg,..> Enable librdkafka debugging (%s).\n" - "\n", - argv0, rd_kafka_get_debug_contexts()); - - if (reason) { - va_list ap; - char reasonbuf[512]; - - va_start(ap, reason); - vsnprintf(reasonbuf, sizeof(reasonbuf), reason, ap); - va_end(ap); - - fprintf(stderr, "ERROR: %s\n", reasonbuf); - } - - exit(reason ? 1 : 0); -} - - -#define fatal(...) \ - do { \ - fprintf(stderr, "ERROR: "); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr, "\n"); \ - exit(2); \ - } while (0) - - -/** - * @brief Set config property. Exit on failure. - */ -static void conf_set(rd_kafka_conf_t *conf, const char *name, const char *val) { - char errstr[512]; - - if (rd_kafka_conf_set(conf, name, val, errstr, sizeof(errstr)) != - RD_KAFKA_CONF_OK) - fatal("Failed to set %s=%s: %s", name, val, errstr); -} - - -static void -print_partition_list(FILE *fp, - const rd_kafka_topic_partition_list_t *partitions, - int print_offset, - const char *prefix) { - int i; - - if (partitions->cnt == 0) { - fprintf(fp, "%sNo partition found", prefix); - } - for (i = 0; i < partitions->cnt; i++) { - char offset_string[512] = {}; - *offset_string = '\0'; - if (print_offset) { - snprintf(offset_string, sizeof(offset_string), - " offset %" PRId64, - partitions->elems[i].offset); - } - fprintf(fp, "%s%s %s [%" PRId32 "]%s error %s", - i > 0 ? "\n" : "", prefix, partitions->elems[i].topic, - partitions->elems[i].partition, offset_string, - rd_kafka_err2str(partitions->elems[i].err)); - } - fprintf(fp, "\n"); -} - -/** - * @brief Parse an integer or fail. - */ -int64_t parse_int(const char *what, const char *str) { - char *end; - unsigned long n = strtoull(str, &end, 0); - - if (end != str + strlen(str)) { - fprintf(stderr, "%% Invalid input for %s: %s: not an integer\n", - what, str); - exit(1); - } - - return (int64_t)n; -} - -static void -cmd_list_consumer_group_offsets(rd_kafka_conf_t *conf, int argc, char **argv) { - char errstr[512]; /* librdkafka API error reporting buffer */ - rd_kafka_t *rk; /* Admin client instance */ - rd_kafka_AdminOptions_t *options; /* (Optional) Options for - * ListConsumerGroupOffsets() */ - rd_kafka_event_t *event; /* ListConsumerGroupOffsets result event */ - const int min_argc = 2; - char *topic; - int partition; - int require_stable_offsets = 0, num_partitions = 0; - rd_kafka_ListConsumerGroupOffsets_t *list_cgrp_offsets; - rd_kafka_error_t *error; - const char *group; - - /* - * Argument validation - */ - if (argc < min_argc || (argc - min_argc) % 2 != 0) - usage("Wrong number of arguments"); - else { - require_stable_offsets = - parse_int("require_stable_offsets", argv[1]); - if (require_stable_offsets < 0 || require_stable_offsets > 1) - usage("Require stable not a 0-1 int"); - } - - num_partitions = (argc - min_argc) / 2; - group = argv[0]; - - /* - * Create an admin client, it can be created using any client type, - * so we choose producer since it requires no extra configuration - * and is more light-weight than the consumer. - * - * NOTE: rd_kafka_new() takes ownership of the conf object - * and the application must not reference it again after - * this call. - */ - rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr)); - if (!rk) { - fprintf(stderr, "%% Failed to create new producer: %s\n", - errstr); - exit(1); - } - - /* The Admin API is completely asynchronous, results are emitted - * on the result queue that is passed to ListConsumerGroupOffsets() */ - queue = rd_kafka_queue_new(rk); - - /* Signal handler for clean shutdown */ - signal(SIGINT, stop); - - /* Set timeout (optional) */ - options = rd_kafka_AdminOptions_new( - rk, RD_KAFKA_ADMIN_OP_LISTCONSUMERGROUPOFFSETS); - if (rd_kafka_AdminOptions_set_request_timeout( - options, 30 * 1000 /* 30s */, errstr, sizeof(errstr))) { - fprintf(stderr, "%% Failed to set timeout: %s\n", errstr); - exit(1); - } - /* Set requested require stable offsets */ - if ((error = rd_kafka_AdminOptions_set_require_stable_offsets( - options, require_stable_offsets))) { - fprintf(stderr, "%% Failed to set require stable offsets: %s\n", - rd_kafka_error_string(error)); - rd_kafka_error_destroy(error); - exit(1); - } - - /* Read passed partition-offsets */ - rd_kafka_topic_partition_list_t *partitions = NULL; - if (num_partitions > 0) { - int i; - partitions = rd_kafka_topic_partition_list_new(num_partitions); - for (i = 0; i < num_partitions; i++) { - topic = argv[min_argc + i * 2]; - partition = - parse_int("partition", argv[min_argc + i * 2 + 1]); - rd_kafka_topic_partition_list_add(partitions, topic, - partition); - } - } - - /* Create argument */ - list_cgrp_offsets = - rd_kafka_ListConsumerGroupOffsets_new(group, partitions); - /* Call ListConsumerGroupOffsets */ - rd_kafka_ListConsumerGroupOffsets(rk, &list_cgrp_offsets, 1, options, - queue); - - /* Clean up input arguments */ - rd_kafka_ListConsumerGroupOffsets_destroy(list_cgrp_offsets); - rd_kafka_AdminOptions_destroy(options); - - - /* Wait for results */ - event = rd_kafka_queue_poll(queue, -1 /* indefinitely but limited by - * the request timeout set - * above (30s) */); - - if (!event) { - /* User hit Ctrl-C, - * see yield call in stop() signal handler */ - fprintf(stderr, "%% Cancelled by user\n"); - - } else if (rd_kafka_event_error(event)) { - /* ListConsumerGroupOffsets request failed */ - fprintf(stderr, "%% ListConsumerGroupOffsets failed: %s\n", - rd_kafka_event_error_string(event)); - exit(1); - - } else { - /* ListConsumerGroupOffsets request succeeded, but individual - * partitions may have errors. */ - const rd_kafka_ListConsumerGroupOffsets_result_t *result; - const rd_kafka_group_result_t **groups; - size_t n_groups, i; - - result = rd_kafka_event_ListConsumerGroupOffsets_result(event); - groups = rd_kafka_ListConsumerGroupOffsets_result_groups( - result, &n_groups); - - printf("ListConsumerGroupOffsets results:\n"); - for (i = 0; i < n_groups; i++) { - const rd_kafka_group_result_t *group = groups[i]; - const rd_kafka_topic_partition_list_t *partitions = - rd_kafka_group_result_partitions(group); - print_partition_list(stderr, partitions, 1, " "); - } - } - - if (partitions) - rd_kafka_topic_partition_list_destroy(partitions); - - /* Destroy event object when we're done with it. - * Note: rd_kafka_event_destroy() allows a NULL event. */ - rd_kafka_event_destroy(event); - - /* Destroy queue */ - rd_kafka_queue_destroy(queue); - - /* Destroy the producer instance */ - rd_kafka_destroy(rk); -} - -int main(int argc, char **argv) { - rd_kafka_conf_t *conf; /**< Client configuration object */ - int opt; - argv0 = argv[0]; - - /* - * Create Kafka client configuration place-holder - */ - conf = rd_kafka_conf_new(); - - - /* - * Parse common options - */ - while ((opt = getopt(argc, argv, "b:X:d:")) != -1) { - switch (opt) { - case 'b': - conf_set(conf, "bootstrap.servers", optarg); - break; - - case 'X': { - char *name = optarg, *val; - - if (!(val = strchr(name, '='))) - fatal("-X expects a name=value argument"); - - *val = '\0'; - val++; - - conf_set(conf, name, val); - break; - } - - case 'd': - conf_set(conf, "debug", optarg); - break; - - default: - usage("Unknown option %c", (char)opt); - } - } - - cmd_list_consumer_group_offsets(conf, argc - optind, &argv[optind]); - - return 0; -} diff --git a/fluent-bit/lib/librdkafka-2.1.0/examples/list_consumer_groups.c b/fluent-bit/lib/librdkafka-2.1.0/examples/list_consumer_groups.c deleted file mode 100644 index 13656cd66..000000000 --- a/fluent-bit/lib/librdkafka-2.1.0/examples/list_consumer_groups.c +++ /dev/null @@ -1,330 +0,0 @@ -/* - * librdkafka - Apache Kafka C library - * - * Copyright (c) 2022, Magnus Edenhill - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/** - * ListConsumerGroups usage example. - */ - -#include <stdio.h> -#include <signal.h> -#include <string.h> -#include <stdlib.h> -#include <stdarg.h> - -#ifdef _WIN32 -#include "../win32/wingetopt.h" -#else -#include <getopt.h> -#endif - - -/* Typical include path would be <librdkafka/rdkafka.h>, but this program - * is builtin from within the librdkafka source tree and thus differs. */ -#include "rdkafka.h" - - -const char *argv0; - -static rd_kafka_queue_t *queue; /** Admin result queue. - * This is a global so we can - * yield in stop() */ -static volatile sig_atomic_t run = 1; - -/** - * @brief Signal termination of program - */ -static void stop(int sig) { - if (!run) { - fprintf(stderr, "%% Forced termination\n"); - exit(2); - } - run = 0; - rd_kafka_queue_yield(queue); -} - - -static void usage(const char *reason, ...) { - - fprintf(stderr, - "List groups usage examples\n" - "\n" - "Usage: %s <options> <state1> <state2> ...\n" - "\n" - "Options:\n" - " -b <brokers> Bootstrap server list to connect to.\n" - " -X <prop=val> Set librdkafka configuration property.\n" - " See CONFIGURATION.md for full list.\n" - " -d <dbg,..> Enable librdkafka debugging (%s).\n" - "\n", - argv0, rd_kafka_get_debug_contexts()); - - if (reason) { - va_list ap; - char reasonbuf[512]; - - va_start(ap, reason); - vsnprintf(reasonbuf, sizeof(reasonbuf), reason, ap); - va_end(ap); - - fprintf(stderr, "ERROR: %s\n", reasonbuf); - } - - exit(reason ? 1 : 0); -} - - -#define fatal(...) \ - do { \ - fprintf(stderr, "ERROR: "); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr, "\n"); \ - exit(2); \ - } while (0) - - -/** - * @brief Set config property. Exit on failure. - */ -static void conf_set(rd_kafka_conf_t *conf, const char *name, const char *val) { - char errstr[512]; - - if (rd_kafka_conf_set(conf, name, val, errstr, sizeof(errstr)) != - RD_KAFKA_CONF_OK) - fatal("Failed to set %s=%s: %s", name, val, errstr); -} - -/** - * @brief Print group information. - */ -static int print_groups_info(const rd_kafka_ListConsumerGroups_result_t *list) { - size_t i; - const rd_kafka_ConsumerGroupListing_t **result_groups; - const rd_kafka_error_t **errors; - size_t result_groups_cnt; - size_t result_error_cnt; - result_groups = - rd_kafka_ListConsumerGroups_result_valid(list, &result_groups_cnt); - errors = - rd_kafka_ListConsumerGroups_result_errors(list, &result_error_cnt); - - if (result_groups_cnt == 0) { - fprintf(stderr, "No matching groups found\n"); - } - - for (i = 0; i < result_groups_cnt; i++) { - const rd_kafka_ConsumerGroupListing_t *group = result_groups[i]; - const char *group_id = - rd_kafka_ConsumerGroupListing_group_id(group); - rd_kafka_consumer_group_state_t state = - rd_kafka_ConsumerGroupListing_state(group); - int is_simple_consumer_group = - rd_kafka_ConsumerGroupListing_is_simple_consumer_group( - group); - - printf("Group \"%s\", is simple %" PRId32 - ", " - "state %s", - group_id, is_simple_consumer_group, - rd_kafka_consumer_group_state_name(state)); - printf("\n"); - } - for (i = 0; i < result_error_cnt; i++) { - const rd_kafka_error_t *error = errors[i]; - printf("Error[%" PRId32 "]: %s\n", rd_kafka_error_code(error), - rd_kafka_error_string(error)); - } - return 0; -} - -/** - * @brief Parse an integer or fail. - */ -int64_t parse_int(const char *what, const char *str) { - char *end; - unsigned long n = strtoull(str, &end, 0); - - if (end != str + strlen(str)) { - fprintf(stderr, "%% Invalid input for %s: %s: not an integer\n", - what, str); - exit(1); - } - - return (int64_t)n; -} - -/** - * @brief Call rd_kafka_ListConsumerGroups() with a list of - * groups. - */ -static void -cmd_list_consumer_groups(rd_kafka_conf_t *conf, int argc, char **argv) { - rd_kafka_t *rk; - const char **states_str = NULL; - char errstr[512]; - rd_kafka_AdminOptions_t *options; - rd_kafka_event_t *event = NULL; - rd_kafka_error_t *error = NULL; - int i; - int retval = 0; - int states_cnt = 0; - rd_kafka_consumer_group_state_t *states; - - - if (argc >= 1) { - states_str = (const char **)&argv[0]; - states_cnt = argc; - } - states = calloc(states_cnt, sizeof(rd_kafka_consumer_group_state_t)); - for (i = 0; i < states_cnt; i++) { - states[i] = parse_int("state code", states_str[i]); - } - - /* - * Create consumer instance - * NOTE: rd_kafka_new() takes ownership of the conf object - * and the application must not reference it again after - * this call. - */ - rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, sizeof(errstr)); - if (!rk) - fatal("Failed to create new consumer: %s", errstr); - - /* - * List consumer groups - */ - queue = rd_kafka_queue_new(rk); - - /* Signal handler for clean shutdown */ - signal(SIGINT, stop); - - options = - rd_kafka_AdminOptions_new(rk, RD_KAFKA_ADMIN_OP_LISTCONSUMERGROUPS); - - if (rd_kafka_AdminOptions_set_request_timeout( - options, 10 * 1000 /* 10s */, errstr, sizeof(errstr))) { - fprintf(stderr, "%% Failed to set timeout: %s\n", errstr); - goto exit; - } - - if ((error = rd_kafka_AdminOptions_set_match_consumer_group_states( - options, states, states_cnt))) { - fprintf(stderr, "%% Failed to set states: %s\n", - rd_kafka_error_string(error)); - rd_kafka_error_destroy(error); - goto exit; - } - free(states); - - rd_kafka_ListConsumerGroups(rk, options, queue); - rd_kafka_AdminOptions_destroy(options); - - /* Wait for results */ - event = rd_kafka_queue_poll(queue, -1 /* indefinitely but limited by - * the request timeout set - * above (10s) */); - - if (!event) { - /* User hit Ctrl-C, - * see yield call in stop() signal handler */ - fprintf(stderr, "%% Cancelled by user\n"); - - } else if (rd_kafka_event_error(event)) { - rd_kafka_resp_err_t err = rd_kafka_event_error(event); - /* ListConsumerGroups request failed */ - fprintf(stderr, - "%% ListConsumerGroups failed[%" PRId32 "]: %s\n", err, - rd_kafka_event_error_string(event)); - goto exit; - - } else { - /* ListConsumerGroups request succeeded, but individual - * groups may have errors. */ - const rd_kafka_ListConsumerGroups_result_t *result; - - result = rd_kafka_event_ListConsumerGroups_result(event); - printf("ListConsumerGroups results:\n"); - retval = print_groups_info(result); - } - - -exit: - if (event) - rd_kafka_event_destroy(event); - rd_kafka_queue_destroy(queue); - /* Destroy the client instance */ - rd_kafka_destroy(rk); - - exit(retval); -} - -int main(int argc, char **argv) { - rd_kafka_conf_t *conf; /**< Client configuration object */ - int opt; - argv0 = argv[0]; - - /* - * Create Kafka client configuration place-holder - */ - conf = rd_kafka_conf_new(); - - - /* - * Parse common options - */ - while ((opt = getopt(argc, argv, "b:X:d:")) != -1) { - switch (opt) { - case 'b': - conf_set(conf, "bootstrap.servers", optarg); - break; - - case 'X': { - char *name = optarg, *val; - - if (!(val = strchr(name, '='))) - fatal("-X expects a name=value argument"); - - *val = '\0'; - val++; - - conf_set(conf, name, val); - break; - } - - case 'd': - conf_set(conf, "debug", optarg); - break; - - default: - usage("Unknown option %c", (char)opt); - } - } - - cmd_list_consumer_groups(conf, argc - optind, &argv[optind]); - - return 0; -} diff --git a/fluent-bit/lib/librdkafka-2.1.0/examples/misc.c b/fluent-bit/lib/librdkafka-2.1.0/examples/misc.c deleted file mode 100644 index b63ab577d..000000000 --- a/fluent-bit/lib/librdkafka-2.1.0/examples/misc.c +++ /dev/null @@ -1,287 +0,0 @@ -/* - * librdkafka - Apache Kafka C library - * - * Copyright (c) 2022, Magnus Edenhill - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/** - * A collection of smaller usage examples - */ - -#include <stdio.h> -#include <signal.h> -#include <string.h> -#include <stdlib.h> -#include <stdarg.h> - -#ifdef _WIN32 -#include "../win32/wingetopt.h" -#else -#include <getopt.h> -#endif - - -/* Typical include path would be <librdkafka/rdkafka.h>, but this program - * is builtin from within the librdkafka source tree and thus differs. */ -#include "rdkafka.h" - - -const char *argv0; - - -static void usage(const char *reason, ...) { - - fprintf(stderr, - "Miscellaneous librdkafka usage examples\n" - "\n" - "Usage: %s <options> <command> [<command arguments>]\n" - "\n" - "Commands:\n" - " List groups:\n" - " %s -b <brokers> list_groups <group>\n" - "\n" - " Show librdkafka version:\n" - " %s version\n" - "\n" - "Common options for all commands:\n" - " -b <brokers> Bootstrap server list to connect to.\n" - " -X <prop=val> Set librdkafka configuration property.\n" - " See CONFIGURATION.md for full list.\n" - " -d <dbg,..> Enable librdkafka debugging (%s).\n" - "\n", - argv0, argv0, argv0, rd_kafka_get_debug_contexts()); - - if (reason) { - va_list ap; - char reasonbuf[512]; - - va_start(ap, reason); - vsnprintf(reasonbuf, sizeof(reasonbuf), reason, ap); - va_end(ap); - - fprintf(stderr, "ERROR: %s\n", reasonbuf); - } - - exit(reason ? 1 : 0); -} - - -#define fatal(...) \ - do { \ - fprintf(stderr, "ERROR: "); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr, "\n"); \ - exit(2); \ - } while (0) - - -/** - * @brief Set config property. Exit on failure. - */ -static void conf_set(rd_kafka_conf_t *conf, const char *name, const char *val) { - char errstr[512]; - - if (rd_kafka_conf_set(conf, name, val, errstr, sizeof(errstr)) != - RD_KAFKA_CONF_OK) - fatal("Failed to set %s=%s: %s", name, val, errstr); -} - - -/** - * Commands - * - */ - -/** - * @brief Just print the librdkafka version - */ -static void cmd_version(rd_kafka_conf_t *conf, int argc, char **argv) { - if (argc) - usage("version command takes no arguments"); - - printf("librdkafka v%s\n", rd_kafka_version_str()); - rd_kafka_conf_destroy(conf); -} - - -/** - * @brief Call rd_kafka_list_groups() with an optional groupid argument. - */ -static void cmd_list_groups(rd_kafka_conf_t *conf, int argc, char **argv) { - rd_kafka_t *rk; - const char *groupid = NULL; - char errstr[512]; - rd_kafka_resp_err_t err; - const struct rd_kafka_group_list *grplist; - int i; - int retval = 0; - - if (argc > 1) - usage("too many arguments to list_groups"); - - if (argc == 1) - groupid = argv[0]; - - /* - * Create consumer instance - * NOTE: rd_kafka_new() takes ownership of the conf object - * and the application must not reference it again after - * this call. - */ - rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, sizeof(errstr)); - if (!rk) - fatal("Failed to create new consumer: %s", errstr); - - /* - * List groups - */ - err = rd_kafka_list_groups(rk, groupid, &grplist, 10 * 1000 /*10s*/); - if (err) - fatal("rd_kafka_list_groups(%s) failed: %s", groupid, - rd_kafka_err2str(err)); - - if (grplist->group_cnt == 0) { - if (groupid) { - fprintf(stderr, "Group %s not found\n", groupid); - retval = 1; - } else { - fprintf(stderr, "No groups in cluster\n"); - } - } - - /* - * Print group information - */ - for (i = 0; i < grplist->group_cnt; i++) { - int j; - const struct rd_kafka_group_info *grp = &grplist->groups[i]; - - printf( - "Group \"%s\" protocol-type %s, protocol %s, " - "state %s, with %d member(s))", - grp->group, grp->protocol_type, grp->protocol, grp->state, - grp->member_cnt); - if (grp->err) - printf(" error: %s", rd_kafka_err2str(grp->err)); - printf("\n"); - for (j = 0; j < grp->member_cnt; j++) { - const struct rd_kafka_group_member_info *mb = - &grp->members[j]; - printf( - " Member \"%s\" with client-id %s, host %s, " - "%d bytes of metadat, %d bytes of assignment\n", - mb->member_id, mb->client_id, mb->client_host, - mb->member_metadata_size, - mb->member_assignment_size); - } - } - - rd_kafka_group_list_destroy(grplist); - - /* Destroy the client instance */ - rd_kafka_destroy(rk); - - exit(retval); -} - - - -int main(int argc, char **argv) { - rd_kafka_conf_t *conf; /**< Client configuration object */ - int opt, i; - const char *cmd; - static const struct { - const char *cmd; - void (*func)(rd_kafka_conf_t *conf, int argc, char **argv); - } cmds[] = { - {"version", cmd_version}, - {"list_groups", cmd_list_groups}, - {NULL}, - }; - - argv0 = argv[0]; - - if (argc == 1) - usage(NULL); - - /* - * Create Kafka client configuration place-holder - */ - conf = rd_kafka_conf_new(); - - - /* - * Parse common options - */ - while ((opt = getopt(argc, argv, "b:X:d:")) != -1) { - switch (opt) { - case 'b': - conf_set(conf, "bootstrap.servers", optarg); - break; - - case 'X': { - char *name = optarg, *val; - - if (!(val = strchr(name, '='))) - fatal("-X expects a name=value argument"); - - *val = '\0'; - val++; - - conf_set(conf, name, val); - break; - } - - case 'd': - conf_set(conf, "debug", optarg); - break; - - default: - usage("Unknown option %c", (char)opt); - } - } - - - if (optind == argc) - usage("No command specified"); - - - cmd = argv[optind++]; - - /* - * Find matching command and run it - */ - for (i = 0; cmds[i].cmd; i++) { - if (!strcmp(cmds[i].cmd, cmd)) { - cmds[i].func(conf, argc - optind, &argv[optind]); - exit(0); - } - } - - usage("Unknown command: %s", cmd); - - /* NOTREACHED */ - return 0; -} diff --git a/fluent-bit/lib/librdkafka-2.1.0/examples/openssl_engine_example.cpp b/fluent-bit/lib/librdkafka-2.1.0/examples/openssl_engine_example.cpp deleted file mode 100644 index 401857e6b..000000000 --- a/fluent-bit/lib/librdkafka-2.1.0/examples/openssl_engine_example.cpp +++ /dev/null @@ -1,249 +0,0 @@ -/* - * librdkafka - Apache Kafka C library - * - * Copyright (c) 2021, Magnus Edenhill - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/** - * OpenSSL engine integration example. This example fetches metadata - * over SSL connection with broker, established using OpenSSL engine. - */ - -#include <iostream> -#include <string> -#include <cstdlib> -#include <cstdio> -#include <csignal> -#include <cstring> - -#ifdef _WIN32 -#include "../win32/wingetopt.h" -#elif _AIX -#include <unistd.h> -#else -#include <getopt.h> -#endif - -/* - * Typically include path in a real application would be - * #include <librdkafka/rdkafkacpp.h> - */ -#include "rdkafkacpp.h" - -static void metadata_print(const RdKafka::Metadata *metadata) { - std::cout << "Number of topics: " << metadata->topics()->size() << std::endl; - - /* Iterate topics */ - RdKafka::Metadata::TopicMetadataIterator it; - for (it = metadata->topics()->begin(); it != metadata->topics()->end(); ++it) - std::cout << " " << (*it)->topic() << " has " - << (*it)->partitions()->size() << " partitions." << std::endl; -} - - -class PrintingSSLVerifyCb : public RdKafka::SslCertificateVerifyCb { - /* This SSL cert verification callback simply prints the incoming - * parameters. It provides no validation, everything is ok. */ - public: - bool ssl_cert_verify_cb(const std::string &broker_name, - int32_t broker_id, - int *x509_error, - int depth, - const char *buf, - size_t size, - std::string &errstr) { - std::cout << "ssl_cert_verify_cb :" - << ": broker_name=" << broker_name << ", broker_id=" << broker_id - << ", x509_error=" << *x509_error << ", depth=" << depth - << ", buf size=" << size << std::endl; - - return true; - } -}; - - -int main(int argc, char **argv) { - std::string brokers; - std::string errstr; - std::string engine_path; - std::string ca_location; - - /* - * Create configuration objects - */ - RdKafka::Conf *conf = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL); - std::string engine_id; - std::string engine_callback_data; - int opt; - - if (conf->set("security.protocol", "ssl", errstr) != RdKafka::Conf::CONF_OK) { - std::cerr << errstr << std::endl; - exit(1); - } - - while ((opt = getopt(argc, argv, "b:p:c:t:d:i:e:X:")) != -1) { - switch (opt) { - case 'b': - brokers = optarg; - break; - case 'p': - engine_path = optarg; - break; - case 'c': - ca_location = optarg; - break; - case 'i': - engine_id = optarg; - break; - case 'e': - engine_callback_data = optarg; - break; - case 'd': - if (conf->set("debug", optarg, errstr) != RdKafka::Conf::CONF_OK) { - std::cerr << errstr << std::endl; - exit(1); - } - break; - case 'X': { - char *name, *val; - - name = optarg; - if (!(val = strchr(name, '='))) { - std::cerr << "%% Expected -X property=value, not " << name << std::endl; - exit(1); - } - - *val = '\0'; - val++; - - if (conf->set(name, val, errstr) != RdKafka::Conf::CONF_OK) { - std::cerr << errstr << std::endl; - exit(1); - } - } break; - - default: - goto usage; - } - } - - if (brokers.empty() || engine_path.empty() || optind != argc) { - usage: - std::string features; - conf->get("builtin.features", features); - fprintf(stderr, - "Usage: %s [options] -b <brokers> -p <engine-path> \n" - "\n" - "OpenSSL engine integration example. This example fetches\n" - "metadata over SSL connection with broker, established using\n" - "OpenSSL engine.\n" - "\n" - "librdkafka version %s (0x%08x, builtin.features \"%s\")\n" - "\n" - " Options:\n" - " -b <brokers> Broker address\n" - " -p <engine-path> Path to OpenSSL engine\n" - " -i <engine-id> OpenSSL engine id\n" - " -e <engine-callback-data> OpenSSL engine callback_data\n" - " -c <ca-cert-location> File path to ca cert\n" - " -d [facs..] Enable debugging contexts: %s\n" - " -X <prop=name> Set arbitrary librdkafka configuration" - " property\n" - "\n", - argv[0], RdKafka::version_str().c_str(), RdKafka::version(), - features.c_str(), RdKafka::get_debug_contexts().c_str()); - exit(1); - } - - if (conf->set("bootstrap.servers", brokers, errstr) != - RdKafka::Conf::CONF_OK) { - std::cerr << errstr << std::endl; - exit(1); - } - - if (conf->set("ssl.engine.location", engine_path, errstr) != - RdKafka::Conf::CONF_OK) { - std::cerr << errstr << std::endl; - exit(1); - } - - if (ca_location.length() > 0 && conf->set("ssl.ca.location", ca_location, - errstr) != RdKafka::Conf::CONF_OK) { - std::cerr << errstr << std::endl; - exit(1); - } - - if (engine_id.length() > 0 && - conf->set("ssl.engine.id", engine_id, errstr) != RdKafka::Conf::CONF_OK) { - std::cerr << errstr << std::endl; - exit(1); - } - - /* engine_callback_data needs to be persistent - * and outlive the lifetime of the Kafka client handle. */ - if (engine_callback_data.length() > 0 && - conf->set_engine_callback_data((void *)engine_callback_data.c_str(), - errstr) != RdKafka::Conf::CONF_OK) { - std::cerr << errstr << std::endl; - exit(1); - } - - /* We use the Certificiate verification callback to print the - * certificate name being used. */ - PrintingSSLVerifyCb ssl_verify_cb; - - if (conf->set("ssl_cert_verify_cb", &ssl_verify_cb, errstr) != - RdKafka::Conf::CONF_OK) { - std::cerr << errstr << std::endl; - exit(1); - } - - /* - * Create producer using accumulated global configuration. - */ - RdKafka::Producer *producer = RdKafka::Producer::create(conf, errstr); - if (!producer) { - std::cerr << "Failed to create producer: " << errstr << std::endl; - exit(1); - } - - std::cout << "% Created producer " << producer->name() << std::endl; - - class RdKafka::Metadata *metadata; - - /* Fetch metadata */ - RdKafka::ErrorCode err = producer->metadata(true, NULL, &metadata, 5000); - if (err != RdKafka::ERR_NO_ERROR) - std::cerr << "%% Failed to acquire metadata: " << RdKafka::err2str(err) - << std::endl; - - metadata_print(metadata); - - delete metadata; - delete producer; - delete conf; - - return 0; -} diff --git a/fluent-bit/lib/librdkafka-2.1.0/examples/producer.c b/fluent-bit/lib/librdkafka-2.1.0/examples/producer.c deleted file mode 100644 index b6fb71150..000000000 --- a/fluent-bit/lib/librdkafka-2.1.0/examples/producer.c +++ /dev/null @@ -1,251 +0,0 @@ -/* - * librdkafka - Apache Kafka C library - * - * Copyright (c) 2017, Magnus Edenhill - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/** - * Simple Apache Kafka producer - * using the Kafka driver from librdkafka - * (https://github.com/edenhill/librdkafka) - */ - -#include <stdio.h> -#include <signal.h> -#include <string.h> - - -/* Typical include path would be <librdkafka/rdkafka.h>, but this program - * is builtin from within the librdkafka source tree and thus differs. */ -#include "rdkafka.h" - - -static volatile sig_atomic_t run = 1; - -/** - * @brief Signal termination of program - */ -static void stop(int sig) { - run = 0; - fclose(stdin); /* abort fgets() */ -} - - -/** - * @brief Message delivery report callback. - * - * This callback is called exactly once per message, indicating if - * the message was succesfully delivered - * (rkmessage->err == RD_KAFKA_RESP_ERR_NO_ERROR) or permanently - * failed delivery (rkmessage->err != RD_KAFKA_RESP_ERR_NO_ERROR). - * - * The callback is triggered from rd_kafka_poll() and executes on - * the application's thread. - */ -static void -dr_msg_cb(rd_kafka_t *rk, const rd_kafka_message_t *rkmessage, void *opaque) { - if (rkmessage->err) - fprintf(stderr, "%% Message delivery failed: %s\n", - rd_kafka_err2str(rkmessage->err)); - else - fprintf(stderr, - "%% Message delivered (%zd bytes, " - "partition %" PRId32 ")\n", - rkmessage->len, rkmessage->partition); - - /* The rkmessage is destroyed automatically by librdkafka */ -} - - - -int main(int argc, char **argv) { - rd_kafka_t *rk; /* Producer instance handle */ - rd_kafka_conf_t *conf; /* Temporary configuration object */ - char errstr[512]; /* librdkafka API error reporting buffer */ - char buf[512]; /* Message value temporary buffer */ - const char *brokers; /* Argument: broker list */ - const char *topic; /* Argument: topic to produce to */ - - /* - * Argument validation - */ - if (argc != 3) { - fprintf(stderr, "%% Usage: %s <broker> <topic>\n", argv[0]); - return 1; - } - - brokers = argv[1]; - topic = argv[2]; - - - /* - * Create Kafka client configuration place-holder - */ - conf = rd_kafka_conf_new(); - - /* Set bootstrap broker(s) as a comma-separated list of - * host or host:port (default port 9092). - * librdkafka will use the bootstrap brokers to acquire the full - * set of brokers from the cluster. */ - if (rd_kafka_conf_set(conf, "bootstrap.servers", brokers, errstr, - sizeof(errstr)) != RD_KAFKA_CONF_OK) { - fprintf(stderr, "%s\n", errstr); - return 1; - } - - /* Set the delivery report callback. - * This callback will be called once per message to inform - * the application if delivery succeeded or failed. - * See dr_msg_cb() above. - * The callback is only triggered from rd_kafka_poll() and - * rd_kafka_flush(). */ - rd_kafka_conf_set_dr_msg_cb(conf, dr_msg_cb); - - /* - * Create producer instance. - * - * NOTE: rd_kafka_new() takes ownership of the conf object - * and the application must not reference it again after - * this call. - */ - rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr)); - if (!rk) { - fprintf(stderr, "%% Failed to create new producer: %s\n", - errstr); - return 1; - } - - /* Signal handler for clean shutdown */ - signal(SIGINT, stop); - - fprintf(stderr, - "%% Type some text and hit enter to produce message\n" - "%% Or just hit enter to only serve delivery reports\n" - "%% Press Ctrl-C or Ctrl-D to exit\n"); - - while (run && fgets(buf, sizeof(buf), stdin)) { - size_t len = strlen(buf); - rd_kafka_resp_err_t err; - - if (buf[len - 1] == '\n') /* Remove newline */ - buf[--len] = '\0'; - - if (len == 0) { - /* Empty line: only serve delivery reports */ - rd_kafka_poll(rk, 0 /*non-blocking */); - continue; - } - - /* - * Send/Produce message. - * This is an asynchronous call, on success it will only - * enqueue the message on the internal producer queue. - * The actual delivery attempts to the broker are handled - * by background threads. - * The previously registered delivery report callback - * (dr_msg_cb) is used to signal back to the application - * when the message has been delivered (or failed). - */ - retry: - err = rd_kafka_producev( - /* Producer handle */ - rk, - /* Topic name */ - RD_KAFKA_V_TOPIC(topic), - /* Make a copy of the payload. */ - RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY), - /* Message value and length */ - RD_KAFKA_V_VALUE(buf, len), - /* Per-Message opaque, provided in - * delivery report callback as - * msg_opaque. */ - RD_KAFKA_V_OPAQUE(NULL), - /* End sentinel */ - RD_KAFKA_V_END); - - if (err) { - /* - * Failed to *enqueue* message for producing. - */ - fprintf(stderr, - "%% Failed to produce to topic %s: %s\n", topic, - rd_kafka_err2str(err)); - - if (err == RD_KAFKA_RESP_ERR__QUEUE_FULL) { - /* If the internal queue is full, wait for - * messages to be delivered and then retry. - * The internal queue represents both - * messages to be sent and messages that have - * been sent or failed, awaiting their - * delivery report callback to be called. - * - * The internal queue is limited by the - * configuration property - * queue.buffering.max.messages and - * queue.buffering.max.kbytes */ - rd_kafka_poll(rk, - 1000 /*block for max 1000ms*/); - goto retry; - } - } else { - fprintf(stderr, - "%% Enqueued message (%zd bytes) " - "for topic %s\n", - len, topic); - } - - - /* A producer application should continually serve - * the delivery report queue by calling rd_kafka_poll() - * at frequent intervals. - * Either put the poll call in your main loop, or in a - * dedicated thread, or call it after every - * rd_kafka_produce() call. - * Just make sure that rd_kafka_poll() is still called - * during periods where you are not producing any messages - * to make sure previously produced messages have their - * delivery report callback served (and any other callbacks - * you register). */ - rd_kafka_poll(rk, 0 /*non-blocking*/); - } - - - /* Wait for final messages to be delivered or fail. - * rd_kafka_flush() is an abstraction over rd_kafka_poll() which - * waits for all messages to be delivered. */ - fprintf(stderr, "%% Flushing final messages..\n"); - rd_kafka_flush(rk, 10 * 1000 /* wait for max 10 seconds */); - - /* If the output queue is still not empty there is an issue - * with producing messages to the clusters. */ - if (rd_kafka_outq_len(rk) > 0) - fprintf(stderr, "%% %d message(s) were not delivered\n", - rd_kafka_outq_len(rk)); - - /* Destroy the producer instance */ - rd_kafka_destroy(rk); - - return 0; -} diff --git a/fluent-bit/lib/librdkafka-2.1.0/examples/producer.cpp b/fluent-bit/lib/librdkafka-2.1.0/examples/producer.cpp deleted file mode 100755 index d4a8a0c49..000000000 --- a/fluent-bit/lib/librdkafka-2.1.0/examples/producer.cpp +++ /dev/null @@ -1,228 +0,0 @@ -/* - * librdkafka - Apache Kafka C library - * - * Copyright (c) 2019, Magnus Edenhill - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/** - * Apache Kafka producer - * using the Kafka driver from librdkafka - * (https://github.com/edenhill/librdkafka) - */ - -#include <iostream> -#include <string> -#include <cstdlib> -#include <cstdio> -#include <csignal> -#include <cstring> - -#if _AIX -#include <unistd.h> -#endif - -/* - * Typical include path in a real application would be - * #include <librdkafka/rdkafkacpp.h> - */ -#include "rdkafkacpp.h" - - -static volatile sig_atomic_t run = 1; - -static void sigterm(int sig) { - run = 0; -} - - -class ExampleDeliveryReportCb : public RdKafka::DeliveryReportCb { - public: - void dr_cb(RdKafka::Message &message) { - /* If message.err() is non-zero the message delivery failed permanently - * for the message. */ - if (message.err()) - std::cerr << "% Message delivery failed: " << message.errstr() - << std::endl; - else - std::cerr << "% Message delivered to topic " << message.topic_name() - << " [" << message.partition() << "] at offset " - << message.offset() << std::endl; - } -}; - -int main(int argc, char **argv) { - if (argc != 3) { - std::cerr << "Usage: " << argv[0] << " <brokers> <topic>\n"; - exit(1); - } - - std::string brokers = argv[1]; - std::string topic = argv[2]; - - /* - * Create configuration object - */ - RdKafka::Conf *conf = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL); - - std::string errstr; - - /* Set bootstrap broker(s) as a comma-separated list of - * host or host:port (default port 9092). - * librdkafka will use the bootstrap brokers to acquire the full - * set of brokers from the cluster. */ - if (conf->set("bootstrap.servers", brokers, errstr) != - RdKafka::Conf::CONF_OK) { - std::cerr << errstr << std::endl; - exit(1); - } - - signal(SIGINT, sigterm); - signal(SIGTERM, sigterm); - - /* Set the delivery report callback. - * This callback will be called once per message to inform - * the application if delivery succeeded or failed. - * See dr_msg_cb() above. - * The callback is only triggered from ::poll() and ::flush(). - * - * IMPORTANT: - * Make sure the DeliveryReport instance outlives the Producer object, - * either by putting it on the heap or as in this case as a stack variable - * that will NOT go out of scope for the duration of the Producer object. - */ - ExampleDeliveryReportCb ex_dr_cb; - - if (conf->set("dr_cb", &ex_dr_cb, errstr) != RdKafka::Conf::CONF_OK) { - std::cerr << errstr << std::endl; - exit(1); - } - - /* - * Create producer instance. - */ - RdKafka::Producer *producer = RdKafka::Producer::create(conf, errstr); - if (!producer) { - std::cerr << "Failed to create producer: " << errstr << std::endl; - exit(1); - } - - delete conf; - - /* - * Read messages from stdin and produce to broker. - */ - std::cout << "% Type message value and hit enter " - << "to produce message." << std::endl; - - for (std::string line; run && std::getline(std::cin, line);) { - if (line.empty()) { - producer->poll(0); - continue; - } - - /* - * Send/Produce message. - * This is an asynchronous call, on success it will only - * enqueue the message on the internal producer queue. - * The actual delivery attempts to the broker are handled - * by background threads. - * The previously registered delivery report callback - * is used to signal back to the application when the message - * has been delivered (or failed permanently after retries). - */ - retry: - RdKafka::ErrorCode err = producer->produce( - /* Topic name */ - topic, - /* Any Partition: the builtin partitioner will be - * used to assign the message to a topic based - * on the message key, or random partition if - * the key is not set. */ - RdKafka::Topic::PARTITION_UA, - /* Make a copy of the value */ - RdKafka::Producer::RK_MSG_COPY /* Copy payload */, - /* Value */ - const_cast<char *>(line.c_str()), line.size(), - /* Key */ - NULL, 0, - /* Timestamp (defaults to current time) */ - 0, - /* Message headers, if any */ - NULL, - /* Per-message opaque value passed to - * delivery report */ - NULL); - - if (err != RdKafka::ERR_NO_ERROR) { - std::cerr << "% Failed to produce to topic " << topic << ": " - << RdKafka::err2str(err) << std::endl; - - if (err == RdKafka::ERR__QUEUE_FULL) { - /* If the internal queue is full, wait for - * messages to be delivered and then retry. - * The internal queue represents both - * messages to be sent and messages that have - * been sent or failed, awaiting their - * delivery report callback to be called. - * - * The internal queue is limited by the - * configuration property - * queue.buffering.max.messages and queue.buffering.max.kbytes */ - producer->poll(1000 /*block for max 1000ms*/); - goto retry; - } - - } else { - std::cerr << "% Enqueued message (" << line.size() << " bytes) " - << "for topic " << topic << std::endl; - } - - /* A producer application should continually serve - * the delivery report queue by calling poll() - * at frequent intervals. - * Either put the poll call in your main loop, or in a - * dedicated thread, or call it after every produce() call. - * Just make sure that poll() is still called - * during periods where you are not producing any messages - * to make sure previously produced messages have their - * delivery report callback served (and any other callbacks - * you register). */ - producer->poll(0); - } - - /* Wait for final messages to be delivered or fail. - * flush() is an abstraction over poll() which - * waits for all messages to be delivered. */ - std::cerr << "% Flushing final messages..." << std::endl; - producer->flush(10 * 1000 /* wait for max 10 seconds */); - - if (producer->outq_len() > 0) - std::cerr << "% " << producer->outq_len() - << " message(s) were not delivered" << std::endl; - - delete producer; - - return 0; -} diff --git a/fluent-bit/lib/librdkafka-2.1.0/examples/rdkafka_complex_consumer_example.c b/fluent-bit/lib/librdkafka-2.1.0/examples/rdkafka_complex_consumer_example.c deleted file mode 100644 index 1632b3030..000000000 --- a/fluent-bit/lib/librdkafka-2.1.0/examples/rdkafka_complex_consumer_example.c +++ /dev/null @@ -1,617 +0,0 @@ -/* - * librdkafka - Apache Kafka C library - * - * Copyright (c) 2015, Magnus Edenhill - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/** - * Apache Kafka high level consumer example program - * using the Kafka driver from librdkafka - * (https://github.com/edenhill/librdkafka) - */ - -#include <ctype.h> -#include <signal.h> -#include <string.h> -#include <unistd.h> -#include <stdlib.h> -#include <syslog.h> -#include <sys/time.h> -#include <errno.h> -#include <getopt.h> - -/* Typical include path would be <librdkafka/rdkafka.h>, but this program - * is builtin from within the librdkafka source tree and thus differs. */ -#include "rdkafka.h" /* for Kafka driver */ - - -static volatile sig_atomic_t run = 1; -static rd_kafka_t *rk; -static int exit_eof = 0; -static int wait_eof = 0; /* number of partitions awaiting EOF */ -static int quiet = 0; -static enum { - OUTPUT_HEXDUMP, - OUTPUT_RAW, -} output = OUTPUT_HEXDUMP; - -static void stop(int sig) { - if (!run) - exit(1); - run = 0; - fclose(stdin); /* abort fgets() */ -} - - -static void hexdump(FILE *fp, const char *name, const void *ptr, size_t len) { - const char *p = (const char *)ptr; - unsigned int of = 0; - - - if (name) - fprintf(fp, "%s hexdump (%zd bytes):\n", name, len); - - for (of = 0; of < len; of += 16) { - char hexen[16 * 3 + 1]; - char charen[16 + 1]; - int hof = 0; - - int cof = 0; - int i; - - for (i = of; i < (int)of + 16 && i < (int)len; i++) { - hof += sprintf(hexen + hof, "%02x ", p[i] & 0xff); - cof += sprintf(charen + cof, "%c", - isprint((int)p[i]) ? p[i] : '.'); - } - fprintf(fp, "%08x: %-48s %-16s\n", of, hexen, charen); - } -} - -/** - * Kafka logger callback (optional) - */ -static void -logger(const rd_kafka_t *rk, int level, const char *fac, const char *buf) { - struct timeval tv; - gettimeofday(&tv, NULL); - fprintf(stdout, "%u.%03u RDKAFKA-%i-%s: %s: %s\n", (int)tv.tv_sec, - (int)(tv.tv_usec / 1000), level, fac, rd_kafka_name(rk), buf); -} - - - -/** - * Handle and print a consumed message. - * Internally crafted messages are also used to propagate state from - * librdkafka to the application. The application needs to check - * the `rkmessage->err` field for this purpose. - */ -static void msg_consume(rd_kafka_message_t *rkmessage) { - if (rkmessage->err) { - if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) { - fprintf(stderr, - "%% Consumer reached end of %s [%" PRId32 - "] " - "message queue at offset %" PRId64 "\n", - rd_kafka_topic_name(rkmessage->rkt), - rkmessage->partition, rkmessage->offset); - - if (exit_eof && --wait_eof == 0) { - fprintf(stderr, - "%% All partition(s) reached EOF: " - "exiting\n"); - run = 0; - } - - return; - } - - if (rkmessage->rkt) - fprintf(stderr, - "%% Consume error for " - "topic \"%s\" [%" PRId32 - "] " - "offset %" PRId64 ": %s\n", - rd_kafka_topic_name(rkmessage->rkt), - rkmessage->partition, rkmessage->offset, - rd_kafka_message_errstr(rkmessage)); - else - fprintf(stderr, "%% Consumer error: %s: %s\n", - rd_kafka_err2str(rkmessage->err), - rd_kafka_message_errstr(rkmessage)); - - if (rkmessage->err == RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION || - rkmessage->err == RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC) - run = 0; - return; - } - - if (!quiet) - fprintf(stdout, - "%% Message (topic %s [%" PRId32 - "], " - "offset %" PRId64 ", %zd bytes):\n", - rd_kafka_topic_name(rkmessage->rkt), - rkmessage->partition, rkmessage->offset, - rkmessage->len); - - if (rkmessage->key_len) { - if (output == OUTPUT_HEXDUMP) - hexdump(stdout, "Message Key", rkmessage->key, - rkmessage->key_len); - else - printf("Key: %.*s\n", (int)rkmessage->key_len, - (char *)rkmessage->key); - } - - if (output == OUTPUT_HEXDUMP) - hexdump(stdout, "Message Payload", rkmessage->payload, - rkmessage->len); - else - printf("%.*s\n", (int)rkmessage->len, - (char *)rkmessage->payload); -} - - -static void -print_partition_list(FILE *fp, - const rd_kafka_topic_partition_list_t *partitions) { - int i; - for (i = 0; i < partitions->cnt; i++) { - fprintf(fp, "%s %s [%" PRId32 "] offset %" PRId64, - i > 0 ? "," : "", partitions->elems[i].topic, - partitions->elems[i].partition, - partitions->elems[i].offset); - } - fprintf(fp, "\n"); -} -static void rebalance_cb(rd_kafka_t *rk, - rd_kafka_resp_err_t err, - rd_kafka_topic_partition_list_t *partitions, - void *opaque) { - rd_kafka_error_t *error = NULL; - rd_kafka_resp_err_t ret_err = RD_KAFKA_RESP_ERR_NO_ERROR; - - fprintf(stderr, "%% Consumer group rebalanced: "); - - switch (err) { - case RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS: - fprintf(stderr, "assigned (%s):\n", - rd_kafka_rebalance_protocol(rk)); - print_partition_list(stderr, partitions); - - if (!strcmp(rd_kafka_rebalance_protocol(rk), "COOPERATIVE")) - error = rd_kafka_incremental_assign(rk, partitions); - else - ret_err = rd_kafka_assign(rk, partitions); - wait_eof += partitions->cnt; - break; - - case RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS: - fprintf(stderr, "revoked (%s):\n", - rd_kafka_rebalance_protocol(rk)); - print_partition_list(stderr, partitions); - - if (!strcmp(rd_kafka_rebalance_protocol(rk), "COOPERATIVE")) { - error = rd_kafka_incremental_unassign(rk, partitions); - wait_eof -= partitions->cnt; - } else { - ret_err = rd_kafka_assign(rk, NULL); - wait_eof = 0; - } - break; - - default: - fprintf(stderr, "failed: %s\n", rd_kafka_err2str(err)); - rd_kafka_assign(rk, NULL); - break; - } - - if (error) { - fprintf(stderr, "incremental assign failure: %s\n", - rd_kafka_error_string(error)); - rd_kafka_error_destroy(error); - } else if (ret_err) { - fprintf(stderr, "assign failure: %s\n", - rd_kafka_err2str(ret_err)); - } -} - - -static int describe_groups(rd_kafka_t *rk, const char *group) { - rd_kafka_resp_err_t err; - const struct rd_kafka_group_list *grplist; - int i; - - err = rd_kafka_list_groups(rk, group, &grplist, 10000); - - if (err) { - fprintf(stderr, "%% Failed to acquire group list: %s\n", - rd_kafka_err2str(err)); - return -1; - } - - for (i = 0; i < grplist->group_cnt; i++) { - const struct rd_kafka_group_info *gi = &grplist->groups[i]; - int j; - - printf("Group \"%s\" in state %s on broker %d (%s:%d)\n", - gi->group, gi->state, gi->broker.id, gi->broker.host, - gi->broker.port); - if (gi->err) - printf(" Error: %s\n", rd_kafka_err2str(gi->err)); - printf( - " Protocol type \"%s\", protocol \"%s\", " - "with %d member(s):\n", - gi->protocol_type, gi->protocol, gi->member_cnt); - - for (j = 0; j < gi->member_cnt; j++) { - const struct rd_kafka_group_member_info *mi; - mi = &gi->members[j]; - - printf(" \"%s\", client id \"%s\" on host %s\n", - mi->member_id, mi->client_id, mi->client_host); - printf(" metadata: %d bytes\n", - mi->member_metadata_size); - printf(" assignment: %d bytes\n", - mi->member_assignment_size); - } - printf("\n"); - } - - if (group && !grplist->group_cnt) - fprintf(stderr, "%% No matching group (%s)\n", group); - - rd_kafka_group_list_destroy(grplist); - - return 0; -} - - - -static void sig_usr1(int sig) { - rd_kafka_dump(stdout, rk); -} - -int main(int argc, char **argv) { - char mode = 'C'; - char *brokers = "localhost:9092"; - int opt; - rd_kafka_conf_t *conf; - char errstr[512]; - const char *debug = NULL; - int do_conf_dump = 0; - char tmp[16]; - rd_kafka_resp_err_t err; - char *group = NULL; - rd_kafka_topic_partition_list_t *topics; - int is_subscription; - int i; - - quiet = !isatty(STDIN_FILENO); - - /* Kafka configuration */ - conf = rd_kafka_conf_new(); - - /* Set logger */ - rd_kafka_conf_set_log_cb(conf, logger); - - /* Quick termination */ - snprintf(tmp, sizeof(tmp), "%i", SIGIO); - rd_kafka_conf_set(conf, "internal.termination.signal", tmp, NULL, 0); - - while ((opt = getopt(argc, argv, "g:b:qd:eX:ADO")) != -1) { - switch (opt) { - case 'b': - brokers = optarg; - break; - case 'g': - group = optarg; - break; - case 'e': - exit_eof = 1; - break; - case 'd': - debug = optarg; - break; - case 'q': - quiet = 1; - break; - case 'A': - output = OUTPUT_RAW; - break; - case 'X': { - char *name, *val; - rd_kafka_conf_res_t res; - - if (!strcmp(optarg, "list") || - !strcmp(optarg, "help")) { - rd_kafka_conf_properties_show(stdout); - exit(0); - } - - if (!strcmp(optarg, "dump")) { - do_conf_dump = 1; - continue; - } - - name = optarg; - if (!(val = strchr(name, '='))) { - fprintf(stderr, - "%% Expected " - "-X property=value, not %s\n", - name); - exit(1); - } - - *val = '\0'; - val++; - - res = rd_kafka_conf_set(conf, name, val, errstr, - sizeof(errstr)); - - if (res != RD_KAFKA_CONF_OK) { - fprintf(stderr, "%% %s\n", errstr); - exit(1); - } - } break; - - case 'D': - case 'O': - mode = opt; - break; - - default: - goto usage; - } - } - - - if (do_conf_dump) { - const char **arr; - size_t cnt; - int pass; - - for (pass = 0; pass < 2; pass++) { - if (pass == 0) { - arr = rd_kafka_conf_dump(conf, &cnt); - printf("# Global config\n"); - } else { - rd_kafka_topic_conf_t *topic_conf = - rd_kafka_conf_get_default_topic_conf(conf); - if (topic_conf) { - printf("# Topic config\n"); - arr = rd_kafka_topic_conf_dump( - topic_conf, &cnt); - } else { - arr = NULL; - } - } - - if (!arr) - continue; - - for (i = 0; i < (int)cnt; i += 2) - printf("%s = %s\n", arr[i], arr[i + 1]); - - printf("\n"); - rd_kafka_conf_dump_free(arr, cnt); - } - - exit(0); - } - - - if (strchr("OC", mode) && optind == argc) { - usage: - fprintf(stderr, - "Usage: %s [options] <topic[:part]> <topic[:part]>..\n" - "\n" - "librdkafka version %s (0x%08x)\n" - "\n" - " Options:\n" - " -g <group> Consumer group (%s)\n" - " -b <brokers> Broker address (%s)\n" - " -e Exit consumer when last message\n" - " in partition has been received.\n" - " -D Describe group.\n" - " -O Get commmitted offset(s)\n" - " -d [facs..] Enable debugging contexts:\n" - " %s\n" - " -q Be quiet\n" - " -A Raw payload output (consumer)\n" - " -X <prop=name> Set arbitrary librdkafka " - "configuration property\n" - " Use '-X list' to see the full list\n" - " of supported properties.\n" - "\n" - "For balanced consumer groups use the 'topic1 topic2..'" - " format\n" - "and for static assignment use " - "'topic1:part1 topic1:part2 topic2:part1..'\n" - "\n", - argv[0], rd_kafka_version_str(), rd_kafka_version(), - group, brokers, RD_KAFKA_DEBUG_CONTEXTS); - exit(1); - } - - - signal(SIGINT, stop); - signal(SIGUSR1, sig_usr1); - - if (debug && rd_kafka_conf_set(conf, "debug", debug, errstr, - sizeof(errstr)) != RD_KAFKA_CONF_OK) { - fprintf(stderr, "%% Debug configuration failed: %s: %s\n", - errstr, debug); - exit(1); - } - - /* - * Client/Consumer group - */ - - if (strchr("CO", mode)) { - /* Consumer groups require a group id */ - if (!group) - group = "rdkafka_consumer_example"; - if (rd_kafka_conf_set(conf, "group.id", group, errstr, - sizeof(errstr)) != RD_KAFKA_CONF_OK) { - fprintf(stderr, "%% %s\n", errstr); - exit(1); - } - - /* Callback called on partition assignment changes */ - rd_kafka_conf_set_rebalance_cb(conf, rebalance_cb); - - rd_kafka_conf_set(conf, "enable.partition.eof", "true", NULL, - 0); - } - - /* Set bootstrap servers */ - if (rd_kafka_conf_set(conf, "bootstrap.servers", brokers, errstr, - sizeof(errstr)) != RD_KAFKA_CONF_OK) { - fprintf(stderr, "%% %s\n", errstr); - exit(1); - } - - /* Create Kafka handle */ - if (!(rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, - sizeof(errstr)))) { - fprintf(stderr, "%% Failed to create new consumer: %s\n", - errstr); - exit(1); - } - - if (mode == 'D') { - int r; - /* Describe groups */ - r = describe_groups(rk, group); - - rd_kafka_destroy(rk); - exit(r == -1 ? 1 : 0); - } - - /* Redirect rd_kafka_poll() to consumer_poll() */ - rd_kafka_poll_set_consumer(rk); - - topics = rd_kafka_topic_partition_list_new(argc - optind); - is_subscription = 1; - for (i = optind; i < argc; i++) { - /* Parse "topic[:part] */ - char *topic = argv[i]; - char *t; - int32_t partition = -1; - - if ((t = strstr(topic, ":"))) { - *t = '\0'; - partition = atoi(t + 1); - is_subscription = 0; /* is assignment */ - wait_eof++; - } - - rd_kafka_topic_partition_list_add(topics, topic, partition); - } - - if (mode == 'O') { - /* Offset query */ - - err = rd_kafka_committed(rk, topics, 5000); - if (err) { - fprintf(stderr, "%% Failed to fetch offsets: %s\n", - rd_kafka_err2str(err)); - exit(1); - } - - for (i = 0; i < topics->cnt; i++) { - rd_kafka_topic_partition_t *p = &topics->elems[i]; - printf("Topic \"%s\" partition %" PRId32, p->topic, - p->partition); - if (p->err) - printf(" error %s", rd_kafka_err2str(p->err)); - else { - printf(" offset %" PRId64 "", p->offset); - - if (p->metadata_size) - printf(" (%d bytes of metadata)", - (int)p->metadata_size); - } - printf("\n"); - } - - goto done; - } - - - if (is_subscription) { - fprintf(stderr, "%% Subscribing to %d topics\n", topics->cnt); - - if ((err = rd_kafka_subscribe(rk, topics))) { - fprintf(stderr, - "%% Failed to start consuming topics: %s\n", - rd_kafka_err2str(err)); - exit(1); - } - } else { - fprintf(stderr, "%% Assigning %d partitions\n", topics->cnt); - - if ((err = rd_kafka_assign(rk, topics))) { - fprintf(stderr, "%% Failed to assign partitions: %s\n", - rd_kafka_err2str(err)); - } - } - - while (run) { - rd_kafka_message_t *rkmessage; - - rkmessage = rd_kafka_consumer_poll(rk, 1000); - if (rkmessage) { - msg_consume(rkmessage); - rd_kafka_message_destroy(rkmessage); - } - } - -done: - err = rd_kafka_consumer_close(rk); - if (err) - fprintf(stderr, "%% Failed to close consumer: %s\n", - rd_kafka_err2str(err)); - else - fprintf(stderr, "%% Consumer closed\n"); - - rd_kafka_topic_partition_list_destroy(topics); - - /* Destroy handle */ - rd_kafka_destroy(rk); - - /* Let background threads clean up and terminate cleanly. */ - run = 5; - while (run-- > 0 && rd_kafka_wait_destroyed(1000) == -1) - printf("Waiting for librdkafka to decommission\n"); - if (run <= 0) - rd_kafka_dump(stdout, rk); - - return 0; -} diff --git a/fluent-bit/lib/librdkafka-2.1.0/examples/rdkafka_complex_consumer_example.cpp b/fluent-bit/lib/librdkafka-2.1.0/examples/rdkafka_complex_consumer_example.cpp deleted file mode 100644 index b4f158cbd..000000000 --- a/fluent-bit/lib/librdkafka-2.1.0/examples/rdkafka_complex_consumer_example.cpp +++ /dev/null @@ -1,467 +0,0 @@ -/* - * librdkafka - Apache Kafka C library - * - * Copyright (c) 2014, Magnus Edenhill - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/** - * Apache Kafka consumer & producer example programs - * using the Kafka driver from librdkafka - * (https://github.com/edenhill/librdkafka) - */ - -#include <iostream> -#include <string> -#include <cstdlib> -#include <cstdio> -#include <csignal> -#include <cstring> - -#ifndef _WIN32 -#include <sys/time.h> -#else -#include <windows.h> /* for GetLocalTime */ -#endif - -#ifdef _MSC_VER -#include "../win32/wingetopt.h" -#elif _AIX -#include <unistd.h> -#else -#include <getopt.h> -#include <unistd.h> -#endif - -/* - * Typically include path in a real application would be - * #include <librdkafka/rdkafkacpp.h> - */ -#include "rdkafkacpp.h" - - - -static volatile sig_atomic_t run = 1; -static bool exit_eof = false; -static int eof_cnt = 0; -static int partition_cnt = 0; -static int verbosity = 1; -static long msg_cnt = 0; -static int64_t msg_bytes = 0; -static void sigterm(int sig) { - run = 0; -} - - -/** - * @brief format a string timestamp from the current time - */ -static void print_time() { -#ifndef _WIN32 - struct timeval tv; - char buf[64]; - gettimeofday(&tv, NULL); - strftime(buf, sizeof(buf) - 1, "%Y-%m-%d %H:%M:%S", localtime(&tv.tv_sec)); - fprintf(stderr, "%s.%03d: ", buf, (int)(tv.tv_usec / 1000)); -#else - SYSTEMTIME lt = {0}; - GetLocalTime(<); - // %Y-%m-%d %H:%M:%S.xxx: - fprintf(stderr, "%04d-%02d-%02d %02d:%02d:%02d.%03d: ", lt.wYear, lt.wMonth, - lt.wDay, lt.wHour, lt.wMinute, lt.wSecond, lt.wMilliseconds); -#endif -} -class ExampleEventCb : public RdKafka::EventCb { - public: - void event_cb(RdKafka::Event &event) { - print_time(); - - switch (event.type()) { - case RdKafka::Event::EVENT_ERROR: - if (event.fatal()) { - std::cerr << "FATAL "; - run = 0; - } - std::cerr << "ERROR (" << RdKafka::err2str(event.err()) - << "): " << event.str() << std::endl; - break; - - case RdKafka::Event::EVENT_STATS: - std::cerr << "\"STATS\": " << event.str() << std::endl; - break; - - case RdKafka::Event::EVENT_LOG: - fprintf(stderr, "LOG-%i-%s: %s\n", event.severity(), event.fac().c_str(), - event.str().c_str()); - break; - - case RdKafka::Event::EVENT_THROTTLE: - std::cerr << "THROTTLED: " << event.throttle_time() << "ms by " - << event.broker_name() << " id " << (int)event.broker_id() - << std::endl; - break; - - default: - std::cerr << "EVENT " << event.type() << " (" - << RdKafka::err2str(event.err()) << "): " << event.str() - << std::endl; - break; - } - } -}; - - -class ExampleRebalanceCb : public RdKafka::RebalanceCb { - private: - static void part_list_print( - const std::vector<RdKafka::TopicPartition *> &partitions) { - for (unsigned int i = 0; i < partitions.size(); i++) - std::cerr << partitions[i]->topic() << "[" << partitions[i]->partition() - << "], "; - std::cerr << "\n"; - } - - public: - void rebalance_cb(RdKafka::KafkaConsumer *consumer, - RdKafka::ErrorCode err, - std::vector<RdKafka::TopicPartition *> &partitions) { - std::cerr << "RebalanceCb: " << RdKafka::err2str(err) << ": "; - - part_list_print(partitions); - - RdKafka::Error *error = NULL; - RdKafka::ErrorCode ret_err = RdKafka::ERR_NO_ERROR; - - if (err == RdKafka::ERR__ASSIGN_PARTITIONS) { - if (consumer->rebalance_protocol() == "COOPERATIVE") - error = consumer->incremental_assign(partitions); - else - ret_err = consumer->assign(partitions); - partition_cnt += (int)partitions.size(); - } else { - if (consumer->rebalance_protocol() == "COOPERATIVE") { - error = consumer->incremental_unassign(partitions); - partition_cnt -= (int)partitions.size(); - } else { - ret_err = consumer->unassign(); - partition_cnt = 0; - } - } - eof_cnt = 0; /* FIXME: Won't work with COOPERATIVE */ - - if (error) { - std::cerr << "incremental assign failed: " << error->str() << "\n"; - delete error; - } else if (ret_err) - std::cerr << "assign failed: " << RdKafka::err2str(ret_err) << "\n"; - } -}; - - -void msg_consume(RdKafka::Message *message, void *opaque) { - switch (message->err()) { - case RdKafka::ERR__TIMED_OUT: - break; - - case RdKafka::ERR_NO_ERROR: - /* Real message */ - msg_cnt++; - msg_bytes += message->len(); - if (verbosity >= 3) - std::cerr << "Read msg at offset " << message->offset() << std::endl; - RdKafka::MessageTimestamp ts; - ts = message->timestamp(); - if (verbosity >= 2 && - ts.type != RdKafka::MessageTimestamp::MSG_TIMESTAMP_NOT_AVAILABLE) { - std::string tsname = "?"; - if (ts.type == RdKafka::MessageTimestamp::MSG_TIMESTAMP_CREATE_TIME) - tsname = "create time"; - else if (ts.type == - RdKafka::MessageTimestamp::MSG_TIMESTAMP_LOG_APPEND_TIME) - tsname = "log append time"; - std::cout << "Timestamp: " << tsname << " " << ts.timestamp << std::endl; - } - if (verbosity >= 2 && message->key()) { - std::cout << "Key: " << *message->key() << std::endl; - } - if (verbosity >= 1) { - printf("%.*s\n", static_cast<int>(message->len()), - static_cast<const char *>(message->payload())); - } - break; - - case RdKafka::ERR__PARTITION_EOF: - /* Last message */ - if (exit_eof && ++eof_cnt == partition_cnt) { - std::cerr << "%% EOF reached for all " << partition_cnt << " partition(s)" - << std::endl; - run = 0; - } - break; - - case RdKafka::ERR__UNKNOWN_TOPIC: - case RdKafka::ERR__UNKNOWN_PARTITION: - std::cerr << "Consume failed: " << message->errstr() << std::endl; - run = 0; - break; - - default: - /* Errors */ - std::cerr << "Consume failed: " << message->errstr() << std::endl; - run = 0; - } -} - -int main(int argc, char **argv) { - std::string brokers = "localhost"; - std::string errstr; - std::string topic_str; - std::string mode; - std::string debug; - std::vector<std::string> topics; - bool do_conf_dump = false; - int opt; - - /* - * Create configuration objects - */ - RdKafka::Conf *conf = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL); - - ExampleRebalanceCb ex_rebalance_cb; - conf->set("rebalance_cb", &ex_rebalance_cb, errstr); - - conf->set("enable.partition.eof", "true", errstr); - - while ((opt = getopt(argc, argv, "g:b:z:qd:eX:AM:qv")) != -1) { - switch (opt) { - case 'g': - if (conf->set("group.id", optarg, errstr) != RdKafka::Conf::CONF_OK) { - std::cerr << errstr << std::endl; - exit(1); - } - break; - case 'b': - brokers = optarg; - break; - case 'z': - if (conf->set("compression.codec", optarg, errstr) != - RdKafka::Conf::CONF_OK) { - std::cerr << errstr << std::endl; - exit(1); - } - break; - case 'e': - exit_eof = true; - break; - case 'd': - debug = optarg; - break; - case 'M': - if (conf->set("statistics.interval.ms", optarg, errstr) != - RdKafka::Conf::CONF_OK) { - std::cerr << errstr << std::endl; - exit(1); - } - break; - case 'X': { - char *name, *val; - - if (!strcmp(optarg, "dump")) { - do_conf_dump = true; - continue; - } - - name = optarg; - if (!(val = strchr(name, '='))) { - std::cerr << "%% Expected -X property=value, not " << name << std::endl; - exit(1); - } - - *val = '\0'; - val++; - - RdKafka::Conf::ConfResult res = conf->set(name, val, errstr); - if (res != RdKafka::Conf::CONF_OK) { - std::cerr << errstr << std::endl; - exit(1); - } - } break; - - case 'q': - verbosity--; - break; - - case 'v': - verbosity++; - break; - - default: - goto usage; - } - } - - for (; optind < argc; optind++) - topics.push_back(std::string(argv[optind])); - - if (topics.empty() || optind != argc) { - usage: - fprintf(stderr, - "Usage: %s -g <group-id> [options] topic1 topic2..\n" - "\n" - "librdkafka version %s (0x%08x)\n" - "\n" - " Options:\n" - " -g <group-id> Consumer group id\n" - " -b <brokers> Broker address (localhost:9092)\n" - " -z <codec> Enable compression:\n" - " none|gzip|snappy\n" - " -e Exit consumer when last message\n" - " in partition has been received.\n" - " -d [facs..] Enable debugging contexts:\n" - " %s\n" - " -M <intervalms> Enable statistics\n" - " -X <prop=name> Set arbitrary librdkafka " - "configuration property\n" - " Use '-X list' to see the full list\n" - " of supported properties.\n" - " -q Quiet / Decrease verbosity\n" - " -v Increase verbosity\n" - "\n" - "\n", - argv[0], RdKafka::version_str().c_str(), RdKafka::version(), - RdKafka::get_debug_contexts().c_str()); - exit(1); - } - - if (exit_eof) { - std::string strategy; - if (conf->get("partition.assignment.strategy", strategy) == - RdKafka::Conf::CONF_OK && - strategy == "cooperative-sticky") { - std::cerr - << "Error: this example has not been modified to " - << "support -e (exit on EOF) when the partition.assignment.strategy " - << "is set to " << strategy << ": remove -e from the command line\n"; - exit(1); - } - } - - /* - * Set configuration properties - */ - conf->set("metadata.broker.list", brokers, errstr); - - if (!debug.empty()) { - if (conf->set("debug", debug, errstr) != RdKafka::Conf::CONF_OK) { - std::cerr << errstr << std::endl; - exit(1); - } - } - - ExampleEventCb ex_event_cb; - conf->set("event_cb", &ex_event_cb, errstr); - - if (do_conf_dump) { - std::list<std::string> *dump; - dump = conf->dump(); - std::cout << "# Global config" << std::endl; - - for (std::list<std::string>::iterator it = dump->begin(); - it != dump->end();) { - std::cout << *it << " = "; - it++; - std::cout << *it << std::endl; - it++; - } - std::cout << std::endl; - - exit(0); - } - - signal(SIGINT, sigterm); - signal(SIGTERM, sigterm); - - - /* - * Consumer mode - */ - - /* - * Create consumer using accumulated global configuration. - */ - RdKafka::KafkaConsumer *consumer = - RdKafka::KafkaConsumer::create(conf, errstr); - if (!consumer) { - std::cerr << "Failed to create consumer: " << errstr << std::endl; - exit(1); - } - - delete conf; - - std::cout << "% Created consumer " << consumer->name() << std::endl; - - - /* - * Subscribe to topics - */ - RdKafka::ErrorCode err = consumer->subscribe(topics); - if (err) { - std::cerr << "Failed to subscribe to " << topics.size() - << " topics: " << RdKafka::err2str(err) << std::endl; - exit(1); - } - - /* - * Consume messages - */ - while (run) { - RdKafka::Message *msg = consumer->consume(1000); - msg_consume(msg, NULL); - delete msg; - } - -#ifndef _WIN32 - alarm(10); -#endif - - /* - * Stop consumer - */ - consumer->close(); - delete consumer; - - std::cerr << "% Consumed " << msg_cnt << " messages (" << msg_bytes - << " bytes)" << std::endl; - - /* - * Wait for RdKafka to decommission. - * This is not strictly needed (with check outq_len() above), but - * allows RdKafka to clean up all its resources before the application - * exits so that memory profilers such as valgrind wont complain about - * memory leaks. - */ - RdKafka::wait_destroyed(5000); - - return 0; -} diff --git a/fluent-bit/lib/librdkafka-2.1.0/examples/rdkafka_consume_batch.cpp b/fluent-bit/lib/librdkafka-2.1.0/examples/rdkafka_consume_batch.cpp deleted file mode 100644 index 576b396f8..000000000 --- a/fluent-bit/lib/librdkafka-2.1.0/examples/rdkafka_consume_batch.cpp +++ /dev/null @@ -1,264 +0,0 @@ -/* - * librdkafka - Apache Kafka C library - * - * Copyright (c) 2018, Magnus Edenhill - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/** - * Apache Kafka consumer & producer example programs - * using the Kafka driver from librdkafka - * (https://github.com/edenhill/librdkafka) - * - * This example shows how to read batches of messages. - * Note that messages are fetched from the broker in batches regardless - * of how the application polls messages from librdkafka, this example - * merely shows how to accumulate a set of messages in the application. - */ - -#include <iostream> -#include <string> -#include <cstdlib> -#include <cstdio> -#include <csignal> -#include <cstring> - -#ifndef _WIN32 -#include <sys/time.h> -#endif - -#ifdef _WIN32 -#include "../win32/wingetopt.h" -#include <atltime.h> -#elif _AIX -#include <unistd.h> -#else -#include <getopt.h> -#include <unistd.h> -#endif - -/* - * Typically include path in a real application would be - * #include <librdkafka/rdkafkacpp.h> - */ -#include "rdkafkacpp.h" - - - -static volatile sig_atomic_t run = 1; - -static void sigterm(int sig) { - run = 0; -} - - - -/** - * @returns the current wall-clock time in milliseconds - */ -static int64_t now() { -#ifndef _WIN32 - struct timeval tv; - gettimeofday(&tv, NULL); - return ((int64_t)tv.tv_sec * 1000) + (tv.tv_usec / 1000); -#else -#error "now() not implemented for Windows, please submit a PR" -#endif -} - - - -/** - * @brief Accumulate a batch of \p batch_size messages, but wait - * no longer than \p batch_tmout milliseconds. - */ -static std::vector<RdKafka::Message *> consume_batch( - RdKafka::KafkaConsumer *consumer, - size_t batch_size, - int batch_tmout) { - std::vector<RdKafka::Message *> msgs; - msgs.reserve(batch_size); - - int64_t end = now() + batch_tmout; - int remaining_timeout = batch_tmout; - - while (msgs.size() < batch_size) { - RdKafka::Message *msg = consumer->consume(remaining_timeout); - - switch (msg->err()) { - case RdKafka::ERR__TIMED_OUT: - delete msg; - return msgs; - - case RdKafka::ERR_NO_ERROR: - msgs.push_back(msg); - break; - - default: - std::cerr << "%% Consumer error: " << msg->errstr() << std::endl; - run = 0; - delete msg; - return msgs; - } - - remaining_timeout = end - now(); - if (remaining_timeout < 0) - break; - } - - return msgs; -} - - -int main(int argc, char **argv) { - std::string errstr; - std::string topic_str; - std::vector<std::string> topics; - int batch_size = 100; - int batch_tmout = 1000; - - /* Create configuration objects */ - RdKafka::Conf *conf = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL); - - if (conf->set("enable.partition.eof", "false", errstr) != - RdKafka::Conf::CONF_OK) { - std::cerr << errstr << std::endl; - exit(1); - } - - /* Read command line arguments */ - int opt; - while ((opt = getopt(argc, argv, "g:B:T:b:X:")) != -1) { - switch (opt) { - case 'g': - if (conf->set("group.id", optarg, errstr) != RdKafka::Conf::CONF_OK) { - std::cerr << errstr << std::endl; - exit(1); - } - break; - - case 'B': - batch_size = atoi(optarg); - break; - - case 'T': - batch_tmout = atoi(optarg); - break; - - case 'b': - if (conf->set("bootstrap.servers", optarg, errstr) != - RdKafka::Conf::CONF_OK) { - std::cerr << errstr << std::endl; - exit(1); - } - break; - - case 'X': { - char *name, *val; - - name = optarg; - if (!(val = strchr(name, '='))) { - std::cerr << "%% Expected -X property=value, not " << name << std::endl; - exit(1); - } - - *val = '\0'; - val++; - - if (conf->set(name, val, errstr) != RdKafka::Conf::CONF_OK) { - std::cerr << errstr << std::endl; - exit(1); - } - } break; - - default: - goto usage; - } - } - - /* Topics to consume */ - for (; optind < argc; optind++) - topics.push_back(std::string(argv[optind])); - - if (topics.empty() || optind != argc) { - usage: - fprintf( - stderr, - "Usage: %s -g <group-id> -B <batch-size> [options] topic1 topic2..\n" - "\n" - "librdkafka version %s (0x%08x)\n" - "\n" - " Options:\n" - " -g <group-id> Consumer group id\n" - " -B <batch-size> How many messages to batch (default: 100).\n" - " -T <batch-tmout> How long to wait for batch-size to accumulate in " - "milliseconds. (default 1000 ms)\n" - " -b <brokers> Broker address (localhost:9092)\n" - " -X <prop=name> Set arbitrary librdkafka configuration property\n" - "\n", - argv[0], RdKafka::version_str().c_str(), RdKafka::version()); - exit(1); - } - - - signal(SIGINT, sigterm); - signal(SIGTERM, sigterm); - - /* Create consumer */ - RdKafka::KafkaConsumer *consumer = - RdKafka::KafkaConsumer::create(conf, errstr); - if (!consumer) { - std::cerr << "Failed to create consumer: " << errstr << std::endl; - exit(1); - } - - delete conf; - - /* Subscribe to topics */ - RdKafka::ErrorCode err = consumer->subscribe(topics); - if (err) { - std::cerr << "Failed to subscribe to " << topics.size() - << " topics: " << RdKafka::err2str(err) << std::endl; - exit(1); - } - - /* Consume messages in batches of \p batch_size */ - while (run) { - auto msgs = consume_batch(consumer, batch_size, batch_tmout); - std::cout << "Accumulated " << msgs.size() << " messages:" << std::endl; - - for (auto &msg : msgs) { - std::cout << " Message in " << msg->topic_name() << " [" - << msg->partition() << "] at offset " << msg->offset() - << std::endl; - delete msg; - } - } - - /* Close and destroy consumer */ - consumer->close(); - delete consumer; - - return 0; -} diff --git a/fluent-bit/lib/librdkafka-2.1.0/examples/rdkafka_example.c b/fluent-bit/lib/librdkafka-2.1.0/examples/rdkafka_example.c deleted file mode 100644 index 91415318a..000000000 --- a/fluent-bit/lib/librdkafka-2.1.0/examples/rdkafka_example.c +++ /dev/null @@ -1,853 +0,0 @@ -/* - * librdkafka - Apache Kafka C library - * - * Copyright (c) 2012, Magnus Edenhill - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/** - * Apache Kafka consumer & producer example programs - * using the Kafka driver from librdkafka - * (https://github.com/edenhill/librdkafka) - */ - -#include <ctype.h> -#include <signal.h> -#include <string.h> -#include <unistd.h> -#include <stdlib.h> -#include <syslog.h> -#include <time.h> -#include <sys/time.h> -#include <getopt.h> - -/* Typical include path would be <librdkafka/rdkafka.h>, but this program - * is builtin from within the librdkafka source tree and thus differs. */ -#include "rdkafka.h" /* for Kafka driver */ - - -static volatile sig_atomic_t run = 1; -static rd_kafka_t *rk; -static int exit_eof = 0; -static int quiet = 0; -static enum { - OUTPUT_HEXDUMP, - OUTPUT_RAW, -} output = OUTPUT_HEXDUMP; - -static void stop(int sig) { - run = 0; - fclose(stdin); /* abort fgets() */ -} - - -static void hexdump(FILE *fp, const char *name, const void *ptr, size_t len) { - const char *p = (const char *)ptr; - size_t of = 0; - - - if (name) - fprintf(fp, "%s hexdump (%zd bytes):\n", name, len); - - for (of = 0; of < len; of += 16) { - char hexen[16 * 3 + 1]; - char charen[16 + 1]; - int hof = 0; - - int cof = 0; - int i; - - for (i = of; i < (int)of + 16 && i < (int)len; i++) { - hof += sprintf(hexen + hof, "%02x ", p[i] & 0xff); - cof += sprintf(charen + cof, "%c", - isprint((int)p[i]) ? p[i] : '.'); - } - fprintf(fp, "%08zx: %-48s %-16s\n", of, hexen, charen); - } -} - -/** - * Kafka logger callback (optional) - */ -static void -logger(const rd_kafka_t *rk, int level, const char *fac, const char *buf) { - struct timeval tv; - gettimeofday(&tv, NULL); - fprintf(stderr, "%u.%03u RDKAFKA-%i-%s: %s: %s\n", (int)tv.tv_sec, - (int)(tv.tv_usec / 1000), level, fac, - rk ? rd_kafka_name(rk) : NULL, buf); -} - - -/** - * Message delivery report callback using the richer rd_kafka_message_t object. - */ -static void msg_delivered(rd_kafka_t *rk, - const rd_kafka_message_t *rkmessage, - void *opaque) { - if (rkmessage->err) - fprintf(stderr, - "%% Message delivery failed (broker %" PRId32 "): %s\n", - rd_kafka_message_broker_id(rkmessage), - rd_kafka_err2str(rkmessage->err)); - else if (!quiet) - fprintf(stderr, - "%% Message delivered (%zd bytes, offset %" PRId64 - ", " - "partition %" PRId32 ", broker %" PRId32 "): %.*s\n", - rkmessage->len, rkmessage->offset, rkmessage->partition, - rd_kafka_message_broker_id(rkmessage), - (int)rkmessage->len, (const char *)rkmessage->payload); -} - - -static void msg_consume(rd_kafka_message_t *rkmessage, void *opaque) { - if (rkmessage->err) { - if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) { - fprintf(stderr, - "%% Consumer reached end of %s [%" PRId32 - "] " - "message queue at offset %" PRId64 "\n", - rd_kafka_topic_name(rkmessage->rkt), - rkmessage->partition, rkmessage->offset); - - if (exit_eof) - run = 0; - - return; - } - - fprintf(stderr, - "%% Consume error for topic \"%s\" [%" PRId32 - "] " - "offset %" PRId64 ": %s\n", - rd_kafka_topic_name(rkmessage->rkt), - rkmessage->partition, rkmessage->offset, - rd_kafka_message_errstr(rkmessage)); - - if (rkmessage->err == RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION || - rkmessage->err == RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC) - run = 0; - return; - } - - if (!quiet) { - rd_kafka_timestamp_type_t tstype; - int64_t timestamp; - rd_kafka_headers_t *hdrs; - - fprintf(stdout, - "%% Message (offset %" PRId64 - ", %zd bytes, " - "broker %" PRId32 "):\n", - rkmessage->offset, rkmessage->len, - rd_kafka_message_broker_id(rkmessage)); - - timestamp = rd_kafka_message_timestamp(rkmessage, &tstype); - if (tstype != RD_KAFKA_TIMESTAMP_NOT_AVAILABLE) { - const char *tsname = "?"; - if (tstype == RD_KAFKA_TIMESTAMP_CREATE_TIME) - tsname = "create time"; - else if (tstype == RD_KAFKA_TIMESTAMP_LOG_APPEND_TIME) - tsname = "log append time"; - - fprintf(stdout, - "%% Message timestamp: %s %" PRId64 - " (%ds ago)\n", - tsname, timestamp, - !timestamp ? 0 - : (int)time(NULL) - - (int)(timestamp / 1000)); - } - - if (!rd_kafka_message_headers(rkmessage, &hdrs)) { - size_t idx = 0; - const char *name; - const void *val; - size_t size; - - fprintf(stdout, "%% Headers:"); - - while (!rd_kafka_header_get_all(hdrs, idx++, &name, - &val, &size)) { - fprintf(stdout, "%s%s=", idx == 1 ? " " : ", ", - name); - if (val) - fprintf(stdout, "\"%.*s\"", (int)size, - (const char *)val); - else - fprintf(stdout, "NULL"); - } - fprintf(stdout, "\n"); - } - } - - if (rkmessage->key_len) { - if (output == OUTPUT_HEXDUMP) - hexdump(stdout, "Message Key", rkmessage->key, - rkmessage->key_len); - else - printf("Key: %.*s\n", (int)rkmessage->key_len, - (char *)rkmessage->key); - } - - if (output == OUTPUT_HEXDUMP) - hexdump(stdout, "Message Payload", rkmessage->payload, - rkmessage->len); - else - printf("%.*s\n", (int)rkmessage->len, - (char *)rkmessage->payload); -} - - -static void metadata_print(const char *topic, - const struct rd_kafka_metadata *metadata) { - int i, j, k; - int32_t controllerid; - - printf("Metadata for %s (from broker %" PRId32 ": %s):\n", - topic ?: "all topics", metadata->orig_broker_id, - metadata->orig_broker_name); - - controllerid = rd_kafka_controllerid(rk, 0); - - - /* Iterate brokers */ - printf(" %i brokers:\n", metadata->broker_cnt); - for (i = 0; i < metadata->broker_cnt; i++) - printf(" broker %" PRId32 " at %s:%i%s\n", - metadata->brokers[i].id, metadata->brokers[i].host, - metadata->brokers[i].port, - controllerid == metadata->brokers[i].id ? " (controller)" - : ""); - - /* Iterate topics */ - printf(" %i topics:\n", metadata->topic_cnt); - for (i = 0; i < metadata->topic_cnt; i++) { - const struct rd_kafka_metadata_topic *t = &metadata->topics[i]; - printf(" topic \"%s\" with %i partitions:", t->topic, - t->partition_cnt); - if (t->err) { - printf(" %s", rd_kafka_err2str(t->err)); - if (t->err == RD_KAFKA_RESP_ERR_LEADER_NOT_AVAILABLE) - printf(" (try again)"); - } - printf("\n"); - - /* Iterate topic's partitions */ - for (j = 0; j < t->partition_cnt; j++) { - const struct rd_kafka_metadata_partition *p; - p = &t->partitions[j]; - printf(" partition %" PRId32 - ", " - "leader %" PRId32 ", replicas: ", - p->id, p->leader); - - /* Iterate partition's replicas */ - for (k = 0; k < p->replica_cnt; k++) - printf("%s%" PRId32, k > 0 ? "," : "", - p->replicas[k]); - - /* Iterate partition's ISRs */ - printf(", isrs: "); - for (k = 0; k < p->isr_cnt; k++) - printf("%s%" PRId32, k > 0 ? "," : "", - p->isrs[k]); - if (p->err) - printf(", %s\n", rd_kafka_err2str(p->err)); - else - printf("\n"); - } - } -} - - -static void sig_usr1(int sig) { - rd_kafka_dump(stdout, rk); -} - -int main(int argc, char **argv) { - rd_kafka_topic_t *rkt; - char *brokers = "localhost:9092"; - char mode = 'C'; - char *topic = NULL; - int partition = RD_KAFKA_PARTITION_UA; - int opt; - rd_kafka_conf_t *conf; - rd_kafka_topic_conf_t *topic_conf; - char errstr[512]; - int64_t start_offset = 0; - int do_conf_dump = 0; - char tmp[16]; - int64_t seek_offset = 0; - int64_t tmp_offset = 0; - int get_wmarks = 0; - rd_kafka_headers_t *hdrs = NULL; - rd_kafka_resp_err_t err; - - /* Kafka configuration */ - conf = rd_kafka_conf_new(); - - /* Set logger */ - rd_kafka_conf_set_log_cb(conf, logger); - - /* Quick termination */ - snprintf(tmp, sizeof(tmp), "%i", SIGIO); - rd_kafka_conf_set(conf, "internal.termination.signal", tmp, NULL, 0); - - /* Topic configuration */ - topic_conf = rd_kafka_topic_conf_new(); - - while ((opt = getopt(argc, argv, "PCLt:p:b:z:qd:o:eX:As:H:")) != -1) { - switch (opt) { - case 'P': - case 'C': - case 'L': - mode = opt; - break; - case 't': - topic = optarg; - break; - case 'p': - partition = atoi(optarg); - break; - case 'b': - brokers = optarg; - break; - case 'z': - if (rd_kafka_conf_set(conf, "compression.codec", optarg, - errstr, sizeof(errstr)) != - RD_KAFKA_CONF_OK) { - fprintf(stderr, "%% %s\n", errstr); - exit(1); - } - break; - case 'o': - case 's': - if (!strcmp(optarg, "end")) - tmp_offset = RD_KAFKA_OFFSET_END; - else if (!strcmp(optarg, "beginning")) - tmp_offset = RD_KAFKA_OFFSET_BEGINNING; - else if (!strcmp(optarg, "stored")) - tmp_offset = RD_KAFKA_OFFSET_STORED; - else if (!strcmp(optarg, "wmark")) - get_wmarks = 1; - else { - tmp_offset = strtoll(optarg, NULL, 10); - - if (tmp_offset < 0) - tmp_offset = - RD_KAFKA_OFFSET_TAIL(-tmp_offset); - } - - if (opt == 'o') - start_offset = tmp_offset; - else if (opt == 's') - seek_offset = tmp_offset; - break; - case 'e': - exit_eof = 1; - break; - case 'd': - if (rd_kafka_conf_set(conf, "debug", optarg, errstr, - sizeof(errstr)) != - RD_KAFKA_CONF_OK) { - fprintf(stderr, - "%% Debug configuration failed: " - "%s: %s\n", - errstr, optarg); - exit(1); - } - break; - case 'q': - quiet = 1; - break; - case 'A': - output = OUTPUT_RAW; - break; - case 'H': { - char *name, *val; - size_t name_sz = -1; - - name = optarg; - val = strchr(name, '='); - if (val) { - name_sz = (size_t)(val - name); - val++; /* past the '=' */ - } - - if (!hdrs) - hdrs = rd_kafka_headers_new(8); - - err = rd_kafka_header_add(hdrs, name, name_sz, val, -1); - if (err) { - fprintf(stderr, - "%% Failed to add header %s: %s\n", - name, rd_kafka_err2str(err)); - exit(1); - } - } break; - - case 'X': { - char *name, *val; - rd_kafka_conf_res_t res; - - if (!strcmp(optarg, "list") || - !strcmp(optarg, "help")) { - rd_kafka_conf_properties_show(stdout); - exit(0); - } - - if (!strcmp(optarg, "dump")) { - do_conf_dump = 1; - continue; - } - - name = optarg; - if (!(val = strchr(name, '='))) { - char dest[512]; - size_t dest_size = sizeof(dest); - /* Return current value for property. */ - - res = RD_KAFKA_CONF_UNKNOWN; - if (!strncmp(name, "topic.", strlen("topic."))) - res = rd_kafka_topic_conf_get( - topic_conf, name + strlen("topic."), - dest, &dest_size); - if (res == RD_KAFKA_CONF_UNKNOWN) - res = rd_kafka_conf_get( - conf, name, dest, &dest_size); - - if (res == RD_KAFKA_CONF_OK) { - printf("%s = %s\n", name, dest); - exit(0); - } else { - fprintf(stderr, "%% %s property\n", - res == RD_KAFKA_CONF_UNKNOWN - ? "Unknown" - : "Invalid"); - exit(1); - } - } - - *val = '\0'; - val++; - - res = RD_KAFKA_CONF_UNKNOWN; - /* Try "topic." prefixed properties on topic - * conf first, and then fall through to global if - * it didnt match a topic configuration property. */ - if (!strncmp(name, "topic.", strlen("topic."))) - res = rd_kafka_topic_conf_set( - topic_conf, name + strlen("topic."), val, - errstr, sizeof(errstr)); - - if (res == RD_KAFKA_CONF_UNKNOWN) - res = rd_kafka_conf_set(conf, name, val, errstr, - sizeof(errstr)); - - if (res != RD_KAFKA_CONF_OK) { - fprintf(stderr, "%% %s\n", errstr); - exit(1); - } - } break; - - default: - goto usage; - } - } - - - if (do_conf_dump) { - const char **arr; - size_t cnt; - int pass; - - for (pass = 0; pass < 2; pass++) { - int i; - - if (pass == 0) { - arr = rd_kafka_conf_dump(conf, &cnt); - printf("# Global config\n"); - } else { - printf("# Topic config\n"); - arr = - rd_kafka_topic_conf_dump(topic_conf, &cnt); - } - - for (i = 0; i < (int)cnt; i += 2) - printf("%s = %s\n", arr[i], arr[i + 1]); - - printf("\n"); - - rd_kafka_conf_dump_free(arr, cnt); - } - - exit(0); - } - - - if (optind != argc || (mode != 'L' && !topic)) { - usage: - fprintf(stderr, - "Usage: %s -C|-P|-L -t <topic> " - "[-p <partition>] [-b <host1:port1,host2:port2,..>]\n" - "\n" - "librdkafka version %s (0x%08x)\n" - "\n" - " Options:\n" - " -C | -P Consumer or Producer mode\n" - " -L Metadata list mode\n" - " -t <topic> Topic to fetch / produce\n" - " -p <num> Partition (random partitioner)\n" - " -b <brokers> Broker address (localhost:9092)\n" - " -z <codec> Enable compression:\n" - " none|gzip|snappy|lz4|zstd\n" - " -o <offset> Start offset (consumer):\n" - " beginning, end, NNNNN or -NNNNN\n" - " wmark returns the current hi&lo " - "watermarks.\n" - " -e Exit consumer when last message\n" - " in partition has been received.\n" - " -d [facs..] Enable debugging contexts:\n" - " %s\n" - " -q Be quiet\n" - " -A Raw payload output (consumer)\n" - " -H <name[=value]> Add header to message (producer)\n" - " -X <prop=name> Set arbitrary librdkafka " - "configuration property\n" - " Properties prefixed with \"topic.\" " - "will be set on topic object.\n" - " -X list Show full list of supported " - "properties.\n" - " -X dump Show configuration\n" - " -X <prop> Get single property value\n" - "\n" - " In Consumer mode:\n" - " writes fetched messages to stdout\n" - " In Producer mode:\n" - " reads messages from stdin and sends to broker\n" - " In List mode:\n" - " queries broker for metadata information, " - "topic is optional.\n" - "\n" - "\n" - "\n", - argv[0], rd_kafka_version_str(), rd_kafka_version(), - RD_KAFKA_DEBUG_CONTEXTS); - exit(1); - } - - if ((mode == 'C' && !isatty(STDIN_FILENO)) || - (mode != 'C' && !isatty(STDOUT_FILENO))) - quiet = 1; - - - signal(SIGINT, stop); - signal(SIGUSR1, sig_usr1); - - /* Set bootstrap servers */ - if (brokers && - rd_kafka_conf_set(conf, "bootstrap.servers", brokers, errstr, - sizeof(errstr)) != RD_KAFKA_CONF_OK) { - fprintf(stderr, "%% %s\n", errstr); - exit(1); - } - - if (mode == 'P') { - /* - * Producer - */ - char buf[2048]; - int sendcnt = 0; - - /* Set up a message delivery report callback. - * It will be called once for each message, either on successful - * delivery to broker, or upon failure to deliver to broker. */ - rd_kafka_conf_set_dr_msg_cb(conf, msg_delivered); - - /* Create Kafka handle */ - if (!(rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, - sizeof(errstr)))) { - fprintf(stderr, - "%% Failed to create new producer: %s\n", - errstr); - exit(1); - } - - /* Create topic */ - rkt = rd_kafka_topic_new(rk, topic, topic_conf); - topic_conf = NULL; /* Now owned by topic */ - - if (!quiet) - fprintf(stderr, - "%% Type stuff and hit enter to send\n"); - - while (run && fgets(buf, sizeof(buf), stdin)) { - size_t len = strlen(buf); - if (buf[len - 1] == '\n') - buf[--len] = '\0'; - - err = RD_KAFKA_RESP_ERR_NO_ERROR; - - /* Send/Produce message. */ - if (hdrs) { - rd_kafka_headers_t *hdrs_copy; - - hdrs_copy = rd_kafka_headers_copy(hdrs); - - err = rd_kafka_producev( - rk, RD_KAFKA_V_RKT(rkt), - RD_KAFKA_V_PARTITION(partition), - RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY), - RD_KAFKA_V_VALUE(buf, len), - RD_KAFKA_V_HEADERS(hdrs_copy), - RD_KAFKA_V_END); - - if (err) - rd_kafka_headers_destroy(hdrs_copy); - - } else { - if (rd_kafka_produce( - rkt, partition, RD_KAFKA_MSG_F_COPY, - /* Payload and length */ - buf, len, - /* Optional key and its length */ - NULL, 0, - /* Message opaque, provided in - * delivery report callback as - * msg_opaque. */ - NULL) == -1) { - err = rd_kafka_last_error(); - } - } - - if (err) { - fprintf(stderr, - "%% Failed to produce to topic %s " - "partition %i: %s\n", - rd_kafka_topic_name(rkt), partition, - rd_kafka_err2str(err)); - - /* Poll to handle delivery reports */ - rd_kafka_poll(rk, 0); - continue; - } - - if (!quiet) - fprintf(stderr, - "%% Sent %zd bytes to topic " - "%s partition %i\n", - len, rd_kafka_topic_name(rkt), - partition); - sendcnt++; - /* Poll to handle delivery reports */ - rd_kafka_poll(rk, 0); - } - - /* Poll to handle delivery reports */ - rd_kafka_poll(rk, 0); - - /* Wait for messages to be delivered */ - while (run && rd_kafka_outq_len(rk) > 0) - rd_kafka_poll(rk, 100); - - /* Destroy topic */ - rd_kafka_topic_destroy(rkt); - - /* Destroy the handle */ - rd_kafka_destroy(rk); - - } else if (mode == 'C') { - /* - * Consumer - */ - - rd_kafka_conf_set(conf, "enable.partition.eof", "true", NULL, - 0); - - /* Create Kafka handle */ - if (!(rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, - sizeof(errstr)))) { - fprintf(stderr, - "%% Failed to create new consumer: %s\n", - errstr); - exit(1); - } - - if (get_wmarks) { - int64_t lo, hi; - - /* Only query for hi&lo partition watermarks */ - - if ((err = rd_kafka_query_watermark_offsets( - rk, topic, partition, &lo, &hi, 5000))) { - fprintf(stderr, - "%% query_watermark_offsets() " - "failed: %s\n", - rd_kafka_err2str(err)); - exit(1); - } - - printf( - "%s [%d]: low - high offsets: " - "%" PRId64 " - %" PRId64 "\n", - topic, partition, lo, hi); - - rd_kafka_destroy(rk); - exit(0); - } - - - /* Create topic */ - rkt = rd_kafka_topic_new(rk, topic, topic_conf); - topic_conf = NULL; /* Now owned by topic */ - - /* Start consuming */ - if (rd_kafka_consume_start(rkt, partition, start_offset) == - -1) { - err = rd_kafka_last_error(); - fprintf(stderr, "%% Failed to start consuming: %s\n", - rd_kafka_err2str(err)); - if (err == RD_KAFKA_RESP_ERR__INVALID_ARG) - fprintf(stderr, - "%% Broker based offset storage " - "requires a group.id, " - "add: -X group.id=yourGroup\n"); - exit(1); - } - - while (run) { - rd_kafka_message_t *rkmessage; - - /* Poll for errors, etc. */ - rd_kafka_poll(rk, 0); - - /* Consume single message. - * See rdkafka_performance.c for high speed - * consuming of messages. */ - rkmessage = rd_kafka_consume(rkt, partition, 1000); - if (!rkmessage) /* timeout */ - continue; - - msg_consume(rkmessage, NULL); - - /* Return message to rdkafka */ - rd_kafka_message_destroy(rkmessage); - - if (seek_offset) { - err = rd_kafka_seek(rkt, partition, seek_offset, - 2000); - if (err) - printf("Seek failed: %s\n", - rd_kafka_err2str(err)); - else - printf("Seeked to %" PRId64 "\n", - seek_offset); - seek_offset = 0; - } - } - - /* Stop consuming */ - rd_kafka_consume_stop(rkt, partition); - - while (rd_kafka_outq_len(rk) > 0) - rd_kafka_poll(rk, 10); - - /* Destroy topic */ - rd_kafka_topic_destroy(rkt); - - /* Destroy handle */ - rd_kafka_destroy(rk); - - } else if (mode == 'L') { - err = RD_KAFKA_RESP_ERR_NO_ERROR; - - /* Create Kafka handle */ - if (!(rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, - sizeof(errstr)))) { - fprintf(stderr, - "%% Failed to create new producer: %s\n", - errstr); - exit(1); - } - - /* Create topic */ - if (topic) { - rkt = rd_kafka_topic_new(rk, topic, topic_conf); - topic_conf = NULL; /* Now owned by topic */ - } else - rkt = NULL; - - while (run) { - const struct rd_kafka_metadata *metadata; - - /* Fetch metadata */ - err = rd_kafka_metadata(rk, rkt ? 0 : 1, rkt, &metadata, - 5000); - if (err != RD_KAFKA_RESP_ERR_NO_ERROR) { - fprintf(stderr, - "%% Failed to acquire metadata: %s\n", - rd_kafka_err2str(err)); - run = 0; - break; - } - - metadata_print(topic, metadata); - - rd_kafka_metadata_destroy(metadata); - run = 0; - } - - /* Destroy topic */ - if (rkt) - rd_kafka_topic_destroy(rkt); - - /* Destroy the handle */ - rd_kafka_destroy(rk); - - if (topic_conf) - rd_kafka_topic_conf_destroy(topic_conf); - - - /* Exit right away, dont wait for background cleanup, we haven't - * done anything important anyway. */ - exit(err ? 2 : 0); - } - - if (hdrs) - rd_kafka_headers_destroy(hdrs); - - if (topic_conf) - rd_kafka_topic_conf_destroy(topic_conf); - - /* Let background threads clean up and terminate cleanly. */ - run = 5; - while (run-- > 0 && rd_kafka_wait_destroyed(1000) == -1) - printf("Waiting for librdkafka to decommission\n"); - if (run <= 0) - rd_kafka_dump(stdout, rk); - - return 0; -} diff --git a/fluent-bit/lib/librdkafka-2.1.0/examples/rdkafka_example.cpp b/fluent-bit/lib/librdkafka-2.1.0/examples/rdkafka_example.cpp deleted file mode 100644 index 91c3440b3..000000000 --- a/fluent-bit/lib/librdkafka-2.1.0/examples/rdkafka_example.cpp +++ /dev/null @@ -1,679 +0,0 @@ -/* - * librdkafka - Apache Kafka C library - * - * Copyright (c) 2014, Magnus Edenhill - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/** - * Apache Kafka consumer & producer example programs - * using the Kafka driver from librdkafka - * (https://github.com/edenhill/librdkafka) - */ - -#include <iostream> -#include <string> -#include <cstdlib> -#include <cstdio> -#include <csignal> -#include <cstring> - -#ifdef _WIN32 -#include "../win32/wingetopt.h" -#elif _AIX -#include <unistd.h> -#else -#include <getopt.h> -#endif - -/* - * Typically include path in a real application would be - * #include <librdkafka/rdkafkacpp.h> - */ -#include "rdkafkacpp.h" - - -static void metadata_print(const std::string &topic, - const RdKafka::Metadata *metadata) { - std::cout << "Metadata for " << (topic.empty() ? "" : "all topics") - << "(from broker " << metadata->orig_broker_id() << ":" - << metadata->orig_broker_name() << std::endl; - - /* Iterate brokers */ - std::cout << " " << metadata->brokers()->size() << " brokers:" << std::endl; - RdKafka::Metadata::BrokerMetadataIterator ib; - for (ib = metadata->brokers()->begin(); ib != metadata->brokers()->end(); - ++ib) { - std::cout << " broker " << (*ib)->id() << " at " << (*ib)->host() << ":" - << (*ib)->port() << std::endl; - } - /* Iterate topics */ - std::cout << metadata->topics()->size() << " topics:" << std::endl; - RdKafka::Metadata::TopicMetadataIterator it; - for (it = metadata->topics()->begin(); it != metadata->topics()->end(); - ++it) { - std::cout << " topic \"" << (*it)->topic() << "\" with " - << (*it)->partitions()->size() << " partitions:"; - - if ((*it)->err() != RdKafka::ERR_NO_ERROR) { - std::cout << " " << err2str((*it)->err()); - if ((*it)->err() == RdKafka::ERR_LEADER_NOT_AVAILABLE) - std::cout << " (try again)"; - } - std::cout << std::endl; - - /* Iterate topic's partitions */ - RdKafka::TopicMetadata::PartitionMetadataIterator ip; - for (ip = (*it)->partitions()->begin(); ip != (*it)->partitions()->end(); - ++ip) { - std::cout << " partition " << (*ip)->id() << ", leader " - << (*ip)->leader() << ", replicas: "; - - /* Iterate partition's replicas */ - RdKafka::PartitionMetadata::ReplicasIterator ir; - for (ir = (*ip)->replicas()->begin(); ir != (*ip)->replicas()->end(); - ++ir) { - std::cout << (ir == (*ip)->replicas()->begin() ? "" : ",") << *ir; - } - - /* Iterate partition's ISRs */ - std::cout << ", isrs: "; - RdKafka::PartitionMetadata::ISRSIterator iis; - for (iis = (*ip)->isrs()->begin(); iis != (*ip)->isrs()->end(); ++iis) - std::cout << (iis == (*ip)->isrs()->begin() ? "" : ",") << *iis; - - if ((*ip)->err() != RdKafka::ERR_NO_ERROR) - std::cout << ", " << RdKafka::err2str((*ip)->err()) << std::endl; - else - std::cout << std::endl; - } - } -} - -static volatile sig_atomic_t run = 1; -static bool exit_eof = false; - -static void sigterm(int sig) { - run = 0; -} - - -class ExampleDeliveryReportCb : public RdKafka::DeliveryReportCb { - public: - void dr_cb(RdKafka::Message &message) { - std::string status_name; - switch (message.status()) { - case RdKafka::Message::MSG_STATUS_NOT_PERSISTED: - status_name = "NotPersisted"; - break; - case RdKafka::Message::MSG_STATUS_POSSIBLY_PERSISTED: - status_name = "PossiblyPersisted"; - break; - case RdKafka::Message::MSG_STATUS_PERSISTED: - status_name = "Persisted"; - break; - default: - status_name = "Unknown?"; - break; - } - std::cout << "Message delivery for (" << message.len() - << " bytes): " << status_name << ": " << message.errstr() - << std::endl; - if (message.key()) - std::cout << "Key: " << *(message.key()) << ";" << std::endl; - } -}; - - -class ExampleEventCb : public RdKafka::EventCb { - public: - void event_cb(RdKafka::Event &event) { - switch (event.type()) { - case RdKafka::Event::EVENT_ERROR: - if (event.fatal()) { - std::cerr << "FATAL "; - run = 0; - } - std::cerr << "ERROR (" << RdKafka::err2str(event.err()) - << "): " << event.str() << std::endl; - break; - - case RdKafka::Event::EVENT_STATS: - std::cerr << "\"STATS\": " << event.str() << std::endl; - break; - - case RdKafka::Event::EVENT_LOG: - fprintf(stderr, "LOG-%i-%s: %s\n", event.severity(), event.fac().c_str(), - event.str().c_str()); - break; - - default: - std::cerr << "EVENT " << event.type() << " (" - << RdKafka::err2str(event.err()) << "): " << event.str() - << std::endl; - break; - } - } -}; - - -/* Use of this partitioner is pretty pointless since no key is provided - * in the produce() call. */ -class MyHashPartitionerCb : public RdKafka::PartitionerCb { - public: - int32_t partitioner_cb(const RdKafka::Topic *topic, - const std::string *key, - int32_t partition_cnt, - void *msg_opaque) { - return djb_hash(key->c_str(), key->size()) % partition_cnt; - } - - private: - static inline unsigned int djb_hash(const char *str, size_t len) { - unsigned int hash = 5381; - for (size_t i = 0; i < len; i++) - hash = ((hash << 5) + hash) + str[i]; - return hash; - } -}; - -void msg_consume(RdKafka::Message *message, void *opaque) { - const RdKafka::Headers *headers; - - switch (message->err()) { - case RdKafka::ERR__TIMED_OUT: - break; - - case RdKafka::ERR_NO_ERROR: - /* Real message */ - std::cout << "Read msg at offset " << message->offset() << std::endl; - if (message->key()) { - std::cout << "Key: " << *message->key() << std::endl; - } - headers = message->headers(); - if (headers) { - std::vector<RdKafka::Headers::Header> hdrs = headers->get_all(); - for (size_t i = 0; i < hdrs.size(); i++) { - const RdKafka::Headers::Header hdr = hdrs[i]; - - if (hdr.value() != NULL) - printf(" Header: %s = \"%.*s\"\n", hdr.key().c_str(), - (int)hdr.value_size(), (const char *)hdr.value()); - else - printf(" Header: %s = NULL\n", hdr.key().c_str()); - } - } - printf("%.*s\n", static_cast<int>(message->len()), - static_cast<const char *>(message->payload())); - break; - - case RdKafka::ERR__PARTITION_EOF: - /* Last message */ - if (exit_eof) { - run = 0; - } - break; - - case RdKafka::ERR__UNKNOWN_TOPIC: - case RdKafka::ERR__UNKNOWN_PARTITION: - std::cerr << "Consume failed: " << message->errstr() << std::endl; - run = 0; - break; - - default: - /* Errors */ - std::cerr << "Consume failed: " << message->errstr() << std::endl; - run = 0; - } -} - - -class ExampleConsumeCb : public RdKafka::ConsumeCb { - public: - void consume_cb(RdKafka::Message &msg, void *opaque) { - msg_consume(&msg, opaque); - } -}; - - - -int main(int argc, char **argv) { - std::string brokers = "localhost"; - std::string errstr; - std::string topic_str; - std::string mode; - std::string debug; - int32_t partition = RdKafka::Topic::PARTITION_UA; - int64_t start_offset = RdKafka::Topic::OFFSET_BEGINNING; - bool do_conf_dump = false; - int opt; - MyHashPartitionerCb hash_partitioner; - int use_ccb = 0; - - /* - * Create configuration objects - */ - RdKafka::Conf *conf = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL); - RdKafka::Conf *tconf = RdKafka::Conf::create(RdKafka::Conf::CONF_TOPIC); - - - while ((opt = getopt(argc, argv, "PCLt:p:b:z:qd:o:eX:AM:f:")) != -1) { - switch (opt) { - case 'P': - case 'C': - case 'L': - mode = opt; - break; - case 't': - topic_str = optarg; - break; - case 'p': - if (!strcmp(optarg, "random")) - /* default */; - else if (!strcmp(optarg, "hash")) { - if (tconf->set("partitioner_cb", &hash_partitioner, errstr) != - RdKafka::Conf::CONF_OK) { - std::cerr << errstr << std::endl; - exit(1); - } - } else - partition = std::atoi(optarg); - break; - case 'b': - brokers = optarg; - break; - case 'z': - if (conf->set("compression.codec", optarg, errstr) != - RdKafka::Conf::CONF_OK) { - std::cerr << errstr << std::endl; - exit(1); - } - break; - case 'o': - if (!strcmp(optarg, "end")) - start_offset = RdKafka::Topic::OFFSET_END; - else if (!strcmp(optarg, "beginning")) - start_offset = RdKafka::Topic::OFFSET_BEGINNING; - else if (!strcmp(optarg, "stored")) - start_offset = RdKafka::Topic::OFFSET_STORED; - else - start_offset = strtoll(optarg, NULL, 10); - break; - case 'e': - exit_eof = true; - break; - case 'd': - debug = optarg; - break; - case 'M': - if (conf->set("statistics.interval.ms", optarg, errstr) != - RdKafka::Conf::CONF_OK) { - std::cerr << errstr << std::endl; - exit(1); - } - break; - case 'X': { - char *name, *val; - - if (!strcmp(optarg, "dump")) { - do_conf_dump = true; - continue; - } - - name = optarg; - if (!(val = strchr(name, '='))) { - std::cerr << "%% Expected -X property=value, not " << name << std::endl; - exit(1); - } - - *val = '\0'; - val++; - - /* Try "topic." prefixed properties on topic - * conf first, and then fall through to global if - * it didnt match a topic configuration property. */ - RdKafka::Conf::ConfResult res; - if (!strncmp(name, "topic.", strlen("topic."))) - res = tconf->set(name + strlen("topic."), val, errstr); - else - res = conf->set(name, val, errstr); - - if (res != RdKafka::Conf::CONF_OK) { - std::cerr << errstr << std::endl; - exit(1); - } - } break; - - case 'f': - if (!strcmp(optarg, "ccb")) - use_ccb = 1; - else { - std::cerr << "Unknown option: " << optarg << std::endl; - exit(1); - } - break; - - default: - goto usage; - } - } - - if (mode.empty() || (topic_str.empty() && mode != "L") || optind != argc) { - usage: - std::string features; - conf->get("builtin.features", features); - fprintf(stderr, - "Usage: %s [-C|-P] -t <topic> " - "[-p <partition>] [-b <host1:port1,host2:port2,..>]\n" - "\n" - "librdkafka version %s (0x%08x, builtin.features \"%s\")\n" - "\n" - " Options:\n" - " -C | -P Consumer or Producer mode\n" - " -L Metadata list mode\n" - " -t <topic> Topic to fetch / produce\n" - " -p <num> Partition (random partitioner)\n" - " -p <func> Use partitioner:\n" - " random (default), hash\n" - " -b <brokers> Broker address (localhost:9092)\n" - " -z <codec> Enable compression:\n" - " none|gzip|snappy|lz4|zstd\n" - " -o <offset> Start offset (consumer)\n" - " -e Exit consumer when last message\n" - " in partition has been received.\n" - " -d [facs..] Enable debugging contexts:\n" - " %s\n" - " -M <intervalms> Enable statistics\n" - " -X <prop=name> Set arbitrary librdkafka " - "configuration property\n" - " Properties prefixed with \"topic.\" " - "will be set on topic object.\n" - " Use '-X list' to see the full list\n" - " of supported properties.\n" - " -f <flag> Set option:\n" - " ccb - use consume_callback\n" - "\n" - " In Consumer mode:\n" - " writes fetched messages to stdout\n" - " In Producer mode:\n" - " reads messages from stdin and sends to broker\n" - "\n" - "\n" - "\n", - argv[0], RdKafka::version_str().c_str(), RdKafka::version(), - features.c_str(), RdKafka::get_debug_contexts().c_str()); - exit(1); - } - - - /* - * Set configuration properties - */ - conf->set("metadata.broker.list", brokers, errstr); - - if (!debug.empty()) { - if (conf->set("debug", debug, errstr) != RdKafka::Conf::CONF_OK) { - std::cerr << errstr << std::endl; - exit(1); - } - } - - ExampleEventCb ex_event_cb; - conf->set("event_cb", &ex_event_cb, errstr); - - if (do_conf_dump) { - int pass; - - for (pass = 0; pass < 2; pass++) { - std::list<std::string> *dump; - if (pass == 0) { - dump = conf->dump(); - std::cout << "# Global config" << std::endl; - } else { - dump = tconf->dump(); - std::cout << "# Topic config" << std::endl; - } - - for (std::list<std::string>::iterator it = dump->begin(); - it != dump->end();) { - std::cout << *it << " = "; - it++; - std::cout << *it << std::endl; - it++; - } - std::cout << std::endl; - } - exit(0); - } - - signal(SIGINT, sigterm); - signal(SIGTERM, sigterm); - - - if (mode == "P") { - /* - * Producer mode - */ - - if (topic_str.empty()) - goto usage; - - ExampleDeliveryReportCb ex_dr_cb; - - /* Set delivery report callback */ - conf->set("dr_cb", &ex_dr_cb, errstr); - - conf->set("default_topic_conf", tconf, errstr); - - /* - * Create producer using accumulated global configuration. - */ - RdKafka::Producer *producer = RdKafka::Producer::create(conf, errstr); - if (!producer) { - std::cerr << "Failed to create producer: " << errstr << std::endl; - exit(1); - } - - std::cout << "% Created producer " << producer->name() << std::endl; - - - /* - * Read messages from stdin and produce to broker. - */ - for (std::string line; run && std::getline(std::cin, line);) { - if (line.empty()) { - producer->poll(0); - continue; - } - - RdKafka::Headers *headers = RdKafka::Headers::create(); - headers->add("my header", "header value"); - headers->add("other header", "yes"); - - /* - * Produce message - */ - RdKafka::ErrorCode resp = - producer->produce(topic_str, partition, - RdKafka::Producer::RK_MSG_COPY /* Copy payload */, - /* Value */ - const_cast<char *>(line.c_str()), line.size(), - /* Key */ - NULL, 0, - /* Timestamp (defaults to now) */ - 0, - /* Message headers, if any */ - headers, - /* Per-message opaque value passed to - * delivery report */ - NULL); - if (resp != RdKafka::ERR_NO_ERROR) { - std::cerr << "% Produce failed: " << RdKafka::err2str(resp) - << std::endl; - delete headers; /* Headers are automatically deleted on produce() - * success. */ - } else { - std::cerr << "% Produced message (" << line.size() << " bytes)" - << std::endl; - } - - producer->poll(0); - } - run = 1; - - while (run && producer->outq_len() > 0) { - std::cerr << "Waiting for " << producer->outq_len() << std::endl; - producer->poll(1000); - } - - delete producer; - - - } else if (mode == "C") { - /* - * Consumer mode - */ - - conf->set("enable.partition.eof", "true", errstr); - - if (topic_str.empty()) - goto usage; - - /* - * Create consumer using accumulated global configuration. - */ - RdKafka::Consumer *consumer = RdKafka::Consumer::create(conf, errstr); - if (!consumer) { - std::cerr << "Failed to create consumer: " << errstr << std::endl; - exit(1); - } - - std::cout << "% Created consumer " << consumer->name() << std::endl; - - /* - * Create topic handle. - */ - RdKafka::Topic *topic = - RdKafka::Topic::create(consumer, topic_str, tconf, errstr); - if (!topic) { - std::cerr << "Failed to create topic: " << errstr << std::endl; - exit(1); - } - - /* - * Start consumer for topic+partition at start offset - */ - RdKafka::ErrorCode resp = consumer->start(topic, partition, start_offset); - if (resp != RdKafka::ERR_NO_ERROR) { - std::cerr << "Failed to start consumer: " << RdKafka::err2str(resp) - << std::endl; - exit(1); - } - - ExampleConsumeCb ex_consume_cb; - - /* - * Consume messages - */ - while (run) { - if (use_ccb) { - consumer->consume_callback(topic, partition, 1000, &ex_consume_cb, - &use_ccb); - } else { - RdKafka::Message *msg = consumer->consume(topic, partition, 1000); - msg_consume(msg, NULL); - delete msg; - } - consumer->poll(0); - } - - /* - * Stop consumer - */ - consumer->stop(topic, partition); - - consumer->poll(1000); - - delete topic; - delete consumer; - } else { - /* Metadata mode */ - - /* - * Create producer using accumulated global configuration. - */ - RdKafka::Producer *producer = RdKafka::Producer::create(conf, errstr); - if (!producer) { - std::cerr << "Failed to create producer: " << errstr << std::endl; - exit(1); - } - - std::cout << "% Created producer " << producer->name() << std::endl; - - /* - * Create topic handle. - */ - RdKafka::Topic *topic = NULL; - if (!topic_str.empty()) { - topic = RdKafka::Topic::create(producer, topic_str, tconf, errstr); - if (!topic) { - std::cerr << "Failed to create topic: " << errstr << std::endl; - exit(1); - } - } - - while (run) { - class RdKafka::Metadata *metadata; - - /* Fetch metadata */ - RdKafka::ErrorCode err = - producer->metadata(!topic, topic, &metadata, 5000); - if (err != RdKafka::ERR_NO_ERROR) { - std::cerr << "%% Failed to acquire metadata: " << RdKafka::err2str(err) - << std::endl; - run = 0; - break; - } - - metadata_print(topic_str, metadata); - - delete metadata; - run = 0; - } - } - - delete conf; - delete tconf; - - /* - * Wait for RdKafka to decommission. - * This is not strictly needed (when check outq_len() above), but - * allows RdKafka to clean up all its resources before the application - * exits so that memory profilers such as valgrind wont complain about - * memory leaks. - */ - RdKafka::wait_destroyed(5000); - - return 0; -} diff --git a/fluent-bit/lib/librdkafka-2.1.0/examples/rdkafka_performance.c b/fluent-bit/lib/librdkafka-2.1.0/examples/rdkafka_performance.c deleted file mode 100644 index a12bb7471..000000000 --- a/fluent-bit/lib/librdkafka-2.1.0/examples/rdkafka_performance.c +++ /dev/null @@ -1,1780 +0,0 @@ -/* - * librdkafka - Apache Kafka C library - * - * Copyright (c) 2012, Magnus Edenhill - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/** - * Apache Kafka consumer & producer performance tester - * using the Kafka driver from librdkafka - * (https://github.com/edenhill/librdkafka) - */ - -#ifdef _MSC_VER -#define _CRT_SECURE_NO_WARNINGS /* Silence nonsense on MSVC */ -#endif - -#include "../src/rd.h" - -#define _GNU_SOURCE /* for strndup() */ -#include <ctype.h> -#include <signal.h> -#include <string.h> -#include <errno.h> - -/* Typical include path would be <librdkafka/rdkafka.h>, but this program - * is built from within the librdkafka source tree and thus differs. */ -#include "rdkafka.h" /* for Kafka driver */ -/* Do not include these defines from your program, they will not be - * provided by librdkafka. */ -#include "rd.h" -#include "rdtime.h" - -#ifdef _WIN32 -#include "../win32/wingetopt.h" -#include "../win32/wintime.h" -#endif - - -static volatile sig_atomic_t run = 1; -static int forever = 1; -static rd_ts_t dispintvl = 1000; -static int do_seq = 0; -static int exit_after = 0; -static int exit_eof = 0; -static FILE *stats_fp; -static int dr_disp_div; -static int verbosity = 1; -static int latency_mode = 0; -static FILE *latency_fp = NULL; -static int msgcnt = -1; -static int incremental_mode = 0; -static int partition_cnt = 0; -static int eof_cnt = 0; -static int with_dr = 1; -static int read_hdrs = 0; - - -static void stop(int sig) { - if (!run) - exit(0); - run = 0; -} - -static long int msgs_wait_cnt = 0; -static long int msgs_wait_produce_cnt = 0; -static rd_ts_t t_end; -static rd_kafka_t *global_rk; - -struct avg { - int64_t val; - int cnt; - uint64_t ts_start; -}; - -static struct { - rd_ts_t t_start; - rd_ts_t t_end; - rd_ts_t t_end_send; - uint64_t msgs; - uint64_t msgs_last; - uint64_t msgs_dr_ok; - uint64_t msgs_dr_err; - uint64_t bytes_dr_ok; - uint64_t bytes; - uint64_t bytes_last; - uint64_t tx; - uint64_t tx_err; - uint64_t avg_rtt; - uint64_t offset; - rd_ts_t t_fetch_latency; - rd_ts_t t_last; - rd_ts_t t_enobufs_last; - rd_ts_t t_total; - rd_ts_t latency_last; - rd_ts_t latency_lo; - rd_ts_t latency_hi; - rd_ts_t latency_sum; - int latency_cnt; - int64_t last_offset; -} cnt; - - -uint64_t wall_clock(void) { - struct timeval tv; - gettimeofday(&tv, NULL); - return ((uint64_t)tv.tv_sec * 1000000LLU) + ((uint64_t)tv.tv_usec); -} - -static void err_cb(rd_kafka_t *rk, int err, const char *reason, void *opaque) { - if (err == RD_KAFKA_RESP_ERR__FATAL) { - char errstr[512]; - err = rd_kafka_fatal_error(rk, errstr, sizeof(errstr)); - printf("%% FATAL ERROR CALLBACK: %s: %s: %s\n", - rd_kafka_name(rk), rd_kafka_err2str(err), errstr); - } else { - printf("%% ERROR CALLBACK: %s: %s: %s\n", rd_kafka_name(rk), - rd_kafka_err2str(err), reason); - } -} - -static void throttle_cb(rd_kafka_t *rk, - const char *broker_name, - int32_t broker_id, - int throttle_time_ms, - void *opaque) { - printf("%% THROTTLED %dms by %s (%" PRId32 ")\n", throttle_time_ms, - broker_name, broker_id); -} - -static void offset_commit_cb(rd_kafka_t *rk, - rd_kafka_resp_err_t err, - rd_kafka_topic_partition_list_t *offsets, - void *opaque) { - int i; - - if (err || verbosity >= 2) - printf("%% Offset commit of %d partition(s): %s\n", - offsets->cnt, rd_kafka_err2str(err)); - - for (i = 0; i < offsets->cnt; i++) { - rd_kafka_topic_partition_t *rktpar = &offsets->elems[i]; - if (rktpar->err || verbosity >= 2) - printf("%% %s [%" PRId32 "] @ %" PRId64 ": %s\n", - rktpar->topic, rktpar->partition, rktpar->offset, - rd_kafka_err2str(err)); - } -} - -/** - * @brief Add latency measurement - */ -static void latency_add(int64_t ts, const char *who) { - if (ts > cnt.latency_hi) - cnt.latency_hi = ts; - if (!cnt.latency_lo || ts < cnt.latency_lo) - cnt.latency_lo = ts; - cnt.latency_last = ts; - cnt.latency_cnt++; - cnt.latency_sum += ts; - if (latency_fp) - fprintf(latency_fp, "%" PRIu64 "\n", ts); -} - - -static void msg_delivered(rd_kafka_t *rk, - const rd_kafka_message_t *rkmessage, - void *opaque) { - static rd_ts_t last; - rd_ts_t now = rd_clock(); - static int msgs; - - msgs++; - - msgs_wait_cnt--; - - if (rkmessage->err) - cnt.msgs_dr_err++; - else { - cnt.msgs_dr_ok++; - cnt.bytes_dr_ok += rkmessage->len; - } - - if (latency_mode) { - /* Extract latency */ - int64_t source_ts; - if (sscanf(rkmessage->payload, "LATENCY:%" SCNd64, - &source_ts) == 1) - latency_add(wall_clock() - source_ts, "producer"); - } - - - if ((rkmessage->err && (cnt.msgs_dr_err < 50 || - !(cnt.msgs_dr_err % (dispintvl / 1000)))) || - !last || msgs_wait_cnt < 5 || !(msgs_wait_cnt % dr_disp_div) || - (now - last) >= dispintvl * 1000 || verbosity >= 3) { - if (rkmessage->err && verbosity >= 2) - printf("%% Message delivery failed (broker %" PRId32 - "): " - "%s [%" PRId32 - "]: " - "%s (%li remain)\n", - rd_kafka_message_broker_id(rkmessage), - rd_kafka_topic_name(rkmessage->rkt), - rkmessage->partition, - rd_kafka_err2str(rkmessage->err), msgs_wait_cnt); - else if (verbosity > 2) - printf("%% Message delivered (offset %" PRId64 - ", broker %" PRId32 - "): " - "%li remain\n", - rkmessage->offset, - rd_kafka_message_broker_id(rkmessage), - msgs_wait_cnt); - if (verbosity >= 3 && do_seq) - printf(" --> \"%.*s\"\n", (int)rkmessage->len, - (const char *)rkmessage->payload); - last = now; - } - - cnt.last_offset = rkmessage->offset; - - if (msgs_wait_produce_cnt == 0 && msgs_wait_cnt == 0 && !forever) { - if (verbosity >= 2 && cnt.msgs > 0) { - double error_percent = - (double)(cnt.msgs - cnt.msgs_dr_ok) / cnt.msgs * - 100; - printf( - "%% Messages delivered with failure " - "percentage of %.5f%%\n", - error_percent); - } - t_end = rd_clock(); - run = 0; - } - - if (exit_after && exit_after <= msgs) { - printf("%% Hard exit after %i messages, as requested\n", - exit_after); - exit(0); - } -} - - -static void msg_consume(rd_kafka_message_t *rkmessage, void *opaque) { - - if (rkmessage->err) { - if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) { - cnt.offset = rkmessage->offset; - - if (verbosity >= 1) - printf( - "%% Consumer reached end of " - "%s [%" PRId32 - "] " - "message queue at offset %" PRId64 "\n", - rd_kafka_topic_name(rkmessage->rkt), - rkmessage->partition, rkmessage->offset); - - if (exit_eof && ++eof_cnt == partition_cnt) - run = 0; - - return; - } - - printf("%% Consume error for topic \"%s\" [%" PRId32 - "] " - "offset %" PRId64 ": %s\n", - rkmessage->rkt ? rd_kafka_topic_name(rkmessage->rkt) - : "", - rkmessage->partition, rkmessage->offset, - rd_kafka_message_errstr(rkmessage)); - - if (rkmessage->err == RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION || - rkmessage->err == RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC) - run = 0; - - cnt.msgs_dr_err++; - return; - } - - /* Start measuring from first message received */ - if (!cnt.t_start) - cnt.t_start = cnt.t_last = rd_clock(); - - cnt.offset = rkmessage->offset; - cnt.msgs++; - cnt.bytes += rkmessage->len; - - if (verbosity >= 3 || (verbosity >= 2 && !(cnt.msgs % 1000000))) - printf("@%" PRId64 ": %.*s: %.*s\n", rkmessage->offset, - (int)rkmessage->key_len, (char *)rkmessage->key, - (int)rkmessage->len, (char *)rkmessage->payload); - - - if (latency_mode) { - int64_t remote_ts, ts; - - if (rkmessage->len > 8 && - !memcmp(rkmessage->payload, "LATENCY:", 8) && - sscanf(rkmessage->payload, "LATENCY:%" SCNd64, - &remote_ts) == 1) { - ts = wall_clock() - remote_ts; - if (ts > 0 && ts < (1000000 * 60 * 5)) { - latency_add(ts, "consumer"); - } else { - if (verbosity >= 1) - printf( - "Received latency timestamp is too " - "far off: %" PRId64 - "us (message offset %" PRId64 - "): ignored\n", - ts, rkmessage->offset); - } - } else if (verbosity > 1) - printf("not a LATENCY payload: %.*s\n", - (int)rkmessage->len, (char *)rkmessage->payload); - } - - if (read_hdrs) { - rd_kafka_headers_t *hdrs; - /* Force parsing of headers but don't do anything with them. */ - rd_kafka_message_headers(rkmessage, &hdrs); - } - - if (msgcnt != -1 && (int)cnt.msgs >= msgcnt) - run = 0; -} - - -static void rebalance_cb(rd_kafka_t *rk, - rd_kafka_resp_err_t err, - rd_kafka_topic_partition_list_t *partitions, - void *opaque) { - rd_kafka_error_t *error = NULL; - rd_kafka_resp_err_t ret_err = RD_KAFKA_RESP_ERR_NO_ERROR; - - if (exit_eof && !strcmp(rd_kafka_rebalance_protocol(rk), "COOPERATIVE")) - fprintf(stderr, - "%% This example has not been modified to " - "support -e (exit on EOF) when " - "partition.assignment.strategy " - "is set to an incremental/cooperative strategy: " - "-e will not behave as expected\n"); - - switch (err) { - case RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS: - fprintf(stderr, - "%% Group rebalanced (%s): " - "%d new partition(s) assigned\n", - rd_kafka_rebalance_protocol(rk), partitions->cnt); - - if (!strcmp(rd_kafka_rebalance_protocol(rk), "COOPERATIVE")) { - error = rd_kafka_incremental_assign(rk, partitions); - } else { - ret_err = rd_kafka_assign(rk, partitions); - eof_cnt = 0; - } - - partition_cnt += partitions->cnt; - break; - - case RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS: - fprintf(stderr, - "%% Group rebalanced (%s): %d partition(s) revoked\n", - rd_kafka_rebalance_protocol(rk), partitions->cnt); - - if (!strcmp(rd_kafka_rebalance_protocol(rk), "COOPERATIVE")) { - error = rd_kafka_incremental_unassign(rk, partitions); - partition_cnt -= partitions->cnt; - } else { - ret_err = rd_kafka_assign(rk, NULL); - partition_cnt = 0; - } - - eof_cnt = 0; /* FIXME: Not correct for incremental case */ - break; - - default: - break; - } - - if (error) { - fprintf(stderr, "%% incremental assign failure: %s\n", - rd_kafka_error_string(error)); - rd_kafka_error_destroy(error); - } else if (ret_err) { - fprintf(stderr, "%% assign failure: %s\n", - rd_kafka_err2str(ret_err)); - } -} - - -/** - * Find and extract single value from a two-level search. - * First find 'field1', then find 'field2' and extract its value. - * Returns 0 on miss else the value. - */ -static uint64_t json_parse_fields(const char *json, - const char **end, - const char *field1, - const char *field2) { - const char *t = json; - const char *t2; - int len1 = (int)strlen(field1); - int len2 = (int)strlen(field2); - - while ((t2 = strstr(t, field1))) { - uint64_t v; - - t = t2; - t += len1; - - /* Find field */ - if (!(t2 = strstr(t, field2))) - continue; - t2 += len2; - - while (isspace((int)*t2)) - t2++; - - v = strtoull(t2, (char **)&t, 10); - if (t2 == t) - continue; - - *end = t; - return v; - } - - *end = t + strlen(t); - return 0; -} - -/** - * Parse various values from rdkafka stats - */ -static void json_parse_stats(const char *json) { - const char *t; -#define MAX_AVGS 100 /* max number of brokers to scan for rtt */ - uint64_t avg_rtt[MAX_AVGS + 1]; - int avg_rtt_i = 0; - - /* Store totals at end of array */ - avg_rtt[MAX_AVGS] = 0; - - /* Extract all broker RTTs */ - t = json; - while (avg_rtt_i < MAX_AVGS && *t) { - avg_rtt[avg_rtt_i] = - json_parse_fields(t, &t, "\"rtt\":", "\"avg\":"); - - /* Skip low RTT values, means no messages are passing */ - if (avg_rtt[avg_rtt_i] < 100 /*0.1ms*/) - continue; - - - avg_rtt[MAX_AVGS] += avg_rtt[avg_rtt_i]; - avg_rtt_i++; - } - - if (avg_rtt_i > 0) - avg_rtt[MAX_AVGS] /= avg_rtt_i; - - cnt.avg_rtt = avg_rtt[MAX_AVGS]; -} - - -static int stats_cb(rd_kafka_t *rk, char *json, size_t json_len, void *opaque) { - - /* Extract values for our own stats */ - json_parse_stats(json); - - if (stats_fp) - fprintf(stats_fp, "%s\n", json); - return 0; -} - -#define _OTYPE_TAB 0x1 /* tabular format */ -#define _OTYPE_SUMMARY 0x2 /* summary format */ -#define _OTYPE_FORCE 0x4 /* force output regardless of interval timing */ -static void -print_stats(rd_kafka_t *rk, int mode, int otype, const char *compression) { - rd_ts_t now = rd_clock(); - rd_ts_t t_total; - static int rows_written = 0; - int print_header; - double latency_avg = 0.0f; - char extra[512]; - int extra_of = 0; - *extra = '\0'; - - if (!(otype & _OTYPE_FORCE) && - (((otype & _OTYPE_SUMMARY) && verbosity == 0) || - cnt.t_last + dispintvl > now)) - return; - - print_header = !rows_written || (verbosity > 0 && !(rows_written % 20)); - - if (cnt.t_end_send) - t_total = cnt.t_end_send - cnt.t_start; - else if (cnt.t_end) - t_total = cnt.t_end - cnt.t_start; - else if (cnt.t_start) - t_total = now - cnt.t_start; - else - t_total = 1; - - if (latency_mode && cnt.latency_cnt) - latency_avg = (double)cnt.latency_sum / (double)cnt.latency_cnt; - - if (mode == 'P') { - - if (otype & _OTYPE_TAB) { -#define ROW_START() \ - do { \ - } while (0) -#define COL_HDR(NAME) printf("| %10.10s ", (NAME)) -#define COL_PR64(NAME, VAL) printf("| %10" PRIu64 " ", (VAL)) -#define COL_PRF(NAME, VAL) printf("| %10.2f ", (VAL)) -#define ROW_END() \ - do { \ - printf("\n"); \ - rows_written++; \ - } while (0) - - if (print_header) { - /* First time, print header */ - ROW_START(); - COL_HDR("elapsed"); - COL_HDR("msgs"); - COL_HDR("bytes"); - COL_HDR("rtt"); - COL_HDR("dr"); - COL_HDR("dr_m/s"); - COL_HDR("dr_MB/s"); - COL_HDR("dr_err"); - COL_HDR("tx_err"); - COL_HDR("outq"); - COL_HDR("offset"); - if (latency_mode) { - COL_HDR("lat_curr"); - COL_HDR("lat_avg"); - COL_HDR("lat_lo"); - COL_HDR("lat_hi"); - } - - ROW_END(); - } - - ROW_START(); - COL_PR64("elapsed", t_total / 1000); - COL_PR64("msgs", cnt.msgs); - COL_PR64("bytes", cnt.bytes); - COL_PR64("rtt", cnt.avg_rtt / 1000); - COL_PR64("dr", cnt.msgs_dr_ok); - COL_PR64("dr_m/s", - ((cnt.msgs_dr_ok * 1000000) / t_total)); - COL_PRF("dr_MB/s", - (float)((cnt.bytes_dr_ok) / (float)t_total)); - COL_PR64("dr_err", cnt.msgs_dr_err); - COL_PR64("tx_err", cnt.tx_err); - COL_PR64("outq", - rk ? (uint64_t)rd_kafka_outq_len(rk) : 0); - COL_PR64("offset", (uint64_t)cnt.last_offset); - if (latency_mode) { - COL_PRF("lat_curr", cnt.latency_last / 1000.0f); - COL_PRF("lat_avg", latency_avg / 1000.0f); - COL_PRF("lat_lo", cnt.latency_lo / 1000.0f); - COL_PRF("lat_hi", cnt.latency_hi / 1000.0f); - } - ROW_END(); - } - - if (otype & _OTYPE_SUMMARY) { - printf("%% %" PRIu64 - " messages produced " - "(%" PRIu64 - " bytes), " - "%" PRIu64 - " delivered " - "(offset %" PRId64 ", %" PRIu64 - " failed) " - "in %" PRIu64 "ms: %" PRIu64 - " msgs/s and " - "%.02f MB/s, " - "%" PRIu64 - " produce failures, %i in queue, " - "%s compression\n", - cnt.msgs, cnt.bytes, cnt.msgs_dr_ok, - cnt.last_offset, cnt.msgs_dr_err, t_total / 1000, - ((cnt.msgs_dr_ok * 1000000) / t_total), - (float)((cnt.bytes_dr_ok) / (float)t_total), - cnt.tx_err, rk ? rd_kafka_outq_len(rk) : 0, - compression); - } - - } else { - - if (otype & _OTYPE_TAB) { - if (print_header) { - /* First time, print header */ - ROW_START(); - COL_HDR("elapsed"); - COL_HDR("msgs"); - COL_HDR("bytes"); - COL_HDR("rtt"); - COL_HDR("m/s"); - COL_HDR("MB/s"); - COL_HDR("rx_err"); - COL_HDR("offset"); - if (latency_mode) { - COL_HDR("lat_curr"); - COL_HDR("lat_avg"); - COL_HDR("lat_lo"); - COL_HDR("lat_hi"); - } - ROW_END(); - } - - ROW_START(); - COL_PR64("elapsed", t_total / 1000); - COL_PR64("msgs", cnt.msgs); - COL_PR64("bytes", cnt.bytes); - COL_PR64("rtt", cnt.avg_rtt / 1000); - COL_PR64("m/s", ((cnt.msgs * 1000000) / t_total)); - COL_PRF("MB/s", (float)((cnt.bytes) / (float)t_total)); - COL_PR64("rx_err", cnt.msgs_dr_err); - COL_PR64("offset", cnt.offset); - if (latency_mode) { - COL_PRF("lat_curr", cnt.latency_last / 1000.0f); - COL_PRF("lat_avg", latency_avg / 1000.0f); - COL_PRF("lat_lo", cnt.latency_lo / 1000.0f); - COL_PRF("lat_hi", cnt.latency_hi / 1000.0f); - } - ROW_END(); - } - - if (otype & _OTYPE_SUMMARY) { - if (latency_avg >= 1.0f) - extra_of += rd_snprintf( - extra + extra_of, sizeof(extra) - extra_of, - ", latency " - "curr/avg/lo/hi " - "%.2f/%.2f/%.2f/%.2fms", - cnt.latency_last / 1000.0f, - latency_avg / 1000.0f, - cnt.latency_lo / 1000.0f, - cnt.latency_hi / 1000.0f); - printf("%% %" PRIu64 " messages (%" PRIu64 - " bytes) " - "consumed in %" PRIu64 "ms: %" PRIu64 - " msgs/s " - "(%.02f MB/s)" - "%s\n", - cnt.msgs, cnt.bytes, t_total / 1000, - ((cnt.msgs * 1000000) / t_total), - (float)((cnt.bytes) / (float)t_total), extra); - } - - if (incremental_mode && now > cnt.t_last) { - uint64_t i_msgs = cnt.msgs - cnt.msgs_last; - uint64_t i_bytes = cnt.bytes - cnt.bytes_last; - uint64_t i_time = cnt.t_last ? now - cnt.t_last : 0; - - printf("%% INTERVAL: %" PRIu64 - " messages " - "(%" PRIu64 - " bytes) " - "consumed in %" PRIu64 "ms: %" PRIu64 - " msgs/s " - "(%.02f MB/s)" - "%s\n", - i_msgs, i_bytes, i_time / 1000, - ((i_msgs * 1000000) / i_time), - (float)((i_bytes) / (float)i_time), extra); - } - } - - cnt.t_last = now; - cnt.msgs_last = cnt.msgs; - cnt.bytes_last = cnt.bytes; -} - - -static void sig_usr1(int sig) { - rd_kafka_dump(stdout, global_rk); -} - - -/** - * @brief Read config from file - * @returns -1 on error, else 0. - */ -static int read_conf_file(rd_kafka_conf_t *conf, const char *path) { - FILE *fp; - char buf[512]; - int line = 0; - char errstr[512]; - - if (!(fp = fopen(path, "r"))) { - fprintf(stderr, "%% Failed to open %s: %s\n", path, - strerror(errno)); - return -1; - } - - while (fgets(buf, sizeof(buf), fp)) { - char *s = buf; - char *t; - rd_kafka_conf_res_t r = RD_KAFKA_CONF_UNKNOWN; - - line++; - - while (isspace((int)*s)) - s++; - - if (!*s || *s == '#') - continue; - - if ((t = strchr(buf, '\n'))) - *t = '\0'; - - t = strchr(buf, '='); - if (!t || t == s || !*(t + 1)) { - fprintf(stderr, "%% %s:%d: expected key=value\n", path, - line); - fclose(fp); - return -1; - } - - *(t++) = '\0'; - - /* Try global config */ - r = rd_kafka_conf_set(conf, s, t, errstr, sizeof(errstr)); - - if (r == RD_KAFKA_CONF_OK) - continue; - - fprintf(stderr, "%% %s:%d: %s=%s: %s\n", path, line, s, t, - errstr); - fclose(fp); - return -1; - } - - fclose(fp); - - return 0; -} - - -static rd_kafka_resp_err_t do_produce(rd_kafka_t *rk, - rd_kafka_topic_t *rkt, - int32_t partition, - int msgflags, - void *payload, - size_t size, - const void *key, - size_t key_size, - const rd_kafka_headers_t *hdrs) { - - /* Send/Produce message. */ - if (hdrs) { - rd_kafka_headers_t *hdrs_copy; - rd_kafka_resp_err_t err; - - hdrs_copy = rd_kafka_headers_copy(hdrs); - - err = rd_kafka_producev( - rk, RD_KAFKA_V_RKT(rkt), RD_KAFKA_V_PARTITION(partition), - RD_KAFKA_V_MSGFLAGS(msgflags), - RD_KAFKA_V_VALUE(payload, size), - RD_KAFKA_V_KEY(key, key_size), - RD_KAFKA_V_HEADERS(hdrs_copy), RD_KAFKA_V_END); - - if (err) - rd_kafka_headers_destroy(hdrs_copy); - - return err; - - } else { - if (rd_kafka_produce(rkt, partition, msgflags, payload, size, - key, key_size, NULL) == -1) - return rd_kafka_last_error(); - } - - return RD_KAFKA_RESP_ERR_NO_ERROR; -} - -/** - * @brief Sleep for \p sleep_us microseconds. - */ -static void do_sleep(int sleep_us) { - if (sleep_us > 100) { -#ifdef _WIN32 - Sleep(sleep_us / 1000); -#else - usleep(sleep_us); -#endif - } else { - rd_ts_t next = rd_clock() + (rd_ts_t)sleep_us; - while (next > rd_clock()) - ; - } -} - - -int main(int argc, char **argv) { - char *brokers = NULL; - char mode = 'C'; - char *topic = NULL; - const char *key = NULL; - int *partitions = NULL; - int opt; - int sendflags = 0; - char *msgpattern = "librdkafka_performance testing!"; - int msgsize = -1; - const char *debug = NULL; - int do_conf_dump = 0; - rd_ts_t now; - char errstr[512]; - uint64_t seq = 0; - int seed = (int)time(NULL); - rd_kafka_t *rk; - rd_kafka_topic_t *rkt; - rd_kafka_conf_t *conf; - rd_kafka_queue_t *rkqu = NULL; - const char *compression = "no"; - int64_t start_offset = 0; - int batch_size = 0; - int idle = 0; - const char *stats_cmd = NULL; - char *stats_intvlstr = NULL; - char tmp[128]; - char *tmp2; - int otype = _OTYPE_SUMMARY; - double dtmp; - int rate_sleep = 0; - rd_kafka_topic_partition_list_t *topics; - int exitcode = 0; - rd_kafka_headers_t *hdrs = NULL; - rd_kafka_resp_err_t err; - - /* Kafka configuration */ - conf = rd_kafka_conf_new(); - rd_kafka_conf_set_error_cb(conf, err_cb); - rd_kafka_conf_set_throttle_cb(conf, throttle_cb); - rd_kafka_conf_set_offset_commit_cb(conf, offset_commit_cb); - -#ifdef SIGIO - /* Quick termination */ - rd_snprintf(tmp, sizeof(tmp), "%i", SIGIO); - rd_kafka_conf_set(conf, "internal.termination.signal", tmp, NULL, 0); -#endif - - /* Producer config */ - rd_kafka_conf_set(conf, "linger.ms", "1000", NULL, 0); - rd_kafka_conf_set(conf, "message.send.max.retries", "3", NULL, 0); - rd_kafka_conf_set(conf, "retry.backoff.ms", "500", NULL, 0); - - /* Consumer config */ - /* Tell rdkafka to (try to) maintain 1M messages - * in its internal receive buffers. This is to avoid - * application -> rdkafka -> broker per-message ping-pong - * latency. - * The larger the local queue, the higher the performance. - * Try other values with: ... -X queued.min.messages=1000 - */ - rd_kafka_conf_set(conf, "queued.min.messages", "1000000", NULL, 0); - rd_kafka_conf_set(conf, "session.timeout.ms", "6000", NULL, 0); - rd_kafka_conf_set(conf, "auto.offset.reset", "earliest", NULL, 0); - - topics = rd_kafka_topic_partition_list_new(1); - - while ((opt = getopt(argc, argv, - "PCG:t:p:b:s:k:c:fi:MDd:m:S:x:" - "R:a:z:o:X:B:eT:Y:qvIur:lA:OwNH:")) != -1) { - switch (opt) { - case 'G': - if (rd_kafka_conf_set(conf, "group.id", optarg, errstr, - sizeof(errstr)) != - RD_KAFKA_CONF_OK) { - fprintf(stderr, "%% %s\n", errstr); - exit(1); - } - /* FALLTHRU */ - case 'P': - case 'C': - mode = opt; - break; - case 't': - rd_kafka_topic_partition_list_add( - topics, optarg, RD_KAFKA_PARTITION_UA); - break; - case 'p': - partition_cnt++; - partitions = realloc(partitions, sizeof(*partitions) * - partition_cnt); - partitions[partition_cnt - 1] = atoi(optarg); - break; - - case 'b': - brokers = optarg; - break; - case 's': - msgsize = atoi(optarg); - break; - case 'k': - key = optarg; - break; - case 'c': - msgcnt = atoi(optarg); - break; - case 'D': - sendflags |= RD_KAFKA_MSG_F_FREE; - break; - case 'i': - dispintvl = atoi(optarg); - break; - case 'm': - msgpattern = optarg; - break; - case 'S': - seq = strtoull(optarg, NULL, 10); - do_seq = 1; - break; - case 'x': - exit_after = atoi(optarg); - break; - case 'R': - seed = atoi(optarg); - break; - case 'a': - if (rd_kafka_conf_set(conf, "acks", optarg, errstr, - sizeof(errstr)) != - RD_KAFKA_CONF_OK) { - fprintf(stderr, "%% %s\n", errstr); - exit(1); - } - break; - case 'B': - batch_size = atoi(optarg); - break; - case 'z': - if (rd_kafka_conf_set(conf, "compression.codec", optarg, - errstr, sizeof(errstr)) != - RD_KAFKA_CONF_OK) { - fprintf(stderr, "%% %s\n", errstr); - exit(1); - } - compression = optarg; - break; - case 'o': - if (!strcmp(optarg, "end")) - start_offset = RD_KAFKA_OFFSET_END; - else if (!strcmp(optarg, "beginning")) - start_offset = RD_KAFKA_OFFSET_BEGINNING; - else if (!strcmp(optarg, "stored")) - start_offset = RD_KAFKA_OFFSET_STORED; - else { - start_offset = strtoll(optarg, NULL, 10); - - if (start_offset < 0) - start_offset = - RD_KAFKA_OFFSET_TAIL(-start_offset); - } - - break; - case 'e': - exit_eof = 1; - break; - case 'd': - debug = optarg; - break; - case 'H': - if (!strcmp(optarg, "parse")) - read_hdrs = 1; - else { - char *name, *val; - size_t name_sz = -1; - - name = optarg; - val = strchr(name, '='); - if (val) { - name_sz = (size_t)(val - name); - val++; /* past the '=' */ - } - - if (!hdrs) - hdrs = rd_kafka_headers_new(8); - - err = rd_kafka_header_add(hdrs, name, name_sz, - val, -1); - if (err) { - fprintf( - stderr, - "%% Failed to add header %s: %s\n", - name, rd_kafka_err2str(err)); - exit(1); - } - } - break; - case 'X': { - char *name, *val; - rd_kafka_conf_res_t res; - - if (!strcmp(optarg, "list") || - !strcmp(optarg, "help")) { - rd_kafka_conf_properties_show(stdout); - exit(0); - } - - if (!strcmp(optarg, "dump")) { - do_conf_dump = 1; - continue; - } - - name = optarg; - if (!(val = strchr(name, '='))) { - fprintf(stderr, - "%% Expected " - "-X property=value, not %s\n", - name); - exit(1); - } - - *val = '\0'; - val++; - - if (!strcmp(name, "file")) { - if (read_conf_file(conf, val) == -1) - exit(1); - break; - } - - res = rd_kafka_conf_set(conf, name, val, errstr, - sizeof(errstr)); - - if (res != RD_KAFKA_CONF_OK) { - fprintf(stderr, "%% %s\n", errstr); - exit(1); - } - } break; - - case 'T': - stats_intvlstr = optarg; - break; - case 'Y': - stats_cmd = optarg; - break; - - case 'q': - verbosity--; - break; - - case 'v': - verbosity++; - break; - - case 'I': - idle = 1; - break; - - case 'u': - otype = _OTYPE_TAB; - verbosity--; /* remove some fluff */ - break; - - case 'r': - dtmp = strtod(optarg, &tmp2); - if (tmp2 == optarg || - (dtmp >= -0.001 && dtmp <= 0.001)) { - fprintf(stderr, "%% Invalid rate: %s\n", - optarg); - exit(1); - } - - rate_sleep = (int)(1000000.0 / dtmp); - break; - - case 'l': - latency_mode = 1; - break; - - case 'A': - if (!(latency_fp = fopen(optarg, "w"))) { - fprintf(stderr, "%% Cant open %s: %s\n", optarg, - strerror(errno)); - exit(1); - } - break; - - case 'M': - incremental_mode = 1; - break; - - case 'N': - with_dr = 0; - break; - - default: - fprintf(stderr, "Unknown option: %c\n", opt); - goto usage; - } - } - - if (topics->cnt == 0 || optind != argc) { - if (optind < argc) - fprintf(stderr, "Unknown argument: %s\n", argv[optind]); - usage: - fprintf( - stderr, - "Usage: %s [-C|-P] -t <topic> " - "[-p <partition>] [-b <broker,broker..>] [options..]\n" - "\n" - "librdkafka version %s (0x%08x)\n" - "\n" - " Options:\n" - " -C | -P | Consumer or Producer mode\n" - " -G <groupid> High-level Kafka Consumer mode\n" - " -t <topic> Topic to consume / produce\n" - " -p <num> Partition (defaults to random). " - "Multiple partitions are allowed in -C consumer mode.\n" - " -M Print consumer interval stats\n" - " -b <brokers> Broker address list (host[:port],..)\n" - " -s <size> Message size (producer)\n" - " -k <key> Message key (producer)\n" - " -H <name[=value]> Add header to message (producer)\n" - " -H parse Read message headers (consumer)\n" - " -c <cnt> Messages to transmit/receive\n" - " -x <cnt> Hard exit after transmitting <cnt> " - "messages (producer)\n" - " -D Copy/Duplicate data buffer (producer)\n" - " -i <ms> Display interval\n" - " -m <msg> Message payload pattern\n" - " -S <start> Send a sequence number starting at " - "<start> as payload\n" - " -R <seed> Random seed value (defaults to time)\n" - " -a <acks> Required acks (producer): " - "-1, 0, 1, >1\n" - " -B <size> Consume batch size (# of msgs)\n" - " -z <codec> Enable compression:\n" - " none|gzip|snappy\n" - " -o <offset> Start offset (consumer)\n" - " beginning, end, NNNNN or -NNNNN\n" - " -d [facs..] Enable debugging contexts:\n" - " %s\n" - " -X <prop=name> Set arbitrary librdkafka " - "configuration property\n" - " -X file=<path> Read config from file.\n" - " -X list Show full list of supported properties.\n" - " -X dump Show configuration\n" - " -T <intvl> Enable statistics from librdkafka at " - "specified interval (ms)\n" - " -Y <command> Pipe statistics to <command>\n" - " -I Idle: dont produce any messages\n" - " -q Decrease verbosity\n" - " -v Increase verbosity (default 1)\n" - " -u Output stats in table format\n" - " -r <rate> Producer msg/s limit\n" - " -l Latency measurement.\n" - " Needs two matching instances, one\n" - " consumer and one producer, both\n" - " running with the -l switch.\n" - " -l Producer: per-message latency stats\n" - " -A <file> Write per-message latency stats to " - "<file>. Requires -l\n" - " -O Report produced offset (producer)\n" - " -N No delivery reports (producer)\n" - "\n" - " In Consumer mode:\n" - " consumes messages and prints thruput\n" - " If -B <..> is supplied the batch consumer\n" - " mode is used, else the callback mode is used.\n" - "\n" - " In Producer mode:\n" - " writes messages of size -s <..> and prints thruput\n" - "\n", - argv[0], rd_kafka_version_str(), rd_kafka_version(), - RD_KAFKA_DEBUG_CONTEXTS); - exit(1); - } - - - dispintvl *= 1000; /* us */ - - if (verbosity > 1) - printf("%% Using random seed %i, verbosity level %i\n", seed, - verbosity); - srand(seed); - signal(SIGINT, stop); -#ifdef SIGUSR1 - signal(SIGUSR1, sig_usr1); -#endif - - - if (debug && rd_kafka_conf_set(conf, "debug", debug, errstr, - sizeof(errstr)) != RD_KAFKA_CONF_OK) { - printf("%% Debug configuration failed: %s: %s\n", errstr, - debug); - exit(1); - } - - /* Always enable stats (for RTT extraction), and if user supplied - * the -T <intvl> option we let her take part of the stats aswell. */ - rd_kafka_conf_set_stats_cb(conf, stats_cb); - - if (!stats_intvlstr) { - /* if no user-desired stats, adjust stats interval - * to the display interval. */ - rd_snprintf(tmp, sizeof(tmp), "%" PRId64, dispintvl / 1000); - } - - if (rd_kafka_conf_set(conf, "statistics.interval.ms", - stats_intvlstr ? stats_intvlstr : tmp, errstr, - sizeof(errstr)) != RD_KAFKA_CONF_OK) { - fprintf(stderr, "%% %s\n", errstr); - exit(1); - } - - if (do_conf_dump) { - const char **arr; - size_t cnt; - int pass; - - for (pass = 0; pass < 2; pass++) { - int i; - - if (pass == 0) { - arr = rd_kafka_conf_dump(conf, &cnt); - printf("# Global config\n"); - } else { - rd_kafka_topic_conf_t *topic_conf = - rd_kafka_conf_get_default_topic_conf(conf); - - if (topic_conf) { - printf("# Topic config\n"); - arr = rd_kafka_topic_conf_dump( - topic_conf, &cnt); - } else { - arr = NULL; - } - } - - if (!arr) - continue; - - for (i = 0; i < (int)cnt; i += 2) - printf("%s = %s\n", arr[i], arr[i + 1]); - - printf("\n"); - - rd_kafka_conf_dump_free(arr, cnt); - } - - exit(0); - } - - if (latency_mode) - do_seq = 0; - - if (stats_intvlstr) { - /* User enabled stats (-T) */ - -#ifndef _WIN32 - if (stats_cmd) { - if (!(stats_fp = popen(stats_cmd, -#ifdef __linux__ - "we" -#else - "w" -#endif - ))) { - fprintf(stderr, - "%% Failed to start stats command: " - "%s: %s", - stats_cmd, strerror(errno)); - exit(1); - } - } else -#endif - stats_fp = stdout; - } - - if (msgcnt != -1) - forever = 0; - - if (msgsize == -1) - msgsize = (int)strlen(msgpattern); - - topic = topics->elems[0].topic; - - if (mode == 'C' || mode == 'G') - rd_kafka_conf_set(conf, "enable.partition.eof", "true", NULL, - 0); - - if (read_hdrs && mode == 'P') { - fprintf(stderr, "%% producer can not read headers\n"); - exit(1); - } - - if (hdrs && mode != 'P') { - fprintf(stderr, "%% consumer can not add headers\n"); - exit(1); - } - - /* Set bootstrap servers */ - if (brokers && - rd_kafka_conf_set(conf, "bootstrap.servers", brokers, errstr, - sizeof(errstr)) != RD_KAFKA_CONF_OK) { - fprintf(stderr, "%% %s\n", errstr); - exit(1); - } - - if (mode == 'P') { - /* - * Producer - */ - char *sbuf; - char *pbuf; - int outq; - int keylen = key ? (int)strlen(key) : 0; - off_t rof = 0; - size_t plen = strlen(msgpattern); - int partition = - partitions ? partitions[0] : RD_KAFKA_PARTITION_UA; - - if (latency_mode) { - int minlen = (int)(strlen("LATENCY:") + - strlen("18446744073709551615 ") + 1); - msgsize = RD_MAX(minlen, msgsize); - sendflags |= RD_KAFKA_MSG_F_COPY; - } else if (do_seq) { - int minlen = (int)strlen("18446744073709551615 ") + 1; - if (msgsize < minlen) - msgsize = minlen; - - /* Force duplication of payload */ - sendflags |= RD_KAFKA_MSG_F_FREE; - } - - sbuf = malloc(msgsize); - - /* Copy payload content to new buffer */ - while (rof < msgsize) { - size_t xlen = RD_MIN((size_t)msgsize - rof, plen); - memcpy(sbuf + rof, msgpattern, xlen); - rof += (off_t)xlen; - } - - if (msgcnt == -1) - printf("%% Sending messages of size %i bytes\n", - msgsize); - else - printf("%% Sending %i messages of size %i bytes\n", - msgcnt, msgsize); - - if (with_dr) - rd_kafka_conf_set_dr_msg_cb(conf, msg_delivered); - - /* Create Kafka handle */ - if (!(rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, - sizeof(errstr)))) { - fprintf(stderr, - "%% Failed to create Kafka producer: %s\n", - errstr); - exit(1); - } - - global_rk = rk; - - /* Explicitly create topic to avoid per-msg lookups. */ - rkt = rd_kafka_topic_new(rk, topic, NULL); - - - if (rate_sleep && verbosity >= 2) - fprintf(stderr, - "%% Inter message rate limiter sleep %ius\n", - rate_sleep); - - dr_disp_div = msgcnt / 50; - if (dr_disp_div == 0) - dr_disp_div = 10; - - cnt.t_start = cnt.t_last = rd_clock(); - - msgs_wait_produce_cnt = msgcnt; - - while (run && (msgcnt == -1 || (int)cnt.msgs < msgcnt)) { - /* Send/Produce message. */ - - if (idle) { - rd_kafka_poll(rk, 1000); - continue; - } - - if (latency_mode) { - rd_snprintf(sbuf, msgsize - 1, - "LATENCY:%" PRIu64, wall_clock()); - } else if (do_seq) { - rd_snprintf(sbuf, msgsize - 1, "%" PRIu64 ": ", - seq); - seq++; - } - - if (sendflags & RD_KAFKA_MSG_F_FREE) { - /* Duplicate memory */ - pbuf = malloc(msgsize); - memcpy(pbuf, sbuf, msgsize); - } else - pbuf = sbuf; - - if (msgsize == 0) - pbuf = NULL; - - cnt.tx++; - while (run && (err = do_produce( - rk, rkt, partition, sendflags, pbuf, - msgsize, key, keylen, hdrs))) { - if (err == RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION) - printf( - "%% No such partition: " - "%" PRId32 "\n", - partition); - else if (verbosity >= 3 || - (err != - RD_KAFKA_RESP_ERR__QUEUE_FULL && - verbosity >= 1)) - printf( - "%% produce error: %s%s\n", - rd_kafka_err2str(err), - err == RD_KAFKA_RESP_ERR__QUEUE_FULL - ? " (backpressure)" - : ""); - - cnt.tx_err++; - if (err != RD_KAFKA_RESP_ERR__QUEUE_FULL) { - run = 0; - break; - } - now = rd_clock(); - if (verbosity >= 2 && - cnt.t_enobufs_last + dispintvl <= now) { - printf( - "%% Backpressure %i " - "(tx %" PRIu64 - ", " - "txerr %" PRIu64 ")\n", - rd_kafka_outq_len(rk), cnt.tx, - cnt.tx_err); - cnt.t_enobufs_last = now; - } - - /* Poll to handle delivery reports */ - rd_kafka_poll(rk, 10); - - print_stats(rk, mode, otype, compression); - } - - msgs_wait_cnt++; - if (msgs_wait_produce_cnt != -1) - msgs_wait_produce_cnt--; - cnt.msgs++; - cnt.bytes += msgsize; - - /* Must poll to handle delivery reports */ - if (rate_sleep) { - rd_ts_t next = rd_clock() + (rd_ts_t)rate_sleep; - do { - rd_kafka_poll( - rk, - (int)RD_MAX(0, (next - rd_clock()) / - 1000)); - } while (next > rd_clock()); - } else if (cnt.msgs % 1000 == 0) { - rd_kafka_poll(rk, 0); - } - - print_stats(rk, mode, otype, compression); - } - - forever = 0; - if (verbosity >= 2) - printf( - "%% All messages produced, " - "now waiting for %li deliveries\n", - msgs_wait_cnt); - - /* Wait for messages to be delivered */ - while (run && rd_kafka_poll(rk, 1000) != -1) - print_stats(rk, mode, otype, compression); - - - outq = rd_kafka_outq_len(rk); - if (verbosity >= 2) - printf("%% %i messages in outq\n", outq); - cnt.msgs -= outq; - cnt.t_end = t_end; - - if (cnt.tx_err > 0) - printf("%% %" PRIu64 " backpressures for %" PRIu64 - " produce calls: %.3f%% backpressure rate\n", - cnt.tx_err, cnt.tx, - ((double)cnt.tx_err / (double)cnt.tx) * 100.0); - - /* Destroy topic */ - rd_kafka_topic_destroy(rkt); - - /* Destroy the handle */ - rd_kafka_destroy(rk); - global_rk = rk = NULL; - - free(sbuf); - - exitcode = cnt.msgs == cnt.msgs_dr_ok ? 0 : 1; - - } else if (mode == 'C') { - /* - * Consumer - */ - - rd_kafka_message_t **rkmessages = NULL; - size_t i = 0; - - /* Create Kafka handle */ - if (!(rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, - sizeof(errstr)))) { - fprintf(stderr, - "%% Failed to create Kafka consumer: %s\n", - errstr); - exit(1); - } - - global_rk = rk; - - /* Create topic to consume from */ - rkt = rd_kafka_topic_new(rk, topic, NULL); - - /* Batch consumer */ - if (batch_size) - rkmessages = malloc(sizeof(*rkmessages) * batch_size); - - /* Start consuming */ - rkqu = rd_kafka_queue_new(rk); - for (i = 0; i < (size_t)partition_cnt; ++i) { - const int r = rd_kafka_consume_start_queue( - rkt, partitions[i], start_offset, rkqu); - - if (r == -1) { - fprintf( - stderr, "%% Error creating queue: %s\n", - rd_kafka_err2str(rd_kafka_last_error())); - exit(1); - } - } - - while (run && (msgcnt == -1 || msgcnt > (int)cnt.msgs)) { - /* Consume messages. - * A message may either be a real message, or - * an error signaling (if rkmessage->err is set). - */ - uint64_t fetch_latency; - ssize_t r; - - fetch_latency = rd_clock(); - - if (batch_size) { - int partition = partitions - ? partitions[0] - : RD_KAFKA_PARTITION_UA; - - /* Batch fetch mode */ - r = rd_kafka_consume_batch(rkt, partition, 1000, - rkmessages, - batch_size); - if (r != -1) { - for (i = 0; (ssize_t)i < r; i++) { - msg_consume(rkmessages[i], - NULL); - rd_kafka_message_destroy( - rkmessages[i]); - } - } - } else { - /* Queue mode */ - r = rd_kafka_consume_callback_queue( - rkqu, 1000, msg_consume, NULL); - } - - cnt.t_fetch_latency += rd_clock() - fetch_latency; - if (r == -1) - fprintf( - stderr, "%% Error: %s\n", - rd_kafka_err2str(rd_kafka_last_error())); - else if (r > 0 && rate_sleep) { - /* Simulate processing time - * if `-r <rate>` was set. */ - do_sleep(rate_sleep); - } - - - print_stats(rk, mode, otype, compression); - - /* Poll to handle stats callbacks */ - rd_kafka_poll(rk, 0); - } - cnt.t_end = rd_clock(); - - /* Stop consuming */ - for (i = 0; i < (size_t)partition_cnt; ++i) { - int r = rd_kafka_consume_stop(rkt, (int32_t)i); - if (r == -1) { - fprintf( - stderr, "%% Error in consume_stop: %s\n", - rd_kafka_err2str(rd_kafka_last_error())); - } - } - rd_kafka_queue_destroy(rkqu); - - /* Destroy topic */ - rd_kafka_topic_destroy(rkt); - - if (batch_size) - free(rkmessages); - - /* Destroy the handle */ - rd_kafka_destroy(rk); - - global_rk = rk = NULL; - - } else if (mode == 'G') { - /* - * High-level balanced Consumer - */ - rd_kafka_message_t **rkmessages = NULL; - - rd_kafka_conf_set_rebalance_cb(conf, rebalance_cb); - - /* Create Kafka handle */ - if (!(rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, - sizeof(errstr)))) { - fprintf(stderr, - "%% Failed to create Kafka consumer: %s\n", - errstr); - exit(1); - } - - /* Forward all events to consumer queue */ - rd_kafka_poll_set_consumer(rk); - - global_rk = rk; - - err = rd_kafka_subscribe(rk, topics); - if (err) { - fprintf(stderr, "%% Subscribe failed: %s\n", - rd_kafka_err2str(err)); - exit(1); - } - fprintf(stderr, "%% Waiting for group rebalance..\n"); - - if (batch_size) { - rkmessages = malloc(sizeof(*rkmessages) * batch_size); - } else { - rkmessages = malloc(sizeof(*rkmessages)); - } - - rkqu = rd_kafka_queue_get_consumer(rk); - - while (run && (msgcnt == -1 || msgcnt > (int)cnt.msgs)) { - /* Consume messages. - * A message may either be a real message, or - * an event (if rkmessage->err is set). - */ - uint64_t fetch_latency; - ssize_t r; - - fetch_latency = rd_clock(); - - if (batch_size) { - /* Batch fetch mode */ - ssize_t i = 0; - r = rd_kafka_consume_batch_queue( - rkqu, 1000, rkmessages, batch_size); - if (r != -1) { - for (i = 0; i < r; i++) { - msg_consume(rkmessages[i], - NULL); - rd_kafka_message_destroy( - rkmessages[i]); - } - } - - if (r == -1) - fprintf(stderr, "%% Error: %s\n", - rd_kafka_err2str( - rd_kafka_last_error())); - else if (r > 0 && rate_sleep) { - /* Simulate processing time - * if `-r <rate>` was set. */ - do_sleep(rate_sleep); - } - - } else { - rkmessages[0] = - rd_kafka_consumer_poll(rk, 1000); - if (rkmessages[0]) { - msg_consume(rkmessages[0], NULL); - rd_kafka_message_destroy(rkmessages[0]); - - /* Simulate processing time - * if `-r <rate>` was set. */ - if (rate_sleep) - do_sleep(rate_sleep); - } - } - - cnt.t_fetch_latency += rd_clock() - fetch_latency; - - print_stats(rk, mode, otype, compression); - } - cnt.t_end = rd_clock(); - - err = rd_kafka_consumer_close(rk); - if (err) - fprintf(stderr, "%% Failed to close consumer: %s\n", - rd_kafka_err2str(err)); - - free(rkmessages); - rd_kafka_queue_destroy(rkqu); - rd_kafka_destroy(rk); - } - - if (hdrs) - rd_kafka_headers_destroy(hdrs); - - print_stats(NULL, mode, otype | _OTYPE_FORCE, compression); - - if (cnt.t_fetch_latency && cnt.msgs) - printf("%% Average application fetch latency: %" PRIu64 "us\n", - cnt.t_fetch_latency / cnt.msgs); - - if (latency_fp) - fclose(latency_fp); - - if (stats_fp) { -#ifndef _WIN32 - pclose(stats_fp); -#endif - stats_fp = NULL; - } - - if (partitions) - free(partitions); - - rd_kafka_topic_partition_list_destroy(topics); - - /* Let background threads clean up and terminate cleanly. */ - rd_kafka_wait_destroyed(2000); - - return exitcode; -} diff --git a/fluent-bit/lib/librdkafka-2.1.0/examples/transactions-older-broker.c b/fluent-bit/lib/librdkafka-2.1.0/examples/transactions-older-broker.c deleted file mode 100644 index e9f8d06f7..000000000 --- a/fluent-bit/lib/librdkafka-2.1.0/examples/transactions-older-broker.c +++ /dev/null @@ -1,668 +0,0 @@ -/* - * librdkafka - Apache Kafka C library - * - * Copyright (c) 2020, Magnus Edenhill - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/** - * @name Transactions example for Apache Kafka <= 2.4.0 (no KIP-447 support). - * - * This example show-cases a simple transactional consume-process-produce - * application that reads messages from an input topic, extracts all - * numbers from the message's value string, adds them up, and sends - * the sum to the output topic as part of a transaction. - * The transaction is committed every 5 seconds or 100 messages, whichever - * comes first. As the transaction is committed a new transaction is started. - * - * @remark This example does not yet support incremental rebalancing and thus - * not the cooperative-sticky partition.assignment.strategy. - */ - -#include <stdio.h> -#include <signal.h> -#include <unistd.h> -#include <string.h> -#include <stdlib.h> -#include <time.h> -#include <ctype.h> - - -/* Typical include path would be <librdkafka/rdkafka.h>, but this program - * is builtin from within the librdkafka source tree and thus differs. */ -#include "rdkafka.h" - - -static volatile sig_atomic_t run = 1; - -static rd_kafka_t *consumer; - -/* From command-line arguments */ -static const char *brokers, *input_topic, *output_topic; - - -/** - * @struct This is the per input partition state, constisting of - * a transactional producer and the in-memory state for the current transaction. - * This demo simply finds all numbers (ascii string numbers) in the message - * payload and adds them. - */ -struct state { - rd_kafka_t *producer; /**< Per-input partition output producer */ - rd_kafka_topic_partition_t *rktpar; /**< Back-pointer to the - * input partition. */ - time_t last_commit; /**< Last transaction commit */ - int msgcnt; /**< Number of messages processed in current txn */ -}; -/* Current assignment for the input consumer. - * The .opaque field of each partition points to an allocated 'struct state'. - */ -static rd_kafka_topic_partition_list_t *assigned_partitions; - - - -/** - * @brief A fatal error has occurred, immediately exit the application. - */ -#define fatal(...) \ - do { \ - fprintf(stderr, "FATAL ERROR: "); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr, "\n"); \ - exit(1); \ - } while (0) - -/** - * @brief Same as fatal() but takes an rd_kafka_error_t object, prints its - * error message, destroys the object and then exits fatally. - */ -#define fatal_error(what, error) \ - do { \ - fprintf(stderr, "FATAL ERROR: %s: %s: %s\n", what, \ - rd_kafka_error_name(error), \ - rd_kafka_error_string(error)); \ - rd_kafka_error_destroy(error); \ - exit(1); \ - } while (0) - -/** - * @brief Signal termination of program - */ -static void stop(int sig) { - run = 0; -} - - -/** - * @brief Message delivery report callback. - * - * This callback is called exactly once per message, indicating if - * the message was succesfully delivered - * (rkmessage->err == RD_KAFKA_RESP_ERR_NO_ERROR) or permanently - * failed delivery (rkmessage->err != RD_KAFKA_RESP_ERR_NO_ERROR). - * - * The callback is triggered from rd_kafka_poll(), rd_kafka_flush(), - * rd_kafka_abort_transaction() and rd_kafka_commit_transaction() and - * executes on the application's thread. - * - * The current transactional will enter the abortable state if any - * message permanently fails delivery and the application must then - * call rd_kafka_abort_transaction(). But it does not need to be done from - * here, this state is checked by all the transactional APIs and it is better - * to perform this error checking when calling - * rd_kafka_send_offsets_to_transaction() and rd_kafka_commit_transaction(). - * In the case of transactional producing the delivery report callback is - * mostly useful for logging the produce failures. - */ -static void -dr_msg_cb(rd_kafka_t *rk, const rd_kafka_message_t *rkmessage, void *opaque) { - if (rkmessage->err) - fprintf(stderr, "%% Message delivery failed: %s\n", - rd_kafka_err2str(rkmessage->err)); - - /* The rkmessage is destroyed automatically by librdkafka */ -} - - - -/** - * @brief Create a transactional producer for the given input pratition - * and begin a new transaction. - */ -static rd_kafka_t * -create_transactional_producer(const rd_kafka_topic_partition_t *rktpar) { - rd_kafka_conf_t *conf = rd_kafka_conf_new(); - rd_kafka_t *rk; - char errstr[256]; - rd_kafka_error_t *error; - char transactional_id[256]; - - snprintf(transactional_id, sizeof(transactional_id), - "librdkafka_transactions_older_example_%s-%d", rktpar->topic, - rktpar->partition); - - if (rd_kafka_conf_set(conf, "bootstrap.servers", brokers, errstr, - sizeof(errstr)) != RD_KAFKA_CONF_OK || - rd_kafka_conf_set(conf, "transactional.id", transactional_id, - errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK || - rd_kafka_conf_set(conf, "transaction.timeout.ms", "60000", errstr, - sizeof(errstr)) != RD_KAFKA_CONF_OK) - fatal("Failed to configure producer: %s", errstr); - - /* This callback will be called once per message to indicate - * final delivery status. */ - rd_kafka_conf_set_dr_msg_cb(conf, dr_msg_cb); - - /* Create producer */ - rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr)); - if (!rk) { - rd_kafka_conf_destroy(conf); - fatal("Failed to create producer: %s", errstr); - } - - /* Initialize transactions, this is only performed once - * per transactional producer to acquire its producer id, et.al. */ - error = rd_kafka_init_transactions(rk, -1); - if (error) - fatal_error("init_transactions()", error); - - - /* Begin a new transaction */ - error = rd_kafka_begin_transaction(rk); - if (error) - fatal_error("begin_transaction()", error); - - return rk; -} - - -/** - * @brief Abort the current transaction and destroy the producer. - */ -static void destroy_transactional_producer(rd_kafka_t *rk) { - rd_kafka_error_t *error; - - fprintf(stdout, "%s: aborting transaction and terminating producer\n", - rd_kafka_name(rk)); - - /* Abort the current transaction, ignore any errors - * since we're terminating the producer anyway. */ - error = rd_kafka_abort_transaction(rk, -1); - if (error) { - fprintf(stderr, - "WARNING: Ignoring abort_transaction() error since " - "producer is being destroyed: %s\n", - rd_kafka_error_string(error)); - rd_kafka_error_destroy(error); - } - - rd_kafka_destroy(rk); -} - - - -/** - * @brief Abort the current transaction and rewind consumer offsets to - * position where the transaction last started, i.e., the committed - * consumer offset. - */ -static void abort_transaction_and_rewind(struct state *state) { - rd_kafka_topic_t *rkt = - rd_kafka_topic_new(consumer, state->rktpar->topic, NULL); - rd_kafka_topic_partition_list_t *offset; - rd_kafka_resp_err_t err; - rd_kafka_error_t *error; - - fprintf(stdout, - "Aborting transaction and rewinding offset for %s [%d]\n", - state->rktpar->topic, state->rktpar->partition); - - /* Abort the current transaction */ - error = rd_kafka_abort_transaction(state->producer, -1); - if (error) - fatal_error("Failed to abort transaction", error); - - /* Begin a new transaction */ - error = rd_kafka_begin_transaction(state->producer); - if (error) - fatal_error("Failed to begin transaction", error); - - /* Get committed offset for this partition */ - offset = rd_kafka_topic_partition_list_new(1); - rd_kafka_topic_partition_list_add(offset, state->rktpar->topic, - state->rktpar->partition); - - /* Note: Timeout must be lower than max.poll.interval.ms */ - err = rd_kafka_committed(consumer, offset, 10 * 1000); - if (err) - fatal("Failed to acquire committed offset for %s [%d]: %s", - state->rktpar->topic, (int)state->rktpar->partition, - rd_kafka_err2str(err)); - - /* Seek to committed offset, or start of partition if no - * no committed offset is available. */ - err = rd_kafka_seek(rkt, state->rktpar->partition, - offset->elems[0].offset < 0 - ? - /* No committed offset, start from beginning */ - RD_KAFKA_OFFSET_BEGINNING - : - /* Use committed offset */ - offset->elems[0].offset, - 0); - - if (err) - fatal("Failed to seek %s [%d]: %s", state->rktpar->topic, - (int)state->rktpar->partition, rd_kafka_err2str(err)); - - rd_kafka_topic_destroy(rkt); -} - - -/** - * @brief Commit the current transaction and start a new transaction. - */ -static void commit_transaction_and_start_new(struct state *state) { - rd_kafka_error_t *error; - rd_kafka_resp_err_t err; - rd_kafka_consumer_group_metadata_t *cgmd; - rd_kafka_topic_partition_list_t *offset; - - fprintf(stdout, "Committing transaction for %s [%d]\n", - state->rktpar->topic, state->rktpar->partition); - - /* Send the input consumer's offset to transaction - * to commit those offsets along with the transaction itself, - * this is what guarantees exactly-once-semantics (EOS), that - * input (offsets) and output (messages) are committed atomically. */ - - /* Get the consumer's current group state */ - cgmd = rd_kafka_consumer_group_metadata(consumer); - - /* Get consumer's current position for this partition */ - offset = rd_kafka_topic_partition_list_new(1); - rd_kafka_topic_partition_list_add(offset, state->rktpar->topic, - state->rktpar->partition); - err = rd_kafka_position(consumer, offset); - if (err) - fatal("Failed to get consumer position for %s [%d]: %s", - state->rktpar->topic, state->rktpar->partition, - rd_kafka_err2str(err)); - - /* Send offsets to transaction coordinator */ - error = rd_kafka_send_offsets_to_transaction(state->producer, offset, - cgmd, -1); - rd_kafka_consumer_group_metadata_destroy(cgmd); - rd_kafka_topic_partition_list_destroy(offset); - if (error) { - if (rd_kafka_error_txn_requires_abort(error)) { - fprintf(stderr, - "WARNING: Failed to send offsets to " - "transaction: %s: %s: aborting transaction\n", - rd_kafka_error_name(error), - rd_kafka_error_string(error)); - rd_kafka_error_destroy(error); - abort_transaction_and_rewind(state); - return; - } else { - fatal_error("Failed to send offsets to transaction", - error); - } - } - - /* Commit the transaction */ - error = rd_kafka_commit_transaction(state->producer, -1); - if (error) { - if (rd_kafka_error_txn_requires_abort(error)) { - fprintf(stderr, - "WARNING: Failed to commit transaction: " - "%s: %s: aborting transaction\n", - rd_kafka_error_name(error), - rd_kafka_error_string(error)); - abort_transaction_and_rewind(state); - rd_kafka_error_destroy(error); - return; - } else { - fatal_error("Failed to commit transaction", error); - } - } - - /* Begin new transaction */ - error = rd_kafka_begin_transaction(state->producer); - if (error) - fatal_error("Failed to begin new transaction", error); -} - -/** - * @brief The rebalance will be triggered (from rd_kafka_consumer_poll()) - * when the consumer's partition assignment is assigned or revoked. - * - * Prior to KIP-447 being supported there must be one transactional output - * producer for each consumed input partition, so we create and destroy - * these producer's from this callback. - */ -static void -consumer_group_rebalance_cb(rd_kafka_t *rk, - rd_kafka_resp_err_t err, - rd_kafka_topic_partition_list_t *partitions, - void *opaque) { - int i; - - if (!strcmp(rd_kafka_rebalance_protocol(rk), "COOPERATIVE")) - fatal( - "This example has not yet been modified to work with " - "cooperative incremental rebalancing " - "(partition.assignment.strategy=cooperative-sticky)"); - - switch (err) { - case RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS: - assigned_partitions = - rd_kafka_topic_partition_list_copy(partitions); - - fprintf(stdout, "Consumer group rebalanced, new assignment:\n"); - - /* Create a transactional producer for each input partition */ - for (i = 0; i < assigned_partitions->cnt; i++) { - /* Store the partition-to-producer mapping - * in the partition's opaque field. */ - rd_kafka_topic_partition_t *rktpar = - &assigned_partitions->elems[i]; - struct state *state = calloc(1, sizeof(*state)); - - state->producer = create_transactional_producer(rktpar); - state->rktpar = rktpar; - rktpar->opaque = state; - state->last_commit = time(NULL); - - fprintf(stdout, - " %s [%d] with transactional producer %s\n", - rktpar->topic, rktpar->partition, - rd_kafka_name(state->producer)); - } - - /* Let the consumer know the rebalance has been handled - * by calling assign. - * This will also tell the consumer to start fetching messages - * for the assigned partitions. */ - rd_kafka_assign(rk, partitions); - break; - - case RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS: - fprintf(stdout, - "Consumer group rebalanced, assignment revoked\n"); - - /* Abort the current transactions and destroy all producers */ - for (i = 0; i < assigned_partitions->cnt; i++) { - /* Store the partition-to-producer mapping - * in the partition's opaque field. */ - struct state *state = - (struct state *)assigned_partitions->elems[i] - .opaque; - - destroy_transactional_producer(state->producer); - free(state); - } - - rd_kafka_topic_partition_list_destroy(assigned_partitions); - assigned_partitions = NULL; - - /* Let the consumer know the rebalance has been handled - * and revoke the current assignment. */ - rd_kafka_assign(rk, NULL); - break; - - default: - /* NOTREACHED */ - fatal("Unexpected rebalance event: %s", rd_kafka_err2name(err)); - } -} - - -/** - * @brief Create the input consumer. - */ -static rd_kafka_t *create_input_consumer(const char *brokers, - const char *input_topic) { - rd_kafka_conf_t *conf = rd_kafka_conf_new(); - rd_kafka_t *rk; - char errstr[256]; - rd_kafka_resp_err_t err; - rd_kafka_topic_partition_list_t *topics; - - if (rd_kafka_conf_set(conf, "bootstrap.servers", brokers, errstr, - sizeof(errstr)) != RD_KAFKA_CONF_OK || - rd_kafka_conf_set(conf, "group.id", - "librdkafka_transactions_older_example_group", - errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK || - /* The input consumer's offsets are explicitly committed with the - * output producer's transaction using - * rd_kafka_send_offsets_to_transaction(), so auto commits - * must be disabled. */ - rd_kafka_conf_set(conf, "enable.auto.commit", "false", errstr, - sizeof(errstr)) != RD_KAFKA_CONF_OK) { - fatal("Failed to configure consumer: %s", errstr); - } - - /* This callback will be called when the consumer group is rebalanced - * and the consumer's partition assignment is assigned or revoked. */ - rd_kafka_conf_set_rebalance_cb(conf, consumer_group_rebalance_cb); - - /* Create consumer */ - rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, sizeof(errstr)); - if (!rk) { - rd_kafka_conf_destroy(conf); - fatal("Failed to create consumer: %s", errstr); - } - - /* Forward all partition messages to the main queue and - * rd_kafka_consumer_poll(). */ - rd_kafka_poll_set_consumer(rk); - - /* Subscribe to the input topic */ - topics = rd_kafka_topic_partition_list_new(1); - rd_kafka_topic_partition_list_add(topics, input_topic, - /* The partition is ignored in - * rd_kafka_subscribe() */ - RD_KAFKA_PARTITION_UA); - err = rd_kafka_subscribe(rk, topics); - rd_kafka_topic_partition_list_destroy(topics); - if (err) { - rd_kafka_destroy(rk); - fatal("Failed to subscribe to %s: %s\n", input_topic, - rd_kafka_err2str(err)); - } - - return rk; -} - - -/** - * @brief Find and parse next integer string in \p start. - * @returns Pointer after found integer string, or NULL if not found. - */ -static const void * -find_next_int(const void *start, const void *end, int *intp) { - const char *p; - int collecting = 0; - int num = 0; - - for (p = (const char *)start; p < (const char *)end; p++) { - if (isdigit((int)(*p))) { - collecting = 1; - num = (num * 10) + ((int)*p - ((int)'0')); - } else if (collecting) - break; - } - - if (!collecting) - return NULL; /* No integer string found */ - - *intp = num; - - return p; -} - - -/** - * @brief Process a message from the input consumer by parsing all - * integer strings, adding them, and then producing the sum - * the output topic using the transactional producer for the given - * inut partition. - */ -static void process_message(struct state *state, - const rd_kafka_message_t *rkmessage) { - int num; - long unsigned sum = 0; - const void *p, *end; - rd_kafka_resp_err_t err; - char value[64]; - - if (rkmessage->len == 0) - return; /* Ignore empty messages */ - - p = rkmessage->payload; - end = ((const char *)rkmessage->payload) + rkmessage->len; - - /* Find and sum all numbers in the message */ - while ((p = find_next_int(p, end, &num))) - sum += num; - - if (sum == 0) - return; /* No integers in message, ignore it. */ - - snprintf(value, sizeof(value), "%lu", sum); - - /* Emit output message on transactional producer */ - while (1) { - err = rd_kafka_producev( - state->producer, RD_KAFKA_V_TOPIC(output_topic), - /* Use same key as input message */ - RD_KAFKA_V_KEY(rkmessage->key, rkmessage->key_len), - /* Value is the current sum of this - * transaction. */ - RD_KAFKA_V_VALUE(value, strlen(value)), - /* Copy value since it is allocated on the stack */ - RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY), RD_KAFKA_V_END); - - if (!err) - break; - else if (err == RD_KAFKA_RESP_ERR__QUEUE_FULL) { - /* If output queue fills up we need to wait for - * some delivery reports and then retry. */ - rd_kafka_poll(state->producer, 100); - continue; - } else { - fprintf(stderr, - "WARNING: Failed to produce message to %s: " - "%s: aborting transaction\n", - output_topic, rd_kafka_err2str(err)); - abort_transaction_and_rewind(state); - return; - } - } -} - - -int main(int argc, char **argv) { - /* - * Argument validation - */ - if (argc != 4) { - fprintf(stderr, - "%% Usage: %s <broker> <input-topic> <output-topic>\n", - argv[0]); - return 1; - } - - brokers = argv[1]; - input_topic = argv[2]; - output_topic = argv[3]; - - /* Signal handler for clean shutdown */ - signal(SIGINT, stop); - - consumer = create_input_consumer(brokers, input_topic); - - fprintf(stdout, - "Expecting integers to sum on input topic %s ...\n" - "To generate input messages you can use:\n" - " $ seq 1 100 | examples/producer %s %s\n", - input_topic, brokers, input_topic); - - while (run) { - rd_kafka_message_t *msg; - struct state *state; - rd_kafka_topic_partition_t *rktpar; - - /* Wait for new mesages or error events */ - msg = rd_kafka_consumer_poll(consumer, 1000 /*1 second*/); - if (!msg) - continue; - - if (msg->err) { - /* Client errors are typically just informational - * since the client will automatically try to recover - * from all types of errors. - * It is thus sufficient for the application to log and - * continue operating when an error is received. */ - fprintf(stderr, "WARNING: Consumer error: %s\n", - rd_kafka_message_errstr(msg)); - rd_kafka_message_destroy(msg); - continue; - } - - /* Find output producer for this input partition */ - rktpar = rd_kafka_topic_partition_list_find( - assigned_partitions, rd_kafka_topic_name(msg->rkt), - msg->partition); - if (!rktpar) - fatal( - "BUG: No output producer for assigned " - "partition %s [%d]", - rd_kafka_topic_name(msg->rkt), (int)msg->partition); - - /* Get state struct for this partition */ - state = (struct state *)rktpar->opaque; - - /* Process message */ - process_message(state, msg); - - rd_kafka_message_destroy(msg); - - /* Commit transaction every 100 messages or 5 seconds */ - if (++state->msgcnt > 100 || - state->last_commit + 5 <= time(NULL)) { - commit_transaction_and_start_new(state); - state->msgcnt = 0; - state->last_commit = time(NULL); - } - } - - fprintf(stdout, "Closing consumer\n"); - rd_kafka_consumer_close(consumer); - - rd_kafka_destroy(consumer); - - return 0; -} diff --git a/fluent-bit/lib/librdkafka-2.1.0/examples/transactions.c b/fluent-bit/lib/librdkafka-2.1.0/examples/transactions.c deleted file mode 100644 index 0a8b9a8cf..000000000 --- a/fluent-bit/lib/librdkafka-2.1.0/examples/transactions.c +++ /dev/null @@ -1,665 +0,0 @@ -/* - * librdkafka - Apache Kafka C library - * - * Copyright (c) 2020, Magnus Edenhill - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/** - * @name Transactions example for Apache Kafka 2.5.0 (KIP-447) and later. - * - * This example show-cases a simple transactional consume-process-produce - * application that reads messages from an input topic, extracts all - * numbers from the message's value string, adds them up, and sends - * the sum to the output topic as part of a transaction. - * The transaction is committed every 5 seconds or 100 messages, whichever - * comes first. As the transaction is committed a new transaction is started. - * - * This example makes use of incremental rebalancing (KIP-429) and the - * cooperative-sticky partition.assignment.strategy on the consumer, providing - * hitless rebalances. - */ - -#include <stdio.h> -#include <signal.h> -#include <unistd.h> -#include <string.h> -#include <stdlib.h> -#include <time.h> -#include <ctype.h> - - -/* Typical include path would be <librdkafka/rdkafka.h>, but this program - * is builtin from within the librdkafka source tree and thus differs. */ -#include "rdkafka.h" - - -static volatile sig_atomic_t run = 1; - -/** - * @brief A fatal error has occurred, immediately exit the application. - */ -#define fatal(...) \ - do { \ - fprintf(stderr, "FATAL ERROR: "); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr, "\n"); \ - exit(1); \ - } while (0) - -/** - * @brief Same as fatal() but takes an rd_kafka_error_t object, prints its - * error message, destroys the object and then exits fatally. - */ -#define fatal_error(what, error) \ - do { \ - fprintf(stderr, "FATAL ERROR: %s: %s: %s\n", what, \ - rd_kafka_error_name(error), \ - rd_kafka_error_string(error)); \ - rd_kafka_error_destroy(error); \ - exit(1); \ - } while (0) - -/** - * @brief Signal termination of program - */ -static void stop(int sig) { - run = 0; -} - - -/** - * @brief Message delivery report callback. - * - * This callback is called exactly once per message, indicating if - * the message was succesfully delivered - * (rkmessage->err == RD_KAFKA_RESP_ERR_NO_ERROR) or permanently - * failed delivery (rkmessage->err != RD_KAFKA_RESP_ERR_NO_ERROR). - * - * The callback is triggered from rd_kafka_poll(), rd_kafka_flush(), - * rd_kafka_abort_transaction() and rd_kafka_commit_transaction() and - * executes on the application's thread. - * - * The current transactional will enter the abortable state if any - * message permanently fails delivery and the application must then - * call rd_kafka_abort_transaction(). But it does not need to be done from - * here, this state is checked by all the transactional APIs and it is better - * to perform this error checking when calling - * rd_kafka_send_offsets_to_transaction() and rd_kafka_commit_transaction(). - * In the case of transactional producing the delivery report callback is - * mostly useful for logging the produce failures. - */ -static void -dr_msg_cb(rd_kafka_t *rk, const rd_kafka_message_t *rkmessage, void *opaque) { - if (rkmessage->err) - fprintf(stderr, "%% Message delivery failed: %s\n", - rd_kafka_err2str(rkmessage->err)); - - /* The rkmessage is destroyed automatically by librdkafka */ -} - - - -/** - * @brief Create a transactional producer. - */ -static rd_kafka_t *create_transactional_producer(const char *brokers, - const char *output_topic) { - rd_kafka_conf_t *conf = rd_kafka_conf_new(); - rd_kafka_t *rk; - char errstr[256]; - rd_kafka_error_t *error; - - if (rd_kafka_conf_set(conf, "bootstrap.servers", brokers, errstr, - sizeof(errstr)) != RD_KAFKA_CONF_OK || - rd_kafka_conf_set(conf, "transactional.id", - "librdkafka_transactions_example", errstr, - sizeof(errstr)) != RD_KAFKA_CONF_OK) - fatal("Failed to configure producer: %s", errstr); - - /* This callback will be called once per message to indicate - * final delivery status. */ - rd_kafka_conf_set_dr_msg_cb(conf, dr_msg_cb); - - /* Create producer */ - rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr)); - if (!rk) { - rd_kafka_conf_destroy(conf); - fatal("Failed to create producer: %s", errstr); - } - - /* Initialize transactions, this is only performed once - * per transactional producer to acquire its producer id, et.al. */ - error = rd_kafka_init_transactions(rk, -1); - if (error) - fatal_error("init_transactions()", error); - - return rk; -} - - -/** - * @brief Rewind consumer's consume position to the last committed offsets - * for the current assignment. - */ -static void rewind_consumer(rd_kafka_t *consumer) { - rd_kafka_topic_partition_list_t *offsets; - rd_kafka_resp_err_t err; - rd_kafka_error_t *error; - int i; - - /* Get committed offsets for the current assignment, if there - * is a current assignment. */ - err = rd_kafka_assignment(consumer, &offsets); - if (err) { - fprintf(stderr, "No current assignment to rewind: %s\n", - rd_kafka_err2str(err)); - return; - } - - if (offsets->cnt == 0) { - fprintf(stderr, "No current assignment to rewind\n"); - rd_kafka_topic_partition_list_destroy(offsets); - return; - } - - /* Note: Timeout must be lower than max.poll.interval.ms */ - err = rd_kafka_committed(consumer, offsets, 10 * 1000); - if (err) - fatal("Failed to acquire committed offsets: %s", - rd_kafka_err2str(err)); - - /* Seek to committed offset, or start of partition if no - * committed offset is available. */ - for (i = 0; i < offsets->cnt; i++) { - /* No committed offset, start from beginning */ - if (offsets->elems[i].offset < 0) - offsets->elems[i].offset = RD_KAFKA_OFFSET_BEGINNING; - } - - /* Perform seek */ - error = rd_kafka_seek_partitions(consumer, offsets, -1); - if (error) - fatal_error("Failed to seek", error); - - rd_kafka_topic_partition_list_destroy(offsets); -} - -/** - * @brief Abort the current transaction and rewind consumer offsets to - * position where the transaction last started, i.e., the committed - * consumer offset, then begin a new transaction. - */ -static void abort_transaction_and_rewind(rd_kafka_t *consumer, - rd_kafka_t *producer) { - rd_kafka_error_t *error; - - fprintf(stdout, "Aborting transaction and rewinding offsets\n"); - - /* Abort the current transaction */ - error = rd_kafka_abort_transaction(producer, -1); - if (error) - fatal_error("Failed to abort transaction", error); - - /* Rewind consumer */ - rewind_consumer(consumer); - - /* Begin a new transaction */ - error = rd_kafka_begin_transaction(producer); - if (error) - fatal_error("Failed to begin transaction", error); -} - - -/** - * @brief Commit the current transaction. - * - * @returns 1 if transaction was successfully committed, or 0 - * if the current transaction was aborted. - */ -static int commit_transaction(rd_kafka_t *consumer, rd_kafka_t *producer) { - rd_kafka_error_t *error; - rd_kafka_resp_err_t err; - rd_kafka_consumer_group_metadata_t *cgmd; - rd_kafka_topic_partition_list_t *offsets; - - fprintf(stdout, "Committing transaction\n"); - - /* Send the input consumer's offset to transaction - * to commit those offsets along with the transaction itself, - * this is what guarantees exactly-once-semantics (EOS), that - * input (offsets) and output (messages) are committed atomically. */ - - /* Get the consumer's current group metadata state */ - cgmd = rd_kafka_consumer_group_metadata(consumer); - - /* Get consumer's current assignment */ - err = rd_kafka_assignment(consumer, &offsets); - if (err || offsets->cnt == 0) { - /* No partition offsets to commit because consumer - * (most likely) lost the assignment, abort transaction. */ - if (err) - fprintf(stderr, - "Failed to get consumer assignment to commit: " - "%s\n", - rd_kafka_err2str(err)); - else - rd_kafka_topic_partition_list_destroy(offsets); - - error = rd_kafka_abort_transaction(producer, -1); - if (error) - fatal_error("Failed to abort transaction", error); - - return 0; - } - - /* Get consumer's current position for this partition */ - err = rd_kafka_position(consumer, offsets); - if (err) - fatal("Failed to get consumer position: %s", - rd_kafka_err2str(err)); - - /* Send offsets to transaction coordinator */ - error = - rd_kafka_send_offsets_to_transaction(producer, offsets, cgmd, -1); - rd_kafka_consumer_group_metadata_destroy(cgmd); - rd_kafka_topic_partition_list_destroy(offsets); - if (error) { - if (rd_kafka_error_txn_requires_abort(error)) { - fprintf(stderr, - "WARNING: Failed to send offsets to " - "transaction: %s: %s: aborting transaction\n", - rd_kafka_error_name(error), - rd_kafka_error_string(error)); - rd_kafka_error_destroy(error); - - /* Abort transaction */ - error = rd_kafka_abort_transaction(producer, -1); - if (error) - fatal_error("Failed to abort transaction", - error); - return 0; - } else { - fatal_error("Failed to send offsets to transaction", - error); - } - } - - /* Commit the transaction */ - error = rd_kafka_commit_transaction(producer, -1); - if (error) { - if (rd_kafka_error_txn_requires_abort(error)) { - fprintf(stderr, - "WARNING: Failed to commit transaction: " - "%s: %s: aborting transaction\n", - rd_kafka_error_name(error), - rd_kafka_error_string(error)); - rd_kafka_error_destroy(error); - - /* Abort transaction */ - error = rd_kafka_abort_transaction(producer, -1); - if (error) - fatal_error("Failed to abort transaction", - error); - return 0; - } else { - fatal_error("Failed to commit transaction", error); - } - } - - return 1; -} - -/** - * @brief Commit the current transaction and start a new transaction. - */ -static void commit_transaction_and_start_new(rd_kafka_t *consumer, - rd_kafka_t *producer) { - rd_kafka_error_t *error; - - /* Commit transaction. - * If commit failed the transaction is aborted and we need - * to rewind the consumer to the last committed offsets. */ - if (!commit_transaction(consumer, producer)) - rewind_consumer(consumer); - - /* Begin new transaction */ - error = rd_kafka_begin_transaction(producer); - if (error) - fatal_error("Failed to begin new transaction", error); -} - -/** - * @brief The rebalance will be triggered (from rd_kafka_consumer_poll()) - * when the consumer's partition assignment is assigned or revoked. - */ -static void -consumer_group_rebalance_cb(rd_kafka_t *consumer, - rd_kafka_resp_err_t err, - rd_kafka_topic_partition_list_t *partitions, - void *opaque) { - rd_kafka_t *producer = (rd_kafka_t *)opaque; - rd_kafka_error_t *error; - - switch (err) { - case RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS: - fprintf(stdout, - "Consumer group rebalanced: " - "%d new partition(s) assigned\n", - partitions->cnt); - - /* Start fetching messages for the assigned partitions - * and add them to the consumer's local assignment. */ - error = rd_kafka_incremental_assign(consumer, partitions); - if (error) - fatal_error("Incremental assign failed", error); - break; - - case RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS: - if (rd_kafka_assignment_lost(consumer)) { - fprintf(stdout, - "Consumer group rebalanced: assignment lost: " - "aborting current transaction\n"); - - error = rd_kafka_abort_transaction(producer, -1); - if (error) - fatal_error("Failed to abort transaction", - error); - } else { - fprintf(stdout, - "Consumer group rebalanced: %d partition(s) " - "revoked: committing current transaction\n", - partitions->cnt); - - commit_transaction(consumer, producer); - } - - /* Begin new transaction */ - error = rd_kafka_begin_transaction(producer); - if (error) - fatal_error("Failed to begin transaction", error); - - /* Stop fetching messages for the revoekd partitions - * and remove them from the consumer's local assignment. */ - error = rd_kafka_incremental_unassign(consumer, partitions); - if (error) - fatal_error("Incremental unassign failed", error); - break; - - default: - /* NOTREACHED */ - fatal("Unexpected rebalance event: %s", rd_kafka_err2name(err)); - } -} - - -/** - * @brief Create the input consumer. - */ -static rd_kafka_t *create_input_consumer(const char *brokers, - const char *input_topic, - rd_kafka_t *producer) { - rd_kafka_conf_t *conf = rd_kafka_conf_new(); - rd_kafka_t *rk; - char errstr[256]; - rd_kafka_resp_err_t err; - rd_kafka_topic_partition_list_t *topics; - - if (rd_kafka_conf_set(conf, "bootstrap.servers", brokers, errstr, - sizeof(errstr)) != RD_KAFKA_CONF_OK || - rd_kafka_conf_set(conf, "group.id", - "librdkafka_transactions_example_group", errstr, - sizeof(errstr)) != RD_KAFKA_CONF_OK || - rd_kafka_conf_set(conf, "partition.assignment.strategy", - "cooperative-sticky", errstr, - sizeof(errstr)) != RD_KAFKA_CONF_OK || - rd_kafka_conf_set(conf, "auto.offset.reset", "earliest", errstr, - sizeof(errstr)) != RD_KAFKA_CONF_OK || - /* The input consumer's offsets are explicitly committed with the - * output producer's transaction using - * rd_kafka_send_offsets_to_transaction(), so auto commits - * must be disabled. */ - rd_kafka_conf_set(conf, "enable.auto.commit", "false", errstr, - sizeof(errstr)) != RD_KAFKA_CONF_OK) { - fatal("Failed to configure consumer: %s", errstr); - } - - /* This callback will be called when the consumer group is rebalanced - * and the consumer's partition assignment is assigned or revoked. */ - rd_kafka_conf_set_rebalance_cb(conf, consumer_group_rebalance_cb); - - /* The producer handle is needed in the consumer's rebalance callback - * to be able to abort and commit transactions, so we pass the - * producer as the consumer's opaque. */ - rd_kafka_conf_set_opaque(conf, producer); - - /* Create consumer */ - rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, sizeof(errstr)); - if (!rk) { - rd_kafka_conf_destroy(conf); - fatal("Failed to create consumer: %s", errstr); - } - - /* Forward all partition messages to the main queue and - * rd_kafka_consumer_poll(). */ - rd_kafka_poll_set_consumer(rk); - - /* Subscribe to the input topic */ - topics = rd_kafka_topic_partition_list_new(1); - rd_kafka_topic_partition_list_add(topics, input_topic, - /* The partition is ignored in - * rd_kafka_subscribe() */ - RD_KAFKA_PARTITION_UA); - err = rd_kafka_subscribe(rk, topics); - rd_kafka_topic_partition_list_destroy(topics); - if (err) { - rd_kafka_destroy(rk); - fatal("Failed to subscribe to %s: %s\n", input_topic, - rd_kafka_err2str(err)); - } - - return rk; -} - - -/** - * @brief Find and parse next integer string in \p start. - * @returns Pointer after found integer string, or NULL if not found. - */ -static const void * -find_next_int(const void *start, const void *end, int *intp) { - const char *p; - int collecting = 0; - int num = 0; - - for (p = (const char *)start; p < (const char *)end; p++) { - if (isdigit((int)(*p))) { - collecting = 1; - num = (num * 10) + ((int)*p - ((int)'0')); - } else if (collecting) - break; - } - - if (!collecting) - return NULL; /* No integer string found */ - - *intp = num; - - return p; -} - - -/** - * @brief Process a message from the input consumer by parsing all - * integer strings, adding them, and then producing the sum - * the output topic using the transactional producer for the given - * inut partition. - */ -static void process_message(rd_kafka_t *consumer, - rd_kafka_t *producer, - const char *output_topic, - const rd_kafka_message_t *rkmessage) { - int num; - long unsigned sum = 0; - const void *p, *end; - rd_kafka_resp_err_t err; - char value[64]; - - if (rkmessage->len == 0) - return; /* Ignore empty messages */ - - p = rkmessage->payload; - end = ((const char *)rkmessage->payload) + rkmessage->len; - - /* Find and sum all numbers in the message */ - while ((p = find_next_int(p, end, &num))) - sum += num; - - if (sum == 0) - return; /* No integers in message, ignore it. */ - - snprintf(value, sizeof(value), "%lu", sum); - - /* Emit output message on transactional producer */ - while (1) { - err = rd_kafka_producev( - producer, RD_KAFKA_V_TOPIC(output_topic), - /* Use same key as input message */ - RD_KAFKA_V_KEY(rkmessage->key, rkmessage->key_len), - /* Value is the current sum of this - * transaction. */ - RD_KAFKA_V_VALUE(value, strlen(value)), - /* Copy value since it is allocated on the stack */ - RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY), RD_KAFKA_V_END); - - if (!err) - break; - else if (err == RD_KAFKA_RESP_ERR__QUEUE_FULL) { - /* If output queue fills up we need to wait for - * some delivery reports and then retry. */ - rd_kafka_poll(producer, 100); - continue; - } else { - fprintf(stderr, - "WARNING: Failed to produce message to %s: " - "%s: aborting transaction\n", - output_topic, rd_kafka_err2str(err)); - abort_transaction_and_rewind(consumer, producer); - return; - } - } -} - - -int main(int argc, char **argv) { - rd_kafka_t *producer, *consumer; - int msgcnt = 0; - time_t last_commit = 0; - const char *brokers, *input_topic, *output_topic; - rd_kafka_error_t *error; - - /* - * Argument validation - */ - if (argc != 4) { - fprintf(stderr, - "%% Usage: %s <broker> <input-topic> <output-topic>\n", - argv[0]); - return 1; - } - - brokers = argv[1]; - input_topic = argv[2]; - output_topic = argv[3]; - - /* Signal handler for clean shutdown */ - signal(SIGINT, stop); - - producer = create_transactional_producer(brokers, output_topic); - - consumer = create_input_consumer(brokers, input_topic, producer); - - fprintf(stdout, - "Expecting integers to sum on input topic %s ...\n" - "To generate input messages you can use:\n" - " $ seq 1 100 | examples/producer %s %s\n" - "Observe summed integers on output topic %s:\n" - " $ examples/consumer %s just-watching %s\n" - "\n", - input_topic, brokers, input_topic, output_topic, brokers, - output_topic); - - /* Begin transaction and start waiting for messages */ - error = rd_kafka_begin_transaction(producer); - if (error) - fatal_error("Failed to begin transaction", error); - - while (run) { - rd_kafka_message_t *msg; - - /* Commit transaction every 100 messages or 5 seconds */ - if (msgcnt > 0 && - (msgcnt > 100 || last_commit + 5 <= time(NULL))) { - printf("msgcnt %d, elapsed %d\n", msgcnt, - (int)(time(NULL) - last_commit)); - commit_transaction_and_start_new(consumer, producer); - msgcnt = 0; - last_commit = time(NULL); - } - - /* Wait for new mesages or error events */ - msg = rd_kafka_consumer_poll(consumer, 1000 /*1 second*/); - if (!msg) - continue; /* Poll timeout */ - - if (msg->err) { - /* Client errors are typically just informational - * since the client will automatically try to recover - * from all types of errors. - * It is thus sufficient for the application to log and - * continue operating when a consumer error is - * encountered. */ - fprintf(stderr, "WARNING: Consumer error: %s\n", - rd_kafka_message_errstr(msg)); - rd_kafka_message_destroy(msg); - continue; - } - - /* Process message */ - process_message(consumer, producer, output_topic, msg); - - rd_kafka_message_destroy(msg); - - msgcnt++; - } - - fprintf(stdout, "Closing consumer\n"); - rd_kafka_consumer_close(consumer); - rd_kafka_destroy(consumer); - - fprintf(stdout, "Closing producer\n"); - rd_kafka_destroy(producer); - - return 0; -} diff --git a/fluent-bit/lib/librdkafka-2.1.0/examples/win_ssl_cert_store.cpp b/fluent-bit/lib/librdkafka-2.1.0/examples/win_ssl_cert_store.cpp deleted file mode 100644 index a80dfea30..000000000 --- a/fluent-bit/lib/librdkafka-2.1.0/examples/win_ssl_cert_store.cpp +++ /dev/null @@ -1,395 +0,0 @@ -/* - * librdkafka - Apache Kafka C library - * - * Copyright (c) 2019, Magnus Edenhill - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/** - * Example of utilizing the Windows Certificate store with SSL. - */ - -#include <iostream> -#include <string> -#include <cstdlib> -#include <cstdio> -#include <csignal> -#include <cstring> -#include <sstream> - -#include "../win32/wingetopt.h" -#include <windows.h> -#include <wincrypt.h> - -/* - * Typically include path in a real application would be - * #include <librdkafka/rdkafkacpp.h> - */ -#include "rdkafkacpp.h" - - - -class ExampleStoreRetriever { - public: - ExampleStoreRetriever(std::string const &subject, std::string const &pass) : - m_cert_subject(subject), - m_password(pass), - m_cert_store(NULL), - m_cert_ctx(NULL) { - load_certificate(); - } - - ~ExampleStoreRetriever() { - if (m_cert_ctx) - CertFreeCertificateContext(m_cert_ctx); - - if (m_cert_store) - CertCloseStore(m_cert_store, 0); - } - - /* @returns the public key in DER format */ - const std::vector<unsigned char> get_public_key() { - std::vector<unsigned char> buf((size_t)m_cert_ctx->cbCertEncoded); - buf.assign((const char *)m_cert_ctx->pbCertEncoded, - (const char *)m_cert_ctx->pbCertEncoded + - (size_t)m_cert_ctx->cbCertEncoded); - return buf; - } - - /* @returns the private key in PCKS#12 format */ - const std::vector<unsigned char> get_private_key() { - ssize_t ret = 0; - /* - * In order to export the private key the certificate - * must first be marked as exportable. - * - * Steps to export the certificate - * 1) Create an in-memory cert store - * 2) Add the certificate to the store - * 3) Export the private key from the in-memory store - */ - - /* Create an in-memory cert store */ - HCERTSTORE hMemStore = - CertOpenStore(CERT_STORE_PROV_MEMORY, 0, NULL, 0, NULL); - if (!hMemStore) - throw "Failed to create in-memory cert store: " + - GetErrorMsg(GetLastError()); - - /* Add certificate to store */ - if (!CertAddCertificateContextToStore(hMemStore, m_cert_ctx, - CERT_STORE_ADD_USE_EXISTING, NULL)) - throw "Failed to add certificate to store: " + - GetErrorMsg(GetLastError()); - - /* - * Export private key from cert - */ - CRYPT_DATA_BLOB db = {NULL}; - - std::wstring w_password(m_password.begin(), m_password.end()); - - /* Acquire output size */ - if (!PFXExportCertStoreEx(hMemStore, &db, w_password.c_str(), NULL, - EXPORT_PRIVATE_KEYS | REPORT_NO_PRIVATE_KEY | - REPORT_NOT_ABLE_TO_EXPORT_PRIVATE_KEY)) - throw "Failed to export private key: " + GetErrorMsg(GetLastError()); - - std::vector<unsigned char> buf; - - buf.resize(db.cbData); - db.pbData = &buf[0]; - - /* Extract key */ - if (!PFXExportCertStoreEx(hMemStore, &db, w_password.c_str(), NULL, - EXPORT_PRIVATE_KEYS | REPORT_NO_PRIVATE_KEY | - REPORT_NOT_ABLE_TO_EXPORT_PRIVATE_KEY)) - throw "Failed to export private key (PFX): " + - GetErrorMsg(GetLastError()); - - CertCloseStore(hMemStore, 0); - - buf.resize(db.cbData); - - return buf; - } - - private: - void load_certificate() { - if (m_cert_ctx) - return; - - m_cert_store = CertOpenStore(CERT_STORE_PROV_SYSTEM, 0, NULL, - CERT_SYSTEM_STORE_CURRENT_USER, L"My"); - if (!m_cert_store) - throw "Failed to open cert store: " + GetErrorMsg(GetLastError()); - - m_cert_ctx = CertFindCertificateInStore( - m_cert_store, X509_ASN_ENCODING, 0, CERT_FIND_SUBJECT_STR, - /* should probally do a better std::string to std::wstring conversion */ - std::wstring(m_cert_subject.begin(), m_cert_subject.end()).c_str(), - NULL); - if (!m_cert_ctx) { - CertCloseStore(m_cert_store, 0); - m_cert_store = NULL; - throw "Certificate " + m_cert_subject + - " not found in cert store: " + GetErrorMsg(GetLastError()); - } - } - - std::string GetErrorMsg(unsigned long error) { - char *message = NULL; - size_t ret = FormatMessageA( - FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, nullptr, - error, 0, (char *)&message, 0, nullptr); - if (ret == 0) { - std::stringstream ss; - - ss << std::string("could not format message for ") << error; - return ss.str(); - } else { - std::string result(message, ret); - LocalFree(message); - return result; - } - } - - private: - std::string m_cert_subject; - std::string m_password; - PCCERT_CONTEXT m_cert_ctx; - HCERTSTORE m_cert_store; -}; - - -class PrintingSSLVerifyCb : public RdKafka::SslCertificateVerifyCb { - /* This SSL cert verification callback simply prints the certificates - * in the certificate chain. - * It provides no validation, everything is ok. */ - public: - bool ssl_cert_verify_cb(const std::string &broker_name, - int32_t broker_id, - int *x509_error, - int depth, - const char *buf, - size_t size, - std::string &errstr) { - PCCERT_CONTEXT ctx = CertCreateCertificateContext( - X509_ASN_ENCODING | PKCS_7_ASN_ENCODING, (const uint8_t *)buf, - static_cast<unsigned long>(size)); - - if (!ctx) - std::cerr << "Failed to parse certificate" << std::endl; - - char subject[256] = "n/a"; - char issuer[256] = "n/a"; - - CertGetNameStringA(ctx, CERT_NAME_FRIENDLY_DISPLAY_TYPE, 0, NULL, subject, - sizeof(subject)); - - CertGetNameStringA(ctx, CERT_NAME_FRIENDLY_DISPLAY_TYPE, - CERT_NAME_ISSUER_FLAG, NULL, issuer, sizeof(issuer)); - - std::cerr << "Broker " << broker_name << " (" << broker_id << "): " - << "certificate depth " << depth << ", X509 error " << *x509_error - << ", subject " << subject << ", issuer " << issuer << std::endl; - - if (ctx) - CertFreeCertificateContext(ctx); - - return true; - } -}; - - -/** - * @brief Print the brokers in the cluster. - */ -static void print_brokers(RdKafka::Handle *handle, - const RdKafka::Metadata *md) { - std::cout << md->brokers()->size() << " broker(s) in cluster " - << handle->clusterid(0) << std::endl; - - /* Iterate brokers */ - RdKafka::Metadata::BrokerMetadataIterator ib; - for (ib = md->brokers()->begin(); ib != md->brokers()->end(); ++ib) - std::cout << " broker " << (*ib)->id() << " at " << (*ib)->host() << ":" - << (*ib)->port() << std::endl; -} - - -int main(int argc, char **argv) { - std::string brokers; - std::string errstr; - std::string cert_subject; - std::string priv_key_pass; - - /* - * Create configuration objects - */ - RdKafka::Conf *conf = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL); - RdKafka::Conf *tconf = RdKafka::Conf::create(RdKafka::Conf::CONF_TOPIC); - - int opt; - while ((opt = getopt(argc, argv, "b:d:X:s:p:")) != -1) { - switch (opt) { - case 'b': - brokers = optarg; - break; - case 'd': - if (conf->set("debug", optarg, errstr) != RdKafka::Conf::CONF_OK) { - std::cerr << errstr << std::endl; - exit(1); - } - break; - case 'X': { - char *name, *val; - - name = optarg; - if (!(val = strchr(name, '='))) { - std::cerr << "%% Expected -X property=value, not " << name << std::endl; - exit(1); - } - - *val = '\0'; - val++; - - if (conf->set(name, val, errstr) != RdKafka::Conf::CONF_OK) { - std::cerr << errstr << std::endl; - exit(1); - } - } break; - - case 's': - cert_subject = optarg; - break; - - case 'p': - priv_key_pass = optarg; - if (conf->set("ssl.key.password", optarg, errstr) != - RdKafka::Conf::CONF_OK) { - std::cerr << errstr << std::endl; - exit(1); - } - - break; - - default: - goto usage; - } - } - - if (brokers.empty() || optind != argc) { - usage: - std::string features; - conf->get("builtin.features", features); - fprintf(stderr, - "Usage: %s [options] -b <brokers> -s <cert-subject> -p " - "<priv-key-password>\n" - "\n" - "Windows Certificate Store integration example.\n" - "Use certlm.msc or mmc to view your certificates.\n" - "\n" - "librdkafka version %s (0x%08x, builtin.features \"%s\")\n" - "\n" - " Options:\n" - " -b <brokers> Broker address\n" - " -s <cert> The subject name of the client's SSL " - "certificate to use\n" - " -p <pass> The private key password\n" - " -d [facs..] Enable debugging contexts: %s\n" - " -X <prop=name> Set arbitrary librdkafka " - "configuration property\n" - "\n", - argv[0], RdKafka::version_str().c_str(), RdKafka::version(), - features.c_str(), RdKafka::get_debug_contexts().c_str()); - exit(1); - } - - if (!cert_subject.empty()) { - try { - /* Load certificates from the Windows store */ - ExampleStoreRetriever certStore(cert_subject, priv_key_pass); - - std::vector<unsigned char> pubkey, privkey; - - pubkey = certStore.get_public_key(); - privkey = certStore.get_private_key(); - - if (conf->set_ssl_cert(RdKafka::CERT_PUBLIC_KEY, RdKafka::CERT_ENC_DER, - &pubkey[0], pubkey.size(), - errstr) != RdKafka::Conf::CONF_OK) - throw "Failed to set public key: " + errstr; - - if (conf->set_ssl_cert(RdKafka::CERT_PRIVATE_KEY, - RdKafka::CERT_ENC_PKCS12, &privkey[0], - privkey.size(), errstr) != RdKafka::Conf::CONF_OK) - throw "Failed to set private key: " + errstr; - - } catch (const std::string &ex) { - std::cerr << ex << std::endl; - exit(1); - } - } - - - /* - * Set configuration properties - */ - conf->set("bootstrap.servers", brokers, errstr); - - /* We use the Certificiate verification callback to print the - * certificate chains being used. */ - PrintingSSLVerifyCb ssl_verify_cb; - - if (conf->set("ssl_cert_verify_cb", &ssl_verify_cb, errstr) != - RdKafka::Conf::CONF_OK) { - std::cerr << errstr << std::endl; - exit(1); - } - - /* Create any type of client, producering being the cheapest. */ - RdKafka::Producer *producer = RdKafka::Producer::create(conf, errstr); - if (!producer) { - std::cerr << "Failed to create producer: " << errstr << std::endl; - exit(1); - } - - RdKafka::Metadata *metadata; - - /* Fetch metadata */ - RdKafka::ErrorCode err = producer->metadata(false, NULL, &metadata, 5000); - if (err != RdKafka::ERR_NO_ERROR) { - std::cerr << "%% Failed to acquire metadata: " << RdKafka::err2str(err) - << std::endl; - exit(1); - } - - print_brokers(producer, metadata); - - delete metadata; - delete producer; - - return 0; -} |