summaryrefslogtreecommitdiffstats
path: root/backends/aws_kinesis
diff options
context:
space:
mode:
Diffstat (limited to 'backends/aws_kinesis')
-rw-r--r--backends/aws_kinesis/Makefile.am12
-rw-r--r--backends/aws_kinesis/README.md53
-rw-r--r--backends/aws_kinesis/aws_kinesis.c94
-rw-r--r--backends/aws_kinesis/aws_kinesis.conf10
-rw-r--r--backends/aws_kinesis/aws_kinesis.h14
-rw-r--r--backends/aws_kinesis/aws_kinesis_put_record.cc87
-rw-r--r--backends/aws_kinesis/aws_kinesis_put_record.h25
7 files changed, 0 insertions, 295 deletions
diff --git a/backends/aws_kinesis/Makefile.am b/backends/aws_kinesis/Makefile.am
deleted file mode 100644
index 1fec72c1f..000000000
--- a/backends/aws_kinesis/Makefile.am
+++ /dev/null
@@ -1,12 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
-
-dist_libconfig_DATA = \
- aws_kinesis.conf \
- $(NULL)
diff --git a/backends/aws_kinesis/README.md b/backends/aws_kinesis/README.md
deleted file mode 100644
index a2b682517..000000000
--- a/backends/aws_kinesis/README.md
+++ /dev/null
@@ -1,53 +0,0 @@
-<!--
-title: "Using Netdata with AWS Kinesis Data Streams"
-custom_edit_url: https://github.com/netdata/netdata/edit/master/backends/aws_kinesis/README.md
--->
-
-# Using Netdata with AWS Kinesis Data Streams
-
-## Prerequisites
-
-To use AWS Kinesis as a backend AWS SDK for C++ should be
-[installed](https://docs.aws.amazon.com/en_us/sdk-for-cpp/v1/developer-guide/setup.html) first. `libcrypto`, `libssl`,
-and `libcurl` are also required to compile Netdata with Kinesis support enabled. Next, Netdata should be re-installed
-from the source. The installer will detect that the required libraries are now available.
-
-If the AWS SDK for C++ is being installed from source, it is useful to set `-DBUILD_ONLY="kinesis"`. Otherwise, the
-building process could take a very long time. Take a note, that the default installation path for the libraries is
-`/usr/local/lib64`. Many Linux distributions don't include this path as the default one for a library search, so it is
-advisable to use the following options to `cmake` while building the AWS SDK:
-
-```sh
-cmake -DCMAKE_INSTALL_LIBDIR=/usr/lib -DCMAKE_INSTALL_INCLUDEDIR=/usr/include -DBUILD_SHARED_LIBS=OFF -DBUILD_ONLY=kinesis <aws-sdk-cpp sources>
-```
-
-## Configuration
-
-To enable data sending to the kinesis backend set the following options in `netdata.conf`:
-
-```conf
-[backend]
- enabled = yes
- type = kinesis
- destination = us-east-1
-```
-
-set the `destination` option to an AWS region.
-
-In the Netdata configuration directory run `./edit-config aws_kinesis.conf` and set AWS credentials and stream name:
-
-```yaml
-# AWS credentials
-aws_access_key_id = your_access_key_id
-aws_secret_access_key = your_secret_access_key
-
-# destination stream
-stream name = your_stream_name
-```
-
-Alternatively, AWS credentials can be set for the `netdata` user using AWS SDK for C++ [standard methods](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html).
-
-A partition key for every record is computed automatically by Netdata with the purpose to distribute records across
-available shards evenly.
-
-[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fbackends%2Faws_kinesis%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/backends/aws_kinesis/aws_kinesis.c b/backends/aws_kinesis/aws_kinesis.c
deleted file mode 100644
index b1ea47862..000000000
--- a/backends/aws_kinesis/aws_kinesis.c
+++ /dev/null
@@ -1,94 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#define BACKENDS_INTERNALS
-#include "aws_kinesis.h"
-
-#define CONFIG_FILE_LINE_MAX ((CONFIG_MAX_NAME + CONFIG_MAX_VALUE + 1024) * 2)
-
-// ----------------------------------------------------------------------------
-// kinesis backend
-
-// read the aws_kinesis.conf file
-int read_kinesis_conf(const char *path, char **access_key_id_p, char **secret_access_key_p, char **stream_name_p)
-{
- char *access_key_id = *access_key_id_p;
- char *secret_access_key = *secret_access_key_p;
- char *stream_name = *stream_name_p;
-
- if(unlikely(access_key_id)) freez(access_key_id);
- if(unlikely(secret_access_key)) freez(secret_access_key);
- if(unlikely(stream_name)) freez(stream_name);
- access_key_id = NULL;
- secret_access_key = NULL;
- stream_name = NULL;
-
- int line = 0;
-
- char filename[FILENAME_MAX + 1];
- snprintfz(filename, FILENAME_MAX, "%s/aws_kinesis.conf", path);
-
- char buffer[CONFIG_FILE_LINE_MAX + 1], *s;
-
- debug(D_BACKEND, "BACKEND: opening config file '%s'", filename);
-
- FILE *fp = fopen(filename, "r");
- if(!fp) {
- return 1;
- }
-
- while(fgets(buffer, CONFIG_FILE_LINE_MAX, fp) != NULL) {
- buffer[CONFIG_FILE_LINE_MAX] = '\0';
- line++;
-
- s = trim(buffer);
- if(!s || *s == '#') {
- debug(D_BACKEND, "BACKEND: ignoring line %d of file '%s', it is empty.", line, filename);
- continue;
- }
-
- char *name = s;
- char *value = strchr(s, '=');
- if(unlikely(!value)) {
- error("BACKEND: ignoring line %d ('%s') of file '%s', there is no = in it.", line, s, filename);
- continue;
- }
- *value = '\0';
- value++;
-
- name = trim(name);
- value = trim(value);
-
- if(unlikely(!name || *name == '#')) {
- error("BACKEND: ignoring line %d of file '%s', name is empty.", line, filename);
- continue;
- }
-
- if(!value)
- value = "";
- else
- value = strip_quotes(value);
-
- if(name[0] == 'a' && name[4] == 'a' && !strcmp(name, "aws_access_key_id")) {
- access_key_id = strdupz(value);
- }
- else if(name[0] == 'a' && name[4] == 's' && !strcmp(name, "aws_secret_access_key")) {
- secret_access_key = strdupz(value);
- }
- else if(name[0] == 's' && !strcmp(name, "stream name")) {
- stream_name = strdupz(value);
- }
- }
-
- fclose(fp);
-
- if(unlikely(!stream_name || !*stream_name)) {
- error("BACKEND: stream name is a mandatory Kinesis parameter but it is not configured");
- return 1;
- }
-
- *access_key_id_p = access_key_id;
- *secret_access_key_p = secret_access_key;
- *stream_name_p = stream_name;
-
- return 0;
-}
diff --git a/backends/aws_kinesis/aws_kinesis.conf b/backends/aws_kinesis/aws_kinesis.conf
deleted file mode 100644
index cc54b5fa2..000000000
--- a/backends/aws_kinesis/aws_kinesis.conf
+++ /dev/null
@@ -1,10 +0,0 @@
-# AWS Kinesis Data Streams backend configuration
-#
-# All options in this file are mandatory
-
-# AWS credentials
-aws_access_key_id =
-aws_secret_access_key =
-
-# destination stream
-stream name = \ No newline at end of file
diff --git a/backends/aws_kinesis/aws_kinesis.h b/backends/aws_kinesis/aws_kinesis.h
deleted file mode 100644
index 50a4631c5..000000000
--- a/backends/aws_kinesis/aws_kinesis.h
+++ /dev/null
@@ -1,14 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#ifndef NETDATA_BACKEND_KINESIS_H
-#define NETDATA_BACKEND_KINESIS_H
-
-#include "backends/backends.h"
-#include "aws_kinesis_put_record.h"
-
-#define KINESIS_PARTITION_KEY_MAX 256
-#define KINESIS_RECORD_MAX 1024 * 1024
-
-extern int read_kinesis_conf(const char *path, char **auth_key_id_p, char **secure_key_p, char **stream_name_p);
-
-#endif //NETDATA_BACKEND_KINESIS_H
diff --git a/backends/aws_kinesis/aws_kinesis_put_record.cc b/backends/aws_kinesis/aws_kinesis_put_record.cc
deleted file mode 100644
index a8ba4aaca..000000000
--- a/backends/aws_kinesis/aws_kinesis_put_record.cc
+++ /dev/null
@@ -1,87 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#include <aws/core/Aws.h>
-#include <aws/core/client/ClientConfiguration.h>
-#include <aws/core/auth/AWSCredentials.h>
-#include <aws/core/utils/Outcome.h>
-#include <aws/kinesis/KinesisClient.h>
-#include <aws/kinesis/model/PutRecordRequest.h>
-#include "aws_kinesis_put_record.h"
-
-using namespace Aws;
-
-static SDKOptions options;
-
-static Kinesis::KinesisClient *client;
-
-struct request_outcome {
- Kinesis::Model::PutRecordOutcomeCallable future_outcome;
- size_t data_len;
-};
-
-static Vector<request_outcome> request_outcomes;
-
-void backends_kinesis_init(const char *region, const char *access_key_id, const char *secret_key, const long timeout) {
- InitAPI(options);
-
- Client::ClientConfiguration config;
-
- config.region = region;
- config.requestTimeoutMs = timeout;
- config.connectTimeoutMs = timeout;
-
- if(access_key_id && *access_key_id && secret_key && *secret_key) {
- client = New<Kinesis::KinesisClient>("client", Auth::AWSCredentials(access_key_id, secret_key), config);
- } else {
- client = New<Kinesis::KinesisClient>("client", config);
- }
-}
-
-void backends_kinesis_shutdown() {
- Delete(client);
-
- ShutdownAPI(options);
-}
-
-int backends_kinesis_put_record(const char *stream_name, const char *partition_key,
- const char *data, size_t data_len) {
- Kinesis::Model::PutRecordRequest request;
-
- request.SetStreamName(stream_name);
- request.SetPartitionKey(partition_key);
- request.SetData(Utils::ByteBuffer((unsigned char*) data, data_len));
-
- request_outcomes.push_back({client->PutRecordCallable(request), data_len});
-
- return 0;
-}
-
-int backends_kinesis_get_result(char *error_message, size_t *sent_bytes, size_t *lost_bytes) {
- Kinesis::Model::PutRecordOutcome outcome;
- *sent_bytes = 0;
- *lost_bytes = 0;
-
- for(auto request_outcome = request_outcomes.begin(); request_outcome != request_outcomes.end(); ) {
- std::future_status status = request_outcome->future_outcome.wait_for(std::chrono::microseconds(100));
-
- if(status == std::future_status::ready || status == std::future_status::deferred) {
- outcome = request_outcome->future_outcome.get();
- *sent_bytes += request_outcome->data_len;
-
- if(!outcome.IsSuccess()) {
- *lost_bytes += request_outcome->data_len;
- outcome.GetError().GetMessage().copy(error_message, ERROR_LINE_MAX);
- }
-
- request_outcomes.erase(request_outcome);
- } else {
- ++request_outcome;
- }
- }
-
- if(*lost_bytes) {
- return 1;
- }
-
- return 0;
-} \ No newline at end of file
diff --git a/backends/aws_kinesis/aws_kinesis_put_record.h b/backends/aws_kinesis/aws_kinesis_put_record.h
deleted file mode 100644
index fa3d03459..000000000
--- a/backends/aws_kinesis/aws_kinesis_put_record.h
+++ /dev/null
@@ -1,25 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#ifndef NETDATA_BACKEND_KINESIS_PUT_RECORD_H
-#define NETDATA_BACKEND_KINESIS_PUT_RECORD_H
-
-#define ERROR_LINE_MAX 1023
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-void backends_kinesis_init(const char *region, const char *access_key_id, const char *secret_key, const long timeout);
-
-void backends_kinesis_shutdown();
-
-int backends_kinesis_put_record(const char *stream_name, const char *partition_key,
- const char *data, size_t data_len);
-
-int backends_kinesis_get_result(char *error_message, size_t *sent_bytes, size_t *lost_bytes);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif //NETDATA_BACKEND_KINESIS_PUT_RECORD_H
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-29 07:30:26 +0000