Adding upstream version 1.17.0.upstream/1.17.0

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 148 insertions, 37 deletions

diff --git a/backends/backends.c b/backends/backends.c
index 15a0cb41b..120c6e703 100644
--- a/backends/backends.c
+++ b/backends/backends.c
@@ -131,7 +131,8 @@ calculated_number backend_calculate_value_from_stored_data(
     }
 */
     for(rd->state->query_ops.init(rd, &handle, after, before) ; !rd->state->query_ops.is_finished(&handle) ; ) {
-        n = rd->state->query_ops.next_metric(&handle);
+        time_t curr_t;
+        n = rd->state->query_ops.next_metric(&handle, &curr_t);
 
         if(unlikely(!does_storage_number_exist(n))) {
             // not collected
@@ -249,12 +250,11 @@ static void backends_main_cleanup(void *ptr) {
 /**
  * Set Kinesis variables
  *
- * Set the variables necessaries to work with this specific backend.
+ * Set the variables necessary to work with this specific backend.
  *
- * @param default_port  the default port of the backend
+ * @param default_port the default port of the backend
  * @param brc function called to check the result.
- * @param brf function called to format the msessage to the backend
- * @param type the backend string selector.
+ * @param brf function called to format the message to the backend
  */
 void backend_set_kinesis_variables(int *default_port,
                                    backend_response_checker_t brc,
@@ -278,12 +278,11 @@ void backend_set_kinesis_variables(int *default_port,
 /**
  * Set Prometheus variables
  *
- * Set the variables necessaries to work with this specific backend.
+ * Set the variables necessary to work with this specific backend.
  *
- * @param default_port  the default port of the backend
+ * @param default_port the default port of the backend
  * @param brc function called to check the result.
- * @param brf function called to format the msessage to the backend
- * @param type the backend string selector.
+ * @param brf function called to format the message to the backend
  */
 void backend_set_prometheus_variables(int *default_port,
                                       backend_response_checker_t brc,
@@ -301,14 +300,41 @@ void backend_set_prometheus_variables(int *default_port,
 }
 
 /**
+ * Set MongoDB variables
+ *
+ * Set the variables necessary to work with this specific backend.
+ *
+ * @param default_port the default port of the backend
+ * @param brc function called to check the result.
+ * @param brf function called to format the message to the backend
+ */
+void backend_set_mongodb_variables(int *default_port,
+                                      backend_response_checker_t brc,
+                                      backend_request_formatter_t brf)
+{
+    (void)default_port;
+#ifndef HAVE_MONGOC
+    (void)brc;
+    (void)brf;
+#endif
+
+#if HAVE_MONGOC
+    *brc = process_json_response;
+    if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
+        *brf = format_dimension_collected_json_plaintext;
+    else
+        *brf = format_dimension_stored_json_plaintext;
+#endif
+}
+
+/**
  * Set JSON variables
  *
- * Set the variables necessaries to work with this specific backend.
+ * Set the variables necessary to work with this specific backend.
  *
- * @param default_port  the default port of the backend
+ * @param default_port the default port of the backend
  * @param brc function called to check the result.
- * @param brf function called to format the msessage to the backend
- * @param type the backend string selector.
+ * @param brf function called to format the message to the backend
  */
 void backend_set_json_variables(int *default_port,
                                 backend_response_checker_t brc,
@@ -326,12 +352,11 @@ void backend_set_json_variables(int *default_port,
 /**
  * Set OpenTSDB HTTP variables
  *
- * Set the variables necessaries to work with this specific backend.
+ * Set the variables necessary to work with this specific backend.
  *
- * @param default_port  the default port of the backend
+ * @param default_port the default port of the backend
  * @param brc function called to check the result.
- * @param brf function called to format the msessage to the backend
- * @param type the backend string selector.
+ * @param brf function called to format the message to the backend
  */
 void backend_set_opentsdb_http_variables(int *default_port,
                                          backend_response_checker_t brc,
@@ -350,12 +375,11 @@ void backend_set_opentsdb_http_variables(int *default_port,
 /**
  * Set OpenTSDB Telnet variables
  *
- * Set the variables necessaries to work with this specific backend.
+ * Set the variables necessary to work with this specific backend.
  *
- * @param default_port  the default port of the backend
+ * @param default_port the default port of the backend
  * @param brc function called to check the result.
- * @param brf function called to format the msessage to the backend
- * @param type the backend string selector.
+ * @param brf function called to format the message to the backend
  */
 void backend_set_opentsdb_telnet_variables(int *default_port,
                                            backend_response_checker_t brc,
@@ -373,12 +397,11 @@ void backend_set_opentsdb_telnet_variables(int *default_port,
 /**
  * Set Graphite variables
  *
- * Set the variables necessaries to work with this specific backend.
+ * Set the variables necessary to work with this specific backend.
  *
- * @param default_port  the default port of the backend
+ * @param default_port the default port of the backend
  * @param brc function called to check the result.
- * @param brf function called to format the msessage to the backend
- * @param type the backend string selector.
+ * @param brf function called to format the message to the backend
  */
 void backend_set_graphite_variables(int *default_port,
                                     backend_response_checker_t brc,
@@ -421,6 +444,9 @@ BACKEND_TYPE backend_select_type(const char *type) {
     else if (!strcmp(type, "kinesis") || !strcmp(type, "kinesis:plaintext")) {
         return BACKEND_TYPE_KINESIS;
     }
+    else if (!strcmp(type, "mongodb") || !strcmp(type, "mongodb:plaintext")) {
+        return BACKEND_TYPE_MONGODB;
+    }
 
     return BACKEND_TYPE_UNKNOWN;
 }
@@ -450,7 +476,18 @@ void *backends_main(void *ptr) {
 
 #if ENABLE_PROMETHEUS_REMOTE_WRITE
     int do_prometheus_remote_write = 0;
-    BUFFER *http_request_header = buffer_create(1);
+    BUFFER *http_request_header = NULL;
+#endif
+
+#if HAVE_MONGOC
+    int do_mongodb = 0;
+    char *mongodb_uri = NULL;
+    char *mongodb_database = NULL;
+    char *mongodb_collection = NULL;
+
+    // set the default socket timeout in ms
+    int32_t mongodb_default_socket_timeout = (int32_t)(global_backend_update_every >= 2)?(global_backend_update_every * MSEC_PER_SEC - 500):1000;
+
 #endif
 
 #ifdef ENABLE_HTTPS
@@ -524,6 +561,7 @@ void *backends_main(void *ptr) {
 #if ENABLE_PROMETHEUS_REMOTE_WRITE
             do_prometheus_remote_write = 1;
 
+            http_request_header = buffer_create(1);
             init_write_request();
 #else
             error("BACKEND: Prometheus remote write support isn't compiled");
@@ -547,6 +585,30 @@ void *backends_main(void *ptr) {
             backend_set_kinesis_variables(&default_port,&backend_response_checker,&backend_request_formatter);
             break;
         }
+        case BACKEND_TYPE_MONGODB: {
+#if HAVE_MONGOC
+            if(unlikely(read_mongodb_conf(netdata_configured_user_config_dir,
+                                          &mongodb_uri,
+                                          &mongodb_database,
+                                          &mongodb_collection))) {
+                error("BACKEND: mongodb backend type is set but cannot read its configuration from %s/mongodb.conf",
+                      netdata_configured_user_config_dir);
+                goto cleanup;
+            }
+
+            if(likely(!mongodb_init(mongodb_uri, mongodb_database, mongodb_collection, mongodb_default_socket_timeout))) {
+                backend_set_mongodb_variables(&default_port, &backend_response_checker, &backend_request_formatter);
+                do_mongodb = 1;
+            }
+            else {
+                error("BACKEND: cannot initialize MongoDB backend");
+                goto cleanup;
+            }
+#else
+            error("BACKEND: MongoDB support isn't compiled");
+#endif // HAVE_MONGOC
+            break;
+        }
         case BACKEND_TYPE_GRAPHITE: {
             backend_set_graphite_variables(&default_port,&backend_response_checker,&backend_request_formatter);
             break;
@@ -658,7 +720,8 @@ void *backends_main(void *ptr) {
         size_t count_dims_total = 0;
 
 #if ENABLE_PROMETHEUS_REMOTE_WRITE
-        clear_write_request();
+        if(do_prometheus_remote_write)
+            clear_write_request();
 #endif
         rrd_rdlock();
         RRDHOST *host;
@@ -833,12 +896,53 @@ void *backends_main(void *ptr) {
                 chart_sent_metrics = chart_buffered_metrics;
 
             buffer_flush(b);
-        }
-        else {
-#else
-        {
+        } else
 #endif /* HAVE_KINESIS */
 
+#if HAVE_MONGOC
+        if(do_mongodb) {
+            size_t buffer_len = buffer_strlen(b);
+            size_t sent = 0;
+
+            while(sent < buffer_len) {
+                const char *first_char = buffer_tostring(b);
+
+                debug(D_BACKEND, "BACKEND: mongodb_insert(): uri = %s, database = %s, collection = %s, \
+                      buffer = %zu", mongodb_uri, mongodb_database, mongodb_collection, buffer_len);
+
+                if(likely(!mongodb_insert((char *)first_char, (size_t)chart_buffered_metrics))) {
+                    sent += buffer_len;
+                    chart_transmission_successes++;
+                    chart_receptions++;
+                }
+                else {
+                    // oops! we couldn't send (all or some of the) data
+                    error("BACKEND: failed to write data to database backend '%s'. Willing to write %zu bytes, wrote %zu bytes.",
+                          mongodb_uri, buffer_len, 0UL);
+
+                    chart_transmission_failures++;
+                    chart_data_lost_events++;
+                    chart_lost_bytes += buffer_len;
+
+                    // estimate the number of lost metrics
+                    chart_lost_metrics += (collected_number)chart_buffered_metrics;
+
+                    break;
+                }
+
+                if(unlikely(netdata_exit)) break;
+            }
+
+            chart_sent_bytes += sent;
+            if(likely(sent == buffer_len))
+                chart_sent_metrics = chart_buffered_metrics;
+
+            buffer_flush(b);
+        } else
+#endif /* HAVE_MONGOC */
+
+        {
+
             // ------------------------------------------------------------------------
             // if we are connected, receive a response, without blocking
 
@@ -1027,14 +1131,14 @@ void *backends_main(void *ptr) {
 
 
 #if ENABLE_PROMETHEUS_REMOTE_WRITE
-        if(failures) {
+        if(do_prometheus_remote_write && failures) {
             (void) buffer_on_failures;
             failures = 0;
             chart_lost_bytes = chart_buffered_bytes = get_write_request_size(); // estimated write request size
             chart_data_lost_events++;
             chart_lost_metrics = chart_buffered_metrics;
-        }
-#else
+        } else
+#endif
         if(failures > buffer_on_failures) {
             // too bad! we are going to lose data
             chart_lost_bytes += buffer_strlen(b);
@@ -1044,7 +1148,6 @@ void *backends_main(void *ptr) {
             chart_data_lost_events++;
             chart_lost_metrics = chart_buffered_metrics;
         }
-#endif /* ENABLE_PROMETHEUS_REMOTE_WRITE */
 
         if(unlikely(netdata_exit)) break;
 
@@ -1101,9 +1204,17 @@ cleanup:
 #endif
 
 #if ENABLE_PROMETHEUS_REMOTE_WRITE
-    if(do_prometheus_remote_write) {
-        buffer_free(http_request_header);
+    buffer_free(http_request_header);
+    if(do_prometheus_remote_write)
         protocol_buffers_shutdown();
+#endif
+
+#if HAVE_MONGOC
+    if(do_mongodb) {
+        mongodb_cleanup();
+        freez(mongodb_uri);
+        freez(mongodb_database);
+        freez(mongodb_collection);
     }
 #endif