diff options
Diffstat (limited to 'web/api')
37 files changed, 266 insertions, 106 deletions
diff --git a/web/api/README.md b/web/api/README.md index 1cc3439c2..89a953002 100644 --- a/web/api/README.md +++ b/web/api/README.md @@ -11,4 +11,4 @@ The complete documentation of the Netdata API is available at the **[Swagger Edi If your prefer it over the Swagger Editor, you can also use **[Swagger UI](https://registry.my-netdata.io/swagger/#!/default/get_data)**. This however does not provide all the information available. -[](<>) + diff --git a/web/api/badges/README.md b/web/api/badges/README.md index b5fc53468..601eae479 100644 --- a/web/api/badges/README.md +++ b/web/api/badges/README.md @@ -360,4 +360,4 @@ You can refresh them from your browser console though. Press F12 to open the web var len = document.images.length; while(len--) { document.images[len].src = document.images[len].src.replace(/\?cacheBuster=\d*/, "") + "?cacheBuster=" + new Date().getTime().toString(); }; ``` -[](<>) + diff --git a/web/api/badges/web_buffer_svg.c b/web/api/badges/web_buffer_svg.c index f8c0a17a6..65ca21d19 100644 --- a/web/api/badges/web_buffer_svg.c +++ b/web/api/badges/web_buffer_svg.c @@ -1102,7 +1102,7 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u // if the collected value is too old, don't calculate its value if (rrdset_last_entry_t(st) >= (now_realtime_sec() - (st->update_every * st->gap_when_lost_iterations_above))) ret = rrdset2value_api_v1(st, w->response.data, &n, (dimensions) ? buffer_tostring(dimensions) : NULL - , points, after, before, group, 0, options, NULL, &latest_timestamp, &value_is_null); + , points, after, before, group, 0, options, NULL, &latest_timestamp, &value_is_null, 0); // if the value cannot be calculated, show empty badge if (ret != HTTP_RESP_OK) { diff --git a/web/api/exporters/README.md b/web/api/exporters/README.md index 401964744..1d517a91e 100644 --- a/web/api/exporters/README.md +++ b/web/api/exporters/README.md @@ -7,4 +7,4 @@ custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/exporter TBD -[](<>) + diff --git a/web/api/exporters/allmetrics.c b/web/api/exporters/allmetrics.c index d10de3d3c..1eba815c3 100644 --- a/web/api/exporters/allmetrics.c +++ b/web/api/exporters/allmetrics.c @@ -25,17 +25,17 @@ inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client if (prometheus_exporter_instance) prometheus_exporting_options = prometheus_exporter_instance->config.options; else - prometheus_exporting_options = global_backend_options; + prometheus_exporting_options = global_exporting_options; PROMETHEUS_OUTPUT_OPTIONS prometheus_output_options = PROMETHEUS_OUTPUT_TIMESTAMPS | - ((prometheus_exporting_options & BACKEND_OPTION_SEND_NAMES) ? PROMETHEUS_OUTPUT_NAMES : 0); + ((prometheus_exporting_options & EXPORTING_OPTION_SEND_NAMES) ? PROMETHEUS_OUTPUT_NAMES : 0); const char *prometheus_prefix; if (prometheus_exporter_instance) prometheus_prefix = prometheus_exporter_instance->config.prefix; else - prometheus_prefix = global_backend_prefix; + prometheus_prefix = global_exporting_prefix; while(url) { char *value = mystrsep(&url, "&"); @@ -64,7 +64,7 @@ inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client prometheus_prefix = value; } else if(!strcmp(name, "data") || !strcmp(name, "source") || !strcmp(name, "data source") || !strcmp(name, "data-source") || !strcmp(name, "data_source") || !strcmp(name, "datasource")) { - prometheus_exporting_options = backend_parse_data_source(value, prometheus_exporting_options); + prometheus_exporting_options = exporting_parse_data_source(value, prometheus_exporting_options); } else { int i; diff --git a/web/api/exporters/prometheus/README.md b/web/api/exporters/prometheus/README.md index 48b85555e..cf7e2caa8 100644 --- a/web/api/exporters/prometheus/README.md +++ b/web/api/exporters/prometheus/README.md @@ -7,4 +7,4 @@ custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/exporter Read the Prometheus exporter documentation: [Using Netdata with Prometheus](/exporting/prometheus/README.md). -[](<>) + diff --git a/web/api/exporters/shell/README.md b/web/api/exporters/shell/README.md index 9d44a3707..a41326c9d 100644 --- a/web/api/exporters/shell/README.md +++ b/web/api/exporters/shell/README.md @@ -66,4 +66,4 @@ NETDATA_${chart_id^^}_${dimension_id^^}="${value}" The value is rounded to the closest integer, since shell script cannot process decimal numbers. -[](<>) + diff --git a/web/api/formatters/README.md b/web/api/formatters/README.md index 1fd2b3029..3e67ac6ee 100644 --- a/web/api/formatters/README.md +++ b/web/api/formatters/README.md @@ -75,4 +75,4 @@ For example, to download a CSV file with CPU utilization of the last hour, This is done by appending `&tqx=outFileName:FILENAME` to any data query. The output will be in the format given with `&format=`. -[](<>) + diff --git a/web/api/formatters/csv/README.md b/web/api/formatters/csv/README.md index 2a859e249..df7c11efa 100644 --- a/web/api/formatters/csv/README.md +++ b/web/api/formatters/csv/README.md @@ -141,4 +141,4 @@ And this is how it looks when formatted: | 2018-10-27 03:44:00 | 7026.9852167 | | 2018-10-27 03:43:00 | 205.9904794 | -[](<>) + diff --git a/web/api/formatters/json/README.md b/web/api/formatters/json/README.md index 685a3f2df..a0f8108e7 100644 --- a/web/api/formatters/json/README.md +++ b/web/api/formatters/json/README.md @@ -153,4 +153,4 @@ google.visualization.Query.setResponse({version:'0.6',reqId:'0',status:'ok',sig: }}); ``` -[](<>) + diff --git a/web/api/formatters/rrd2json.c b/web/api/formatters/rrd2json.c index 29bb4beb5..1a8b07c7c 100644 --- a/web/api/formatters/rrd2json.c +++ b/web/api/formatters/rrd2json.c @@ -167,9 +167,10 @@ int rrdset2value_api_v1( , time_t *db_after , time_t *db_before , int *value_is_null + , int timeout ) { - RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions, NULL); + RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions, NULL, timeout); if(!r) { if(value_is_null) *value_is_null = 1; @@ -197,7 +198,7 @@ int rrdset2value_api_v1( if(db_before) *db_before = r->before; long i = (!(options & RRDR_OPTION_REVERSED))?rrdr_rows(r) - 1:0; - *n = rrdr2value(r, i, options, value_is_null); + *n = rrdr2value(r, i, options, value_is_null, NULL); rrdr_free(r); return HTTP_RESP_OK; @@ -217,17 +218,27 @@ int rrdset2anything_api_v1( , time_t *latest_timestamp , struct context_param *context_param_list , char *chart_label_key -) { - + , int max_anomaly_rates + , int timeout +) +{ if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) st->last_accessed_time = now_realtime_sec(); - RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions?buffer_tostring(dimensions):NULL, context_param_list); + RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions?buffer_tostring(dimensions):NULL, context_param_list, timeout); if(!r) { buffer_strcat(wb, "Cannot generate output with these parameters on this chart."); return HTTP_RESP_INTERNAL_SERVER_ERROR; } + if (r->result_options & RRDR_RESULT_OPTION_CANCEL) { + rrdr_free(r); + return HTTP_RESP_BACKEND_FETCH_FAILED; + } + + if (st && st->state && st->state->is_ar_chart) + ml_process_rrdr(r, max_anomaly_rates); + RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL; if(r->result_options & RRDR_RESULT_OPTION_RELATIVE) @@ -243,12 +254,12 @@ int rrdset2anything_api_v1( if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key); - rrdr2ssv(r, wb, options, "", " ", ""); + rrdr2ssv(r, wb, options, "", " ", "", temp_rd); rrdr_json_wrapper_end(r, wb, format, options, 1); } else { wb->contenttype = CT_TEXT_PLAIN; - rrdr2ssv(r, wb, options, "", " ", ""); + rrdr2ssv(r, wb, options, "", " ", "", temp_rd); } break; @@ -256,12 +267,12 @@ int rrdset2anything_api_v1( if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key); - rrdr2ssv(r, wb, options, "", ",", ""); + rrdr2ssv(r, wb, options, "", ",", "", temp_rd); rrdr_json_wrapper_end(r, wb, format, options, 1); } else { wb->contenttype = CT_TEXT_PLAIN; - rrdr2ssv(r, wb, options, "", ",", ""); + rrdr2ssv(r, wb, options, "", ",", "", temp_rd); } break; @@ -269,12 +280,12 @@ int rrdset2anything_api_v1( if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key); - rrdr2ssv(r, wb, options, "[", ",", "]"); + rrdr2ssv(r, wb, options, "[", ",", "]", temp_rd); rrdr_json_wrapper_end(r, wb, format, options, 0); } else { wb->contenttype = CT_APPLICATION_JSON; - rrdr2ssv(r, wb, options, "[", ",", "]"); + rrdr2ssv(r, wb, options, "[", ",", "]", temp_rd); } break; diff --git a/web/api/formatters/rrd2json.h b/web/api/formatters/rrd2json.h index 3dc598973..af809c54f 100644 --- a/web/api/formatters/rrd2json.h +++ b/web/api/formatters/rrd2json.h @@ -67,6 +67,8 @@ extern int rrdset2anything_api_v1( , time_t *latest_timestamp , struct context_param *context_param_list , char *chart_label_key + , int max_anomaly_rates + , int timeout ); extern int rrdset2value_api_v1( @@ -83,6 +85,7 @@ extern int rrdset2value_api_v1( , time_t *db_after , time_t *db_before , int *value_is_null + , int timeout ); extern void build_context_param_list(struct context_param **param_list, RRDSET *st); diff --git a/web/api/formatters/ssv/README.md b/web/api/formatters/ssv/README.md index d439949a5..d9e193d66 100644 --- a/web/api/formatters/ssv/README.md +++ b/web/api/formatters/ssv/README.md @@ -56,4 +56,4 @@ in a JSON array: [278,258,268,239,259,260,243,266,278,318,264,258] ``` -[](<>) + diff --git a/web/api/formatters/ssv/ssv.c b/web/api/formatters/ssv/ssv.c index eeba0283d..8d3ddbfdf 100644 --- a/web/api/formatters/ssv/ssv.c +++ b/web/api/formatters/ssv/ssv.c @@ -2,7 +2,7 @@ #include "ssv.h" -void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, const char *separator, const char *suffix) { +void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, const char *separator, const char *suffix, RRDDIM *temp_rd) { //info("RRD2SSV(): %s: BEGIN", r->st->id); long i; @@ -17,7 +17,7 @@ void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, con // for each line in the array for(i = start; i != end ;i += step) { int all_values_are_null = 0; - calculated_number v = rrdr2value(r, i, options, &all_values_are_null); + calculated_number v = rrdr2value(r, i, options, &all_values_are_null, temp_rd); if(likely(i != start)) { if(r->min > v) r->min = v; diff --git a/web/api/formatters/ssv/ssv.h b/web/api/formatters/ssv/ssv.h index 6963dcf6e..66716b9c9 100644 --- a/web/api/formatters/ssv/ssv.h +++ b/web/api/formatters/ssv/ssv.h @@ -5,6 +5,6 @@ #include "../rrd2json.h" -extern void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, const char *separator, const char *suffix); +extern void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, const char *separator, const char *suffix, RRDDIM *temp_rd); #endif //NETDATA_API_FORMATTER_SSV_H diff --git a/web/api/formatters/value/README.md b/web/api/formatters/value/README.md index 21c937080..a51e32de7 100644 --- a/web/api/formatters/value/README.md +++ b/web/api/formatters/value/README.md @@ -21,4 +21,4 @@ The Value formatter is not exposed by the API by itself. Instead it is used by the [`ssv`](/web/api/formatters/ssv/README.md) formatter and [health monitoring queries](/health/README.md). -[](<>) + diff --git a/web/api/formatters/value/value.c b/web/api/formatters/value/value.c index a69af6165..9ac91f509 100644 --- a/web/api/formatters/value/value.c +++ b/web/api/formatters/value/value.c @@ -3,7 +3,7 @@ #include "value.h" -inline calculated_number rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null) { +inline calculated_number rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, RRDDIM *temp_rd) { if (r->st_needs_lock) rrdset_check_rdlock(r->st); @@ -20,7 +20,7 @@ inline calculated_number rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int * int set_min_max = 0; if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { total = 0; - for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) { + for (c = 0, d = temp_rd ? temp_rd : r->st->dimensions; d && c < r->d; c++, d = d->next) { calculated_number n = cn[c]; if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) @@ -34,7 +34,7 @@ inline calculated_number rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int * } // for each dimension - for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) { + for (c = 0, d = temp_rd ? temp_rd : r->st->dimensions; d && c < r->d; c++, d = d->next) { if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; diff --git a/web/api/formatters/value/value.h b/web/api/formatters/value/value.h index d9e981f88..2d6bd1242 100644 --- a/web/api/formatters/value/value.h +++ b/web/api/formatters/value/value.h @@ -5,6 +5,6 @@ #include "../rrd2json.h" -extern calculated_number rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null); +extern calculated_number rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, RRDDIM *temp_rd); #endif //NETDATA_API_FORMATTER_VALUE_H diff --git a/web/api/health/README.md b/web/api/health/README.md index b6e8b5c23..9ec8f31c0 100644 --- a/web/api/health/README.md +++ b/web/api/health/README.md @@ -222,4 +222,4 @@ The file's location is configurable in `netdata.conf`. The default is shown belo The test script under [tests/health_mgmtapi](/tests/health_mgmtapi/README.md) contains a series of tests that you can either run or read through to understand the various calls and responses better. -[](<>) + diff --git a/web/api/netdata-swagger.json b/web/api/netdata-swagger.json index 7786b333a..4952e6116 100644 --- a/web/api/netdata-swagger.json +++ b/web/api/netdata-swagger.json @@ -3,13 +3,13 @@ "info": { "title": "Netdata API", "description": "Real-time performance and health monitoring.", - "version": "1.11.1_rolling" + "version": "1.33.1" }, "paths": { "/info": { "get": { "summary": "Get netdata basic information", - "description": "The info endpoint returns basic information about netdata. It provides:\n* netdata version\n* netdata unique id\n* list of hosts mirrored (includes itself)\n* Operating System, Virtualization, K8s nodes and Container technology information\n* List of active collector plugins and modules\n* number of alarms in the host\n * number of alarms in normal state\n * number of alarms in warning state\n * number of alarms in critical state\n", + "description": "The info endpoint returns basic information about netdata. It provides:\n* netdata version\n* netdata unique id\n* list of hosts mirrored (includes itself)\n* Operating System, Virtualization, K8s nodes and Container technology information\n* List of active collector plugins and modules\n* Streaming information\n* number of alarms in the host\n * number of alarms in normal state\n * number of alarms in warning state\n * number of alarms in critical state\n", "responses": { "200": { "description": "netdata basic information.", @@ -200,6 +200,28 @@ } }, { + "name": "chart_label_key", + "in": "query", + "description": "Specify the chart label keys that need to match for context queries as comma separated values. At least one matching key is needed to match the corresponding chart.", + "required": false, + "allowEmptyValue": false, + "schema": { + "type": "string", + "format": "key1,key2,key3" + } + }, + { + "name": "chart_labels_filter", + "in": "query", + "description": "Specify the chart label keys and values to match for context queries. All keys/values need to match for the chart to be included in the query. The labels are specified as key1:value1,key2:value2", + "required": false, + "allowEmptyValue": false, + "schema": { + "type": "string", + "format": "key1:value1,key2:value2,key3:value3" + } + }, + { "name": "group", "in": "query", "description": "The grouping method. If multiple collected values are to be grouped in order to return fewer points, this parameters defines the method of grouping. methods supported \"min\", \"max\", \"average\", \"sum\", \"incremental-sum\". \"max\" is actually calculated on the absolute value collected (so it works for both positive and negative dimensions to return the most extreme value in either direction).", @@ -232,6 +254,18 @@ } }, { + "name": "timeout", + "in": "query", + "description": "Specify a timeout value in milliseconds after which the agent will abort the query and return a 503 error. A value of 0 indicates no timeout.", + "required": false, + "allowEmptyValue": false, + "schema": { + "type": "number", + "format": "integer", + "default": 0 + } + }, + { "name": "format", "in": "query", "description": "The format of the data to be returned.", @@ -1220,6 +1254,11 @@ "description": "Container technology detection method.", "example": "dockerenv" }, + "stream_compression": { + "type": "boolean", + "description": "Stream transmission compression method.", + "example": "true" + }, "labels": { "type": "object", "description": "List of host labels.", @@ -2085,21 +2124,20 @@ "type": "boolean", "description": "Describes whether this agent is capable of connection to the Cloud. False means agent has been built without ACLK component either on purpose (user choice) or due to missing dependency." }, - "aclk-implementation": { - "type": "string", - "description": "Describes which ACLK implementation is currently used.", - "enum": [ - "Next Generation", - "Legacy" - ] + "aclk-version": { + "type": "integer", + "description": "Describes which ACLK version is currently used." }, - "new-cloud-protocol-supported": { - "type": "boolean", - "description": "Informs about new protobuf based Cloud/Agent protocol support of this agent. If false agent has to be compiled with protobuf and protoc available." + "protocols-supported": { + "type": "array", + "description": "List of supported protocols for communication with Cloud.", + "items": { + "type": "string" + } }, "agent-claimed": { "type": "boolean", - "description": "Informs whether this agent has been added to a space in the cloud (User has to perform claiming). If false (user didnt perform claiming) agent will never attempt any cloud connection." + "description": "Informs whether this agent has been added to a space in the cloud (User has to perform claiming). If false (user didn't perform claiming) agent will never attempt any cloud connection." }, "claimed-id": { "type": "string", @@ -2122,4 +2160,4 @@ } } } -}
\ No newline at end of file +} diff --git a/web/api/netdata-swagger.yaml b/web/api/netdata-swagger.yaml index 83487bda8..1e20ad0f5 100644 --- a/web/api/netdata-swagger.yaml +++ b/web/api/netdata-swagger.yaml @@ -2,7 +2,7 @@ openapi: 3.0.0 info: title: Netdata API description: Real-time performance and health monitoring. - version: 1.11.1_rolling + version: 1.33.1 paths: /info: get: @@ -14,6 +14,7 @@ paths: * list of hosts mirrored (includes itself) * Operating System, Virtualization, K8s nodes and Container technology information * List of active collector plugins and modules + * Streaming information * number of alarms in the host * number of alarms in normal state * number of alarms in warning state @@ -169,6 +170,24 @@ paths: type: number format: integer default: 20 + - name: chart_label_key + in: query + description: Specify the chart label keys that need to match for context queries as comma separated values. + At least one matching key is needed to match the corresponding chart. + required: false + allowEmptyValue: false + schema: + type: string + format: key1,key2,key3 + - name: chart_labels_filter + in: query + description: Specify the chart label keys and values to match for context queries. All keys/values need to + match for the chart to be included in the query. The labels are specified as key1:value1,key2:value2 + required: false + allowEmptyValue: false + schema: + type: string + format: key1:value1,key2:value2,key3:value3 - name: group in: query description: The grouping method. If multiple collected values are to be grouped @@ -201,6 +220,16 @@ paths: type: number format: integer default: 0 + - name: timeout + in: query + description: Specify a timeout value in milliseconds after which the agent will + abort the query and return a 503 error. A value of 0 indicates no timeout. + required: false + allowEmptyValue: false + schema: + type: number + format: integer + default: 0 - name: format in: query description: The format of the data to be returned. @@ -984,6 +1013,10 @@ components: type: string description: Container technology detection method. example: dockerenv + stream_compression: + type: boolean + description: Stream transmission compression method. + example: true labels: type: object description: List of host labels. @@ -1628,20 +1661,18 @@ components: type: string description: Describes whether this agent is capable of connection to the Cloud. False means agent has been built without ACLK component either on purpose (user choice) or due to missing dependency. - aclk-implementation: - type: string - description: Describes which ACLK implementation is currently used. - enum: - - Next Generation - - Legacy - new-cloud-protocol-supported: - type: boolean - description: Informs about new protobuf based Cloud/Agent protocol support of this agent. - If false agent has to be compiled with protobuf and protoc available. + aclk-version: + type: integer + description: Describes which ACLK version is currently used. + protocols-supported: + type: array + description: List of supported protocols for communication with Cloud. + items: + type: string agent-claimed: type: boolean description: Informs whether this agent has been added to a space in the cloud (User has to perform claiming). - If false (user didnt perform claiming) agent will never attempt any cloud connection. + If false (user didn't perform claiming) agent will never attempt any cloud connection. claimed_id: type: string format: uuid diff --git a/web/api/queries/README.md b/web/api/queries/README.md index 31ec49655..44cdd05b4 100644 --- a/web/api/queries/README.md +++ b/web/api/queries/README.md @@ -173,4 +173,4 @@ So, the proper way to query the database is to also set at least `after`. The fo When you keep calling this URL, you will see that it returns one new value every 10 seconds, and the timestamp always ends with zero. Similarly, if you say `points=1&after=-5` it will always return timestamps ending with 0 or 5. -[](<>) + diff --git a/web/api/queries/average/README.md b/web/api/queries/average/README.md index f32a67571..b8d4ba7e7 100644 --- a/web/api/queries/average/README.md +++ b/web/api/queries/average/README.md @@ -43,4 +43,4 @@ Examining last 1 minute `successful` web server responses: - <https://en.wikipedia.org/wiki/Average>. -[](<>) + diff --git a/web/api/queries/des/README.md b/web/api/queries/des/README.md index 5505de5a6..33c5f1a0c 100644 --- a/web/api/queries/des/README.md +++ b/web/api/queries/des/README.md @@ -70,4 +70,4 @@ Examining last 1 minute `successful` web server responses: - <https://en.wikipedia.org/wiki/Exponential_smoothing>. -[](<>) + diff --git a/web/api/queries/incremental_sum/README.md b/web/api/queries/incremental_sum/README.md index e5f3dfc0c..44301172e 100644 --- a/web/api/queries/incremental_sum/README.md +++ b/web/api/queries/incremental_sum/README.md @@ -38,4 +38,4 @@ Examining last 1 minute `successful` web server responses: - none -[](<>) + diff --git a/web/api/queries/max/README.md b/web/api/queries/max/README.md index 32b1d434c..48da7cf08 100644 --- a/web/api/queries/max/README.md +++ b/web/api/queries/max/README.md @@ -35,4 +35,4 @@ Examining last 1 minute `successful` web server responses: - <https://en.wikipedia.org/wiki/Sample_maximum_and_minimum>. -[](<>) + diff --git a/web/api/queries/median/README.md b/web/api/queries/median/README.md index 25ce8b8d6..bb7d4c66b 100644 --- a/web/api/queries/median/README.md +++ b/web/api/queries/median/README.md @@ -42,4 +42,4 @@ Examining last 1 minute `successful` web server responses: - <https://en.wikipedia.org/wiki/Median>. -[](<>) + diff --git a/web/api/queries/min/README.md b/web/api/queries/min/README.md index 69ef4ea12..495523c04 100644 --- a/web/api/queries/min/README.md +++ b/web/api/queries/min/README.md @@ -35,4 +35,4 @@ Examining last 1 minute `successful` web server responses: - <https://en.wikipedia.org/wiki/Sample_maximum_and_minimum>. -[](<>) + diff --git a/web/api/queries/query.c b/web/api/queries/query.c index 216417ae8..c55a97060 100644 --- a/web/api/queries/query.c +++ b/web/api/queries/query.c @@ -347,11 +347,11 @@ static void rrdr_disable_not_selected_dimensions(RRDR *r, RRDR_OPTIONS options, // ---------------------------------------------------------------------------- // helpers to find our way in RRDR -static inline RRDR_VALUE_FLAGS *rrdr_line_options(RRDR *r, long rrdr_line) { +static inline RRDR_VALUE_FLAGS *UNUSED_FUNCTION(rrdr_line_options)(RRDR *r, long rrdr_line) { return &r->o[ rrdr_line * r->d ]; } -static inline calculated_number *rrdr_line_values(RRDR *r, long rrdr_line) { +static inline calculated_number *UNUSED_FUNCTION(rrdr_line_values)(RRDR *r, long rrdr_line) { return &r->v[ rrdr_line * r->d ]; } @@ -562,7 +562,7 @@ static inline void do_dimension_fixedstep( calculated_number min = r->min, max = r->max; size_t db_points_read = 0; time_t db_now = now; - + time_t first_time_t = rrddim_first_entry_t(rd); for(rd->state->query_ops.init(rd, &handle, now, before_wanted) ; points_added < points_wanted ; now += dt) { // make sure we return data in the proper time range if(unlikely(now > before_wanted)) { @@ -586,7 +586,11 @@ static inline void do_dimension_fixedstep( } #endif db_now = now; // this is needed to set db_now in case the next_metric implementation does not set it - storage_number n = rd->state->query_ops.next_metric(&handle, &db_now); + storage_number n; + if (rd->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE && now <= first_time_t) + n = SN_EMPTY_SLOT; + else + n = rd->state->query_ops.next_metric(&handle, &db_now); if(unlikely(db_now > before_wanted)) { #ifdef NETDATA_INTERNAL_CHECKS r->internal.log = "stopped, because attempted to access the db after 'wanted before'"; @@ -840,6 +844,7 @@ static RRDR *rrd2rrdr_fixedstep( , time_t last_entry_t , int absolute_period_requested , struct context_param *context_param_list + , int timeout ) { int aligned = !(options & RRDR_OPTION_NOT_ALIGNED); @@ -1093,6 +1098,10 @@ static RRDR *rrd2rrdr_fixedstep( RRDDIM *rd; long c, dimensions_used = 0, dimensions_nonzero = 0; + struct timeval query_start_time; + struct timeval query_current_time; + if (timeout) + now_realtime_timeval(&query_start_time); for(rd = temp_rd?temp_rd:st->dimensions, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) { // if we need a percentage, we need to calculate all dimensions @@ -1114,6 +1123,8 @@ static RRDR *rrd2rrdr_fixedstep( , before_wanted , options ); + if (timeout) + now_realtime_timeval(&query_current_time); if(r->od[c] & RRDR_DIMENSION_NONZERO) dimensions_nonzero++; @@ -1151,6 +1162,12 @@ static RRDR *rrd2rrdr_fixedstep( } dimensions_used++; + if (timeout && (dt_usec(&query_start_time, &query_current_time) / 1000.0) > timeout) { + log_access("QUERY CANCELED RUNTIME EXCEEDED %0.2f ms (LIMIT %d ms)", + dt_usec(&query_start_time, &query_current_time) / 1000.0, timeout); + r->result_options |= RRDR_RESULT_OPTION_CANCEL; + break; + } } #ifdef NETDATA_INTERNAL_CHECKS @@ -1184,7 +1201,7 @@ static RRDR *rrd2rrdr_fixedstep( r->internal.grouping_free(r); // when all the dimensions are zero, we should return all of them - if(unlikely(options & RRDR_OPTION_NONZERO && !dimensions_nonzero)) { + if(unlikely(options & RRDR_OPTION_NONZERO && !dimensions_nonzero && !(r->result_options & RRDR_RESULT_OPTION_CANCEL))) { // all the dimensions are zero // mark them as NONZERO to send them all for(rd = temp_rd?temp_rd:st->dimensions, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) { @@ -1213,6 +1230,7 @@ static RRDR *rrd2rrdr_variablestep( , int absolute_period_requested , struct rrdeng_region_info *region_info_array , struct context_param *context_param_list + , int timeout ) { int aligned = !(options & RRDR_OPTION_NOT_ALIGNED); @@ -1470,6 +1488,10 @@ static RRDR *rrd2rrdr_variablestep( RRDDIM *rd; long c, dimensions_used = 0, dimensions_nonzero = 0; + struct timeval query_start_time; + struct timeval query_current_time; + if (timeout) + now_realtime_timeval(&query_start_time); for(rd = temp_rd?temp_rd:st->dimensions, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) { // if we need a percentage, we need to calculate all dimensions @@ -1491,6 +1513,8 @@ static RRDR *rrd2rrdr_variablestep( , before_wanted , options ); + if (timeout) + now_realtime_timeval(&query_current_time); if(r->od[c] & RRDR_DIMENSION_NONZERO) dimensions_nonzero++; @@ -1528,6 +1552,12 @@ static RRDR *rrd2rrdr_variablestep( } dimensions_used++; + if (timeout && (dt_usec(&query_start_time, &query_current_time) / 1000.0) > timeout) { + log_access("QUERY CANCELED RUNTIME EXCEEDED %0.2f ms (LIMIT %d ms)", + dt_usec(&query_start_time, &query_current_time) / 1000.0, timeout); + r->result_options |= RRDR_RESULT_OPTION_CANCEL; + break; + } } #ifdef NETDATA_INTERNAL_CHECKS @@ -1562,7 +1592,7 @@ static RRDR *rrd2rrdr_variablestep( r->internal.grouping_free(r); // when all the dimensions are zero, we should return all of them - if(unlikely(options & RRDR_OPTION_NONZERO && !dimensions_nonzero)) { + if(unlikely(options & RRDR_OPTION_NONZERO && !dimensions_nonzero && !(r->result_options & RRDR_RESULT_OPTION_CANCEL))) { // all the dimensions are zero // mark them as NONZERO to send them all for(rd = temp_rd?temp_rd:st->dimensions, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) { @@ -1587,6 +1617,7 @@ RRDR *rrd2rrdr( , RRDR_OPTIONS options , const char *dimensions , struct context_param *context_param_list + , int timeout ) { int rrd_update_every; @@ -1640,7 +1671,7 @@ RRDR *rrd2rrdr( } return rrd2rrdr_fixedstep(st, points_requested, after_requested, before_requested, group_method, resampling_time_requested, options, dimensions, rrd_update_every, - first_entry_t, last_entry_t, absolute_period_requested, context_param_list); + first_entry_t, last_entry_t, absolute_period_requested, context_param_list, timeout); } else { if (rrd_update_every != (uint16_t)max_interval) { rrd_update_every = (uint16_t) max_interval; @@ -1651,11 +1682,11 @@ RRDR *rrd2rrdr( } return rrd2rrdr_variablestep(st, points_requested, after_requested, before_requested, group_method, resampling_time_requested, options, dimensions, rrd_update_every, - first_entry_t, last_entry_t, absolute_period_requested, region_info_array, context_param_list); + first_entry_t, last_entry_t, absolute_period_requested, region_info_array, context_param_list, timeout); } } #endif return rrd2rrdr_fixedstep(st, points_requested, after_requested, before_requested, group_method, resampling_time_requested, options, dimensions, - rrd_update_every, first_entry_t, last_entry_t, absolute_period_requested, context_param_list); + rrd_update_every, first_entry_t, last_entry_t, absolute_period_requested, context_param_list, timeout); } diff --git a/web/api/queries/rrdr.h b/web/api/queries/rrdr.h index 3637df687..bd94e56e2 100644 --- a/web/api/queries/rrdr.h +++ b/web/api/queries/rrdr.h @@ -47,6 +47,7 @@ typedef enum rrdr_result_flags { RRDR_RESULT_OPTION_RELATIVE = 0x00000002, // the query uses relative time-frames // (should not to be cached by browsers and proxies) RRDR_RESULT_OPTION_VARIABLE_STEP = 0x00000004, // the query uses variable-step time-frames + RRDR_RESULT_OPTION_CANCEL = 0x00000008, // the query needs to be cancelled } RRDR_RESULT_FLAGS; typedef struct rrdresult { @@ -110,7 +111,7 @@ extern RRDR *rrdr_create(struct rrdset *st, long n, struct context_param *contex extern RRDR *rrd2rrdr( RRDSET *st, long points_requested, long long after_requested, long long before_requested, RRDR_GROUPING group_method, long resampling_time_requested, RRDR_OPTIONS options, const char *dimensions, - struct context_param *context_param_list); + struct context_param *context_param_list, int timeout); #include "query.h" diff --git a/web/api/queries/ses/README.md b/web/api/queries/ses/README.md index c27970135..b835b8120 100644 --- a/web/api/queries/ses/README.md +++ b/web/api/queries/ses/README.md @@ -58,4 +58,4 @@ Examining last 1 minute `successful` web server responses: - <https://en.wikipedia.org/wiki/Moving_average#exponential-moving-average> - <https://en.wikipedia.org/wiki/Exponential_smoothing>. -[](<>) + diff --git a/web/api/queries/stddev/README.md b/web/api/queries/stddev/README.md index 7cd7d62af..2fca47d5e 100644 --- a/web/api/queries/stddev/README.md +++ b/web/api/queries/stddev/README.md @@ -90,4 +90,4 @@ Examining last 1 minute `successful` web server responses: Check <https://en.wikipedia.org/wiki/Coefficient_of_variation>. -[](<>) + diff --git a/web/api/queries/sum/README.md b/web/api/queries/sum/README.md index aeace0a16..d4465bd82 100644 --- a/web/api/queries/sum/README.md +++ b/web/api/queries/sum/README.md @@ -38,4 +38,4 @@ Examining last 1 minute `successful` web server responses: - <https://en.wikipedia.org/wiki/Summation>. -[](<>) + diff --git a/web/api/tests/valid_urls.c b/web/api/tests/valid_urls.c index 30ae23247..30dc29828 100644 --- a/web/api/tests/valid_urls.c +++ b/web/api/tests/valid_urls.c @@ -75,9 +75,9 @@ void __wrap_finished_web_request_statistics( char *__wrap_config_get(struct config *root, const char *section, const char *name, const char *default_value) { - if (!strcmp(section, CONFIG_SECTION_WEB) && !strcmp(name, "web files owner")) - return "netdata"; (void)root; + (void)section; + (void)name; (void)default_value; return "UNKNOWN FIX ME"; } diff --git a/web/api/tests/web_api.c b/web/api/tests/web_api.c index b96213255..df4efdabd 100644 --- a/web/api/tests/web_api.c +++ b/web/api/tests/web_api.c @@ -75,9 +75,9 @@ void __wrap_finished_web_request_statistics( char *__wrap_config_get(struct config *root, const char *section, const char *name, const char *default_value) { - if (!strcmp(section, CONFIG_SECTION_WEB) && !strcmp(name, "web files owner")) - return "netdata"; (void)root; + (void)section; + (void)name; (void)default_value; return "UNKNOWN FIX ME"; } diff --git a/web/api/web_api_v1.c b/web/api/web_api_v1.c index 6361f9970..8cf89d38d 100644 --- a/web/api/web_api_v1.c +++ b/web/api/web_api_v1.c @@ -137,16 +137,25 @@ char *get_mgmt_api_key(void) { // save it fd = open(api_key_filename, O_WRONLY|O_CREAT|O_TRUNC, 444); - if(fd == -1) - fatal("Cannot create unique management API key file '%s'. Please fix this.", api_key_filename); + if(fd == -1) { + error("Cannot create unique management API key file '%s'. Please adjust config parameter 'netdata management api key file' to a proper path and file.", api_key_filename); + goto temp_key; + } - if(write(fd, guid, GUID_LEN) != GUID_LEN) - fatal("Cannot write the unique management API key file '%s'. Please fix this.", api_key_filename); + if(write(fd, guid, GUID_LEN) != GUID_LEN) { + error("Cannot write the unique management API key file '%s'. Please adjust config parameter 'netdata management api key file' to a proper path and file with enough space left.", api_key_filename); + close(fd); + goto temp_key; + } close(fd); } return guid; + +temp_key: + info("You can still continue to use the alarm management API using the authorization token %s during this Netdata session only.", guid); + return guid; } void web_client_api_v1_management_init(void) { @@ -401,13 +410,16 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c time_t last_timestamp_in_data = 0, google_timestamp = 0; - char *chart = NULL - , *before_str = NULL - , *after_str = NULL - , *group_time_str = NULL - , *points_str = NULL - , *context = NULL - , *chart_label_key = NULL; + char *chart = NULL; + char *before_str = NULL; + char *after_str = NULL; + char *group_time_str = NULL; + char *points_str = NULL; + char *timeout_str = NULL; + char *max_anomaly_rates_str = NULL; + char *context = NULL; + char *chart_label_key = NULL; + char *chart_labels_filter = NULL; int group = RRDR_GROUPING_AVERAGE; uint32_t format = DATASOURCE_JSON; @@ -428,6 +440,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c if(!strcmp(name, "context")) context = value; else if(!strcmp(name, "chart_label_key")) chart_label_key = value; + else if(!strcmp(name, "chart_labels_filter")) chart_labels_filter = value; else if(!strcmp(name, "chart")) chart = value; else if(!strcmp(name, "dimension") || !strcmp(name, "dim") || !strcmp(name, "dimensions") || !strcmp(name, "dims")) { if(!dimensions) dimensions = buffer_create(100); @@ -437,6 +450,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c else if(!strcmp(name, "after")) after_str = value; else if(!strcmp(name, "before")) before_str = value; else if(!strcmp(name, "points")) points_str = value; + else if(!strcmp(name, "timeout")) timeout_str = value; else if(!strcmp(name, "gtime")) group_time_str = value; else if(!strcmp(name, "group")) { group = web_client_api_request_v1_data_group(value, RRDR_GROUPING_AVERAGE); @@ -484,6 +498,9 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c outFileName = tqx_value; } } + else if(!strcmp(name, "max_anomaly_rates")) { + max_anomaly_rates_str = value; + } } // validate the google parameters given @@ -507,16 +524,21 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c uint32_t context_hash = simple_hash(context); rrdhost_rdlock(host); + char *words[MAX_CHART_LABELS_FILTER]; + uint32_t hash_key_list[MAX_CHART_LABELS_FILTER]; + int word_count = 0; rrdset_foreach_read(st1, host) { if (st1->hash_context == context_hash && !strcmp(st1->context, context) && - (!chart_label_key || rrdset_contains_label_keylist(st1, chart_label_key))) - build_context_param_list(&context_param_list, st1); + (!chart_label_key || rrdset_contains_label_keylist(st1, chart_label_key)) && + (!chart_labels_filter || + rrdset_matches_label_keys(st1, chart_labels_filter, words, hash_key_list, &word_count, MAX_CHART_LABELS_FILTER))) + build_context_param_list(&context_param_list, st1); } rrdhost_unlock(host); if (likely(context_param_list && context_param_list->rd)) // Just set the first one st = context_param_list->rd->rrdset; else { - if (!chart_label_key) + if (!chart_label_key && !chart_labels_filter) sql_build_context_param_list(&context_param_list, host, context, NULL); } } @@ -563,7 +585,9 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c long long before = (before_str && *before_str)?str2l(before_str):0; long long after = (after_str && *after_str) ?str2l(after_str):-600; int points = (points_str && *points_str)?str2i(points_str):0; + int timeout = (timeout_str && *timeout_str)?str2i(timeout_str): 0; long group_time = (group_time_str && *group_time_str)?str2l(group_time_str):0; + int max_anomaly_rates = (max_anomaly_rates_str && *max_anomaly_rates_str) ? str2i(max_anomaly_rates_str) : 0; debug(D_WEB_CLIENT, "%llu: API command 'data' for chart '%s', dimensions '%s', after '%lld', before '%lld', points '%d', group '%d', format '%u', options '0x%08x'" , w->id @@ -606,8 +630,10 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c buffer_strcat(w->response.data, "("); } - ret = rrdset2anything_api_v1(st, w->response.data, dimensions, format, points, after, before, group, group_time - , options, &last_timestamp_in_data, context_param_list, chart_label_key); + ret = rrdset2anything_api_v1(st, w->response.data, dimensions, format, + points, after, before, group, group_time, + options, &last_timestamp_in_data, context_param_list, + chart_label_key, max_anomaly_rates, timeout); free_context_param_list(&context_param_list); @@ -879,11 +905,18 @@ static inline void web_client_api_request_v1_info_mirrored_hosts(BUFFER *wb) { rrdhost_aclk_state_lock(host); if (host->aclk_state.claimed_id) - buffer_sprintf(wb, "\"%s\" }", host->aclk_state.claimed_id); + buffer_sprintf(wb, "\"%s\", ", host->aclk_state.claimed_id); else - buffer_strcat(wb, "null }"); + buffer_strcat(wb, "null, "); rrdhost_aclk_state_unlock(host); + if (host->node_id) { + char node_id_str[GUID_LEN + 1]; + uuid_unparse_lower(*host->node_id, node_id_str); + buffer_sprintf(wb, "\"node_id\": \"%s\" }", node_id_str); + } else + buffer_strcat(wb, "\"node_id\": null }"); + count++; } rrd_unlock(); @@ -968,6 +1001,13 @@ inline int web_client_api_request_v1_info_fill_buffer(RRDHOST *host, BUFFER *wb) buffer_sprintf(wb, "\t\"container\": \"%s\",\n", (host->system_info->container) ? host->system_info->container : ""); buffer_sprintf(wb, "\t\"container_detection\": \"%s\",\n", (host->system_info->container_detection) ? host->system_info->container_detection : ""); + if (host->system_info->cloud_provider_type) + buffer_sprintf(wb, "\t\"cloud_provider_type\": \"%s\",\n", host->system_info->cloud_provider_type); + if (host->system_info->cloud_instance_type) + buffer_sprintf(wb, "\t\"cloud_instance_type\": \"%s\",\n", host->system_info->cloud_instance_type); + if (host->system_info->cloud_instance_region) + buffer_sprintf(wb, "\t\"cloud_instance_region\": \"%s\",\n", host->system_info->cloud_instance_region); + buffer_strcat(wb, "\t\"host_labels\": {\n"); host_labels2json(host, wb, 2); buffer_strcat(wb, "\t},\n"); @@ -1036,11 +1076,15 @@ inline int web_client_api_request_v1_info_fill_buffer(RRDHOST *host, BUFFER *wb) buffer_strcat(wb, ",\n"); #ifdef ENABLE_COMPRESSION - buffer_strcat(wb, "\t\"stream-compression\": "); - buffer_strcat(wb, (default_compression_enabled ? "\"enabled\"" : "\"disabled\"")); - buffer_strcat(wb, ",\n"); + if(host->sender){ + buffer_strcat(wb, "\t\"stream-compression\": "); + buffer_strcat(wb, (host->sender->rrdpush_compression ? "true" : "false")); + buffer_strcat(wb, ",\n"); + }else{ + buffer_strcat(wb, "\t\"stream-compression\": null,\n"); + } #else - buffer_strcat(wb, "\t\"stream-compression\": \"N/A\",\n"); + buffer_strcat(wb, "\t\"stream-compression\": null,\n"); #endif //ENABLE_COMPRESSION buffer_strcat(wb, "\t\"hosts-available\": "); diff --git a/web/api/web_api_v1.h b/web/api/web_api_v1.h index 445b0e4a5..a88c511ad 100644 --- a/web/api/web_api_v1.h +++ b/web/api/web_api_v1.h @@ -8,6 +8,7 @@ #include "web/api/formatters/rrd2json.h" #include "web/api/health/health_cmdapi.h" +#define MAX_CHART_LABELS_FILTER (32) extern uint32_t web_client_api_request_v1_data_options(char *o); extern uint32_t web_client_api_request_v1_data_format(char *name); extern uint32_t web_client_api_request_v1_data_google_format(char *name); |