diff options
Diffstat (limited to 'web/api/formatters')
26 files changed, 0 insertions, 3568 deletions
diff --git a/web/api/formatters/Makefile.am b/web/api/formatters/Makefile.am deleted file mode 100644 index 11f239ccf..000000000 --- a/web/api/formatters/Makefile.am +++ /dev/null @@ -1,15 +0,0 @@ -# SPDX-License-Identifier: GPL-3.0-or-later - -AUTOMAKE_OPTIONS = subdir-objects -MAINTAINERCLEANFILES = $(srcdir)/Makefile.in - -SUBDIRS = \ - csv \ - json \ - ssv \ - value \ - $(NULL) - -dist_noinst_DATA = \ - README.md \ - $(NULL) diff --git a/web/api/formatters/README.md b/web/api/formatters/README.md deleted file mode 100644 index ddc70d90f..000000000 --- a/web/api/formatters/README.md +++ /dev/null @@ -1,82 +0,0 @@ -<!-- -title: "Query formatting" -custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/formatters/README.md -sidebar_label: "Query formatting" -learn_status: "Published" -learn_topic_type: "References" -learn_rel_path: "Developers/Web/Api/Formatters" ---> - -# Query formatting - -API data queries need to be formatted before returned to the caller. -Using API parameters, the caller may define the format he/she wishes to get back. - -The following formats are supported: - -| format|module|content type|description| -|:----:|:----:|:----------:|:----------| -| `array`|[ssv](https://github.com/netdata/netdata/blob/master/web/api/formatters/ssv/README.md)|application/json|a JSON array| -| `csv`|[csv](https://github.com/netdata/netdata/blob/master/web/api/formatters/csv/README.md)|text/plain|a text table, comma separated, with a header line (dimension names) and `\r\n` at the end of the lines| -| `csvjsonarray`|[csv](https://github.com/netdata/netdata/blob/master/web/api/formatters/csv/README.md)|application/json|a JSON array, with each row as another array (the first row has the dimension names)| -| `datasource`|[json](https://github.com/netdata/netdata/blob/master/web/api/formatters/json/README.md)|application/json|a Google Visualization Provider `datasource` javascript callback| -| `datatable`|[json](https://github.com/netdata/netdata/blob/master/web/api/formatters/json/README.md)|application/json|a Google `datatable`| -| `html`|[csv](https://github.com/netdata/netdata/blob/master/web/api/formatters/csv/README.md)|text/html|an html table| -| `json`|[json](https://github.com/netdata/netdata/blob/master/web/api/formatters/json/README.md)|application/json|a JSON object| -| `jsonp`|[json](https://github.com/netdata/netdata/blob/master/web/api/formatters/json/README.md)|application/json|a JSONP javascript callback| -| `markdown`|[csv](https://github.com/netdata/netdata/blob/master/web/api/formatters/csv/README.md)|text/plain|a markdown table| -| `ssv`|[ssv](https://github.com/netdata/netdata/blob/master/web/api/formatters/ssv/README.md)|text/plain|a space separated list of values| -| `ssvcomma`|[ssv](https://github.com/netdata/netdata/blob/master/web/api/formatters/ssv/README.md)|text/plain|a comma separated list of values| -| `tsv`|[csv](https://github.com/netdata/netdata/blob/master/web/api/formatters/csv/README.md)|text/plain|a TAB delimited `csv` (MS Excel flavor)| - -For examples of each format, check the relative module documentation. - -## Metadata with the `jsonwrap` option - -All data queries can be encapsulated to JSON object having metadata about the query and the results. - -This is done by adding the `options=jsonwrap` to the API URL (if there are other `options` append -`,jsonwrap` to the existing ones). - -This is such an object: - -```bash -# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=system.cpu&after=-3600&points=6&group=average&format=csv&options=nonzero,jsonwrap' -{ - "api": 1, - "id": "system.cpu", - "name": "system.cpu", - "view_update_every": 600, - "update_every": 1, - "first_entry": 1540387074, - "last_entry": 1540647070, - "before": 1540647000, - "after": 1540644000, - "dimension_names": ["steal", "softirq", "user", "system", "iowait"], - "dimension_ids": ["steal", "softirq", "user", "system", "iowait"], - "latest_values": [0, 0.2493766, 1.745636, 0.4987531, 0], - "view_latest_values": [0.0158314, 0.0516506, 0.866549, 0.7196127, 0.0050002], - "dimensions": 5, - "points": 6, - "format": "csv", - "result": "time,steal,softirq,user,system,iowait\n2018-10-27 13:30:00,0.0158314,0.0516506,0.866549,0.7196127,0.0050002\n2018-10-27 13:20:00,0.0149856,0.0529183,0.8673155,0.7121144,0.0049979\n2018-10-27 13:10:00,0.0137501,0.053315,0.8578097,0.7197613,0.0054209\n2018-10-27 13:00:00,0.0154252,0.0554688,0.899432,0.7200638,0.0067252\n2018-10-27 12:50:00,0.0145866,0.0495922,0.8404341,0.7011141,0.0041688\n2018-10-27 12:40:00,0.0162366,0.0595954,0.8827475,0.7020573,0.0041636\n", - "min": 0, - "max": 0 -} -``` - -## Downloading data query result files - -Following the [Google Visualization Provider guidelines](https://developers.google.com/chart/interactive/docs/dev/implementing_data_source), -Netdata supports parsing `tqx` options. - -Using these options, any Netdata data query can instruct the web browser to download -the result and save it under a given filename. - -For example, to download a CSV file with CPU utilization of the last hour, -[click here](https://registry.my-netdata.io/api/v1/data?chart=system.cpu&after=-3600&format=csv&options=nonzero&tqx=outFileName:system+cpu+utilization+of+the+last_hour.csv). - -This is done by appending `&tqx=outFileName:FILENAME` to any data query. -The output will be in the format given with `&format=`. - - diff --git a/web/api/formatters/charts2json.c b/web/api/formatters/charts2json.c deleted file mode 100644 index cab4debae..000000000 --- a/web/api/formatters/charts2json.c +++ /dev/null @@ -1,104 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#include "charts2json.h" - -// generate JSON for the /api/v1/charts API call - -const char* get_release_channel() { - static int use_stable = -1; - - if (use_stable == -1) { - char filename[FILENAME_MAX + 1]; - snprintfz(filename, FILENAME_MAX, "%s/.environment", netdata_configured_user_config_dir); - procfile *ff = procfile_open(filename, "=", PROCFILE_FLAG_NO_ERROR_ON_FILE_IO); - if (ff) { - procfile_set_quotes(ff, "'\""); - ff = procfile_readall(ff); - if (ff) { - unsigned int i; - for (i = 0; i < procfile_lines(ff); i++) { - if (!procfile_linewords(ff, i)) - continue; - if (!strcmp(procfile_lineword(ff, i, 0), "RELEASE_CHANNEL")) { - if (!strcmp(procfile_lineword(ff, i, 1), "stable")) - use_stable = 1; - else if (!strcmp(procfile_lineword(ff, i, 1), "nightly")) - use_stable = 0; - break; - } - } - procfile_close(ff); - } - } - if (use_stable == -1) - use_stable = strchr(program_version, '-') ? 0 : 1; - } - return (use_stable)?"stable":"nightly"; -} - -void charts2json(RRDHOST *host, BUFFER *wb) { - static char *custom_dashboard_info_js_filename = NULL; - size_t c = 0, dimensions = 0, memory = 0, alarms = 0; - RRDSET *st; - - time_t now = now_realtime_sec(); - - if(unlikely(!custom_dashboard_info_js_filename)) - custom_dashboard_info_js_filename = config_get(CONFIG_SECTION_WEB, "custom dashboard_info.js", ""); - - buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT); - - buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(host)); - buffer_json_member_add_string(wb, "version", rrdhost_program_version(host)); - buffer_json_member_add_string(wb, "release_channel", get_release_channel()); - buffer_json_member_add_string(wb, "os", rrdhost_os(host)); - buffer_json_member_add_string(wb, "timezone", rrdhost_timezone(host)); - buffer_json_member_add_int64(wb, "update_every", host->rrd_update_every); - buffer_json_member_add_int64(wb, "history", host->rrd_history_entries); - buffer_json_member_add_string(wb, "memory_mode", rrd_memory_mode_name(host->rrd_memory_mode)); - buffer_json_member_add_string(wb, "custom_info", custom_dashboard_info_js_filename); - - buffer_json_member_add_object(wb, "charts"); - rrdset_foreach_read(st, host) { - if (rrdset_is_available_for_viewers(st)) { - - buffer_json_member_add_object(wb, rrdset_id(st)); - rrdset2json(st, wb, &dimensions, &memory); - buffer_json_object_close(wb); - st->last_accessed_time_s = now; - c++; - } - } - rrdset_foreach_done(st); - buffer_json_object_close(wb); - - RRDCALC *rc; - foreach_rrdcalc_in_rrdhost_read(host, rc) { - if(rc->rrdset) - alarms++; - } - foreach_rrdcalc_in_rrdhost_done(rc); - - buffer_json_member_add_int64(wb, "charts_count", (int64_t) c); - buffer_json_member_add_int64(wb, "dimensions_count", (int64_t) dimensions); - buffer_json_member_add_int64(wb, "alarms_count", (int64_t)alarms); - buffer_json_member_add_int64(wb, "rrd_memory_bytes", (int64_t)memory); - buffer_json_member_add_int64(wb, "hosts_count", (int64_t) rrdhost_hosts_available()); - - buffer_json_member_add_array(wb, "hosts"); - { - rrd_rdlock(); - RRDHOST *h; - rrdhost_foreach_read(h) { - if(!rrdhost_should_be_removed(h, host, now) /*&& !rrdhost_flag_check(h, RRDHOST_FLAG_ARCHIVED) */) { - buffer_json_add_array_item_object(wb); - buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(h)); - buffer_json_object_close(wb); - } - } - rrd_unlock(); - } - buffer_json_array_close(wb); - - buffer_json_finalize(wb); -} diff --git a/web/api/formatters/charts2json.h b/web/api/formatters/charts2json.h deleted file mode 100644 index 7b07af5a0..000000000 --- a/web/api/formatters/charts2json.h +++ /dev/null @@ -1,11 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#ifndef NETDATA_API_FORMATTER_CHARTS2JSON_H -#define NETDATA_API_FORMATTER_CHARTS2JSON_H - -#include "rrd2json.h" - -void charts2json(RRDHOST *host, BUFFER *wb); -const char* get_release_channel(); - -#endif //NETDATA_API_FORMATTER_CHARTS2JSON_H diff --git a/web/api/formatters/csv/Makefile.am b/web/api/formatters/csv/Makefile.am deleted file mode 100644 index 161784b8f..000000000 --- a/web/api/formatters/csv/Makefile.am +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-License-Identifier: GPL-3.0-or-later - -AUTOMAKE_OPTIONS = subdir-objects -MAINTAINERCLEANFILES = $(srcdir)/Makefile.in - -dist_noinst_DATA = \ - README.md \ - $(NULL) diff --git a/web/api/formatters/csv/README.md b/web/api/formatters/csv/README.md deleted file mode 100644 index 4585710b4..000000000 --- a/web/api/formatters/csv/README.md +++ /dev/null @@ -1,148 +0,0 @@ -<!-- -title: "CSV formatter" -custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/formatters/csv/README.md -sidebar_label: "CSV formatter" -learn_status: "Published" -learn_topic_type: "References" -learn_rel_path: "Developers/Web/Api/Formatters" ---> - -# CSV formatter - -The CSV formatter presents [results of database queries](https://github.com/netdata/netdata/blob/master/web/api/queries/README.md) in the following formats: - -| format|content type|description| -| :----:|:----------:|:----------| -| `csv`|text/plain|a text table, comma separated, with a header line (dimension names) and `\r\n` at the end of the lines| -| `csvjsonarray`|application/json|a JSON array, with each row as another array (the first row has the dimension names)| -| `tsv`|text/plain|like `csv` but TAB is used instead of comma to separate values (MS Excel flavor)| -| `html`|text/html|an html table| -| `markdown`|text/plain|markdown table| - -In all formats the date and time is the first column. - -The CSV formatter respects the following API `&options=`: - -| option|supported|description| -|:----:|:-------:|:----------| -| `nonzero`|yes|to return only the dimensions that have at least a non-zero value| -| `flip`|yes|to return the rows older to newer (the default is newer to older)| -| `seconds`|yes|to return the date and time in unix timestamp| -| `ms`|yes|to return the date and time in unit timestamp as milliseconds| -| `percent`|yes|to replace all values with their percentage over the row total| -| `abs`|yes|to turn all values positive| -| `null2zero`|yes|to replace gaps with zeros (the default prints the string `null`| - -## Examples - -Get the system total bandwidth for all physical network interfaces, over the last hour, -in 6 rows (one for every 10 minutes), in `csv` format: - -Netdata always returns bandwidth in `kilobits`. - -```bash -# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=system.net&format=csv&after=-3600&group=sum&points=6&options=abs' -time,received,sent -2018-10-26 23:50:00,90214.67847,215137.79762 -2018-10-26 23:40:00,90126.32286,238587.57522 -2018-10-26 23:30:00,86061.22688,213389.23526 -2018-10-26 23:20:00,85590.75164,206129.01608 -2018-10-26 23:10:00,83163.30691,194311.77384 -2018-10-26 23:00:00,85167.29657,197538.07773 -``` - ---- - -Get the max RAM used by the SQL server and any cron jobs, over the last hour, in 2 rows (one for every 30 -minutes), in `tsv` format, and format the date and time as unix timestamp: - -Netdata always returns memory in `MB`. - -```bash -# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=apps.mem&format=tsv&after=-3600&group=max&points=2&options=nonzero,seconds&dimensions=sql,cron' -time sql cron -1540598400 61.95703 0.25 -1540596600 61.95703 0.25 -``` - ---- - -Get an HTML table of the last 4 values (4 seconds) of system CPU utilization: - -Netdata always returns CPU utilization as `%`. - -```bash -# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=system.cpu&format=html&after=-4&options=nonzero' -<html> -<center> -<table border="0" cellpadding="5" cellspacing="5"> -<tr><td>time</td><td>softirq</td><td>user</td><td>system</td></tr> -<tr><td>2018-10-27 00:16:07</td><td>0.25</td><td>1</td><td>0.75</td></tr> -<tr><td>2018-10-27 00:16:06</td><td>0</td><td>1.0025063</td><td>0.5012531</td></tr> -<tr><td>2018-10-27 00:16:05</td><td>0</td><td>1</td><td>0.75</td></tr> -<tr><td>2018-10-27 00:16:04</td><td>0</td><td>1.0025063</td><td>0.7518797</td></tr> -</table> -</center> -</html> -``` - -This is how it looks when rendered by a web browser: - - - ---- - -Get a JSON array with the average bandwidth rate of the mysql server, over the last hour, in 6 values -(one every 10 minutes), and return the date and time in milliseconds: - -Netdata always returns bandwidth rates in `kilobits/s`. - -```bash -# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=mysql_local.net&format=csvjsonarray&after=-3600&points=6&group=average&options=abs,ms' -[ -["time","in","out"], -[1540599600000,0.7499986,120.2810185], -[1540599000000,0.7500019,120.2815509], -[1540598400000,0.7499999,120.2812319], -[1540597800000,0.7500044,120.2819634], -[1540597200000,0.7499968,120.2807337], -[1540596600000,0.7499988,120.2810527] -] -``` - ---- - -Get the number of processes started per minute, for the last 10 minutes, in `markdown` format: - -```bash -# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=system.forks&format=markdown&after=-600&points=10&group=sum' -time | started -:---: |:---: -2018-10-27 03:52:00| 245.1706149 -2018-10-27 03:51:00| 152.6654636 -2018-10-27 03:50:00| 163.1755789 -2018-10-27 03:49:00| 176.1574766 -2018-10-27 03:48:00| 178.0137076 -2018-10-27 03:47:00| 183.8306543 -2018-10-27 03:46:00| 264.1635621 -2018-10-27 03:45:00| 205.001551 -2018-10-27 03:44:00| 7026.9852167 -2018-10-27 03:43:00| 205.9904794 -``` - -And this is how it looks when formatted: - -| time | started | -|:--:|:-----:| -| 2018-10-27 03:52:00 | 245.1706149 | -| 2018-10-27 03:51:00 | 152.6654636 | -| 2018-10-27 03:50:00 | 163.1755789 | -| 2018-10-27 03:49:00 | 176.1574766 | -| 2018-10-27 03:48:00 | 178.0137076 | -| 2018-10-27 03:47:00 | 183.8306543 | -| 2018-10-27 03:46:00 | 264.1635621 | -| 2018-10-27 03:45:00 | 205.001551 | -| 2018-10-27 03:44:00 | 7026.9852167 | -| 2018-10-27 03:43:00 | 205.9904794 | - - diff --git a/web/api/formatters/csv/csv.c b/web/api/formatters/csv/csv.c deleted file mode 100644 index d81ddb34e..000000000 --- a/web/api/formatters/csv/csv.c +++ /dev/null @@ -1,109 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#include "libnetdata/libnetdata.h" -#include "csv.h" - -void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines) { - //netdata_log_info("RRD2CSV(): %s: BEGIN", r->st->id); - long c, i; - const long used = (long)r->d; - - // print the csv header - for(c = 0, i = 0; c < used ; c++) { - if(!rrdr_dimension_should_be_exposed(r->od[c], options)) - continue; - - if(!i) { - buffer_strcat(wb, startline); - if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); - buffer_strcat(wb, "time"); - if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); - } - buffer_strcat(wb, separator); - if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); - buffer_strcat(wb, string2str(r->dn[c])); - if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); - i++; - } - buffer_strcat(wb, endline); - - if(format == DATASOURCE_CSV_MARKDOWN) { - // print the --- line after header - for(c = 0, i = 0; c < used ;c++) { - if(!rrdr_dimension_should_be_exposed(r->od[c], options)) - continue; - - if(!i) { - buffer_strcat(wb, startline); - if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); - buffer_strcat(wb, ":---:"); - if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); - } - buffer_strcat(wb, separator); - if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); - buffer_strcat(wb, ":---:"); - if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); - i++; - } - buffer_strcat(wb, endline); - } - - if(!i) { - // no dimensions present - return; - } - - long start = 0, end = rrdr_rows(r), step = 1; - if(!(options & RRDR_OPTION_REVERSED)) { - start = rrdr_rows(r) - 1; - end = -1; - step = -1; - } - - // for each line in the array - for(i = start; i != end ;i += step) { - NETDATA_DOUBLE *cn = &r->v[ i * r->d ]; - RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ]; - - buffer_strcat(wb, betweenlines); - buffer_strcat(wb, startline); - - time_t now = r->t[i]; - - if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) { - // print the timestamp of the line - buffer_print_netdata_double(wb, (NETDATA_DOUBLE) now); - // in ms - if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000"); - } - else { - // generate the local date time - struct tm tmbuf, *tm = localtime_r(&now, &tmbuf); - if(!tm) { - netdata_log_error("localtime() failed."); continue; } - buffer_date(wb, tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec); - } - - // for each dimension - for(c = 0; c < used ;c++) { - if(!rrdr_dimension_should_be_exposed(r->od[c], options)) - continue; - - buffer_strcat(wb, separator); - - NETDATA_DOUBLE n = cn[c]; - - if(co[c] & RRDR_VALUE_EMPTY) { - if(options & RRDR_OPTION_NULL2ZERO) - buffer_strcat(wb, "0"); - else - buffer_strcat(wb, "null"); - } - else - buffer_print_netdata_double(wb, n); - } - - buffer_strcat(wb, endline); - } - //netdata_log_info("RRD2CSV(): %s: END", r->st->id); -} diff --git a/web/api/formatters/csv/csv.h b/web/api/formatters/csv/csv.h deleted file mode 100644 index 666d4c660..000000000 --- a/web/api/formatters/csv/csv.h +++ /dev/null @@ -1,12 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#ifndef NETDATA_API_FORMATTER_CSV_H -#define NETDATA_API_FORMATTER_CSV_H - -#include "web/api/queries/rrdr.h" - -void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines); - -#include "../rrd2json.h" - -#endif //NETDATA_API_FORMATTER_CSV_H diff --git a/web/api/formatters/json/Makefile.am b/web/api/formatters/json/Makefile.am deleted file mode 100644 index 161784b8f..000000000 --- a/web/api/formatters/json/Makefile.am +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-License-Identifier: GPL-3.0-or-later - -AUTOMAKE_OPTIONS = subdir-objects -MAINTAINERCLEANFILES = $(srcdir)/Makefile.in - -dist_noinst_DATA = \ - README.md \ - $(NULL) diff --git a/web/api/formatters/json/README.md b/web/api/formatters/json/README.md deleted file mode 100644 index bc70aec02..000000000 --- a/web/api/formatters/json/README.md +++ /dev/null @@ -1,160 +0,0 @@ -<!-- -title: "JSON formatter" -custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/formatters/json/README.md -sidebar_label: "JSON formatter" -learn_status: "Published" -learn_topic_type: "References" -learn_rel_path: "Developers/Web/Api/Formatters" ---> - -# JSON formatter - -The CSV formatter presents [results of database queries](https://github.com/netdata/netdata/blob/master/web/api/queries/README.md) in the following formats: - -| format | content type | description| -|:----:|:----------:|:----------| -| `json` | application/json | return the query result as a json object| -| `jsonp` | application/json | return the query result as a JSONP javascript callback| -| `datatable` | application/json | return the query result as a Google `datatable`| -| `datasource` | application/json | return the query result as a Google Visualization Provider `datasource` javascript callback| - -The CSV formatter respects the following API `&options=`: - -| option | supported | description| -|:----:|:-------:|:----------| -| `google_json` | yes | enable the Google flavor of JSON (using double quotes for strings and `Date()` function for dates| -| `objectrows` | yes | return each row as an object, instead of an array| -| `nonzero` | yes | to return only the dimensions that have at least a non-zero value| -| `flip` | yes | to return the rows older to newer (the default is newer to older)| -| `seconds` | yes | to return the date and time in unix timestamp| -| `ms` | yes | to return the date and time in unit timestamp as milliseconds| -| `percent` | yes | to replace all values with their percentage over the row total| -| `abs` | yes | to turn all values positive| -| `null2zero` | yes | to replace gaps with zeros (the default prints the string `null`| - -## Examples - -To show the differences between each format, in the following examples we query the same -chart (having just one dimension called `active`), changing only the query `format` and its `options`. - -> Using `format=json` and `options=` - -```bash -# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=nginx_local.connections&after=-3600&points=6&group=average&format=json&options=' -{ - "labels": ["time", "active"], - "data": - [ - [ 1540644600, 224.2516667], - [ 1540644000, 229.29], - [ 1540643400, 222.41], - [ 1540642800, 226.6816667], - [ 1540642200, 246.4083333], - [ 1540641600, 241.0966667] - ] -} -``` - -> Using `format=json` and `options=objectrows` - -```bash -# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=nginx_local.connections&after=-3600&points=6&group=average&format=json&options=objectrows' -{ - "labels": ["time", "active"], - "data": - [ - { "time": 1540644600, "active": 224.2516667}, - { "time": 1540644000, "active": 229.29}, - { "time": 1540643400, "active": 222.41}, - { "time": 1540642800, "active": 226.6816667}, - { "time": 1540642200, "active": 246.4083333}, - { "time": 1540641600, "active": 241.0966667} - ] -} -``` - -> Using `format=json` and `options=objectrows,google_json` - -```bash -# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=nginx_local.connections&after=-3600&points=6&group=average&formatjson&options=objectrows,google_json' -{ - "labels": ["time", "active"], - "data": - [ - { "time": new Date(2018,9,27,12,50,0), "active": 224.2516667}, - { "time": new Date(2018,9,27,12,40,0), "active": 229.29}, - { "time": new Date(2018,9,27,12,30,0), "active": 222.41}, - { "time": new Date(2018,9,27,12,20,0), "active": 226.6816667}, - { "time": new Date(2018,9,27,12,10,0), "active": 246.4083333}, - { "time": new Date(2018,9,27,12,0,0), "active": 241.0966667} - ] -} -``` - -> Using `format=jsonp` and `options=` - -```bash -curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=nginx_local.connections&after=-3600&points=6&group=average&formjsonp&options=' -callback({ - "labels": ["time", "active"], - "data": - [ - [ 1540645200, 235.885], - [ 1540644600, 224.2516667], - [ 1540644000, 229.29], - [ 1540643400, 222.41], - [ 1540642800, 226.6816667], - [ 1540642200, 246.4083333] - ] -}); -``` - -> Using `format=datatable` and `options=` - -```bash -curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=nginx_local.connections&after=-3600&points=6&group=average&formdatatable&options=' -{ - "cols": - [ - {"id":"","label":"time","pattern":"","type":"datetime"}, - {"id":"","label":"","pattern":"","type":"string","p":{"role":"annotation"}}, - {"id":"","label":"","pattern":"","type":"string","p":{"role":"annotationText"}}, - {"id":"","label":"active","pattern":"","type":"number"} - ], - "rows": - [ - {"c":[{"v":"Date(2018,9,27,13,0,0)"},{"v":null},{"v":null},{"v":235.885}]}, - {"c":[{"v":"Date(2018,9,27,12,50,0)"},{"v":null},{"v":null},{"v":224.2516667}]}, - {"c":[{"v":"Date(2018,9,27,12,40,0)"},{"v":null},{"v":null},{"v":229.29}]}, - {"c":[{"v":"Date(2018,9,27,12,30,0)"},{"v":null},{"v":null},{"v":222.41}]}, - {"c":[{"v":"Date(2018,9,27,12,20,0)"},{"v":null},{"v":null},{"v":226.6816667}]}, - {"c":[{"v":"Date(2018,9,27,12,10,0)"},{"v":null},{"v":null},{"v":246.4083333}]} - ] -} -``` - -> Using `format=datasource` and `options=` - -```bash -curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=nginx_local.connections&after=-3600&points=6&group=average&format=datasource&options=' -google.visualization.Query.setResponse({version:'0.6',reqId:'0',status:'ok',sig:'1540645368',table:{ - "cols": - [ - {"id":"","label":"time","pattern":"","type":"datetime"}, - {"id":"","label":"","pattern":"","type":"string","p":{"role":"annotation"}}, - {"id":"","label":"","pattern":"","type":"string","p":{"role":"annotationText"}}, - {"id":"","label":"active","pattern":"","type":"number"} - ], - "rows": - [ - {"c":[{"v":"Date(2018,9,27,13,0,0)"},{"v":null},{"v":null},{"v":235.885}]}, - {"c":[{"v":"Date(2018,9,27,12,50,0)"},{"v":null},{"v":null},{"v":224.2516667}]}, - {"c":[{"v":"Date(2018,9,27,12,40,0)"},{"v":null},{"v":null},{"v":229.29}]}, - {"c":[{"v":"Date(2018,9,27,12,30,0)"},{"v":null},{"v":null},{"v":222.41}]}, - {"c":[{"v":"Date(2018,9,27,12,20,0)"},{"v":null},{"v":null},{"v":226.6816667}]}, - {"c":[{"v":"Date(2018,9,27,12,10,0)"},{"v":null},{"v":null},{"v":246.4083333}]} - ] -}}); -``` - - diff --git a/web/api/formatters/json/json.c b/web/api/formatters/json/json.c deleted file mode 100644 index 7e3f400e9..000000000 --- a/web/api/formatters/json/json.c +++ /dev/null @@ -1,350 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#include "json.h" - -#define JSON_DATES_JS 1 -#define JSON_DATES_TIMESTAMP 2 - -void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) { - //netdata_log_info("RRD2JSON(): %s: BEGIN", r->st->id); - int row_annotations = 0, dates, dates_with_new = 0; - char kq[2] = "", // key quote - sq[2] = "", // string quote - pre_label[101] = "", // before each label - post_label[101] = "", // after each label - pre_date[101] = "", // the beginning of line, to the date - post_date[101] = "", // closing the date - pre_value[101] = "", // before each value - post_value[101] = "", // after each value - post_line[101] = "", // at the end of each row - normal_annotation[201] = "", // default row annotation - overflow_annotation[201] = "", // overflow row annotation - data_begin[101] = "", // between labels and values - finish[101] = "", // at the end of everything - object_rows_time[101] = ""; - - if(datatable) { - dates = JSON_DATES_JS; - if( options & RRDR_OPTION_GOOGLE_JSON ) { - kq[0] = '\0'; - sq[0] = '\''; - } - else { - kq[0] = '"'; - sq[0] = '"'; - } - row_annotations = 1; - snprintfz(pre_date, 100, " {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq); - snprintfz(post_date, 100, "%s}", sq); - snprintfz(pre_label, 100, ",\n {%sid%s:%s%s,%slabel%s:%s", kq, kq, sq, sq, kq, kq, sq); - snprintfz(post_label, 100, "%s,%spattern%s:%s%s,%stype%s:%snumber%s}", sq, kq, kq, sq, sq, kq, kq, sq, sq); - snprintfz(pre_value, 100, ",{%sv%s:", kq, kq); - strcpy(post_value, "}"); - strcpy(post_line, "]}"); - snprintfz(data_begin, 100, "\n ],\n %srows%s:\n [\n", kq, kq); - strcpy(finish, "\n ]\n }"); - - snprintfz(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq); - snprintfz(normal_annotation, 200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq); - - buffer_sprintf(wb, "{\n %scols%s:\n [\n", kq, kq); - buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq); - buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq); - buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq); - - // remove the valueobjects flag - // google wants its own keys - if(options & RRDR_OPTION_OBJECTSROWS) - options &= ~RRDR_OPTION_OBJECTSROWS; - } - else { - kq[0] = '"'; - sq[0] = '"'; - if(options & RRDR_OPTION_GOOGLE_JSON) { - dates = JSON_DATES_JS; - dates_with_new = 1; - } - else { - dates = JSON_DATES_TIMESTAMP; - dates_with_new = 0; - } - if( options & RRDR_OPTION_OBJECTSROWS ) - strcpy(pre_date, " {"); - else - strcpy(pre_date, " ["); - strcpy(pre_label, ",\""); - strcpy(post_label, "\""); - strcpy(pre_value, ","); - if( options & RRDR_OPTION_OBJECTSROWS ) - strcpy(post_line, "}"); - else - strcpy(post_line, "]"); - snprintfz(data_begin, 100, "],\n %sdata%s:[\n", kq, kq); - strcpy(finish, "\n ]\n }"); - - buffer_sprintf(wb, "{\n %slabels%s:[", kq, kq); - buffer_sprintf(wb, "%stime%s", sq, sq); - - if( options & RRDR_OPTION_OBJECTSROWS ) - snprintfz(object_rows_time, 100, "%stime%s: ", kq, kq); - - } - - size_t pre_value_len = strlen(pre_value); - size_t post_value_len = strlen(post_value); - size_t pre_label_len = strlen(pre_label); - size_t post_label_len = strlen(post_label); - size_t pre_date_len = strlen(pre_date); - size_t post_date_len = strlen(post_date); - size_t post_line_len = strlen(post_line); - size_t normal_annotation_len = strlen(normal_annotation); - size_t overflow_annotation_len = strlen(overflow_annotation); - size_t object_rows_time_len = strlen(object_rows_time); - - // ------------------------------------------------------------------------- - // print the JSON header - - long c, i; - const long used = (long)r->d; - - // print the header lines - for(c = 0, i = 0; c < used ; c++) { - if(!rrdr_dimension_should_be_exposed(r->od[c], options)) - continue; - - buffer_fast_strcat(wb, pre_label, pre_label_len); - buffer_strcat(wb, string2str(r->dn[c])); - buffer_fast_strcat(wb, post_label, post_label_len); - i++; - } - - if(!i) { - buffer_fast_strcat(wb, pre_label, pre_label_len); - buffer_fast_strcat(wb, "no data", 7); - buffer_fast_strcat(wb, post_label, post_label_len); - } - size_t total_number_of_dimensions = i; - - // print the beginning of row data - buffer_strcat(wb, data_begin); - - // if all dimensions are hidden, print a null - if(!i) { - buffer_strcat(wb, finish); - return; - } - - long start = 0, end = rrdr_rows(r), step = 1; - if(!(options & RRDR_OPTION_REVERSED)) { - start = rrdr_rows(r) - 1; - end = -1; - step = -1; - } - - // pre-allocate a large enough buffer for us - // this does not need to be accurate - it is just a hint to avoid multiple realloc(). - buffer_need_bytes(wb, - ( 20 * rrdr_rows(r)) // timestamp + json overhead - + ( (pre_value_len + post_value_len + 4) * total_number_of_dimensions * rrdr_rows(r) ) // number - ); - - // for each line in the array - for(i = start; i != end ;i += step) { - NETDATA_DOUBLE *cn = &r->v[ i * r->d ]; - RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ]; - NETDATA_DOUBLE *ar = &r->ar[ i * r->d ]; - - time_t now = r->t[i]; - - if(dates == JSON_DATES_JS) { - // generate the local date time - struct tm tmbuf, *tm = localtime_r(&now, &tmbuf); - if(!tm) { - netdata_log_error("localtime_r() failed."); continue; } - - if(likely(i != start)) buffer_fast_strcat(wb, ",\n", 2); - buffer_fast_strcat(wb, pre_date, pre_date_len); - - if( options & RRDR_OPTION_OBJECTSROWS ) - buffer_fast_strcat(wb, object_rows_time, object_rows_time_len); - - if(unlikely(dates_with_new)) - buffer_fast_strcat(wb, "new ", 4); - - buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec); - - buffer_fast_strcat(wb, post_date, post_date_len); - - if(unlikely(row_annotations)) { - // google supports one annotation per row - int annotation_found = 0; - for(c = 0; c < used ; c++) { - if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue; - - if(unlikely(co[c] & RRDR_VALUE_RESET)) { - buffer_fast_strcat(wb, overflow_annotation, overflow_annotation_len); - annotation_found = 1; - break; - } - } - if(likely(!annotation_found)) - buffer_fast_strcat(wb, normal_annotation, normal_annotation_len); - } - } - else { - // print the timestamp of the line - if(likely(i != start)) - buffer_fast_strcat(wb, ",\n", 2); - - buffer_fast_strcat(wb, pre_date, pre_date_len); - - if(unlikely( options & RRDR_OPTION_OBJECTSROWS )) - buffer_fast_strcat(wb, object_rows_time, object_rows_time_len); - - buffer_print_netdata_double(wb, (NETDATA_DOUBLE) r->t[i]); - - // in ms - if(unlikely(options & RRDR_OPTION_MILLISECONDS)) - buffer_fast_strcat(wb, "000", 3); - - buffer_fast_strcat(wb, post_date, post_date_len); - } - - // for each dimension - for(c = 0; c < used ;c++) { - if(!rrdr_dimension_should_be_exposed(r->od[c], options)) - continue; - - NETDATA_DOUBLE n; - if(unlikely(options & RRDR_OPTION_INTERNAL_AR)) - n = ar[c]; - else - n = cn[c]; - - buffer_fast_strcat(wb, pre_value, pre_value_len); - - if(unlikely( options & RRDR_OPTION_OBJECTSROWS )) - buffer_sprintf(wb, "%s%s%s: ", kq, string2str(r->dn[c]), kq); - - if(co[c] & RRDR_VALUE_EMPTY && !(options & (RRDR_OPTION_INTERNAL_AR))) { - if(unlikely(options & RRDR_OPTION_NULL2ZERO)) - buffer_fast_strcat(wb, "0", 1); - else - buffer_fast_strcat(wb, "null", 4); - } - else - buffer_print_netdata_double(wb, n); - - buffer_fast_strcat(wb, post_value, post_value_len); - } - - buffer_fast_strcat(wb, post_line, post_line_len); - } - - buffer_strcat(wb, finish); - //netdata_log_info("RRD2JSON(): %s: END", r->st->id); -} - -void rrdr2json_v2(RRDR *r, BUFFER *wb) { - QUERY_TARGET *qt = r->internal.qt; - RRDR_OPTIONS options = qt->window.options; - - bool send_count = query_target_aggregatable(qt); - bool send_hidden = send_count && r->vh && query_has_group_by_aggregation_percentage(qt); - - buffer_json_member_add_object(wb, "result"); - - buffer_json_member_add_array(wb, "labels"); - buffer_json_add_array_item_string(wb, "time"); - long d, i; - const long used = (long)r->d; - for(d = 0, i = 0; d < used ; d++) { - if(!rrdr_dimension_should_be_exposed(r->od[d], options)) - continue; - - buffer_json_add_array_item_string(wb, string2str(r->di[d])); - i++; - } - buffer_json_array_close(wb); // labels - - buffer_json_member_add_object(wb, "point"); - { - size_t point_count = 0; - buffer_json_member_add_uint64(wb, "value", point_count++); - buffer_json_member_add_uint64(wb, "arp", point_count++); - buffer_json_member_add_uint64(wb, "pa", point_count++); - if (send_count) - buffer_json_member_add_uint64(wb, "count", point_count++); - if (send_hidden) - buffer_json_member_add_uint64(wb, "hidden", point_count++); - } - buffer_json_object_close(wb); // point - - buffer_json_member_add_array(wb, "data"); - if(i) { - long start = 0, end = rrdr_rows(r), step = 1; - if (!(options & RRDR_OPTION_REVERSED)) { - start = rrdr_rows(r) - 1; - end = -1; - step = -1; - } - - // for each line in the array - for (i = start; i != end; i += step) { - NETDATA_DOUBLE *cn = &r->v[ i * r->d ]; - NETDATA_DOUBLE *ch = send_hidden ? &r->vh[i * r->d ] : NULL; - RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ]; - NETDATA_DOUBLE *ar = &r->ar[ i * r->d ]; - uint32_t *gbc = &r->gbc [ i * r->d ]; - time_t now = r->t[i]; - - buffer_json_add_array_item_array(wb); // row - - if (options & RRDR_OPTION_MILLISECONDS) - buffer_json_add_array_item_time_ms(wb, now); // the time - else - buffer_json_add_array_item_time_t(wb, now); // the time - - for (d = 0; d < used; d++) { - if (!rrdr_dimension_should_be_exposed(r->od[d], options)) - continue; - - RRDR_VALUE_FLAGS o = co[d]; - - buffer_json_add_array_item_array(wb); // point - - // add the value - NETDATA_DOUBLE n = cn[d]; - - if(o & RRDR_VALUE_EMPTY) { - if (unlikely(options & RRDR_OPTION_NULL2ZERO)) - buffer_json_add_array_item_double(wb, 0); - else - buffer_json_add_array_item_double(wb, NAN); - } - else - buffer_json_add_array_item_double(wb, n); - - // add the anomaly - buffer_json_add_array_item_double(wb, ar[d]); - - // add the point annotations - buffer_json_add_array_item_uint64(wb, o); - - // add the count - if(send_count) - buffer_json_add_array_item_uint64(wb, gbc[d]); - if(send_hidden) - buffer_json_add_array_item_double(wb, ch[d]); - - buffer_json_array_close(wb); // point - } - - buffer_json_array_close(wb); // row - } - } - - buffer_json_array_close(wb); // data - - buffer_json_object_close(wb); // annotations -} diff --git a/web/api/formatters/json/json.h b/web/api/formatters/json/json.h deleted file mode 100644 index d1ab4f901..000000000 --- a/web/api/formatters/json/json.h +++ /dev/null @@ -1,11 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#ifndef NETDATA_API_FORMATTER_JSON_H -#define NETDATA_API_FORMATTER_JSON_H - -#include "../rrd2json.h" - -void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable); -void rrdr2json_v2(RRDR *r, BUFFER *wb); - -#endif //NETDATA_API_FORMATTER_JSON_H diff --git a/web/api/formatters/json_wrapper.c b/web/api/formatters/json_wrapper.c deleted file mode 100644 index 708a0f1f1..000000000 --- a/web/api/formatters/json_wrapper.c +++ /dev/null @@ -1,1578 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#include "json_wrapper.h" - -static void jsonwrap_query_metric_plan(BUFFER *wb, QUERY_METRIC *qm) { - buffer_json_member_add_array(wb, "plans"); - for (size_t p = 0; p < qm->plan.used; p++) { - QUERY_PLAN_ENTRY *qp = &qm->plan.array[p]; - - buffer_json_add_array_item_object(wb); - buffer_json_member_add_uint64(wb, "tr", qp->tier); - buffer_json_member_add_time_t(wb, "af", qp->after); - buffer_json_member_add_time_t(wb, "bf", qp->before); - buffer_json_object_close(wb); - } - buffer_json_array_close(wb); - - buffer_json_member_add_array(wb, "tiers"); - for (size_t tier = 0; tier < storage_tiers; tier++) { - buffer_json_add_array_item_object(wb); - buffer_json_member_add_uint64(wb, "tr", tier); - buffer_json_member_add_time_t(wb, "fe", qm->tiers[tier].db_first_time_s); - buffer_json_member_add_time_t(wb, "le", qm->tiers[tier].db_last_time_s); - buffer_json_member_add_int64(wb, "wg", qm->tiers[tier].weight); - buffer_json_object_close(wb); - } - buffer_json_array_close(wb); -} - -void jsonwrap_query_plan(RRDR *r, BUFFER *wb) { - QUERY_TARGET *qt = r->internal.qt; - - buffer_json_member_add_object(wb, "query_plan"); - for(size_t m = 0; m < qt->query.used; m++) { - QUERY_METRIC *qm = query_metric(qt, m); - buffer_json_member_add_object(wb, query_metric_id(qt, qm)); - jsonwrap_query_metric_plan(wb, qm); - buffer_json_object_close(wb); - } - buffer_json_object_close(wb); -} - -static inline size_t rrdr_dimension_names(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { - const size_t dimensions = r->d; - size_t c, i; - - buffer_json_member_add_array(wb, key); - for(c = 0, i = 0; c < dimensions ; c++) { - if(!rrdr_dimension_should_be_exposed(r->od[c], options)) - continue; - - buffer_json_add_array_item_string(wb, string2str(r->dn[c])); - i++; - } - buffer_json_array_close(wb); - - return i; -} - -static inline size_t rrdr_dimension_ids(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { - const size_t dimensions = r->d; - size_t c, i; - - buffer_json_member_add_array(wb, key); - for(c = 0, i = 0; c < dimensions ; c++) { - if(!rrdr_dimension_should_be_exposed(r->od[c], options)) - continue; - - buffer_json_add_array_item_string(wb, string2str(r->di[c])); - i++; - } - buffer_json_array_close(wb); - - return i; -} - -static inline long jsonwrap_v1_chart_ids(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { - QUERY_TARGET *qt = r->internal.qt; - const long query_used = qt->query.used; - long c, i; - - buffer_json_member_add_array(wb, key); - for (c = 0, i = 0; c < query_used; c++) { - if(!rrdr_dimension_should_be_exposed(r->od[c], options)) - continue; - - QUERY_METRIC *qm = query_metric(qt, c); - QUERY_INSTANCE *qi = query_instance(qt, qm->link.query_instance_id); - buffer_json_add_array_item_string(wb, rrdinstance_acquired_id(qi->ria)); - i++; - } - buffer_json_array_close(wb); - - return i; -} - -struct summary_total_counts { - size_t selected; - size_t excluded; - size_t queried; - size_t failed; -}; - -static inline void aggregate_into_summary_totals(struct summary_total_counts *totals, QUERY_METRICS_COUNTS *metrics) { - if(unlikely(!totals || !metrics)) - return; - - if(metrics->selected) { - totals->selected++; - - if(metrics->queried) - totals->queried++; - - else if(metrics->failed) - totals->failed++; - } - else - totals->excluded++; -} - -static inline void query_target_total_counts(BUFFER *wb, const char *key, struct summary_total_counts *totals) { - if(!totals->selected && !totals->queried && !totals->failed && !totals->excluded) - return; - - buffer_json_member_add_object(wb, key); - - if(totals->selected) - buffer_json_member_add_uint64(wb, "sl", totals->selected); - - if(totals->excluded) - buffer_json_member_add_uint64(wb, "ex", totals->excluded); - - if(totals->queried) - buffer_json_member_add_uint64(wb, "qr", totals->queried); - - if(totals->failed) - buffer_json_member_add_uint64(wb, "fl", totals->failed); - - buffer_json_object_close(wb); -} - -static inline void query_target_metric_counts(BUFFER *wb, QUERY_METRICS_COUNTS *metrics) { - if(!metrics->selected && !metrics->queried && !metrics->failed && !metrics->excluded) - return; - - buffer_json_member_add_object(wb, "ds"); - - if(metrics->selected) - buffer_json_member_add_uint64(wb, "sl", metrics->selected); - - if(metrics->excluded) - buffer_json_member_add_uint64(wb, "ex", metrics->excluded); - - if(metrics->queried) - buffer_json_member_add_uint64(wb, "qr", metrics->queried); - - if(metrics->failed) - buffer_json_member_add_uint64(wb, "fl", metrics->failed); - - buffer_json_object_close(wb); -} - -static inline void query_target_instance_counts(BUFFER *wb, QUERY_INSTANCES_COUNTS *instances) { - if(!instances->selected && !instances->queried && !instances->failed && !instances->excluded) - return; - - buffer_json_member_add_object(wb, "is"); - - if(instances->selected) - buffer_json_member_add_uint64(wb, "sl", instances->selected); - - if(instances->excluded) - buffer_json_member_add_uint64(wb, "ex", instances->excluded); - - if(instances->queried) - buffer_json_member_add_uint64(wb, "qr", instances->queried); - - if(instances->failed) - buffer_json_member_add_uint64(wb, "fl", instances->failed); - - buffer_json_object_close(wb); -} - -static inline void query_target_alerts_counts(BUFFER *wb, QUERY_ALERTS_COUNTS *alerts, const char *name, bool array) { - if(!alerts->clear && !alerts->other && !alerts->critical && !alerts->warning) - return; - - if(array) - buffer_json_add_array_item_object(wb); - else - buffer_json_member_add_object(wb, "al"); - - if(name) - buffer_json_member_add_string(wb, "nm", name); - - if(alerts->clear) - buffer_json_member_add_uint64(wb, "cl", alerts->clear); - - if(alerts->warning) - buffer_json_member_add_uint64(wb, "wr", alerts->warning); - - if(alerts->critical) - buffer_json_member_add_uint64(wb, "cr", alerts->critical); - - if(alerts->other) - buffer_json_member_add_uint64(wb, "ot", alerts->other); - - buffer_json_object_close(wb); -} - -static inline void query_target_points_statistics(BUFFER *wb, QUERY_TARGET *qt, STORAGE_POINT *sp) { - if(!sp->count) - return; - - buffer_json_member_add_object(wb, "sts"); - - buffer_json_member_add_double(wb, "min", sp->min); - buffer_json_member_add_double(wb, "max", sp->max); - - if(query_target_aggregatable(qt)) { - buffer_json_member_add_uint64(wb, "cnt", sp->count); - - if(sp->sum != 0.0) { - buffer_json_member_add_double(wb, "sum", sp->sum); - buffer_json_member_add_double(wb, "vol", sp->sum * (NETDATA_DOUBLE) query_view_update_every(qt)); - } - - if(sp->anomaly_count != 0) - buffer_json_member_add_uint64(wb, "arc", sp->anomaly_count); - } - else { - NETDATA_DOUBLE avg = (sp->count) ? sp->sum / (NETDATA_DOUBLE)sp->count : 0.0; - if(avg != 0.0) - buffer_json_member_add_double(wb, "avg", avg); - - NETDATA_DOUBLE arp = storage_point_anomaly_rate(*sp); - if(arp != 0.0) - buffer_json_member_add_double(wb, "arp", arp); - - NETDATA_DOUBLE con = (qt->query_points.sum > 0.0) ? sp->sum * 100.0 / qt->query_points.sum : 0.0; - if(con != 0.0) - buffer_json_member_add_double(wb, "con", con); - } - buffer_json_object_close(wb); -} - -static void query_target_summary_nodes_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key, struct summary_total_counts *totals) { - buffer_json_member_add_array(wb, key); - for (size_t c = 0; c < qt->nodes.used; c++) { - QUERY_NODE *qn = query_node(qt, c); - RRDHOST *host = qn->rrdhost; - buffer_json_add_array_item_object(wb); - buffer_json_node_add_v2(wb, host, qn->slot, qn->duration_ut, true); - query_target_instance_counts(wb, &qn->instances); - query_target_metric_counts(wb, &qn->metrics); - query_target_alerts_counts(wb, &qn->alerts, NULL, false); - query_target_points_statistics(wb, qt, &qn->query_points); - buffer_json_object_close(wb); - - aggregate_into_summary_totals(totals, &qn->metrics); - } - buffer_json_array_close(wb); -} - -static size_t query_target_summary_contexts_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key, struct summary_total_counts *totals) { - buffer_json_member_add_array(wb, key); - DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE); - - struct { - STORAGE_POINT query_points; - QUERY_INSTANCES_COUNTS instances; - QUERY_METRICS_COUNTS metrics; - QUERY_ALERTS_COUNTS alerts; - } *z; - - for (long c = 0; c < (long) qt->contexts.used; c++) { - QUERY_CONTEXT *qc = query_context(qt, c); - - z = dictionary_set(dict, rrdcontext_acquired_id(qc->rca), NULL, sizeof(*z)); - - z->instances.selected += qc->instances.selected; - z->instances.excluded += qc->instances.selected; - z->instances.queried += qc->instances.queried; - z->instances.failed += qc->instances.failed; - - z->metrics.selected += qc->metrics.selected; - z->metrics.excluded += qc->metrics.excluded; - z->metrics.queried += qc->metrics.queried; - z->metrics.failed += qc->metrics.failed; - - z->alerts.clear += qc->alerts.clear; - z->alerts.warning += qc->alerts.warning; - z->alerts.critical += qc->alerts.critical; - - storage_point_merge_to(z->query_points, qc->query_points); - } - - size_t unique_contexts = dictionary_entries(dict); - dfe_start_read(dict, z) { - buffer_json_add_array_item_object(wb); - buffer_json_member_add_string(wb, "id", z_dfe.name); - query_target_instance_counts(wb, &z->instances); - query_target_metric_counts(wb, &z->metrics); - query_target_alerts_counts(wb, &z->alerts, NULL, false); - query_target_points_statistics(wb, qt, &z->query_points); - buffer_json_object_close(wb); - - aggregate_into_summary_totals(totals, &z->metrics); - } - dfe_done(z); - buffer_json_array_close(wb); - dictionary_destroy(dict); - - return unique_contexts; -} - -static void query_target_summary_instances_v1(BUFFER *wb, QUERY_TARGET *qt, const char *key) { - char name[RRD_ID_LENGTH_MAX * 2 + 2]; - - buffer_json_member_add_array(wb, key); - DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE); - for (long c = 0; c < (long) qt->instances.used; c++) { - QUERY_INSTANCE *qi = query_instance(qt, c); - - snprintfz(name, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s", - rrdinstance_acquired_id(qi->ria), - rrdinstance_acquired_name(qi->ria)); - - bool *set = dictionary_set(dict, name, NULL, sizeof(*set)); - if (!*set) { - *set = true; - buffer_json_add_array_item_array(wb); - buffer_json_add_array_item_string(wb, rrdinstance_acquired_id(qi->ria)); - buffer_json_add_array_item_string(wb, rrdinstance_acquired_name(qi->ria)); - buffer_json_array_close(wb); - } - } - dictionary_destroy(dict); - buffer_json_array_close(wb); -} - -static void query_target_summary_instances_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key, struct summary_total_counts *totals) { - buffer_json_member_add_array(wb, key); - for (long c = 0; c < (long) qt->instances.used; c++) { - QUERY_INSTANCE *qi = query_instance(qt, c); -// QUERY_HOST *qh = query_host(qt, qi->query_host_id); - - buffer_json_add_array_item_object(wb); - buffer_json_member_add_string(wb, "id", rrdinstance_acquired_id(qi->ria)); - - if(!rrdinstance_acquired_id_and_name_are_same(qi->ria)) - buffer_json_member_add_string(wb, "nm", rrdinstance_acquired_name(qi->ria)); - - buffer_json_member_add_uint64(wb, "ni", qi->query_host_id); -// buffer_json_member_add_string(wb, "id", string2str(qi->id_fqdn)); -// buffer_json_member_add_string(wb, "nm", string2str(qi->name_fqdn)); -// buffer_json_member_add_string(wb, "lc", rrdinstance_acquired_name(qi->ria)); -// buffer_json_member_add_string(wb, "mg", qh->host->machine_guid); -// if(qh->node_id[0]) -// buffer_json_member_add_string(wb, "nd", qh->node_id); - query_target_metric_counts(wb, &qi->metrics); - query_target_alerts_counts(wb, &qi->alerts, NULL, false); - query_target_points_statistics(wb, qt, &qi->query_points); - buffer_json_object_close(wb); - - aggregate_into_summary_totals(totals, &qi->metrics); - } - buffer_json_array_close(wb); -} - -struct dimensions_sorted_walkthrough_data { - BUFFER *wb; - struct summary_total_counts *totals; - QUERY_TARGET *qt; -}; - -struct dimensions_sorted_entry { - const char *id; - const char *name; - STORAGE_POINT query_points; - QUERY_METRICS_COUNTS metrics; - uint32_t priority; -}; - -static int dimensions_sorted_walktrhough_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) { - struct dimensions_sorted_walkthrough_data *sdwd = data; - BUFFER *wb = sdwd->wb; - struct summary_total_counts *totals = sdwd->totals; - QUERY_TARGET *qt = sdwd->qt; - struct dimensions_sorted_entry *z = value; - - buffer_json_add_array_item_object(wb); - buffer_json_member_add_string(wb, "id", z->id); - if (z->id != z->name && z->name) - buffer_json_member_add_string(wb, "nm", z->name); - - query_target_metric_counts(wb, &z->metrics); - query_target_points_statistics(wb, qt, &z->query_points); - buffer_json_member_add_uint64(wb, "pri", z->priority); - buffer_json_object_close(wb); - - aggregate_into_summary_totals(totals, &z->metrics); - - return 1; -} - -int dimensions_sorted_compar(const DICTIONARY_ITEM **item1, const DICTIONARY_ITEM **item2) { - struct dimensions_sorted_entry *z1 = dictionary_acquired_item_value(*item1); - struct dimensions_sorted_entry *z2 = dictionary_acquired_item_value(*item2); - - if(z1->priority == z2->priority) - return strcmp(dictionary_acquired_item_name(*item1), dictionary_acquired_item_name(*item2)); - else if(z1->priority < z2->priority) - return -1; - else - return 1; -} - -static void query_target_summary_dimensions_v12(BUFFER *wb, QUERY_TARGET *qt, const char *key, bool v2, struct summary_total_counts *totals) { - char buf[RRD_ID_LENGTH_MAX * 2 + 2]; - - buffer_json_member_add_array(wb, key); - DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE); - struct dimensions_sorted_entry *z; - size_t q = 0; - for (long c = 0; c < (long) qt->dimensions.used; c++) { - QUERY_DIMENSION * qd = query_dimension(qt, c); - RRDMETRIC_ACQUIRED *rma = qd->rma; - - QUERY_METRIC *qm = NULL; - for( ; q < qt->query.used ;q++) { - QUERY_METRIC *tqm = query_metric(qt, q); - QUERY_DIMENSION *tqd = query_dimension(qt, tqm->link.query_dimension_id); - if(tqd->rma != rma) break; - qm = tqm; - } - - const char *key, *id, *name; - - if(v2) { - key = rrdmetric_acquired_name(rma); - id = key; - name = key; - } - else { - snprintfz(buf, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s", - rrdmetric_acquired_id(rma), - rrdmetric_acquired_name(rma)); - key = buf; - id = rrdmetric_acquired_id(rma); - name = rrdmetric_acquired_name(rma); - } - - z = dictionary_set(dict, key, NULL, sizeof(*z)); - if(!z->id) { - z->id = id; - z->name = name; - z->priority = qd->priority; - } - else { - if(qd->priority < z->priority) - z->priority = qd->priority; - } - - if(qm) { - z->metrics.selected += (qm->status & RRDR_DIMENSION_SELECTED) ? 1 : 0; - z->metrics.failed += (qm->status & RRDR_DIMENSION_FAILED) ? 1 : 0; - - if(qm->status & RRDR_DIMENSION_QUERIED) { - z->metrics.queried++; - storage_point_merge_to(z->query_points, qm->query_points); - } - } - else - z->metrics.excluded++; - } - - if(v2) { - struct dimensions_sorted_walkthrough_data t = { - .wb = wb, - .totals = totals, - .qt = qt, - }; - dictionary_sorted_walkthrough_rw(dict, DICTIONARY_LOCK_READ, dimensions_sorted_walktrhough_cb, - &t, dimensions_sorted_compar); - } - else { - // v1 - dfe_start_read(dict, z) { - buffer_json_add_array_item_array(wb); - buffer_json_add_array_item_string(wb, z->id); - buffer_json_add_array_item_string(wb, z->name); - buffer_json_array_close(wb); - } - dfe_done(z); - } - dictionary_destroy(dict); - buffer_json_array_close(wb); -} - -struct rrdlabels_formatting_v2 { - DICTIONARY *keys; - QUERY_INSTANCE *qi; - bool v2; -}; - -struct rrdlabels_keys_dict_entry { - const char *name; - DICTIONARY *values; - STORAGE_POINT query_points; - QUERY_METRICS_COUNTS metrics; -}; - -struct rrdlabels_key_value_dict_entry { - const char *key; - const char *value; - STORAGE_POINT query_points; - QUERY_METRICS_COUNTS metrics; -}; - -static int rrdlabels_formatting_v2(const char *name, const char *value, RRDLABEL_SRC ls __maybe_unused, void *data) { - struct rrdlabels_formatting_v2 *t = data; - - struct rrdlabels_keys_dict_entry *d = dictionary_set(t->keys, name, NULL, sizeof(*d)); - if(!d->values) { - d->name = name; - d->values = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE); - } - - char n[RRD_ID_LENGTH_MAX * 2 + 2]; - snprintfz(n, RRD_ID_LENGTH_MAX * 2, "%s:%s", name, value); - - struct rrdlabels_key_value_dict_entry *z = dictionary_set(d->values, n, NULL, sizeof(*z)); - if(!z->key) { - z->key = name; - z->value = value; - } - - if(t->v2) { - QUERY_INSTANCE *qi = t->qi; - - z->metrics.selected += qi->metrics.selected; - z->metrics.excluded += qi->metrics.excluded; - z->metrics.queried += qi->metrics.queried; - z->metrics.failed += qi->metrics.failed; - - d->metrics.selected += qi->metrics.selected; - d->metrics.excluded += qi->metrics.excluded; - d->metrics.queried += qi->metrics.queried; - d->metrics.failed += qi->metrics.failed; - - storage_point_merge_to(z->query_points, qi->query_points); - storage_point_merge_to(d->query_points, qi->query_points); - } - - return 1; -} - -static void query_target_summary_labels_v12(BUFFER *wb, QUERY_TARGET *qt, const char *key, bool v2, struct summary_total_counts *key_totals, struct summary_total_counts *value_totals) { - buffer_json_member_add_array(wb, key); - struct rrdlabels_formatting_v2 t = { - .keys = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE), - .v2 = v2, - }; - for (long c = 0; c < (long) qt->instances.used; c++) { - QUERY_INSTANCE *qi = query_instance(qt, c); - RRDINSTANCE_ACQUIRED *ria = qi->ria; - t.qi = qi; - rrdlabels_walkthrough_read(rrdinstance_acquired_labels(ria), rrdlabels_formatting_v2, &t); - } - struct rrdlabels_keys_dict_entry *d; - dfe_start_read(t.keys, d) { - if(v2) { - buffer_json_add_array_item_object(wb); - buffer_json_member_add_string(wb, "id", d_dfe.name); - query_target_metric_counts(wb, &d->metrics); - query_target_points_statistics(wb, qt, &d->query_points); - aggregate_into_summary_totals(key_totals, &d->metrics); - buffer_json_member_add_array(wb, "vl"); - } - struct rrdlabels_key_value_dict_entry *z; - dfe_start_read(d->values, z){ - if (v2) { - buffer_json_add_array_item_object(wb); - buffer_json_member_add_string(wb, "id", z->value); - query_target_metric_counts(wb, &z->metrics); - query_target_points_statistics(wb, qt, &z->query_points); - buffer_json_object_close(wb); - aggregate_into_summary_totals(value_totals, &z->metrics); - } else { - buffer_json_add_array_item_array(wb); - buffer_json_add_array_item_string(wb, z->key); - buffer_json_add_array_item_string(wb, z->value); - buffer_json_array_close(wb); - } - } - dfe_done(z); - dictionary_destroy(d->values); - if(v2) { - buffer_json_array_close(wb); - buffer_json_object_close(wb); - } - } - dfe_done(d); - dictionary_destroy(t.keys); - buffer_json_array_close(wb); -} - -static void query_target_summary_alerts_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key) { - buffer_json_member_add_array(wb, key); - QUERY_ALERTS_COUNTS *z; - - DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE); - for (long c = 0; c < (long) qt->instances.used; c++) { - QUERY_INSTANCE *qi = query_instance(qt, c); - RRDSET *st = rrdinstance_acquired_rrdset(qi->ria); - if (st) { - rw_spinlock_read_lock(&st->alerts.spinlock); - if (st->alerts.base) { - for (RRDCALC *rc = st->alerts.base; rc; rc = rc->next) { - z = dictionary_set(dict, string2str(rc->name), NULL, sizeof(*z)); - - switch(rc->status) { - case RRDCALC_STATUS_CLEAR: - z->clear++; - break; - - case RRDCALC_STATUS_WARNING: - z->warning++; - break; - - case RRDCALC_STATUS_CRITICAL: - z->critical++; - break; - - default: - case RRDCALC_STATUS_UNINITIALIZED: - case RRDCALC_STATUS_UNDEFINED: - case RRDCALC_STATUS_REMOVED: - z->other++; - break; - } - } - } - rw_spinlock_read_unlock(&st->alerts.spinlock); - } - } - dfe_start_read(dict, z) - query_target_alerts_counts(wb, z, z_dfe.name, true); - dfe_done(z); - dictionary_destroy(dict); - buffer_json_array_close(wb); // alerts -} - -static inline void query_target_functions(BUFFER *wb, const char *key, RRDR *r) { - QUERY_TARGET *qt = r->internal.qt; - const long query_used = qt->query.used; - - DICTIONARY *funcs = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE); - RRDINSTANCE_ACQUIRED *ria = NULL; - for (long c = 0; c < query_used ; c++) { - QUERY_METRIC *qm = query_metric(qt, c); - QUERY_INSTANCE *qi = query_instance(qt, qm->link.query_instance_id); - if(qi->ria == ria) - continue; - - ria = qi->ria; - chart_functions_to_dict(rrdinstance_acquired_functions(ria), funcs, NULL, 0); - } - - buffer_json_member_add_array(wb, key); - void *t; (void)t; - dfe_start_read(funcs, t) - buffer_json_add_array_item_string(wb, t_dfe.name); - dfe_done(t); - dictionary_destroy(funcs); - buffer_json_array_close(wb); -} - -static inline long query_target_chart_labels_filter_v1(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { - QUERY_TARGET *qt = r->internal.qt; - const long query_used = qt->query.used; - long c, i = 0; - - buffer_json_member_add_object(wb, key); - - SIMPLE_PATTERN *pattern = qt->instances.chart_label_key_pattern; - char *label_key = NULL; - while (pattern && (label_key = simple_pattern_iterate(&pattern))) { - buffer_json_member_add_array(wb, label_key); - - for (c = 0, i = 0; c < query_used; c++) { - if(!rrdr_dimension_should_be_exposed(r->od[c], options)) - continue; - - QUERY_METRIC *qm = query_metric(qt, c); - QUERY_INSTANCE *qi = query_instance(qt, qm->link.query_instance_id); - rrdlabels_value_to_buffer_array_item_or_null(rrdinstance_acquired_labels(qi->ria), wb, label_key); - i++; - } - buffer_json_array_close(wb); - } - - buffer_json_object_close(wb); - - return i; -} - -static inline long query_target_metrics_latest_values(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { - QUERY_TARGET *qt = r->internal.qt; - const long query_used = qt->query.used; - long c, i; - - buffer_json_member_add_array(wb, key); - - for(c = 0, i = 0; c < query_used ;c++) { - if(!rrdr_dimension_should_be_exposed(r->od[c], options)) - continue; - - QUERY_METRIC *qm = query_metric(qt, c); - QUERY_DIMENSION *qd = query_dimension(qt, qm->link.query_dimension_id); - buffer_json_add_array_item_double(wb, rrdmetric_acquired_last_stored_value(qd->rma)); - i++; - } - - buffer_json_array_close(wb); - - return i; -} - -static inline size_t rrdr_dimension_view_latest_values(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { - buffer_json_member_add_array(wb, key); - - size_t c, i; - for(c = 0, i = 0; c < r->d ; c++) { - if(!rrdr_dimension_should_be_exposed(r->od[c], options)) - continue; - - i++; - - NETDATA_DOUBLE *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ]; - RRDR_VALUE_FLAGS *co = &r->o[ (rrdr_rows(r) - 1) * r->d ]; - NETDATA_DOUBLE n = cn[c]; - - if(co[c] & RRDR_VALUE_EMPTY) { - if(options & RRDR_OPTION_NULL2ZERO) - buffer_json_add_array_item_double(wb, 0.0); - else - buffer_json_add_array_item_double(wb, NAN); - } - else - buffer_json_add_array_item_double(wb, n); - } - - buffer_json_array_close(wb); - - return i; -} - -static inline void rrdr_dimension_query_points_statistics(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options, bool dview) { - STORAGE_POINT *sp = (dview) ? r->dview : r->dqp; - NETDATA_DOUBLE anomaly_rate_multiplier = (dview) ? RRDR_DVIEW_ANOMALY_COUNT_MULTIPLIER : 1.0; - - if(unlikely(!sp)) - return; - - if(key) - buffer_json_member_add_object(wb, key); - - buffer_json_member_add_array(wb, "min"); - for(size_t c = 0; c < r->d ; c++) { - if (!rrdr_dimension_should_be_exposed(r->od[c], options)) - continue; - - buffer_json_add_array_item_double(wb, sp[c].min); - } - buffer_json_array_close(wb); - - buffer_json_member_add_array(wb, "max"); - for(size_t c = 0; c < r->d ; c++) { - if (!rrdr_dimension_should_be_exposed(r->od[c], options)) - continue; - - buffer_json_add_array_item_double(wb, sp[c].max); - } - buffer_json_array_close(wb); - - if(options & RRDR_OPTION_RETURN_RAW) { - buffer_json_member_add_array(wb, "sum"); - for(size_t c = 0; c < r->d ; c++) { - if (!rrdr_dimension_should_be_exposed(r->od[c], options)) - continue; - - buffer_json_add_array_item_double(wb, sp[c].sum); - } - buffer_json_array_close(wb); - - buffer_json_member_add_array(wb, "cnt"); - for(size_t c = 0; c < r->d ; c++) { - if (!rrdr_dimension_should_be_exposed(r->od[c], options)) - continue; - - buffer_json_add_array_item_uint64(wb, sp[c].count); - } - buffer_json_array_close(wb); - - buffer_json_member_add_array(wb, "arc"); - for(size_t c = 0; c < r->d ; c++) { - if (!rrdr_dimension_should_be_exposed(r->od[c], options)) - continue; - - buffer_json_add_array_item_uint64(wb, storage_point_anomaly_rate(sp[c]) / anomaly_rate_multiplier / 100.0 * sp[c].count); - } - buffer_json_array_close(wb); - } - else { - NETDATA_DOUBLE sum = 0.0; - for(size_t c = 0; c < r->d ; c++) { - if(!rrdr_dimension_should_be_exposed(r->od[c], options)) - continue; - - sum += ABS(sp[c].sum); - } - - buffer_json_member_add_array(wb, "avg"); - for(size_t c = 0; c < r->d ; c++) { - if (!rrdr_dimension_should_be_exposed(r->od[c], options)) - continue; - - buffer_json_add_array_item_double(wb, storage_point_average_value(sp[c])); - } - buffer_json_array_close(wb); - - buffer_json_member_add_array(wb, "arp"); - for(size_t c = 0; c < r->d ; c++) { - if (!rrdr_dimension_should_be_exposed(r->od[c], options)) - continue; - - buffer_json_add_array_item_double(wb, storage_point_anomaly_rate(sp[c]) / anomaly_rate_multiplier); - } - buffer_json_array_close(wb); - - buffer_json_member_add_array(wb, "con"); - for(size_t c = 0; c < r->d ; c++) { - if (!rrdr_dimension_should_be_exposed(r->od[c], options)) - continue; - - NETDATA_DOUBLE con = (sum > 0.0) ? ABS(sp[c].sum) * 100.0 / sum : 0.0; - buffer_json_add_array_item_double(wb, con); - } - buffer_json_array_close(wb); - } - - if(key) - buffer_json_object_close(wb); -} - -void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb) { - QUERY_TARGET *qt = r->internal.qt; - DATASOURCE_FORMAT format = qt->request.format; - RRDR_OPTIONS options = qt->window.options; - - long rows = rrdr_rows(r); - - char kq[2] = "", // key quote - sq[2] = ""; // string quote - - if( options & RRDR_OPTION_GOOGLE_JSON ) { - kq[0] = '\0'; - sq[0] = '\''; - } - else { - kq[0] = '"'; - sq[0] = '"'; - } - - buffer_json_initialize( - wb, kq, sq, 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT); - - buffer_json_member_add_uint64(wb, "api", 1); - buffer_json_member_add_string(wb, "id", qt->id); - buffer_json_member_add_string(wb, "name", qt->id); - buffer_json_member_add_time_t(wb, "view_update_every", r->view.update_every); - buffer_json_member_add_time_t(wb, "update_every", qt->db.minimum_latest_update_every_s); - buffer_json_member_add_time_t(wb, "first_entry", qt->db.first_time_s); - buffer_json_member_add_time_t(wb, "last_entry", qt->db.last_time_s); - buffer_json_member_add_time_t(wb, "after", r->view.after); - buffer_json_member_add_time_t(wb, "before", r->view.before); - buffer_json_member_add_string(wb, "group", time_grouping_tostring(qt->request.time_group_method)); - web_client_api_request_v1_data_options_to_buffer_json_array(wb, "options", options); - - if(!rrdr_dimension_names(wb, "dimension_names", r, options)) - rows = 0; - - if(!rrdr_dimension_ids(wb, "dimension_ids", r, options)) - rows = 0; - - if (options & RRDR_OPTION_ALL_DIMENSIONS) { - query_target_summary_instances_v1(wb, qt, "full_chart_list"); - query_target_summary_dimensions_v12(wb, qt, "full_dimension_list", false, NULL); - query_target_summary_labels_v12(wb, qt, "full_chart_labels", false, NULL, NULL); - } - - query_target_functions(wb, "functions", r); - - if (!qt->request.st && !jsonwrap_v1_chart_ids(wb, "chart_ids", r, options)) - rows = 0; - - if (qt->instances.chart_label_key_pattern && !query_target_chart_labels_filter_v1(wb, "chart_labels", r, options)) - rows = 0; - - if(!query_target_metrics_latest_values(wb, "latest_values", r, options)) - rows = 0; - - size_t dimensions = rrdr_dimension_view_latest_values(wb, "view_latest_values", r, options); - if(!dimensions) - rows = 0; - - buffer_json_member_add_uint64(wb, "dimensions", dimensions); - buffer_json_member_add_uint64(wb, "points", rows); - buffer_json_member_add_string(wb, "format", rrdr_format_to_string(format)); - - buffer_json_member_add_array(wb, "db_points_per_tier"); - for(size_t tier = 0; tier < storage_tiers ; tier++) - buffer_json_add_array_item_uint64(wb, qt->db.tiers[tier].points); - buffer_json_array_close(wb); - - if(options & RRDR_OPTION_DEBUG) - jsonwrap_query_plan(r, wb); -} - -static void rrdset_rrdcalc_entries_v2(BUFFER *wb, RRDINSTANCE_ACQUIRED *ria) { - RRDSET *st = rrdinstance_acquired_rrdset(ria); - if(st) { - rw_spinlock_read_lock(&st->alerts.spinlock); - if(st->alerts.base) { - buffer_json_member_add_object(wb, "alerts"); - for(RRDCALC *rc = st->alerts.base; rc ;rc = rc->next) { - if(rc->status < RRDCALC_STATUS_CLEAR) - continue; - - buffer_json_member_add_object(wb, string2str(rc->name)); - buffer_json_member_add_string(wb, "st", rrdcalc_status2string(rc->status)); - buffer_json_member_add_double(wb, "vl", rc->value); - buffer_json_member_add_string(wb, "un", string2str(rc->units)); - buffer_json_object_close(wb); - } - buffer_json_object_close(wb); - } - rw_spinlock_read_unlock(&st->alerts.spinlock); - } -} - -static void query_target_combined_units_v2(BUFFER *wb, QUERY_TARGET *qt, size_t contexts, bool ignore_percentage) { - if(!ignore_percentage && query_target_has_percentage_units(qt)) { - buffer_json_member_add_string(wb, "units", "%"); - } - else if(contexts == 1) { - buffer_json_member_add_string(wb, "units", rrdcontext_acquired_units(qt->contexts.array[0].rca)); - } - else if(contexts > 1) { - DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE); - for(size_t c = 0; c < qt->contexts.used ;c++) - dictionary_set(dict, rrdcontext_acquired_units(qt->contexts.array[c].rca), NULL, 0); - - if(dictionary_entries(dict) == 1) - buffer_json_member_add_string(wb, "units", rrdcontext_acquired_units(qt->contexts.array[0].rca)); - else { - buffer_json_member_add_array(wb, "units"); - const char *s; - dfe_start_read(dict, s) - buffer_json_add_array_item_string(wb, s_dfe.name); - dfe_done(s); - buffer_json_array_close(wb); - } - dictionary_destroy(dict); - } -} - -static void query_target_combined_chart_type(BUFFER *wb, QUERY_TARGET *qt, size_t contexts) { - if(contexts >= 1) - buffer_json_member_add_string(wb, "chart_type", rrdset_type_name(rrdcontext_acquired_chart_type(qt->contexts.array[0].rca))); -} - -static void rrdr_grouped_by_array_v2(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options __maybe_unused) { - QUERY_TARGET *qt = r->internal.qt; - - buffer_json_member_add_array(wb, key); - - // find the deeper group-by - ssize_t g = 0; - for(g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) { - if(qt->request.group_by[g].group_by == RRDR_GROUP_BY_NONE) - break; - } - - if(g > 0) - g--; - - RRDR_GROUP_BY group_by = qt->request.group_by[g].group_by; - - if(group_by & RRDR_GROUP_BY_SELECTED) - buffer_json_add_array_item_string(wb, "selected"); - - else if(group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE) - buffer_json_add_array_item_string(wb, "percentage-of-instance"); - - else { - - if(group_by & RRDR_GROUP_BY_DIMENSION) - buffer_json_add_array_item_string(wb, "dimension"); - - if(group_by & RRDR_GROUP_BY_INSTANCE) - buffer_json_add_array_item_string(wb, "instance"); - - if(group_by & RRDR_GROUP_BY_LABEL) { - BUFFER *b = buffer_create(0, NULL); - for (size_t l = 0; l < qt->group_by[g].used; l++) { - buffer_flush(b); - buffer_fast_strcat(b, "label:", 6); - buffer_strcat(b, qt->group_by[g].label_keys[l]); - buffer_json_add_array_item_string(wb, buffer_tostring(b)); - } - buffer_free(b); - } - - if(group_by & RRDR_GROUP_BY_NODE) - buffer_json_add_array_item_string(wb, "node"); - - if(group_by & RRDR_GROUP_BY_CONTEXT) - buffer_json_add_array_item_string(wb, "context"); - - if(group_by & RRDR_GROUP_BY_UNITS) - buffer_json_add_array_item_string(wb, "units"); - } - - buffer_json_array_close(wb); // group_by_order -} - -static void rrdr_dimension_units_array_v2(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options, bool ignore_percentage) { - if(!r->du) - return; - - bool percentage = !ignore_percentage && query_target_has_percentage_units(r->internal.qt); - - buffer_json_member_add_array(wb, key); - for(size_t c = 0; c < r->d ; c++) { - if(!rrdr_dimension_should_be_exposed(r->od[c], options)) - continue; - - if(percentage) - buffer_json_add_array_item_string(wb, "%"); - else - buffer_json_add_array_item_string(wb, string2str(r->du[c])); - } - buffer_json_array_close(wb); -} - -static void rrdr_dimension_priority_array_v2(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { - if(!r->dp) - return; - - buffer_json_member_add_array(wb, key); - for(size_t c = 0; c < r->d ; c++) { - if(!rrdr_dimension_should_be_exposed(r->od[c], options)) - continue; - - buffer_json_add_array_item_uint64(wb, r->dp[c]); - } - buffer_json_array_close(wb); -} - -static void rrdr_dimension_aggregated_array_v2(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { - if(!r->dgbc) - return; - - buffer_json_member_add_array(wb, key); - for(size_t c = 0; c < r->d ;c++) { - if(!rrdr_dimension_should_be_exposed(r->od[c], options)) - continue; - - buffer_json_add_array_item_uint64(wb, r->dgbc[c]); - } - buffer_json_array_close(wb); -} - -static void query_target_title(BUFFER *wb, QUERY_TARGET *qt, size_t contexts) { - if(contexts == 1) { - buffer_json_member_add_string(wb, "title", rrdcontext_acquired_title(qt->contexts.array[0].rca)); - } - else if(contexts > 1) { - BUFFER *t = buffer_create(0, NULL); - DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE); - - buffer_strcat(t, "Chart for contexts: "); - - size_t added = 0; - for(size_t c = 0; c < qt->contexts.used ;c++) { - bool *set = dictionary_set(dict, rrdcontext_acquired_id(qt->contexts.array[c].rca), NULL, sizeof(*set)); - if(!*set) { - *set = true; - if(added) - buffer_fast_strcat(t, ", ", 2); - - buffer_strcat(t, rrdcontext_acquired_id(qt->contexts.array[c].rca)); - added++; - } - } - buffer_json_member_add_string(wb, "title", buffer_tostring(t)); - dictionary_destroy(dict); - buffer_free(t); - } -} - -static void query_target_detailed_objects_tree(BUFFER *wb, RRDR *r, RRDR_OPTIONS options) { - QUERY_TARGET *qt = r->internal.qt; - buffer_json_member_add_object(wb, "nodes"); - - time_t now_s = now_realtime_sec(); - RRDHOST *last_host = NULL; - RRDCONTEXT_ACQUIRED *last_rca = NULL; - RRDINSTANCE_ACQUIRED *last_ria = NULL; - - size_t h = 0, c = 0, i = 0, m = 0, q = 0; - for(; h < qt->nodes.used ; h++) { - QUERY_NODE *qn = query_node(qt, h); - RRDHOST *host = qn->rrdhost; - - for( ;c < qt->contexts.used ;c++) { - QUERY_CONTEXT *qc = query_context(qt, c); - RRDCONTEXT_ACQUIRED *rca = qc->rca; - if(!rrdcontext_acquired_belongs_to_host(rca, host)) break; - - for( ;i < qt->instances.used ;i++) { - QUERY_INSTANCE *qi = query_instance(qt, i); - RRDINSTANCE_ACQUIRED *ria = qi->ria; - if(!rrdinstance_acquired_belongs_to_context(ria, rca)) break; - - for( ; m < qt->dimensions.used ; m++) { - QUERY_DIMENSION *qd = query_dimension(qt, m); - RRDMETRIC_ACQUIRED *rma = qd->rma; - if(!rrdmetric_acquired_belongs_to_instance(rma, ria)) break; - - QUERY_METRIC *qm = NULL; - bool queried = false; - for( ; q < qt->query.used ;q++) { - QUERY_METRIC *tqm = query_metric(qt, q); - QUERY_DIMENSION *tqd = query_dimension(qt, tqm->link.query_dimension_id); - if(tqd->rma != rma) break; - - queried = tqm->status & RRDR_DIMENSION_QUERIED; - qm = tqm; - } - - if(!queried & !(options & RRDR_OPTION_ALL_DIMENSIONS)) - continue; - - if(host != last_host) { - if(last_host) { - if(last_rca) { - if(last_ria) { - buffer_json_object_close(wb); // dimensions - buffer_json_object_close(wb); // instance - last_ria = NULL; - } - buffer_json_object_close(wb); // instances - buffer_json_object_close(wb); // context - last_rca = NULL; - } - buffer_json_object_close(wb); // contexts - buffer_json_object_close(wb); // host - last_host = NULL; - } - - buffer_json_member_add_object(wb, host->machine_guid); - if(qn->node_id[0]) - buffer_json_member_add_string(wb, "nd", qn->node_id); - buffer_json_member_add_uint64(wb, "ni", qn->slot); - buffer_json_member_add_string(wb, "nm", rrdhost_hostname(host)); - buffer_json_member_add_object(wb, "contexts"); - - last_host = host; - } - - if(rca != last_rca) { - if(last_rca) { - if(last_ria) { - buffer_json_object_close(wb); // dimensions - buffer_json_object_close(wb); // instance - last_ria = NULL; - } - buffer_json_object_close(wb); // instances - buffer_json_object_close(wb); // context - last_rca = NULL; - } - - buffer_json_member_add_object(wb, rrdcontext_acquired_id(rca)); - buffer_json_member_add_object(wb, "instances"); - - last_rca = rca; - } - - if(ria != last_ria) { - if(last_ria) { - buffer_json_object_close(wb); // dimensions - buffer_json_object_close(wb); // instance - last_ria = NULL; - } - - buffer_json_member_add_object(wb, rrdinstance_acquired_id(ria)); - buffer_json_member_add_string(wb, "nm", rrdinstance_acquired_name(ria)); - buffer_json_member_add_time_t(wb, "ue", rrdinstance_acquired_update_every(ria)); - RRDLABELS *labels = rrdinstance_acquired_labels(ria); - if(labels) { - buffer_json_member_add_object(wb, "labels"); - rrdlabels_to_buffer_json_members(labels, wb); - buffer_json_object_close(wb); - } - rrdset_rrdcalc_entries_v2(wb, ria); - buffer_json_member_add_object(wb, "dimensions"); - - last_ria = ria; - } - - buffer_json_member_add_object(wb, rrdmetric_acquired_id(rma)); - { - buffer_json_member_add_string(wb, "nm", rrdmetric_acquired_name(rma)); - buffer_json_member_add_uint64(wb, "qr", queried ? 1 : 0); - time_t first_entry_s = rrdmetric_acquired_first_entry(rma); - time_t last_entry_s = rrdmetric_acquired_last_entry(rma); - buffer_json_member_add_time_t(wb, "fe", first_entry_s); - buffer_json_member_add_time_t(wb, "le", last_entry_s ? last_entry_s : now_s); - - if(qm) { - if(qm->status & RRDR_DIMENSION_GROUPED) { - // buffer_json_member_add_string(wb, "grouped_as_id", string2str(qm->grouped_as.id)); - buffer_json_member_add_string(wb, "as", string2str(qm->grouped_as.name)); - } - - query_target_points_statistics(wb, qt, &qm->query_points); - - if(options & RRDR_OPTION_DEBUG) - jsonwrap_query_metric_plan(wb, qm); - } - } - buffer_json_object_close(wb); // metric - } - } - } - } - - if(last_host) { - if(last_rca) { - if(last_ria) { - buffer_json_object_close(wb); // dimensions - buffer_json_object_close(wb); // instance - last_ria = NULL; - } - buffer_json_object_close(wb); // instances - buffer_json_object_close(wb); // context - last_rca = NULL; - } - buffer_json_object_close(wb); // contexts - buffer_json_object_close(wb); // host - last_host = NULL; - } - buffer_json_object_close(wb); // hosts -} - -void version_hashes_api_v2(BUFFER *wb, struct query_versions *versions) { - buffer_json_member_add_object(wb, "versions"); - buffer_json_member_add_uint64(wb, "routing_hard_hash", 1); - buffer_json_member_add_uint64(wb, "nodes_hard_hash", dictionary_version(rrdhost_root_index)); - buffer_json_member_add_uint64(wb, "contexts_hard_hash", versions->contexts_hard_hash); - buffer_json_member_add_uint64(wb, "contexts_soft_hash", versions->contexts_soft_hash); - buffer_json_member_add_uint64(wb, "alerts_hard_hash", versions->alerts_hard_hash); - buffer_json_member_add_uint64(wb, "alerts_soft_hash", versions->alerts_soft_hash); - buffer_json_object_close(wb); -} - -void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb) { - QUERY_TARGET *qt = r->internal.qt; - RRDR_OPTIONS options = qt->window.options; - - char kq[2] = "\"", // key quote - sq[2] = "\""; // string quote - - if(unlikely(options & RRDR_OPTION_GOOGLE_JSON)) { - kq[0] = '\0'; - sq[0] = '\''; - } - - buffer_json_initialize( - wb, kq, sq, 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT); - buffer_json_member_add_uint64(wb, "api", 2); - - if(options & RRDR_OPTION_DEBUG) { - buffer_json_member_add_string(wb, "id", qt->id); - buffer_json_member_add_object(wb, "request"); - { - buffer_json_member_add_string(wb, "format", rrdr_format_to_string(qt->request.format)); - web_client_api_request_v1_data_options_to_buffer_json_array(wb, "options", qt->request.options); - - buffer_json_member_add_object(wb, "scope"); - buffer_json_member_add_string(wb, "scope_nodes", qt->request.scope_nodes); - buffer_json_member_add_string(wb, "scope_contexts", qt->request.scope_contexts); - buffer_json_object_close(wb); // scope - - buffer_json_member_add_object(wb, "selectors"); - if (qt->request.host) - buffer_json_member_add_string(wb, "nodes", rrdhost_hostname(qt->request.host)); - else - buffer_json_member_add_string(wb, "nodes", qt->request.nodes); - buffer_json_member_add_string(wb, "contexts", qt->request.contexts); - buffer_json_member_add_string(wb, "instances", qt->request.instances); - buffer_json_member_add_string(wb, "dimensions", qt->request.dimensions); - buffer_json_member_add_string(wb, "labels", qt->request.labels); - buffer_json_member_add_string(wb, "alerts", qt->request.alerts); - buffer_json_object_close(wb); // selectors - - buffer_json_member_add_object(wb, "window"); - buffer_json_member_add_time_t(wb, "after", qt->request.after); - buffer_json_member_add_time_t(wb, "before", qt->request.before); - buffer_json_member_add_uint64(wb, "points", qt->request.points); - if (qt->request.options & RRDR_OPTION_SELECTED_TIER) - buffer_json_member_add_uint64(wb, "tier", qt->request.tier); - else - buffer_json_member_add_string(wb, "tier", NULL); - buffer_json_object_close(wb); // window - - buffer_json_member_add_object(wb, "aggregations"); - { - buffer_json_member_add_object(wb, "time"); - buffer_json_member_add_string(wb, "time_group", time_grouping_tostring(qt->request.time_group_method)); - buffer_json_member_add_string(wb, "time_group_options", qt->request.time_group_options); - if (qt->request.resampling_time > 0) - buffer_json_member_add_time_t(wb, "time_resampling", qt->request.resampling_time); - else - buffer_json_member_add_string(wb, "time_resampling", NULL); - buffer_json_object_close(wb); // time - - buffer_json_member_add_array(wb, "metrics"); - for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) { - if(qt->request.group_by[g].group_by == RRDR_GROUP_BY_NONE) - break; - - buffer_json_add_array_item_object(wb); - { - buffer_json_member_add_array(wb, "group_by"); - buffer_json_group_by_to_array(wb, qt->request.group_by[g].group_by); - buffer_json_array_close(wb); - - buffer_json_member_add_array(wb, "group_by_label"); - for (size_t l = 0; l < qt->group_by[g].used; l++) - buffer_json_add_array_item_string(wb, qt->group_by[g].label_keys[l]); - buffer_json_array_close(wb); - - buffer_json_member_add_string( - wb, "aggregation",group_by_aggregate_function_to_string(qt->request.group_by[g].aggregation)); - } - buffer_json_object_close(wb); - } - buffer_json_array_close(wb); // group_by - } - buffer_json_object_close(wb); // aggregations - - buffer_json_member_add_uint64(wb, "timeout", qt->request.timeout_ms); - } - buffer_json_object_close(wb); // request - } - - version_hashes_api_v2(wb, &qt->versions); - - buffer_json_member_add_object(wb, "summary"); - struct summary_total_counts - nodes_totals = { 0 }, - contexts_totals = { 0 }, - instances_totals = { 0 }, - metrics_totals = { 0 }, - label_key_totals = { 0 }, - label_key_value_totals = { 0 }; - { - query_target_summary_nodes_v2(wb, qt, "nodes", &nodes_totals); - r->internal.contexts = query_target_summary_contexts_v2(wb, qt, "contexts", &contexts_totals); - query_target_summary_instances_v2(wb, qt, "instances", &instances_totals); - query_target_summary_dimensions_v12(wb, qt, "dimensions", true, &metrics_totals); - query_target_summary_labels_v12(wb, qt, "labels", true, &label_key_totals, &label_key_value_totals); - query_target_summary_alerts_v2(wb, qt, "alerts"); - } - if(query_target_aggregatable(qt)) { - buffer_json_member_add_object(wb, "globals"); - query_target_points_statistics(wb, qt, &qt->query_points); - buffer_json_object_close(wb); // globals - } - buffer_json_object_close(wb); // summary - - buffer_json_member_add_object(wb, "totals"); - query_target_total_counts(wb, "nodes", &nodes_totals); - query_target_total_counts(wb, "contexts", &contexts_totals); - query_target_total_counts(wb, "instances", &instances_totals); - query_target_total_counts(wb, "dimensions", &metrics_totals); - query_target_total_counts(wb, "label_keys", &label_key_totals); - query_target_total_counts(wb, "label_key_values", &label_key_value_totals); - buffer_json_object_close(wb); // totals - - if(options & RRDR_OPTION_SHOW_DETAILS) { - buffer_json_member_add_object(wb, "detailed"); - query_target_detailed_objects_tree(wb, r, options); - buffer_json_object_close(wb); // detailed - } - - query_target_functions(wb, "functions", r); -} - -//static void annotations_range_for_value_flags(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format __maybe_unused, RRDR_OPTIONS options, RRDR_VALUE_FLAGS flags, const char *type) { -// const size_t dims = r->d, rows = r->rows; -// size_t next_d_idx = 0; -// for(size_t d = 0; d < dims ; d++) { -// if(!rrdr_dimension_should_be_exposed(r->od[d], options)) -// continue; -// -// size_t d_idx = next_d_idx++; -// -// size_t t = 0; -// while(t < rows) { -// -// // find the beginning -// time_t started = 0; -// for(; t < rows ;t++) { -// RRDR_VALUE_FLAGS o = r->o[t * r->d + d]; -// if(o & flags) { -// started = r->t[t]; -// break; -// } -// } -// -// if(started) { -// time_t ended = 0; -// for(; t < rows ;t++) { -// RRDR_VALUE_FLAGS o = r->o[t * r->d + d]; -// if(!(o & flags)) { -// ended = r->t[t]; -// break; -// } -// } -// -// if(!ended) -// ended = r->t[rows - 1]; -// -// buffer_json_add_array_item_object(wb); -// buffer_json_member_add_string(wb, "t", type); -// // buffer_json_member_add_string(wb, "d", string2str(r->dn[d])); -// buffer_json_member_add_uint64(wb, "d", d_idx); -// if(started == ended) { -// if(options & RRDR_OPTION_MILLISECONDS) -// buffer_json_member_add_time_t2ms(wb, "x", started); -// else -// buffer_json_member_add_time_t(wb, "x", started); -// } -// else { -// buffer_json_member_add_array(wb, "x"); -// if(options & RRDR_OPTION_MILLISECONDS) { -// buffer_json_add_array_item_time_t2ms(wb, started); -// buffer_json_add_array_item_time_t2ms(wb, ended); -// } -// else { -// buffer_json_add_array_item_time_t(wb, started); -// buffer_json_add_array_item_time_t(wb, ended); -// } -// buffer_json_array_close(wb); -// } -// buffer_json_object_close(wb); -// } -// } -// } -//} -// -//void rrdr_json_wrapper_annotations(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format __maybe_unused, RRDR_OPTIONS options) { -// buffer_json_member_add_array(wb, "annotations"); -// -// annotations_range_for_value_flags(r, wb, format, options, RRDR_VALUE_EMPTY, "G"); // Gap -// annotations_range_for_value_flags(r, wb, format, options, RRDR_VALUE_RESET, "O"); // Overflow -// annotations_range_for_value_flags(r, wb, format, options, RRDR_VALUE_PARTIAL, "P"); // Partial -// -// buffer_json_array_close(wb); // annotations -//} - -void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb) { - buffer_json_member_add_double(wb, "min", r->view.min); - buffer_json_member_add_double(wb, "max", r->view.max); - - buffer_json_query_timings(wb, "timings", &r->internal.qt->timings); - buffer_json_finalize(wb); -} - -void rrdr_json_wrapper_end2(RRDR *r, BUFFER *wb) { - QUERY_TARGET *qt = r->internal.qt; - DATASOURCE_FORMAT format = qt->request.format; - RRDR_OPTIONS options = qt->window.options; - - buffer_json_member_add_object(wb, "db"); - { - buffer_json_member_add_uint64(wb, "tiers", storage_tiers); - buffer_json_member_add_time_t(wb, "update_every", qt->db.minimum_latest_update_every_s); - buffer_json_member_add_time_t(wb, "first_entry", qt->db.first_time_s); - buffer_json_member_add_time_t(wb, "last_entry", qt->db.last_time_s); - - query_target_combined_units_v2(wb, qt, r->internal.contexts, true); - buffer_json_member_add_object(wb, "dimensions"); - { - rrdr_dimension_ids(wb, "ids", r, options); - rrdr_dimension_units_array_v2(wb, "units", r, options, true); - rrdr_dimension_query_points_statistics(wb, "sts", r, options, false); - } - buffer_json_object_close(wb); // dimensions - - buffer_json_member_add_array(wb, "per_tier"); - for(size_t tier = 0; tier < storage_tiers ; tier++) { - buffer_json_add_array_item_object(wb); - buffer_json_member_add_uint64(wb, "tier", tier); - buffer_json_member_add_uint64(wb, "queries", qt->db.tiers[tier].queries); - buffer_json_member_add_uint64(wb, "points", qt->db.tiers[tier].points); - buffer_json_member_add_time_t(wb, "update_every", qt->db.tiers[tier].update_every); - buffer_json_member_add_time_t(wb, "first_entry", qt->db.tiers[tier].retention.first_time_s); - buffer_json_member_add_time_t(wb, "last_entry", qt->db.tiers[tier].retention.last_time_s); - buffer_json_object_close(wb); - } - buffer_json_array_close(wb); - } - buffer_json_object_close(wb); - - buffer_json_member_add_object(wb, "view"); - { - query_target_title(wb, qt, r->internal.contexts); - buffer_json_member_add_time_t(wb, "update_every", r->view.update_every); - buffer_json_member_add_time_t(wb, "after", r->view.after); - buffer_json_member_add_time_t(wb, "before", r->view.before); - - if(options & RRDR_OPTION_DEBUG) { - buffer_json_member_add_string(wb, "format", rrdr_format_to_string(format)); - web_client_api_request_v1_data_options_to_buffer_json_array(wb, "options", options); - buffer_json_member_add_string(wb, "time_group", time_grouping_tostring(qt->request.time_group_method)); - } - - if(options & RRDR_OPTION_DEBUG) { - buffer_json_member_add_object(wb, "partial_data_trimming"); - buffer_json_member_add_time_t(wb, "max_update_every", r->partial_data_trimming.max_update_every); - buffer_json_member_add_time_t(wb, "expected_after", r->partial_data_trimming.expected_after); - buffer_json_member_add_time_t(wb, "trimmed_after", r->partial_data_trimming.trimmed_after); - buffer_json_object_close(wb); - } - - if(options & RRDR_OPTION_RETURN_RAW) - buffer_json_member_add_uint64(wb, "points", rrdr_rows(r)); - - query_target_combined_units_v2(wb, qt, r->internal.contexts, false); - query_target_combined_chart_type(wb, qt, r->internal.contexts); - buffer_json_member_add_object(wb, "dimensions"); - { - rrdr_grouped_by_array_v2(wb, "grouped_by", r, options); - rrdr_dimension_ids(wb, "ids", r, options); - rrdr_dimension_names(wb, "names", r, options); - rrdr_dimension_units_array_v2(wb, "units", r, options, false); - rrdr_dimension_priority_array_v2(wb, "priorities", r, options); - rrdr_dimension_aggregated_array_v2(wb, "aggregated", r, options); - rrdr_dimension_query_points_statistics(wb, "sts", r, options, true); - rrdr_json_group_by_labels(wb, "labels", r, options); - } - buffer_json_object_close(wb); // dimensions - buffer_json_member_add_double(wb, "min", r->view.min); - buffer_json_member_add_double(wb, "max", r->view.max); - } - buffer_json_object_close(wb); // view - - buffer_json_agents_v2(wb, &r->internal.qt->timings, 0, false, true); - buffer_json_cloud_timings(wb, "timings", &r->internal.qt->timings); - buffer_json_finalize(wb); -} diff --git a/web/api/formatters/json_wrapper.h b/web/api/formatters/json_wrapper.h deleted file mode 100644 index a702f3a5c..000000000 --- a/web/api/formatters/json_wrapper.h +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#ifndef NETDATA_API_FORMATTER_JSON_WRAPPER_H -#define NETDATA_API_FORMATTER_JSON_WRAPPER_H - -#include "rrd2json.h" -#include "web/api/queries/query.h" - -typedef void (*wrapper_begin_t)(RRDR *r, BUFFER *wb); -typedef void (*wrapper_end_t)(RRDR *r, BUFFER *wb); - -void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb); -void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb); - -void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb); -void rrdr_json_wrapper_end2(RRDR *r, BUFFER *wb); - -struct query_versions; -void version_hashes_api_v2(BUFFER *wb, struct query_versions *versions); - -#endif //NETDATA_API_FORMATTER_JSON_WRAPPER_H diff --git a/web/api/formatters/rrd2json.c b/web/api/formatters/rrd2json.c deleted file mode 100644 index 81c9ad5c7..000000000 --- a/web/api/formatters/rrd2json.c +++ /dev/null @@ -1,391 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#include "web/api/web_api_v1.h" -#include "database/storage_engine.h" - -void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) -{ - buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT); - rrdset2json(st, wb, NULL, NULL); - buffer_json_finalize(wb); -} - -const char *rrdr_format_to_string(DATASOURCE_FORMAT format) { - switch(format) { - case DATASOURCE_JSON: - return DATASOURCE_FORMAT_JSON; - - case DATASOURCE_JSON2: - return DATASOURCE_FORMAT_JSON2; - - case DATASOURCE_DATATABLE_JSON: - return DATASOURCE_FORMAT_DATATABLE_JSON; - - case DATASOURCE_DATATABLE_JSONP: - return DATASOURCE_FORMAT_DATATABLE_JSONP; - - case DATASOURCE_JSONP: - return DATASOURCE_FORMAT_JSONP; - - case DATASOURCE_SSV: - return DATASOURCE_FORMAT_SSV; - - case DATASOURCE_CSV: - return DATASOURCE_FORMAT_CSV; - - case DATASOURCE_TSV: - return DATASOURCE_FORMAT_TSV; - - case DATASOURCE_HTML: - return DATASOURCE_FORMAT_HTML; - - case DATASOURCE_JS_ARRAY: - return DATASOURCE_FORMAT_JS_ARRAY; - - case DATASOURCE_SSV_COMMA: - return DATASOURCE_FORMAT_SSV_COMMA; - - default: - return "unknown"; - } -} - -int rrdset2value_api_v1( - RRDSET *st - , BUFFER *wb - , NETDATA_DOUBLE *n - , const char *dimensions - , size_t points - , time_t after - , time_t before - , RRDR_TIME_GROUPING group_method - , const char *group_options - , time_t resampling_time - , uint32_t options - , time_t *db_after - , time_t *db_before - , size_t *db_points_read - , size_t *db_points_per_tier - , size_t *result_points_generated - , int *value_is_null - , NETDATA_DOUBLE *anomaly_rate - , time_t timeout - , size_t tier - , QUERY_SOURCE query_source - , STORAGE_PRIORITY priority -) { - int ret = HTTP_RESP_INTERNAL_SERVER_ERROR; - - ONEWAYALLOC *owa = onewayalloc_create(0); - RRDR *r = rrd2rrdr_legacy( - owa, - st, - points, - after, - before, - group_method, - resampling_time, - options, - dimensions, - group_options, - timeout, - tier, - query_source, - priority); - - if(!r) { - if(value_is_null) *value_is_null = 1; - ret = HTTP_RESP_INTERNAL_SERVER_ERROR; - goto cleanup; - } - - if(db_points_read) - *db_points_read += r->stats.db_points_read; - - if(db_points_per_tier) { - for(size_t t = 0; t < storage_tiers ;t++) - db_points_per_tier[t] += r->internal.qt->db.tiers[t].points; - } - - if(result_points_generated) - *result_points_generated += r->stats.result_points_generated; - - if(rrdr_rows(r) == 0) { - if(db_after) *db_after = 0; - if(db_before) *db_before = 0; - if(value_is_null) *value_is_null = 1; - - ret = HTTP_RESP_BAD_REQUEST; - goto cleanup; - } - - if(wb) { - if (r->view.flags & RRDR_RESULT_FLAG_RELATIVE) - buffer_no_cacheable(wb); - else if (r->view.flags & RRDR_RESULT_FLAG_ABSOLUTE) - buffer_cacheable(wb); - } - - if(db_after) *db_after = r->view.after; - if(db_before) *db_before = r->view.before; - - long i = (!(options & RRDR_OPTION_REVERSED))?(long)rrdr_rows(r) - 1:0; - *n = rrdr2value(r, i, options, value_is_null, anomaly_rate); - ret = HTTP_RESP_OK; - -cleanup: - rrdr_free(owa, r); - onewayalloc_destroy(owa); - return ret; -} - -static inline void buffer_json_member_add_key_only(BUFFER *wb, const char *key) { - buffer_print_json_comma_newline_spacing(wb); - buffer_print_json_key(wb, key); - buffer_fast_strcat(wb, ":", 1); - wb->json.stack[wb->json.depth].count++; -} - -static inline void buffer_json_member_add_string_open(BUFFER *wb, const char *key) { - buffer_json_member_add_key_only(wb, key); - buffer_strcat(wb, wb->json.value_quote); -} - -static inline void buffer_json_member_add_string_close(BUFFER *wb) { - buffer_strcat(wb, wb->json.value_quote); -} - -int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *latest_timestamp) { - wrapper_begin_t wrapper_begin = rrdr_json_wrapper_begin; - wrapper_end_t wrapper_end = rrdr_json_wrapper_end; - - if(qt->request.version == 2) { - wrapper_begin = rrdr_json_wrapper_begin2; - wrapper_end = rrdr_json_wrapper_end2; - } - - RRDR *r = rrd2rrdr(owa, qt); - - if(!r) { - buffer_strcat(wb, "Cannot generate output with these parameters on this chart."); - return HTTP_RESP_INTERNAL_SERVER_ERROR; - } - - if (r->view.flags & RRDR_RESULT_FLAG_CANCEL) { - rrdr_free(owa, r); - return HTTP_RESP_CLIENT_CLOSED_REQUEST; - } - - if(r->view.flags & RRDR_RESULT_FLAG_RELATIVE) - buffer_no_cacheable(wb); - else if(r->view.flags & RRDR_RESULT_FLAG_ABSOLUTE) - buffer_cacheable(wb); - - if(latest_timestamp && rrdr_rows(r) > 0) - *latest_timestamp = r->view.before; - - DATASOURCE_FORMAT format = qt->request.format; - RRDR_OPTIONS options = qt->window.options; - - switch(format) { - case DATASOURCE_SSV: - if(options & RRDR_OPTION_JSON_WRAP) { - wb->content_type = CT_APPLICATION_JSON; - wrapper_begin(r, wb); - buffer_json_member_add_string_open(wb, "result"); - rrdr2ssv(r, wb, options, "", " ", ""); - buffer_json_member_add_string_close(wb); - wrapper_end(r, wb); - } - else { - wb->content_type = CT_TEXT_PLAIN; - rrdr2ssv(r, wb, options, "", " ", ""); - } - break; - - case DATASOURCE_SSV_COMMA: - if(options & RRDR_OPTION_JSON_WRAP) { - wb->content_type = CT_APPLICATION_JSON; - wrapper_begin(r, wb); - buffer_json_member_add_string_open(wb, "result"); - rrdr2ssv(r, wb, options, "", ",", ""); - buffer_json_member_add_string_close(wb); - wrapper_end(r, wb); - } - else { - wb->content_type = CT_TEXT_PLAIN; - rrdr2ssv(r, wb, options, "", ",", ""); - } - break; - - case DATASOURCE_JS_ARRAY: - if(options & RRDR_OPTION_JSON_WRAP) { - wb->content_type = CT_APPLICATION_JSON; - wrapper_begin(r, wb); - buffer_json_member_add_array(wb, "result"); - rrdr2ssv(r, wb, options, "", ",", ""); - buffer_json_array_close(wb); - wrapper_end(r, wb); - } - else { - wb->content_type = CT_APPLICATION_JSON; - rrdr2ssv(r, wb, options, "[", ",", "]"); - } - break; - - case DATASOURCE_CSV: - if(options & RRDR_OPTION_JSON_WRAP) { - wb->content_type = CT_APPLICATION_JSON; - wrapper_begin(r, wb); - buffer_json_member_add_string_open(wb, "result"); - rrdr2csv(r, wb, format, options, "", ",", "\\n", ""); - buffer_json_member_add_string_close(wb); - wrapper_end(r, wb); - } - else { - wb->content_type = CT_TEXT_PLAIN; - rrdr2csv(r, wb, format, options, "", ",", "\r\n", ""); - } - break; - - case DATASOURCE_CSV_MARKDOWN: - if(options & RRDR_OPTION_JSON_WRAP) { - wb->content_type = CT_APPLICATION_JSON; - wrapper_begin(r, wb); - buffer_json_member_add_string_open(wb, "result"); - rrdr2csv(r, wb, format, options, "", "|", "\\n", ""); - buffer_json_member_add_string_close(wb); - wrapper_end(r, wb); - } - else { - wb->content_type = CT_TEXT_PLAIN; - rrdr2csv(r, wb, format, options, "", "|", "\r\n", ""); - } - break; - - case DATASOURCE_CSV_JSON_ARRAY: - wb->content_type = CT_APPLICATION_JSON; - if(options & RRDR_OPTION_JSON_WRAP) { - wrapper_begin(r, wb); - buffer_json_member_add_array(wb, "result"); - rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n"); - buffer_json_array_close(wb); - wrapper_end(r, wb); - } - else { - wb->content_type = CT_APPLICATION_JSON; - buffer_strcat(wb, "[\n"); - rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n"); - buffer_strcat(wb, "\n]"); - } - break; - - case DATASOURCE_TSV: - if(options & RRDR_OPTION_JSON_WRAP) { - wb->content_type = CT_APPLICATION_JSON; - wrapper_begin(r, wb); - buffer_json_member_add_string_open(wb, "result"); - rrdr2csv(r, wb, format, options, "", "\t", "\\n", ""); - buffer_json_member_add_string_close(wb); - wrapper_end(r, wb); - } - else { - wb->content_type = CT_TEXT_PLAIN; - rrdr2csv(r, wb, format, options, "", "\t", "\r\n", ""); - } - break; - - case DATASOURCE_HTML: - if(options & RRDR_OPTION_JSON_WRAP) { - wb->content_type = CT_APPLICATION_JSON; - wrapper_begin(r, wb); - buffer_json_member_add_string_open(wb, "result"); - buffer_strcat(wb, "<html>\\n<center>\\n<table border=\\\"0\\\" cellpadding=\\\"5\\\" cellspacing=\\\"5\\\">\\n"); - rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\\n", ""); - buffer_strcat(wb, "</table>\\n</center>\\n</html>\\n"); - buffer_json_member_add_string_close(wb); - wrapper_end(r, wb); - } - else { - wb->content_type = CT_TEXT_HTML; - buffer_strcat(wb, "<html>\n<center>\n<table border=\"0\" cellpadding=\"5\" cellspacing=\"5\">\n"); - rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\n", ""); - buffer_strcat(wb, "</table>\n</center>\n</html>\n"); - } - break; - - case DATASOURCE_DATATABLE_JSONP: - wb->content_type = CT_APPLICATION_X_JAVASCRIPT; - - if(options & RRDR_OPTION_JSON_WRAP) { - wrapper_begin(r, wb); - buffer_json_member_add_key_only(wb, "result"); - } - - rrdr2json(r, wb, options, 1); - - if(options & RRDR_OPTION_JSON_WRAP) - wrapper_end(r, wb); - - break; - - case DATASOURCE_DATATABLE_JSON: - wb->content_type = CT_APPLICATION_JSON; - - if(options & RRDR_OPTION_JSON_WRAP) { - wrapper_begin(r, wb); - buffer_json_member_add_key_only(wb, "result"); - } - - rrdr2json(r, wb, options, 1); - - if(options & RRDR_OPTION_JSON_WRAP) - wrapper_end(r, wb); - - break; - - case DATASOURCE_JSONP: - wb->content_type = CT_APPLICATION_X_JAVASCRIPT; - if(options & RRDR_OPTION_JSON_WRAP) { - wrapper_begin(r, wb); - buffer_json_member_add_key_only(wb, "result"); - } - - rrdr2json(r, wb, options, 0); - - if(options & RRDR_OPTION_JSON_WRAP) - wrapper_end(r, wb); - - break; - - case DATASOURCE_JSON: - default: - wb->content_type = CT_APPLICATION_JSON; - - if(options & RRDR_OPTION_JSON_WRAP) { - wrapper_begin(r, wb); - buffer_json_member_add_key_only(wb, "result"); - } - - rrdr2json(r, wb, options, 0); - - if(options & RRDR_OPTION_JSON_WRAP) { - if (options & RRDR_OPTION_RETURN_JWAR) { - buffer_json_member_add_key_only(wb, "anomaly_rates"); - rrdr2json(r, wb, options | RRDR_OPTION_INTERNAL_AR, false); - } - wrapper_end(r, wb); - } - break; - - case DATASOURCE_JSON2: - wb->content_type = CT_APPLICATION_JSON; - wrapper_begin(r, wb); - rrdr2json_v2(r, wb); - wrapper_end(r, wb); - break; - } - - rrdr_free(owa, r); - return HTTP_RESP_OK; -} diff --git a/web/api/formatters/rrd2json.h b/web/api/formatters/rrd2json.h deleted file mode 100644 index f0c0c39ba..000000000 --- a/web/api/formatters/rrd2json.h +++ /dev/null @@ -1,98 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#ifndef NETDATA_RRD2JSON_H -#define NETDATA_RRD2JSON_H 1 - -// type of JSON generations -typedef enum { - DATASOURCE_JSON = 0, - DATASOURCE_DATATABLE_JSON = 1, - DATASOURCE_DATATABLE_JSONP = 2, - DATASOURCE_SSV = 3, - DATASOURCE_CSV = 4, - DATASOURCE_JSONP = 5, - DATASOURCE_TSV = 6, - DATASOURCE_HTML = 7, - DATASOURCE_JS_ARRAY = 8, - DATASOURCE_SSV_COMMA = 9, - DATASOURCE_CSV_JSON_ARRAY = 10, - DATASOURCE_CSV_MARKDOWN = 11, - DATASOURCE_JSON2 = 12, -} DATASOURCE_FORMAT; - -#include "web/api/web_api_v1.h" - -#include "web/api/exporters/allmetrics.h" -#include "web/api/queries/rrdr.h" - -#include "web/api/formatters/csv/csv.h" -#include "web/api/formatters/ssv/ssv.h" -#include "web/api/formatters/json/json.h" -#include "web/api/formatters/value/value.h" - -#include "web/api/formatters/rrdset2json.h" -#include "web/api/formatters/charts2json.h" -#include "web/api/formatters/json_wrapper.h" - -#include "web/server/web_client.h" - -#define HOSTNAME_MAX 1024 - -#define DATASOURCE_FORMAT_JSON "json" -#define DATASOURCE_FORMAT_JSON2 "json2" -#define DATASOURCE_FORMAT_DATATABLE_JSON "datatable" -#define DATASOURCE_FORMAT_DATATABLE_JSONP "datasource" -#define DATASOURCE_FORMAT_JSONP "jsonp" -#define DATASOURCE_FORMAT_SSV "ssv" -#define DATASOURCE_FORMAT_CSV "csv" -#define DATASOURCE_FORMAT_TSV "tsv" -#define DATASOURCE_FORMAT_HTML "html" -#define DATASOURCE_FORMAT_JS_ARRAY "array" -#define DATASOURCE_FORMAT_SSV_COMMA "ssvcomma" -#define DATASOURCE_FORMAT_CSV_JSON_ARRAY "csvjsonarray" -#define DATASOURCE_FORMAT_CSV_MARKDOWN "markdown" - -void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb); -const char *rrdr_format_to_string(DATASOURCE_FORMAT format); - -int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, struct query_target *qt, time_t *latest_timestamp); - -void rrdr_json_group_by_labels(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options); - -int rrdset2value_api_v1( - RRDSET *st - , BUFFER *wb - , NETDATA_DOUBLE *n - , const char *dimensions - , size_t points - , time_t after - , time_t before - , RRDR_TIME_GROUPING group_method - , const char *group_options - , time_t resampling_time - , uint32_t options - , time_t *db_after - , time_t *db_before - , size_t *db_points_read - , size_t *db_points_per_tier - , size_t *result_points_generated - , int *value_is_null - , NETDATA_DOUBLE *anomaly_rate - , time_t timeout - , size_t tier - , QUERY_SOURCE query_source - , STORAGE_PRIORITY priority -); - -static inline bool rrdr_dimension_should_be_exposed(RRDR_DIMENSION_FLAGS rrdr_dim_flags, RRDR_OPTIONS options) { - if(unlikely((options & RRDR_OPTION_RETURN_RAW) && (rrdr_dim_flags & RRDR_DIMENSION_QUERIED))) - return true; - - if(unlikely(rrdr_dim_flags & RRDR_DIMENSION_HIDDEN)) return false; - if(unlikely(!(rrdr_dim_flags & RRDR_DIMENSION_QUERIED))) return false; - if(unlikely((options & RRDR_OPTION_NONZERO) && !(rrdr_dim_flags & RRDR_DIMENSION_NONZERO))) return false; - - return true; -} - -#endif /* NETDATA_RRD2JSON_H */ diff --git a/web/api/formatters/rrdset2json.c b/web/api/formatters/rrdset2json.c deleted file mode 100644 index 9ada35336..000000000 --- a/web/api/formatters/rrdset2json.c +++ /dev/null @@ -1,106 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#include "rrdset2json.h" - -static int process_label_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) { - BUFFER *wb = data; - buffer_json_member_add_string_or_empty(wb, name, value); - return 1; -} - -void chart_labels2json(RRDSET *st, BUFFER *wb) -{ - if(unlikely(!st->rrdlabels)) - return; - - rrdlabels_walkthrough_read(st->rrdlabels, process_label_callback, wb); -} - -// generate JSON for the /api/v1/chart API call -void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memory_used) -{ - time_t first_entry_t = rrdset_first_entry_s(st); - time_t last_entry_t = rrdset_last_entry_s(st); - char buf[RRD_ID_LENGTH_MAX + 16]; - - buffer_json_member_add_string(wb, "id", rrdset_id(st)); - buffer_json_member_add_string(wb, "name", rrdset_name(st)); - buffer_json_member_add_string(wb, "type", rrdset_parts_type(st)); - buffer_json_member_add_string(wb, "family", rrdset_family(st)); - buffer_json_member_add_string(wb, "context", rrdset_context(st)); - snprintfz(buf, RRD_ID_LENGTH_MAX + 15, "%s (%s)", rrdset_title(st), rrdset_name(st)); - buffer_json_member_add_string(wb, "title", buf); - buffer_json_member_add_int64(wb, "priority", st->priority); - buffer_json_member_add_string(wb, "plugin", rrdset_plugin_name(st)); - buffer_json_member_add_string(wb, "module", rrdset_module_name(st)); - buffer_json_member_add_string(wb, "units", rrdset_units(st)); - - snprintfz(buf, RRD_ID_LENGTH_MAX + 15, "/api/v1/data?chart=%s", rrdset_name(st)); - buffer_json_member_add_string(wb, "data_url", buf); - - buffer_json_member_add_string(wb, "chart_type", rrdset_type_name(st->chart_type)); - buffer_json_member_add_int64(wb, "duration", (int64_t)(last_entry_t - first_entry_t + st->update_every)); - buffer_json_member_add_int64(wb, "first_entry", (int64_t)first_entry_t); - buffer_json_member_add_int64(wb, "last_entry", (int64_t)last_entry_t); - buffer_json_member_add_int64(wb, "update_every", (int64_t)st->update_every); - - unsigned long memory = sizeof(RRDSET); - - size_t dimensions = 0; - buffer_json_member_add_object(wb, "dimensions"); - { - RRDDIM *rd; - rrddim_foreach_read(rd, st) - { - if (rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN) || rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) - continue; - - memory += rrddim_size() + rd->db.memsize; - - buffer_json_member_add_object(wb, rrddim_id(rd)); - buffer_json_member_add_string(wb, "name", rrddim_name(rd)); - buffer_json_object_close(wb); - - dimensions++; - } - rrddim_foreach_done(rd); - } - buffer_json_object_close(wb); - - if(dimensions_count) *dimensions_count += dimensions; - if(memory_used) *memory_used += memory; - - buffer_json_member_add_object(wb, "chart_variables"); - health_api_v1_chart_custom_variables2json(st, wb); - buffer_json_object_close(wb); - - buffer_json_member_add_double(wb, "green", st->green); - buffer_json_member_add_double(wb, "red", st->red); - - { - buffer_json_member_add_object(wb, "alarms"); - RRDCALC *rc; - rw_spinlock_read_lock(&st->alerts.spinlock); - DOUBLE_LINKED_LIST_FOREACH_FORWARD(st->alerts.base, rc, prev, next) - { - { - buffer_json_member_add_object(wb, rrdcalc_name(rc)); - buffer_json_member_add_string_or_empty(wb, "id", rrdcalc_name(rc)); - buffer_json_member_add_string_or_empty(wb, "status", rrdcalc_status2string(rc->status)); - buffer_json_member_add_string_or_empty(wb, "units", rrdcalc_units(rc)); - buffer_json_member_add_int64(wb, "duration", (int64_t)rc->update_every); - buffer_json_object_close(wb); - } - } - rw_spinlock_read_unlock(&st->alerts.spinlock); - buffer_json_object_close(wb); - } - - buffer_json_member_add_object(wb, "chart_labels"); - chart_labels2json(st, wb); - buffer_json_object_close(wb); - - buffer_json_member_add_object(wb, "functions"); - chart_functions2json(st, wb); - buffer_json_object_close(wb); -} diff --git a/web/api/formatters/rrdset2json.h b/web/api/formatters/rrdset2json.h deleted file mode 100644 index 8b325c65d..000000000 --- a/web/api/formatters/rrdset2json.h +++ /dev/null @@ -1,10 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#ifndef NETDATA_API_FORMATTER_RRDSET2JSON_H -#define NETDATA_API_FORMATTER_RRDSET2JSON_H - -#include "rrd2json.h" - -void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memory_used); - -#endif //NETDATA_API_FORMATTER_RRDSET2JSON_H diff --git a/web/api/formatters/ssv/Makefile.am b/web/api/formatters/ssv/Makefile.am deleted file mode 100644 index 161784b8f..000000000 --- a/web/api/formatters/ssv/Makefile.am +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-License-Identifier: GPL-3.0-or-later - -AUTOMAKE_OPTIONS = subdir-objects -MAINTAINERCLEANFILES = $(srcdir)/Makefile.in - -dist_noinst_DATA = \ - README.md \ - $(NULL) diff --git a/web/api/formatters/ssv/README.md b/web/api/formatters/ssv/README.md deleted file mode 100644 index 434d56721..000000000 --- a/web/api/formatters/ssv/README.md +++ /dev/null @@ -1,63 +0,0 @@ -<!-- -title: "SSV formatter" -custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/formatters/ssv/README.md -sidebar_label: "SSV formatter" -learn_status: "Published" -learn_topic_type: "References" -learn_rel_path: "Developers/Web/Api/Formatters" ---> - -# SSV formatter - -The SSV formatter sums all dimensions in [results of database queries](https://github.com/netdata/netdata/blob/master/web/api/queries/README.md) -to a single value and returns a list of such values showing how it changes through time. - -It supports the following formats: - -| format | content type | description | -|:----:|:----------:|:----------| -| `ssv` | text/plain | a space separated list of values | -| `ssvcomma` | text/plain | a comma separated list of values | -| `array` | application/json | a JSON array | - -The SSV formatter respects the following API `&options=`: - -| option | supported | description | -| :----:|:-------:|:----------| -| `nonzero` | yes | to return only the dimensions that have at least a non-zero value | -| `flip` | yes | to return the numbers older to newer (the default is newer to older) | -| `percent` | yes | to replace all values with their percentage over the row total | -| `abs` | yes | to turn all values positive, before using them | -| `min2max` | yes | to return the delta from the minimum value to the maximum value (across dimensions) | - -## Examples - -Get the average system CPU utilization of the last hour, in 6 values (one every 10 minutes): - -```bash -# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=system.cpu&format=ssv&after=-3600&points=6&group=average' -1.741352 1.6800467 1.769411 1.6761112 1.629862 1.6807968 -``` - ---- - -Get the total mysql bandwidth (in + out) for the last hour, in 6 values (one every 10 minutes): - -Netdata returns bandwidth in `kilobits`. - -```bash -# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=mysql_local.net&format=ssvcomma&after=-3600&points=6&group=sum&options=abs' -72618.7936215,72618.778889,72618.788084,72618.9195918,72618.7760612,72618.6712421 -``` - ---- - -Get the web server max connections for the last hour, in 12 values (one every 5 minutes) -in a JSON array: - -```bash -# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=nginx_local.connections&format=array&after=-3600&points=12&group=max' -[278,258,268,239,259,260,243,266,278,318,264,258] -``` - - diff --git a/web/api/formatters/ssv/ssv.c b/web/api/formatters/ssv/ssv.c deleted file mode 100644 index 2eb26b459..000000000 --- a/web/api/formatters/ssv/ssv.c +++ /dev/null @@ -1,45 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#include "ssv.h" - -void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, const char *separator, const char *suffix) { - //netdata_log_info("RRD2SSV(): %s: BEGIN", r->st->id); - long i; - - buffer_strcat(wb, prefix); - long start = 0, end = rrdr_rows(r), step = 1; - if(!(options & RRDR_OPTION_REVERSED)) { - start = rrdr_rows(r) - 1; - end = -1; - step = -1; - } - - // for each line in the array - for(i = start; i != end ;i += step) { - int all_values_are_null = 0; - NETDATA_DOUBLE v = rrdr2value(r, i, options, &all_values_are_null, NULL); - - if(likely(i != start)) { - if(r->view.min > v) r->view.min = v; - if(r->view.max < v) r->view.max = v; - } - else { - r->view.min = v; - r->view.max = v; - } - - if(likely(i != start)) - buffer_strcat(wb, separator); - - if(all_values_are_null) { - if(options & RRDR_OPTION_NULL2ZERO) - buffer_strcat(wb, "0"); - else - buffer_strcat(wb, "null"); - } - else - buffer_print_netdata_double(wb, v); - } - buffer_strcat(wb, suffix); - //netdata_log_info("RRD2SSV(): %s: END", r->st->id); -} diff --git a/web/api/formatters/ssv/ssv.h b/web/api/formatters/ssv/ssv.h deleted file mode 100644 index f7d4a9548..000000000 --- a/web/api/formatters/ssv/ssv.h +++ /dev/null @@ -1,10 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#ifndef NETDATA_API_FORMATTER_SSV_H -#define NETDATA_API_FORMATTER_SSV_H - -#include "../rrd2json.h" - -void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, const char *separator, const char *suffix); - -#endif //NETDATA_API_FORMATTER_SSV_H diff --git a/web/api/formatters/value/Makefile.am b/web/api/formatters/value/Makefile.am deleted file mode 100644 index 161784b8f..000000000 --- a/web/api/formatters/value/Makefile.am +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-License-Identifier: GPL-3.0-or-later - -AUTOMAKE_OPTIONS = subdir-objects -MAINTAINERCLEANFILES = $(srcdir)/Makefile.in - -dist_noinst_DATA = \ - README.md \ - $(NULL) diff --git a/web/api/formatters/value/README.md b/web/api/formatters/value/README.md deleted file mode 100644 index 5631d8207..000000000 --- a/web/api/formatters/value/README.md +++ /dev/null @@ -1,28 +0,0 @@ -<!-- -title: "Value formatter" -custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/formatters/value/README.md -sidebar_label: "Value formatter" -learn_status: "Published" -learn_topic_type: "References" -learn_rel_path: "Developers/Web/Api/Formatters" ---> - -# Value formatter - -The Value formatter presents [results of database queries](https://github.com/netdata/netdata/blob/master/web/api/queries/README.md) as a single value. - -To calculate the single value to be returned, it sums the values of all dimensions. - -The Value formatter respects the following API `&options=`: - -| option | supported | description | -|:----: |:-------: |:---------- | -| `percent` | yes | to replace all values with their percentage over the row total| -| `abs` | yes | to turn all values positive, before using them | -| `min2max` | yes | to return the delta from the minimum value to the maximum value (across dimensions)| - -The Value formatter is not exposed by the API by itself. -Instead it is used by the [`ssv`](https://github.com/netdata/netdata/blob/master/web/api/formatters/ssv/README.md) formatter -and [health monitoring queries](https://github.com/netdata/netdata/blob/master/health/README.md). - - diff --git a/web/api/formatters/value/value.c b/web/api/formatters/value/value.c deleted file mode 100644 index 1d07f62f6..000000000 --- a/web/api/formatters/value/value.c +++ /dev/null @@ -1,151 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#include "value.h" - - -inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, NETDATA_DOUBLE *anomaly_rate) { - size_t c; - - NETDATA_DOUBLE *cn = &r->v[ i * r->d ]; - RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ]; - NETDATA_DOUBLE *ar = &r->ar[ i * r->d ]; - - NETDATA_DOUBLE sum = 0, min = 0, max = 0, v; - int all_null = 1, init = 1; - - NETDATA_DOUBLE total_anomaly_rate = 0; - - // for each dimension - for (c = 0; c < r->d ; c++) { - if(!rrdr_dimension_should_be_exposed(r->od[c], options)) - continue; - - NETDATA_DOUBLE n = cn[c]; - - if(unlikely(init)) { - if(n > 0) { - min = 0; - max = n; - } - else { - min = n; - max = 0; - } - init = 0; - } - - if(likely(!(co[c] & RRDR_VALUE_EMPTY))) { - all_null = 0; - sum += n; - } - - if(n < min) min = n; - if(n > max) max = n; - - total_anomaly_rate += ar[c]; - } - - if(anomaly_rate) { - if(!r->d) *anomaly_rate = 0; - else *anomaly_rate = total_anomaly_rate / (NETDATA_DOUBLE)r->d; - } - - if(unlikely(all_null)) { - if(likely(all_values_are_null)) - *all_values_are_null = 1; - return 0; - } - else { - if(likely(all_values_are_null)) - *all_values_are_null = 0; - } - - if(options & RRDR_OPTION_MIN2MAX) - v = max - min; - else - v = sum; - - return v; -} - -QUERY_VALUE rrdmetric2value(RRDHOST *host, - struct rrdcontext_acquired *rca, struct rrdinstance_acquired *ria, struct rrdmetric_acquired *rma, - time_t after, time_t before, - RRDR_OPTIONS options, RRDR_TIME_GROUPING time_group_method, const char *time_group_options, - size_t tier, time_t timeout, QUERY_SOURCE query_source, STORAGE_PRIORITY priority -) { - QUERY_TARGET_REQUEST qtr = { - .version = 1, - .host = host, - .rca = rca, - .ria = ria, - .rma = rma, - .after = after, - .before = before, - .points = 1, - .options = options, - .time_group_method = time_group_method, - .time_group_options = time_group_options, - .tier = tier, - .timeout_ms = timeout, - .query_source = query_source, - .priority = priority, - }; - - ONEWAYALLOC *owa = onewayalloc_create(16 * 1024); - QUERY_TARGET *qt = query_target_create(&qtr); - RRDR *r = rrd2rrdr(owa, qt); - - QUERY_VALUE qv; - - if(!r || rrdr_rows(r) == 0) { - qv = (QUERY_VALUE) { - .value = NAN, - .anomaly_rate = NAN, - .sp = { - .count = 0, - .min = NAN, - .max = NAN, - .sum = NAN, - .anomaly_count = 0, - }, - .duration_ut = (r) ? r->internal.qt->timings.executed_ut - r->internal.qt->timings.received_ut : 0, - }; - } - else { - qv = (QUERY_VALUE) { - .after = r->view.after, - .before = r->view.before, - .points_read = r->stats.db_points_read, - .result_points = r->stats.result_points_generated, - .sp = { - .count = 0, - }, - .duration_ut = r->internal.qt->timings.executed_ut - r->internal.qt->timings.received_ut, - }; - - for(size_t d = 0; d < r->internal.qt->query.used ;d++) { - if(!rrdr_dimension_should_be_exposed(r->internal.qt->query.array[d].status, options)) - continue; - - storage_point_merge_to(qv.sp, r->internal.qt->query.array[d].query_points); - } - - for(size_t t = 0; t < storage_tiers ;t++) - qv.storage_points_per_tier[t] = r->internal.qt->db.tiers[t].points; - - long i = (!(options & RRDR_OPTION_REVERSED))?(long)rrdr_rows(r) - 1:0; - int all_values_are_null = 0; - qv.value = rrdr2value(r, i, options, &all_values_are_null, &qv.anomaly_rate); - if(all_values_are_null) { - qv.value = NAN; - qv.anomaly_rate = NAN; - } - } - - rrdr_free(owa, r); - query_target_release(qt); - onewayalloc_destroy(owa); - - return qv; -} diff --git a/web/api/formatters/value/value.h b/web/api/formatters/value/value.h deleted file mode 100644 index 072ca14f8..000000000 --- a/web/api/formatters/value/value.h +++ /dev/null @@ -1,33 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#ifndef NETDATA_API_FORMATTER_VALUE_H -#define NETDATA_API_FORMATTER_VALUE_H - -#include "../rrd2json.h" - -typedef struct storage_value { - NETDATA_DOUBLE value; - NETDATA_DOUBLE anomaly_rate; - time_t after; - time_t before; - size_t points_read; - size_t storage_points_per_tier[RRD_STORAGE_TIERS]; - size_t result_points; - STORAGE_POINT sp; - usec_t duration_ut; -} QUERY_VALUE; - -struct rrdmetric_acquired; -struct rrdinstance_acquired; -struct rrdcontext_acquired; - -QUERY_VALUE rrdmetric2value(RRDHOST *host, - struct rrdcontext_acquired *rca, struct rrdinstance_acquired *ria, struct rrdmetric_acquired *rma, - time_t after, time_t before, - RRDR_OPTIONS options, RRDR_TIME_GROUPING time_group_method, const char *time_group_options, - size_t tier, time_t timeout, QUERY_SOURCE query_source, STORAGE_PRIORITY priority -); - -NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, NETDATA_DOUBLE *anomaly_rate); - -#endif //NETDATA_API_FORMATTER_VALUE_H |