From b5f8ee61a7f7e9bd291dd26b0585d03eb686c941 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 5 May 2024 13:19:16 +0200 Subject: Adding upstream version 1.46.3. Signed-off-by: Daniel Baumann --- src/web/api/formatters/README.md | 82 ++ src/web/api/formatters/charts2json.c | 104 +++ src/web/api/formatters/charts2json.h | 11 + src/web/api/formatters/csv/README.md | 148 +++ src/web/api/formatters/csv/csv.c | 109 +++ src/web/api/formatters/csv/csv.h | 12 + src/web/api/formatters/json/README.md | 160 ++++ src/web/api/formatters/json/json.c | 350 +++++++ src/web/api/formatters/json/json.h | 11 + src/web/api/formatters/json_wrapper.c | 1578 ++++++++++++++++++++++++++++++++ src/web/api/formatters/json_wrapper.h | 21 + src/web/api/formatters/rrd2json.c | 391 ++++++++ src/web/api/formatters/rrd2json.h | 98 ++ src/web/api/formatters/rrdset2json.c | 106 +++ src/web/api/formatters/rrdset2json.h | 10 + src/web/api/formatters/ssv/README.md | 63 ++ src/web/api/formatters/ssv/ssv.c | 45 + src/web/api/formatters/ssv/ssv.h | 10 + src/web/api/formatters/value/README.md | 28 + src/web/api/formatters/value/value.c | 153 ++++ src/web/api/formatters/value/value.h | 33 + 21 files changed, 3523 insertions(+) create mode 100644 src/web/api/formatters/README.md create mode 100644 src/web/api/formatters/charts2json.c create mode 100644 src/web/api/formatters/charts2json.h create mode 100644 src/web/api/formatters/csv/README.md create mode 100644 src/web/api/formatters/csv/csv.c create mode 100644 src/web/api/formatters/csv/csv.h create mode 100644 src/web/api/formatters/json/README.md create mode 100644 src/web/api/formatters/json/json.c create mode 100644 src/web/api/formatters/json/json.h create mode 100644 src/web/api/formatters/json_wrapper.c create mode 100644 src/web/api/formatters/json_wrapper.h create mode 100644 src/web/api/formatters/rrd2json.c create mode 100644 src/web/api/formatters/rrd2json.h create mode 100644 src/web/api/formatters/rrdset2json.c create mode 100644 src/web/api/formatters/rrdset2json.h create mode 100644 src/web/api/formatters/ssv/README.md create mode 100644 src/web/api/formatters/ssv/ssv.c create mode 100644 src/web/api/formatters/ssv/ssv.h create mode 100644 src/web/api/formatters/value/README.md create mode 100644 src/web/api/formatters/value/value.c create mode 100644 src/web/api/formatters/value/value.h (limited to 'src/web/api/formatters') diff --git a/src/web/api/formatters/README.md b/src/web/api/formatters/README.md new file mode 100644 index 000000000..6347f5fb4 --- /dev/null +++ b/src/web/api/formatters/README.md @@ -0,0 +1,82 @@ + + +# Query formatting + +API data queries need to be formatted before returned to the caller. +Using API parameters, the caller may define the format he/she wishes to get back. + +The following formats are supported: + +| format|module|content type|description| +|:----:|:----:|:----------:|:----------| +| `array`|[ssv](/src/web/api/formatters/ssv/README.md)|application/json|a JSON array| +| `csv`|[csv](/src/web/api/formatters/csv/README.md)|text/plain|a text table, comma separated, with a header line (dimension names) and `\r\n` at the end of the lines| +| `csvjsonarray`|[csv](/src/web/api/formatters/csv/README.md)|application/json|a JSON array, with each row as another array (the first row has the dimension names)| +| `datasource`|[json](/src/web/api/formatters/json/README.md)|application/json|a Google Visualization Provider `datasource` javascript callback| +| `datatable`|[json](/src/web/api/formatters/json/README.md)|application/json|a Google `datatable`| +| `html`|[csv](/src/web/api/formatters/csv/README.md)|text/html|an html table| +| `json`|[json](/src/web/api/formatters/json/README.md)|application/json|a JSON object| +| `jsonp`|[json](/src/web/api/formatters/json/README.md)|application/json|a JSONP javascript callback| +| `markdown`|[csv](/src/web/api/formatters/csv/README.md)|text/plain|a markdown table| +| `ssv`|[ssv](/src/web/api/formatters/ssv/README.md)|text/plain|a space separated list of values| +| `ssvcomma`|[ssv](/src/web/api/formatters/ssv/README.md)|text/plain|a comma separated list of values| +| `tsv`|[csv](/src/web/api/formatters/csv/README.md)|text/plain|a TAB delimited `csv` (MS Excel flavor)| + +For examples of each format, check the relative module documentation. + +## Metadata with the `jsonwrap` option + +All data queries can be encapsulated to JSON object having metadata about the query and the results. + +This is done by adding the `options=jsonwrap` to the API URL (if there are other `options` append +`,jsonwrap` to the existing ones). + +This is such an object: + +```bash +# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=system.cpu&after=-3600&points=6&group=average&format=csv&options=nonzero,jsonwrap' +{ + "api": 1, + "id": "system.cpu", + "name": "system.cpu", + "view_update_every": 600, + "update_every": 1, + "first_entry": 1540387074, + "last_entry": 1540647070, + "before": 1540647000, + "after": 1540644000, + "dimension_names": ["steal", "softirq", "user", "system", "iowait"], + "dimension_ids": ["steal", "softirq", "user", "system", "iowait"], + "latest_values": [0, 0.2493766, 1.745636, 0.4987531, 0], + "view_latest_values": [0.0158314, 0.0516506, 0.866549, 0.7196127, 0.0050002], + "dimensions": 5, + "points": 6, + "format": "csv", + "result": "time,steal,softirq,user,system,iowait\n2018-10-27 13:30:00,0.0158314,0.0516506,0.866549,0.7196127,0.0050002\n2018-10-27 13:20:00,0.0149856,0.0529183,0.8673155,0.7121144,0.0049979\n2018-10-27 13:10:00,0.0137501,0.053315,0.8578097,0.7197613,0.0054209\n2018-10-27 13:00:00,0.0154252,0.0554688,0.899432,0.7200638,0.0067252\n2018-10-27 12:50:00,0.0145866,0.0495922,0.8404341,0.7011141,0.0041688\n2018-10-27 12:40:00,0.0162366,0.0595954,0.8827475,0.7020573,0.0041636\n", + "min": 0, + "max": 0 +} +``` + +## Downloading data query result files + +Following the [Google Visualization Provider guidelines](https://developers.google.com/chart/interactive/docs/dev/implementing_data_source), +Netdata supports parsing `tqx` options. + +Using these options, any Netdata data query can instruct the web browser to download +the result and save it under a given filename. + +For example, to download a CSV file with CPU utilization of the last hour, +[click here](https://registry.my-netdata.io/api/v1/data?chart=system.cpu&after=-3600&format=csv&options=nonzero&tqx=outFileName:system+cpu+utilization+of+the+last_hour.csv). + +This is done by appending `&tqx=outFileName:FILENAME` to any data query. +The output will be in the format given with `&format=`. + + diff --git a/src/web/api/formatters/charts2json.c b/src/web/api/formatters/charts2json.c new file mode 100644 index 000000000..0b45d77c4 --- /dev/null +++ b/src/web/api/formatters/charts2json.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "charts2json.h" + +// generate JSON for the /api/v1/charts API call + +const char* get_release_channel() { + static int use_stable = -1; + + if (use_stable == -1) { + char filename[FILENAME_MAX + 1]; + snprintfz(filename, FILENAME_MAX, "%s/.environment", netdata_configured_user_config_dir); + procfile *ff = procfile_open(filename, "=", PROCFILE_FLAG_NO_ERROR_ON_FILE_IO); + if (ff) { + procfile_set_quotes(ff, "'\""); + ff = procfile_readall(ff); + if (ff) { + unsigned int i; + for (i = 0; i < procfile_lines(ff); i++) { + if (!procfile_linewords(ff, i)) + continue; + if (!strcmp(procfile_lineword(ff, i, 0), "RELEASE_CHANNEL")) { + if (!strcmp(procfile_lineword(ff, i, 1), "stable")) + use_stable = 1; + else if (!strcmp(procfile_lineword(ff, i, 1), "nightly")) + use_stable = 0; + break; + } + } + procfile_close(ff); + } + } + if (use_stable == -1) + use_stable = strchr(NETDATA_VERSION, '-') ? 0 : 1; + } + return (use_stable)?"stable":"nightly"; +} + +void charts2json(RRDHOST *host, BUFFER *wb) { + static char *custom_dashboard_info_js_filename = NULL; + size_t c = 0, dimensions = 0, memory = 0, alarms = 0; + RRDSET *st; + + time_t now = now_realtime_sec(); + + if(unlikely(!custom_dashboard_info_js_filename)) + custom_dashboard_info_js_filename = config_get(CONFIG_SECTION_WEB, "custom dashboard_info.js", ""); + + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT); + + buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(host)); + buffer_json_member_add_string(wb, "version", rrdhost_program_version(host)); + buffer_json_member_add_string(wb, "release_channel", get_release_channel()); + buffer_json_member_add_string(wb, "os", rrdhost_os(host)); + buffer_json_member_add_string(wb, "timezone", rrdhost_timezone(host)); + buffer_json_member_add_int64(wb, "update_every", host->rrd_update_every); + buffer_json_member_add_int64(wb, "history", host->rrd_history_entries); + buffer_json_member_add_string(wb, "memory_mode", rrd_memory_mode_name(host->rrd_memory_mode)); + buffer_json_member_add_string(wb, "custom_info", custom_dashboard_info_js_filename); + + buffer_json_member_add_object(wb, "charts"); + rrdset_foreach_read(st, host) { + if (rrdset_is_available_for_viewers(st)) { + + buffer_json_member_add_object(wb, rrdset_id(st)); + rrdset2json(st, wb, &dimensions, &memory); + buffer_json_object_close(wb); + st->last_accessed_time_s = now; + c++; + } + } + rrdset_foreach_done(st); + buffer_json_object_close(wb); + + RRDCALC *rc; + foreach_rrdcalc_in_rrdhost_read(host, rc) { + if(rc->rrdset) + alarms++; + } + foreach_rrdcalc_in_rrdhost_done(rc); + + buffer_json_member_add_int64(wb, "charts_count", (int64_t) c); + buffer_json_member_add_int64(wb, "dimensions_count", (int64_t) dimensions); + buffer_json_member_add_int64(wb, "alarms_count", (int64_t)alarms); + buffer_json_member_add_int64(wb, "rrd_memory_bytes", (int64_t)memory); + buffer_json_member_add_int64(wb, "hosts_count", (int64_t) rrdhost_hosts_available()); + + buffer_json_member_add_array(wb, "hosts"); + { + rrd_rdlock(); + RRDHOST *h; + rrdhost_foreach_read(h) { + if(!rrdhost_should_be_removed(h, host, now) /*&& !rrdhost_flag_check(h, RRDHOST_FLAG_ARCHIVED) */) { + buffer_json_add_array_item_object(wb); + buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(h)); + buffer_json_object_close(wb); + } + } + rrd_rdunlock(); + } + buffer_json_array_close(wb); + + buffer_json_finalize(wb); +} diff --git a/src/web/api/formatters/charts2json.h b/src/web/api/formatters/charts2json.h new file mode 100644 index 000000000..7b07af5a0 --- /dev/null +++ b/src/web/api/formatters/charts2json.h @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_API_FORMATTER_CHARTS2JSON_H +#define NETDATA_API_FORMATTER_CHARTS2JSON_H + +#include "rrd2json.h" + +void charts2json(RRDHOST *host, BUFFER *wb); +const char* get_release_channel(); + +#endif //NETDATA_API_FORMATTER_CHARTS2JSON_H diff --git a/src/web/api/formatters/csv/README.md b/src/web/api/formatters/csv/README.md new file mode 100644 index 000000000..e60aab57b --- /dev/null +++ b/src/web/api/formatters/csv/README.md @@ -0,0 +1,148 @@ + + +# CSV formatter + +The CSV formatter presents [results of database queries](/src/web/api/queries/README.md) in the following formats: + +| format|content type|description| +| :----:|:----------:|:----------| +| `csv`|text/plain|a text table, comma separated, with a header line (dimension names) and `\r\n` at the end of the lines| +| `csvjsonarray`|application/json|a JSON array, with each row as another array (the first row has the dimension names)| +| `tsv`|text/plain|like `csv` but TAB is used instead of comma to separate values (MS Excel flavor)| +| `html`|text/html|an html table| +| `markdown`|text/plain|markdown table| + +In all formats the date and time is the first column. + +The CSV formatter respects the following API `&options=`: + +| option|supported|description| +|:----:|:-------:|:----------| +| `nonzero`|yes|to return only the dimensions that have at least a non-zero value| +| `flip`|yes|to return the rows older to newer (the default is newer to older)| +| `seconds`|yes|to return the date and time in unix timestamp| +| `ms`|yes|to return the date and time in unit timestamp as milliseconds| +| `percent`|yes|to replace all values with their percentage over the row total| +| `abs`|yes|to turn all values positive| +| `null2zero`|yes|to replace gaps with zeros (the default prints the string `null`| + +## Examples + +Get the system total bandwidth for all physical network interfaces, over the last hour, +in 6 rows (one for every 10 minutes), in `csv` format: + +Netdata always returns bandwidth in `kilobits`. + +```bash +# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=system.net&format=csv&after=-3600&group=sum&points=6&options=abs' +time,received,sent +2018-10-26 23:50:00,90214.67847,215137.79762 +2018-10-26 23:40:00,90126.32286,238587.57522 +2018-10-26 23:30:00,86061.22688,213389.23526 +2018-10-26 23:20:00,85590.75164,206129.01608 +2018-10-26 23:10:00,83163.30691,194311.77384 +2018-10-26 23:00:00,85167.29657,197538.07773 +``` + +--- + +Get the max RAM used by the SQL server and any cron jobs, over the last hour, in 2 rows (one for every 30 +minutes), in `tsv` format, and format the date and time as unix timestamp: + +Netdata always returns memory in `MB`. + +```bash +# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=apps.mem&format=tsv&after=-3600&group=max&points=2&options=nonzero,seconds&dimensions=sql,cron' +time sql cron +1540598400 61.95703 0.25 +1540596600 61.95703 0.25 +``` + +--- + +Get an HTML table of the last 4 values (4 seconds) of system CPU utilization: + +Netdata always returns CPU utilization as `%`. + +```bash +# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=system.cpu&format=html&after=-4&options=nonzero' + +
+ + + + + + +
timesoftirqusersystem
2018-10-27 00:16:070.2510.75
2018-10-27 00:16:0601.00250630.5012531
2018-10-27 00:16:05010.75
2018-10-27 00:16:0401.00250630.7518797
+
+ +``` + +This is how it looks when rendered by a web browser: + +![image](https://user-images.githubusercontent.com/2662304/47597887-bafbf480-d99c-11e8-864a-d880bb8d2e5b.png) + +--- + +Get a JSON array with the average bandwidth rate of the mysql server, over the last hour, in 6 values +(one every 10 minutes), and return the date and time in milliseconds: + +Netdata always returns bandwidth rates in `kilobits/s`. + +```bash +# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=mysql_local.net&format=csvjsonarray&after=-3600&points=6&group=average&options=abs,ms' +[ +["time","in","out"], +[1540599600000,0.7499986,120.2810185], +[1540599000000,0.7500019,120.2815509], +[1540598400000,0.7499999,120.2812319], +[1540597800000,0.7500044,120.2819634], +[1540597200000,0.7499968,120.2807337], +[1540596600000,0.7499988,120.2810527] +] +``` + +--- + +Get the number of processes started per minute, for the last 10 minutes, in `markdown` format: + +```bash +# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=system.forks&format=markdown&after=-600&points=10&group=sum' +time | started +:---: |:---: +2018-10-27 03:52:00| 245.1706149 +2018-10-27 03:51:00| 152.6654636 +2018-10-27 03:50:00| 163.1755789 +2018-10-27 03:49:00| 176.1574766 +2018-10-27 03:48:00| 178.0137076 +2018-10-27 03:47:00| 183.8306543 +2018-10-27 03:46:00| 264.1635621 +2018-10-27 03:45:00| 205.001551 +2018-10-27 03:44:00| 7026.9852167 +2018-10-27 03:43:00| 205.9904794 +``` + +And this is how it looks when formatted: + +| time | started | +|:--:|:-----:| +| 2018-10-27 03:52:00 | 245.1706149 | +| 2018-10-27 03:51:00 | 152.6654636 | +| 2018-10-27 03:50:00 | 163.1755789 | +| 2018-10-27 03:49:00 | 176.1574766 | +| 2018-10-27 03:48:00 | 178.0137076 | +| 2018-10-27 03:47:00 | 183.8306543 | +| 2018-10-27 03:46:00 | 264.1635621 | +| 2018-10-27 03:45:00 | 205.001551 | +| 2018-10-27 03:44:00 | 7026.9852167 | +| 2018-10-27 03:43:00 | 205.9904794 | + + diff --git a/src/web/api/formatters/csv/csv.c b/src/web/api/formatters/csv/csv.c new file mode 100644 index 000000000..d81ddb34e --- /dev/null +++ b/src/web/api/formatters/csv/csv.c @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "libnetdata/libnetdata.h" +#include "csv.h" + +void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines) { + //netdata_log_info("RRD2CSV(): %s: BEGIN", r->st->id); + long c, i; + const long used = (long)r->d; + + // print the csv header + for(c = 0, i = 0; c < used ; c++) { + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + if(!i) { + buffer_strcat(wb, startline); + if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); + buffer_strcat(wb, "time"); + if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); + } + buffer_strcat(wb, separator); + if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); + buffer_strcat(wb, string2str(r->dn[c])); + if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); + i++; + } + buffer_strcat(wb, endline); + + if(format == DATASOURCE_CSV_MARKDOWN) { + // print the --- line after header + for(c = 0, i = 0; c < used ;c++) { + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + if(!i) { + buffer_strcat(wb, startline); + if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); + buffer_strcat(wb, ":---:"); + if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); + } + buffer_strcat(wb, separator); + if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); + buffer_strcat(wb, ":---:"); + if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); + i++; + } + buffer_strcat(wb, endline); + } + + if(!i) { + // no dimensions present + return; + } + + long start = 0, end = rrdr_rows(r), step = 1; + if(!(options & RRDR_OPTION_REVERSED)) { + start = rrdr_rows(r) - 1; + end = -1; + step = -1; + } + + // for each line in the array + for(i = start; i != end ;i += step) { + NETDATA_DOUBLE *cn = &r->v[ i * r->d ]; + RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ]; + + buffer_strcat(wb, betweenlines); + buffer_strcat(wb, startline); + + time_t now = r->t[i]; + + if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) { + // print the timestamp of the line + buffer_print_netdata_double(wb, (NETDATA_DOUBLE) now); + // in ms + if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000"); + } + else { + // generate the local date time + struct tm tmbuf, *tm = localtime_r(&now, &tmbuf); + if(!tm) { + netdata_log_error("localtime() failed."); continue; } + buffer_date(wb, tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec); + } + + // for each dimension + for(c = 0; c < used ;c++) { + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + buffer_strcat(wb, separator); + + NETDATA_DOUBLE n = cn[c]; + + if(co[c] & RRDR_VALUE_EMPTY) { + if(options & RRDR_OPTION_NULL2ZERO) + buffer_strcat(wb, "0"); + else + buffer_strcat(wb, "null"); + } + else + buffer_print_netdata_double(wb, n); + } + + buffer_strcat(wb, endline); + } + //netdata_log_info("RRD2CSV(): %s: END", r->st->id); +} diff --git a/src/web/api/formatters/csv/csv.h b/src/web/api/formatters/csv/csv.h new file mode 100644 index 000000000..666d4c660 --- /dev/null +++ b/src/web/api/formatters/csv/csv.h @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_API_FORMATTER_CSV_H +#define NETDATA_API_FORMATTER_CSV_H + +#include "web/api/queries/rrdr.h" + +void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines); + +#include "../rrd2json.h" + +#endif //NETDATA_API_FORMATTER_CSV_H diff --git a/src/web/api/formatters/json/README.md b/src/web/api/formatters/json/README.md new file mode 100644 index 000000000..4137b0372 --- /dev/null +++ b/src/web/api/formatters/json/README.md @@ -0,0 +1,160 @@ + + +# JSON formatter + +The CSV formatter presents [results of database queries](/src/web/api/queries/README.md) in the following formats: + +| format | content type | description| +|:----:|:----------:|:----------| +| `json` | application/json | return the query result as a json object| +| `jsonp` | application/json | return the query result as a JSONP javascript callback| +| `datatable` | application/json | return the query result as a Google `datatable`| +| `datasource` | application/json | return the query result as a Google Visualization Provider `datasource` javascript callback| + +The CSV formatter respects the following API `&options=`: + +| option | supported | description| +|:----:|:-------:|:----------| +| `google_json` | yes | enable the Google flavor of JSON (using double quotes for strings and `Date()` function for dates| +| `objectrows` | yes | return each row as an object, instead of an array| +| `nonzero` | yes | to return only the dimensions that have at least a non-zero value| +| `flip` | yes | to return the rows older to newer (the default is newer to older)| +| `seconds` | yes | to return the date and time in unix timestamp| +| `ms` | yes | to return the date and time in unit timestamp as milliseconds| +| `percent` | yes | to replace all values with their percentage over the row total| +| `abs` | yes | to turn all values positive| +| `null2zero` | yes | to replace gaps with zeros (the default prints the string `null`| + +## Examples + +To show the differences between each format, in the following examples we query the same +chart (having just one dimension called `active`), changing only the query `format` and its `options`. + +> Using `format=json` and `options=` + +```bash +# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=nginx_local.connections&after=-3600&points=6&group=average&format=json&options=' +{ + "labels": ["time", "active"], + "data": + [ + [ 1540644600, 224.2516667], + [ 1540644000, 229.29], + [ 1540643400, 222.41], + [ 1540642800, 226.6816667], + [ 1540642200, 246.4083333], + [ 1540641600, 241.0966667] + ] +} +``` + +> Using `format=json` and `options=objectrows` + +```bash +# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=nginx_local.connections&after=-3600&points=6&group=average&format=json&options=objectrows' +{ + "labels": ["time", "active"], + "data": + [ + { "time": 1540644600, "active": 224.2516667}, + { "time": 1540644000, "active": 229.29}, + { "time": 1540643400, "active": 222.41}, + { "time": 1540642800, "active": 226.6816667}, + { "time": 1540642200, "active": 246.4083333}, + { "time": 1540641600, "active": 241.0966667} + ] +} +``` + +> Using `format=json` and `options=objectrows,google_json` + +```bash +# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=nginx_local.connections&after=-3600&points=6&group=average&formatjson&options=objectrows,google_json' +{ + "labels": ["time", "active"], + "data": + [ + { "time": new Date(2018,9,27,12,50,0), "active": 224.2516667}, + { "time": new Date(2018,9,27,12,40,0), "active": 229.29}, + { "time": new Date(2018,9,27,12,30,0), "active": 222.41}, + { "time": new Date(2018,9,27,12,20,0), "active": 226.6816667}, + { "time": new Date(2018,9,27,12,10,0), "active": 246.4083333}, + { "time": new Date(2018,9,27,12,0,0), "active": 241.0966667} + ] +} +``` + +> Using `format=jsonp` and `options=` + +```bash +curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=nginx_local.connections&after=-3600&points=6&group=average&formjsonp&options=' +callback({ + "labels": ["time", "active"], + "data": + [ + [ 1540645200, 235.885], + [ 1540644600, 224.2516667], + [ 1540644000, 229.29], + [ 1540643400, 222.41], + [ 1540642800, 226.6816667], + [ 1540642200, 246.4083333] + ] +}); +``` + +> Using `format=datatable` and `options=` + +```bash +curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=nginx_local.connections&after=-3600&points=6&group=average&formdatatable&options=' +{ + "cols": + [ + {"id":"","label":"time","pattern":"","type":"datetime"}, + {"id":"","label":"","pattern":"","type":"string","p":{"role":"annotation"}}, + {"id":"","label":"","pattern":"","type":"string","p":{"role":"annotationText"}}, + {"id":"","label":"active","pattern":"","type":"number"} + ], + "rows": + [ + {"c":[{"v":"Date(2018,9,27,13,0,0)"},{"v":null},{"v":null},{"v":235.885}]}, + {"c":[{"v":"Date(2018,9,27,12,50,0)"},{"v":null},{"v":null},{"v":224.2516667}]}, + {"c":[{"v":"Date(2018,9,27,12,40,0)"},{"v":null},{"v":null},{"v":229.29}]}, + {"c":[{"v":"Date(2018,9,27,12,30,0)"},{"v":null},{"v":null},{"v":222.41}]}, + {"c":[{"v":"Date(2018,9,27,12,20,0)"},{"v":null},{"v":null},{"v":226.6816667}]}, + {"c":[{"v":"Date(2018,9,27,12,10,0)"},{"v":null},{"v":null},{"v":246.4083333}]} + ] +} +``` + +> Using `format=datasource` and `options=` + +```bash +curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=nginx_local.connections&after=-3600&points=6&group=average&format=datasource&options=' +google.visualization.Query.setResponse({version:'0.6',reqId:'0',status:'ok',sig:'1540645368',table:{ + "cols": + [ + {"id":"","label":"time","pattern":"","type":"datetime"}, + {"id":"","label":"","pattern":"","type":"string","p":{"role":"annotation"}}, + {"id":"","label":"","pattern":"","type":"string","p":{"role":"annotationText"}}, + {"id":"","label":"active","pattern":"","type":"number"} + ], + "rows": + [ + {"c":[{"v":"Date(2018,9,27,13,0,0)"},{"v":null},{"v":null},{"v":235.885}]}, + {"c":[{"v":"Date(2018,9,27,12,50,0)"},{"v":null},{"v":null},{"v":224.2516667}]}, + {"c":[{"v":"Date(2018,9,27,12,40,0)"},{"v":null},{"v":null},{"v":229.29}]}, + {"c":[{"v":"Date(2018,9,27,12,30,0)"},{"v":null},{"v":null},{"v":222.41}]}, + {"c":[{"v":"Date(2018,9,27,12,20,0)"},{"v":null},{"v":null},{"v":226.6816667}]}, + {"c":[{"v":"Date(2018,9,27,12,10,0)"},{"v":null},{"v":null},{"v":246.4083333}]} + ] +}}); +``` + + diff --git a/src/web/api/formatters/json/json.c b/src/web/api/formatters/json/json.c new file mode 100644 index 000000000..7e3f400e9 --- /dev/null +++ b/src/web/api/formatters/json/json.c @@ -0,0 +1,350 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "json.h" + +#define JSON_DATES_JS 1 +#define JSON_DATES_TIMESTAMP 2 + +void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) { + //netdata_log_info("RRD2JSON(): %s: BEGIN", r->st->id); + int row_annotations = 0, dates, dates_with_new = 0; + char kq[2] = "", // key quote + sq[2] = "", // string quote + pre_label[101] = "", // before each label + post_label[101] = "", // after each label + pre_date[101] = "", // the beginning of line, to the date + post_date[101] = "", // closing the date + pre_value[101] = "", // before each value + post_value[101] = "", // after each value + post_line[101] = "", // at the end of each row + normal_annotation[201] = "", // default row annotation + overflow_annotation[201] = "", // overflow row annotation + data_begin[101] = "", // between labels and values + finish[101] = "", // at the end of everything + object_rows_time[101] = ""; + + if(datatable) { + dates = JSON_DATES_JS; + if( options & RRDR_OPTION_GOOGLE_JSON ) { + kq[0] = '\0'; + sq[0] = '\''; + } + else { + kq[0] = '"'; + sq[0] = '"'; + } + row_annotations = 1; + snprintfz(pre_date, 100, " {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq); + snprintfz(post_date, 100, "%s}", sq); + snprintfz(pre_label, 100, ",\n {%sid%s:%s%s,%slabel%s:%s", kq, kq, sq, sq, kq, kq, sq); + snprintfz(post_label, 100, "%s,%spattern%s:%s%s,%stype%s:%snumber%s}", sq, kq, kq, sq, sq, kq, kq, sq, sq); + snprintfz(pre_value, 100, ",{%sv%s:", kq, kq); + strcpy(post_value, "}"); + strcpy(post_line, "]}"); + snprintfz(data_begin, 100, "\n ],\n %srows%s:\n [\n", kq, kq); + strcpy(finish, "\n ]\n }"); + + snprintfz(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq); + snprintfz(normal_annotation, 200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq); + + buffer_sprintf(wb, "{\n %scols%s:\n [\n", kq, kq); + buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq); + buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq); + buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq); + + // remove the valueobjects flag + // google wants its own keys + if(options & RRDR_OPTION_OBJECTSROWS) + options &= ~RRDR_OPTION_OBJECTSROWS; + } + else { + kq[0] = '"'; + sq[0] = '"'; + if(options & RRDR_OPTION_GOOGLE_JSON) { + dates = JSON_DATES_JS; + dates_with_new = 1; + } + else { + dates = JSON_DATES_TIMESTAMP; + dates_with_new = 0; + } + if( options & RRDR_OPTION_OBJECTSROWS ) + strcpy(pre_date, " {"); + else + strcpy(pre_date, " ["); + strcpy(pre_label, ",\""); + strcpy(post_label, "\""); + strcpy(pre_value, ","); + if( options & RRDR_OPTION_OBJECTSROWS ) + strcpy(post_line, "}"); + else + strcpy(post_line, "]"); + snprintfz(data_begin, 100, "],\n %sdata%s:[\n", kq, kq); + strcpy(finish, "\n ]\n }"); + + buffer_sprintf(wb, "{\n %slabels%s:[", kq, kq); + buffer_sprintf(wb, "%stime%s", sq, sq); + + if( options & RRDR_OPTION_OBJECTSROWS ) + snprintfz(object_rows_time, 100, "%stime%s: ", kq, kq); + + } + + size_t pre_value_len = strlen(pre_value); + size_t post_value_len = strlen(post_value); + size_t pre_label_len = strlen(pre_label); + size_t post_label_len = strlen(post_label); + size_t pre_date_len = strlen(pre_date); + size_t post_date_len = strlen(post_date); + size_t post_line_len = strlen(post_line); + size_t normal_annotation_len = strlen(normal_annotation); + size_t overflow_annotation_len = strlen(overflow_annotation); + size_t object_rows_time_len = strlen(object_rows_time); + + // ------------------------------------------------------------------------- + // print the JSON header + + long c, i; + const long used = (long)r->d; + + // print the header lines + for(c = 0, i = 0; c < used ; c++) { + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + buffer_fast_strcat(wb, pre_label, pre_label_len); + buffer_strcat(wb, string2str(r->dn[c])); + buffer_fast_strcat(wb, post_label, post_label_len); + i++; + } + + if(!i) { + buffer_fast_strcat(wb, pre_label, pre_label_len); + buffer_fast_strcat(wb, "no data", 7); + buffer_fast_strcat(wb, post_label, post_label_len); + } + size_t total_number_of_dimensions = i; + + // print the beginning of row data + buffer_strcat(wb, data_begin); + + // if all dimensions are hidden, print a null + if(!i) { + buffer_strcat(wb, finish); + return; + } + + long start = 0, end = rrdr_rows(r), step = 1; + if(!(options & RRDR_OPTION_REVERSED)) { + start = rrdr_rows(r) - 1; + end = -1; + step = -1; + } + + // pre-allocate a large enough buffer for us + // this does not need to be accurate - it is just a hint to avoid multiple realloc(). + buffer_need_bytes(wb, + ( 20 * rrdr_rows(r)) // timestamp + json overhead + + ( (pre_value_len + post_value_len + 4) * total_number_of_dimensions * rrdr_rows(r) ) // number + ); + + // for each line in the array + for(i = start; i != end ;i += step) { + NETDATA_DOUBLE *cn = &r->v[ i * r->d ]; + RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ]; + NETDATA_DOUBLE *ar = &r->ar[ i * r->d ]; + + time_t now = r->t[i]; + + if(dates == JSON_DATES_JS) { + // generate the local date time + struct tm tmbuf, *tm = localtime_r(&now, &tmbuf); + if(!tm) { + netdata_log_error("localtime_r() failed."); continue; } + + if(likely(i != start)) buffer_fast_strcat(wb, ",\n", 2); + buffer_fast_strcat(wb, pre_date, pre_date_len); + + if( options & RRDR_OPTION_OBJECTSROWS ) + buffer_fast_strcat(wb, object_rows_time, object_rows_time_len); + + if(unlikely(dates_with_new)) + buffer_fast_strcat(wb, "new ", 4); + + buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec); + + buffer_fast_strcat(wb, post_date, post_date_len); + + if(unlikely(row_annotations)) { + // google supports one annotation per row + int annotation_found = 0; + for(c = 0; c < used ; c++) { + if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue; + + if(unlikely(co[c] & RRDR_VALUE_RESET)) { + buffer_fast_strcat(wb, overflow_annotation, overflow_annotation_len); + annotation_found = 1; + break; + } + } + if(likely(!annotation_found)) + buffer_fast_strcat(wb, normal_annotation, normal_annotation_len); + } + } + else { + // print the timestamp of the line + if(likely(i != start)) + buffer_fast_strcat(wb, ",\n", 2); + + buffer_fast_strcat(wb, pre_date, pre_date_len); + + if(unlikely( options & RRDR_OPTION_OBJECTSROWS )) + buffer_fast_strcat(wb, object_rows_time, object_rows_time_len); + + buffer_print_netdata_double(wb, (NETDATA_DOUBLE) r->t[i]); + + // in ms + if(unlikely(options & RRDR_OPTION_MILLISECONDS)) + buffer_fast_strcat(wb, "000", 3); + + buffer_fast_strcat(wb, post_date, post_date_len); + } + + // for each dimension + for(c = 0; c < used ;c++) { + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + NETDATA_DOUBLE n; + if(unlikely(options & RRDR_OPTION_INTERNAL_AR)) + n = ar[c]; + else + n = cn[c]; + + buffer_fast_strcat(wb, pre_value, pre_value_len); + + if(unlikely( options & RRDR_OPTION_OBJECTSROWS )) + buffer_sprintf(wb, "%s%s%s: ", kq, string2str(r->dn[c]), kq); + + if(co[c] & RRDR_VALUE_EMPTY && !(options & (RRDR_OPTION_INTERNAL_AR))) { + if(unlikely(options & RRDR_OPTION_NULL2ZERO)) + buffer_fast_strcat(wb, "0", 1); + else + buffer_fast_strcat(wb, "null", 4); + } + else + buffer_print_netdata_double(wb, n); + + buffer_fast_strcat(wb, post_value, post_value_len); + } + + buffer_fast_strcat(wb, post_line, post_line_len); + } + + buffer_strcat(wb, finish); + //netdata_log_info("RRD2JSON(): %s: END", r->st->id); +} + +void rrdr2json_v2(RRDR *r, BUFFER *wb) { + QUERY_TARGET *qt = r->internal.qt; + RRDR_OPTIONS options = qt->window.options; + + bool send_count = query_target_aggregatable(qt); + bool send_hidden = send_count && r->vh && query_has_group_by_aggregation_percentage(qt); + + buffer_json_member_add_object(wb, "result"); + + buffer_json_member_add_array(wb, "labels"); + buffer_json_add_array_item_string(wb, "time"); + long d, i; + const long used = (long)r->d; + for(d = 0, i = 0; d < used ; d++) { + if(!rrdr_dimension_should_be_exposed(r->od[d], options)) + continue; + + buffer_json_add_array_item_string(wb, string2str(r->di[d])); + i++; + } + buffer_json_array_close(wb); // labels + + buffer_json_member_add_object(wb, "point"); + { + size_t point_count = 0; + buffer_json_member_add_uint64(wb, "value", point_count++); + buffer_json_member_add_uint64(wb, "arp", point_count++); + buffer_json_member_add_uint64(wb, "pa", point_count++); + if (send_count) + buffer_json_member_add_uint64(wb, "count", point_count++); + if (send_hidden) + buffer_json_member_add_uint64(wb, "hidden", point_count++); + } + buffer_json_object_close(wb); // point + + buffer_json_member_add_array(wb, "data"); + if(i) { + long start = 0, end = rrdr_rows(r), step = 1; + if (!(options & RRDR_OPTION_REVERSED)) { + start = rrdr_rows(r) - 1; + end = -1; + step = -1; + } + + // for each line in the array + for (i = start; i != end; i += step) { + NETDATA_DOUBLE *cn = &r->v[ i * r->d ]; + NETDATA_DOUBLE *ch = send_hidden ? &r->vh[i * r->d ] : NULL; + RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ]; + NETDATA_DOUBLE *ar = &r->ar[ i * r->d ]; + uint32_t *gbc = &r->gbc [ i * r->d ]; + time_t now = r->t[i]; + + buffer_json_add_array_item_array(wb); // row + + if (options & RRDR_OPTION_MILLISECONDS) + buffer_json_add_array_item_time_ms(wb, now); // the time + else + buffer_json_add_array_item_time_t(wb, now); // the time + + for (d = 0; d < used; d++) { + if (!rrdr_dimension_should_be_exposed(r->od[d], options)) + continue; + + RRDR_VALUE_FLAGS o = co[d]; + + buffer_json_add_array_item_array(wb); // point + + // add the value + NETDATA_DOUBLE n = cn[d]; + + if(o & RRDR_VALUE_EMPTY) { + if (unlikely(options & RRDR_OPTION_NULL2ZERO)) + buffer_json_add_array_item_double(wb, 0); + else + buffer_json_add_array_item_double(wb, NAN); + } + else + buffer_json_add_array_item_double(wb, n); + + // add the anomaly + buffer_json_add_array_item_double(wb, ar[d]); + + // add the point annotations + buffer_json_add_array_item_uint64(wb, o); + + // add the count + if(send_count) + buffer_json_add_array_item_uint64(wb, gbc[d]); + if(send_hidden) + buffer_json_add_array_item_double(wb, ch[d]); + + buffer_json_array_close(wb); // point + } + + buffer_json_array_close(wb); // row + } + } + + buffer_json_array_close(wb); // data + + buffer_json_object_close(wb); // annotations +} diff --git a/src/web/api/formatters/json/json.h b/src/web/api/formatters/json/json.h new file mode 100644 index 000000000..d1ab4f901 --- /dev/null +++ b/src/web/api/formatters/json/json.h @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_API_FORMATTER_JSON_H +#define NETDATA_API_FORMATTER_JSON_H + +#include "../rrd2json.h" + +void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable); +void rrdr2json_v2(RRDR *r, BUFFER *wb); + +#endif //NETDATA_API_FORMATTER_JSON_H diff --git a/src/web/api/formatters/json_wrapper.c b/src/web/api/formatters/json_wrapper.c new file mode 100644 index 000000000..fca5a0b83 --- /dev/null +++ b/src/web/api/formatters/json_wrapper.c @@ -0,0 +1,1578 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "json_wrapper.h" + +static void jsonwrap_query_metric_plan(BUFFER *wb, QUERY_METRIC *qm) { + buffer_json_member_add_array(wb, "plans"); + for (size_t p = 0; p < qm->plan.used; p++) { + QUERY_PLAN_ENTRY *qp = &qm->plan.array[p]; + + buffer_json_add_array_item_object(wb); + buffer_json_member_add_uint64(wb, "tr", qp->tier); + buffer_json_member_add_time_t(wb, "af", qp->after); + buffer_json_member_add_time_t(wb, "bf", qp->before); + buffer_json_object_close(wb); + } + buffer_json_array_close(wb); + + buffer_json_member_add_array(wb, "tiers"); + for (size_t tier = 0; tier < storage_tiers; tier++) { + buffer_json_add_array_item_object(wb); + buffer_json_member_add_uint64(wb, "tr", tier); + buffer_json_member_add_time_t(wb, "fe", qm->tiers[tier].db_first_time_s); + buffer_json_member_add_time_t(wb, "le", qm->tiers[tier].db_last_time_s); + buffer_json_member_add_int64(wb, "wg", qm->tiers[tier].weight); + buffer_json_object_close(wb); + } + buffer_json_array_close(wb); +} + +void jsonwrap_query_plan(RRDR *r, BUFFER *wb) { + QUERY_TARGET *qt = r->internal.qt; + + buffer_json_member_add_object(wb, "query_plan"); + for(size_t m = 0; m < qt->query.used; m++) { + QUERY_METRIC *qm = query_metric(qt, m); + buffer_json_member_add_object(wb, query_metric_id(qt, qm)); + jsonwrap_query_metric_plan(wb, qm); + buffer_json_object_close(wb); + } + buffer_json_object_close(wb); +} + +static inline size_t rrdr_dimension_names(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { + const size_t dimensions = r->d; + size_t c, i; + + buffer_json_member_add_array(wb, key); + for(c = 0, i = 0; c < dimensions ; c++) { + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + buffer_json_add_array_item_string(wb, string2str(r->dn[c])); + i++; + } + buffer_json_array_close(wb); + + return i; +} + +static inline size_t rrdr_dimension_ids(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { + const size_t dimensions = r->d; + size_t c, i; + + buffer_json_member_add_array(wb, key); + for(c = 0, i = 0; c < dimensions ; c++) { + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + buffer_json_add_array_item_string(wb, string2str(r->di[c])); + i++; + } + buffer_json_array_close(wb); + + return i; +} + +static inline long jsonwrap_v1_chart_ids(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { + QUERY_TARGET *qt = r->internal.qt; + const long query_used = qt->query.used; + long c, i; + + buffer_json_member_add_array(wb, key); + for (c = 0, i = 0; c < query_used; c++) { + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + QUERY_METRIC *qm = query_metric(qt, c); + QUERY_INSTANCE *qi = query_instance(qt, qm->link.query_instance_id); + buffer_json_add_array_item_string(wb, rrdinstance_acquired_id(qi->ria)); + i++; + } + buffer_json_array_close(wb); + + return i; +} + +struct summary_total_counts { + size_t selected; + size_t excluded; + size_t queried; + size_t failed; +}; + +static inline void aggregate_into_summary_totals(struct summary_total_counts *totals, QUERY_METRICS_COUNTS *metrics) { + if(unlikely(!totals || !metrics)) + return; + + if(metrics->selected) { + totals->selected++; + + if(metrics->queried) + totals->queried++; + + else if(metrics->failed) + totals->failed++; + } + else + totals->excluded++; +} + +static inline void query_target_total_counts(BUFFER *wb, const char *key, struct summary_total_counts *totals) { + if(!totals->selected && !totals->queried && !totals->failed && !totals->excluded) + return; + + buffer_json_member_add_object(wb, key); + + if(totals->selected) + buffer_json_member_add_uint64(wb, "sl", totals->selected); + + if(totals->excluded) + buffer_json_member_add_uint64(wb, "ex", totals->excluded); + + if(totals->queried) + buffer_json_member_add_uint64(wb, "qr", totals->queried); + + if(totals->failed) + buffer_json_member_add_uint64(wb, "fl", totals->failed); + + buffer_json_object_close(wb); +} + +static inline void query_target_metric_counts(BUFFER *wb, QUERY_METRICS_COUNTS *metrics) { + if(!metrics->selected && !metrics->queried && !metrics->failed && !metrics->excluded) + return; + + buffer_json_member_add_object(wb, "ds"); + + if(metrics->selected) + buffer_json_member_add_uint64(wb, "sl", metrics->selected); + + if(metrics->excluded) + buffer_json_member_add_uint64(wb, "ex", metrics->excluded); + + if(metrics->queried) + buffer_json_member_add_uint64(wb, "qr", metrics->queried); + + if(metrics->failed) + buffer_json_member_add_uint64(wb, "fl", metrics->failed); + + buffer_json_object_close(wb); +} + +static inline void query_target_instance_counts(BUFFER *wb, QUERY_INSTANCES_COUNTS *instances) { + if(!instances->selected && !instances->queried && !instances->failed && !instances->excluded) + return; + + buffer_json_member_add_object(wb, "is"); + + if(instances->selected) + buffer_json_member_add_uint64(wb, "sl", instances->selected); + + if(instances->excluded) + buffer_json_member_add_uint64(wb, "ex", instances->excluded); + + if(instances->queried) + buffer_json_member_add_uint64(wb, "qr", instances->queried); + + if(instances->failed) + buffer_json_member_add_uint64(wb, "fl", instances->failed); + + buffer_json_object_close(wb); +} + +static inline void query_target_alerts_counts(BUFFER *wb, QUERY_ALERTS_COUNTS *alerts, const char *name, bool array) { + if(!alerts->clear && !alerts->other && !alerts->critical && !alerts->warning) + return; + + if(array) + buffer_json_add_array_item_object(wb); + else + buffer_json_member_add_object(wb, "al"); + + if(name) + buffer_json_member_add_string(wb, "nm", name); + + if(alerts->clear) + buffer_json_member_add_uint64(wb, "cl", alerts->clear); + + if(alerts->warning) + buffer_json_member_add_uint64(wb, "wr", alerts->warning); + + if(alerts->critical) + buffer_json_member_add_uint64(wb, "cr", alerts->critical); + + if(alerts->other) + buffer_json_member_add_uint64(wb, "ot", alerts->other); + + buffer_json_object_close(wb); +} + +static inline void query_target_points_statistics(BUFFER *wb, QUERY_TARGET *qt, STORAGE_POINT *sp) { + if(!sp->count) + return; + + buffer_json_member_add_object(wb, "sts"); + + buffer_json_member_add_double(wb, "min", sp->min); + buffer_json_member_add_double(wb, "max", sp->max); + + if(query_target_aggregatable(qt)) { + buffer_json_member_add_uint64(wb, "cnt", sp->count); + + if(sp->sum != 0.0) { + buffer_json_member_add_double(wb, "sum", sp->sum); + buffer_json_member_add_double(wb, "vol", sp->sum * (NETDATA_DOUBLE) query_view_update_every(qt)); + } + + if(sp->anomaly_count != 0) + buffer_json_member_add_uint64(wb, "arc", sp->anomaly_count); + } + else { + NETDATA_DOUBLE avg = (sp->count) ? sp->sum / (NETDATA_DOUBLE)sp->count : 0.0; + if(avg != 0.0) + buffer_json_member_add_double(wb, "avg", avg); + + NETDATA_DOUBLE arp = storage_point_anomaly_rate(*sp); + if(arp != 0.0) + buffer_json_member_add_double(wb, "arp", arp); + + NETDATA_DOUBLE con = (qt->query_points.sum > 0.0) ? sp->sum * 100.0 / qt->query_points.sum : 0.0; + if(con != 0.0) + buffer_json_member_add_double(wb, "con", con); + } + buffer_json_object_close(wb); +} + +static void query_target_summary_nodes_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key, struct summary_total_counts *totals) { + buffer_json_member_add_array(wb, key); + for (size_t c = 0; c < qt->nodes.used; c++) { + QUERY_NODE *qn = query_node(qt, c); + RRDHOST *host = qn->rrdhost; + buffer_json_add_array_item_object(wb); + buffer_json_node_add_v2(wb, host, qn->slot, qn->duration_ut, true); + query_target_instance_counts(wb, &qn->instances); + query_target_metric_counts(wb, &qn->metrics); + query_target_alerts_counts(wb, &qn->alerts, NULL, false); + query_target_points_statistics(wb, qt, &qn->query_points); + buffer_json_object_close(wb); + + aggregate_into_summary_totals(totals, &qn->metrics); + } + buffer_json_array_close(wb); +} + +static size_t query_target_summary_contexts_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key, struct summary_total_counts *totals) { + buffer_json_member_add_array(wb, key); + DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE); + + struct { + STORAGE_POINT query_points; + QUERY_INSTANCES_COUNTS instances; + QUERY_METRICS_COUNTS metrics; + QUERY_ALERTS_COUNTS alerts; + } *z; + + for (long c = 0; c < (long) qt->contexts.used; c++) { + QUERY_CONTEXT *qc = query_context(qt, c); + + z = dictionary_set(dict, rrdcontext_acquired_id(qc->rca), NULL, sizeof(*z)); + + z->instances.selected += qc->instances.selected; + z->instances.excluded += qc->instances.selected; + z->instances.queried += qc->instances.queried; + z->instances.failed += qc->instances.failed; + + z->metrics.selected += qc->metrics.selected; + z->metrics.excluded += qc->metrics.excluded; + z->metrics.queried += qc->metrics.queried; + z->metrics.failed += qc->metrics.failed; + + z->alerts.clear += qc->alerts.clear; + z->alerts.warning += qc->alerts.warning; + z->alerts.critical += qc->alerts.critical; + + storage_point_merge_to(z->query_points, qc->query_points); + } + + size_t unique_contexts = dictionary_entries(dict); + dfe_start_read(dict, z) { + buffer_json_add_array_item_object(wb); + buffer_json_member_add_string(wb, "id", z_dfe.name); + query_target_instance_counts(wb, &z->instances); + query_target_metric_counts(wb, &z->metrics); + query_target_alerts_counts(wb, &z->alerts, NULL, false); + query_target_points_statistics(wb, qt, &z->query_points); + buffer_json_object_close(wb); + + aggregate_into_summary_totals(totals, &z->metrics); + } + dfe_done(z); + buffer_json_array_close(wb); + dictionary_destroy(dict); + + return unique_contexts; +} + +static void query_target_summary_instances_v1(BUFFER *wb, QUERY_TARGET *qt, const char *key) { + char name[RRD_ID_LENGTH_MAX * 2 + 2]; + + buffer_json_member_add_array(wb, key); + DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE); + for (long c = 0; c < (long) qt->instances.used; c++) { + QUERY_INSTANCE *qi = query_instance(qt, c); + + snprintfz(name, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s", + rrdinstance_acquired_id(qi->ria), + rrdinstance_acquired_name(qi->ria)); + + bool *set = dictionary_set(dict, name, NULL, sizeof(*set)); + if (!*set) { + *set = true; + buffer_json_add_array_item_array(wb); + buffer_json_add_array_item_string(wb, rrdinstance_acquired_id(qi->ria)); + buffer_json_add_array_item_string(wb, rrdinstance_acquired_name(qi->ria)); + buffer_json_array_close(wb); + } + } + dictionary_destroy(dict); + buffer_json_array_close(wb); +} + +static void query_target_summary_instances_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key, struct summary_total_counts *totals) { + buffer_json_member_add_array(wb, key); + for (long c = 0; c < (long) qt->instances.used; c++) { + QUERY_INSTANCE *qi = query_instance(qt, c); +// QUERY_HOST *qh = query_host(qt, qi->query_host_id); + + buffer_json_add_array_item_object(wb); + buffer_json_member_add_string(wb, "id", rrdinstance_acquired_id(qi->ria)); + + if(!rrdinstance_acquired_id_and_name_are_same(qi->ria)) + buffer_json_member_add_string(wb, "nm", rrdinstance_acquired_name(qi->ria)); + + buffer_json_member_add_uint64(wb, "ni", qi->query_host_id); +// buffer_json_member_add_string(wb, "id", string2str(qi->id_fqdn)); +// buffer_json_member_add_string(wb, "nm", string2str(qi->name_fqdn)); +// buffer_json_member_add_string(wb, "lc", rrdinstance_acquired_name(qi->ria)); +// buffer_json_member_add_string(wb, "mg", qh->host->machine_guid); +// if(qh->node_id[0]) +// buffer_json_member_add_string(wb, "nd", qh->node_id); + query_target_metric_counts(wb, &qi->metrics); + query_target_alerts_counts(wb, &qi->alerts, NULL, false); + query_target_points_statistics(wb, qt, &qi->query_points); + buffer_json_object_close(wb); + + aggregate_into_summary_totals(totals, &qi->metrics); + } + buffer_json_array_close(wb); +} + +struct dimensions_sorted_walkthrough_data { + BUFFER *wb; + struct summary_total_counts *totals; + QUERY_TARGET *qt; +}; + +struct dimensions_sorted_entry { + const char *id; + const char *name; + STORAGE_POINT query_points; + QUERY_METRICS_COUNTS metrics; + uint32_t priority; +}; + +static int dimensions_sorted_walktrhough_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) { + struct dimensions_sorted_walkthrough_data *sdwd = data; + BUFFER *wb = sdwd->wb; + struct summary_total_counts *totals = sdwd->totals; + QUERY_TARGET *qt = sdwd->qt; + struct dimensions_sorted_entry *z = value; + + buffer_json_add_array_item_object(wb); + buffer_json_member_add_string(wb, "id", z->id); + if (z->id != z->name && z->name) + buffer_json_member_add_string(wb, "nm", z->name); + + query_target_metric_counts(wb, &z->metrics); + query_target_points_statistics(wb, qt, &z->query_points); + buffer_json_member_add_uint64(wb, "pri", z->priority); + buffer_json_object_close(wb); + + aggregate_into_summary_totals(totals, &z->metrics); + + return 1; +} + +int dimensions_sorted_compar(const DICTIONARY_ITEM **item1, const DICTIONARY_ITEM **item2) { + struct dimensions_sorted_entry *z1 = dictionary_acquired_item_value(*item1); + struct dimensions_sorted_entry *z2 = dictionary_acquired_item_value(*item2); + + if(z1->priority == z2->priority) + return strcmp(dictionary_acquired_item_name(*item1), dictionary_acquired_item_name(*item2)); + else if(z1->priority < z2->priority) + return -1; + else + return 1; +} + +static void query_target_summary_dimensions_v12(BUFFER *wb, QUERY_TARGET *qt, const char *key, bool v2, struct summary_total_counts *totals) { + char buf[RRD_ID_LENGTH_MAX * 2 + 2]; + + buffer_json_member_add_array(wb, key); + DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE); + struct dimensions_sorted_entry *z; + size_t q = 0; + for (long c = 0; c < (long) qt->dimensions.used; c++) { + QUERY_DIMENSION * qd = query_dimension(qt, c); + RRDMETRIC_ACQUIRED *rma = qd->rma; + + QUERY_METRIC *qm = NULL; + for( ; q < qt->query.used ;q++) { + QUERY_METRIC *tqm = query_metric(qt, q); + QUERY_DIMENSION *tqd = query_dimension(qt, tqm->link.query_dimension_id); + if(tqd->rma != rma) break; + qm = tqm; + } + + const char *k, *id, *name; + + if(v2) { + k = rrdmetric_acquired_name(rma); + id = k; + name = k; + } + else { + snprintfz(buf, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s", + rrdmetric_acquired_id(rma), + rrdmetric_acquired_name(rma)); + k = buf; + id = rrdmetric_acquired_id(rma); + name = rrdmetric_acquired_name(rma); + } + + z = dictionary_set(dict, k, NULL, sizeof(*z)); + if(!z->id) { + z->id = id; + z->name = name; + z->priority = qd->priority; + } + else { + if(qd->priority < z->priority) + z->priority = qd->priority; + } + + if(qm) { + z->metrics.selected += (qm->status & RRDR_DIMENSION_SELECTED) ? 1 : 0; + z->metrics.failed += (qm->status & RRDR_DIMENSION_FAILED) ? 1 : 0; + + if(qm->status & RRDR_DIMENSION_QUERIED) { + z->metrics.queried++; + storage_point_merge_to(z->query_points, qm->query_points); + } + } + else + z->metrics.excluded++; + } + + if(v2) { + struct dimensions_sorted_walkthrough_data t = { + .wb = wb, + .totals = totals, + .qt = qt, + }; + dictionary_sorted_walkthrough_rw(dict, DICTIONARY_LOCK_READ, dimensions_sorted_walktrhough_cb, + &t, dimensions_sorted_compar); + } + else { + // v1 + dfe_start_read(dict, z) { + buffer_json_add_array_item_array(wb); + buffer_json_add_array_item_string(wb, z->id); + buffer_json_add_array_item_string(wb, z->name); + buffer_json_array_close(wb); + } + dfe_done(z); + } + dictionary_destroy(dict); + buffer_json_array_close(wb); +} + +struct rrdlabels_formatting_v2 { + DICTIONARY *keys; + QUERY_INSTANCE *qi; + bool v2; +}; + +struct rrdlabels_keys_dict_entry { + const char *name; + DICTIONARY *values; + STORAGE_POINT query_points; + QUERY_METRICS_COUNTS metrics; +}; + +struct rrdlabels_key_value_dict_entry { + const char *key; + const char *value; + STORAGE_POINT query_points; + QUERY_METRICS_COUNTS metrics; +}; + +static int rrdlabels_formatting_v2(const char *name, const char *value, RRDLABEL_SRC ls __maybe_unused, void *data) { + struct rrdlabels_formatting_v2 *t = data; + + struct rrdlabels_keys_dict_entry *d = dictionary_set(t->keys, name, NULL, sizeof(*d)); + if(!d->values) { + d->name = name; + d->values = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE); + } + + char n[RRD_ID_LENGTH_MAX * 2 + 2]; + snprintfz(n, RRD_ID_LENGTH_MAX * 2, "%s:%s", name, value); + + struct rrdlabels_key_value_dict_entry *z = dictionary_set(d->values, n, NULL, sizeof(*z)); + if(!z->key) { + z->key = name; + z->value = value; + } + + if(t->v2) { + QUERY_INSTANCE *qi = t->qi; + + z->metrics.selected += qi->metrics.selected; + z->metrics.excluded += qi->metrics.excluded; + z->metrics.queried += qi->metrics.queried; + z->metrics.failed += qi->metrics.failed; + + d->metrics.selected += qi->metrics.selected; + d->metrics.excluded += qi->metrics.excluded; + d->metrics.queried += qi->metrics.queried; + d->metrics.failed += qi->metrics.failed; + + storage_point_merge_to(z->query_points, qi->query_points); + storage_point_merge_to(d->query_points, qi->query_points); + } + + return 1; +} + +static void query_target_summary_labels_v12(BUFFER *wb, QUERY_TARGET *qt, const char *key, bool v2, struct summary_total_counts *key_totals, struct summary_total_counts *value_totals) { + buffer_json_member_add_array(wb, key); + struct rrdlabels_formatting_v2 t = { + .keys = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE), + .v2 = v2, + }; + for (long c = 0; c < (long) qt->instances.used; c++) { + QUERY_INSTANCE *qi = query_instance(qt, c); + RRDINSTANCE_ACQUIRED *ria = qi->ria; + t.qi = qi; + rrdlabels_walkthrough_read(rrdinstance_acquired_labels(ria), rrdlabels_formatting_v2, &t); + } + struct rrdlabels_keys_dict_entry *d; + dfe_start_read(t.keys, d) { + if(v2) { + buffer_json_add_array_item_object(wb); + buffer_json_member_add_string(wb, "id", d_dfe.name); + query_target_metric_counts(wb, &d->metrics); + query_target_points_statistics(wb, qt, &d->query_points); + aggregate_into_summary_totals(key_totals, &d->metrics); + buffer_json_member_add_array(wb, "vl"); + } + struct rrdlabels_key_value_dict_entry *z; + dfe_start_read(d->values, z){ + if (v2) { + buffer_json_add_array_item_object(wb); + buffer_json_member_add_string(wb, "id", z->value); + query_target_metric_counts(wb, &z->metrics); + query_target_points_statistics(wb, qt, &z->query_points); + buffer_json_object_close(wb); + aggregate_into_summary_totals(value_totals, &z->metrics); + } else { + buffer_json_add_array_item_array(wb); + buffer_json_add_array_item_string(wb, z->key); + buffer_json_add_array_item_string(wb, z->value); + buffer_json_array_close(wb); + } + } + dfe_done(z); + dictionary_destroy(d->values); + if(v2) { + buffer_json_array_close(wb); + buffer_json_object_close(wb); + } + } + dfe_done(d); + dictionary_destroy(t.keys); + buffer_json_array_close(wb); +} + +static void query_target_summary_alerts_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key) { + buffer_json_member_add_array(wb, key); + QUERY_ALERTS_COUNTS *z; + + DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE); + for (long c = 0; c < (long) qt->instances.used; c++) { + QUERY_INSTANCE *qi = query_instance(qt, c); + RRDSET *st = rrdinstance_acquired_rrdset(qi->ria); + if (st) { + rw_spinlock_read_lock(&st->alerts.spinlock); + if (st->alerts.base) { + for (RRDCALC *rc = st->alerts.base; rc; rc = rc->next) { + z = dictionary_set(dict, string2str(rc->config.name), NULL, sizeof(*z)); + + switch(rc->status) { + case RRDCALC_STATUS_CLEAR: + z->clear++; + break; + + case RRDCALC_STATUS_WARNING: + z->warning++; + break; + + case RRDCALC_STATUS_CRITICAL: + z->critical++; + break; + + default: + case RRDCALC_STATUS_UNINITIALIZED: + case RRDCALC_STATUS_UNDEFINED: + case RRDCALC_STATUS_REMOVED: + z->other++; + break; + } + } + } + rw_spinlock_read_unlock(&st->alerts.spinlock); + } + } + dfe_start_read(dict, z) + query_target_alerts_counts(wb, z, z_dfe.name, true); + dfe_done(z); + dictionary_destroy(dict); + buffer_json_array_close(wb); // alerts +} + +static inline void query_target_functions(BUFFER *wb, const char *key, RRDR *r) { + QUERY_TARGET *qt = r->internal.qt; + const long query_used = qt->query.used; + + DICTIONARY *funcs = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE); + RRDINSTANCE_ACQUIRED *ria = NULL; + for (long c = 0; c < query_used ; c++) { + QUERY_METRIC *qm = query_metric(qt, c); + QUERY_INSTANCE *qi = query_instance(qt, qm->link.query_instance_id); + if(qi->ria == ria) + continue; + + ria = qi->ria; + chart_functions_to_dict(rrdinstance_acquired_functions(ria), funcs, NULL, 0); + } + + buffer_json_member_add_array(wb, key); + void *t; (void)t; + dfe_start_read(funcs, t) + buffer_json_add_array_item_string(wb, t_dfe.name); + dfe_done(t); + dictionary_destroy(funcs); + buffer_json_array_close(wb); +} + +static inline long query_target_chart_labels_filter_v1(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { + QUERY_TARGET *qt = r->internal.qt; + const long query_used = qt->query.used; + long c, i = 0; + + buffer_json_member_add_object(wb, key); + + SIMPLE_PATTERN *pattern = qt->instances.chart_label_key_pattern; + char *label_key = NULL; + while (pattern && (label_key = simple_pattern_iterate(&pattern))) { + buffer_json_member_add_array(wb, label_key); + + for (c = 0, i = 0; c < query_used; c++) { + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + QUERY_METRIC *qm = query_metric(qt, c); + QUERY_INSTANCE *qi = query_instance(qt, qm->link.query_instance_id); + rrdlabels_value_to_buffer_array_item_or_null(rrdinstance_acquired_labels(qi->ria), wb, label_key); + i++; + } + buffer_json_array_close(wb); + } + + buffer_json_object_close(wb); + + return i; +} + +static inline long query_target_metrics_latest_values(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { + QUERY_TARGET *qt = r->internal.qt; + const long query_used = qt->query.used; + long c, i; + + buffer_json_member_add_array(wb, key); + + for(c = 0, i = 0; c < query_used ;c++) { + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + QUERY_METRIC *qm = query_metric(qt, c); + QUERY_DIMENSION *qd = query_dimension(qt, qm->link.query_dimension_id); + buffer_json_add_array_item_double(wb, rrdmetric_acquired_last_stored_value(qd->rma)); + i++; + } + + buffer_json_array_close(wb); + + return i; +} + +static inline size_t rrdr_dimension_view_latest_values(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { + buffer_json_member_add_array(wb, key); + + size_t c, i; + for(c = 0, i = 0; c < r->d ; c++) { + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + i++; + + NETDATA_DOUBLE *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ]; + RRDR_VALUE_FLAGS *co = &r->o[ (rrdr_rows(r) - 1) * r->d ]; + NETDATA_DOUBLE n = cn[c]; + + if(co[c] & RRDR_VALUE_EMPTY) { + if(options & RRDR_OPTION_NULL2ZERO) + buffer_json_add_array_item_double(wb, 0.0); + else + buffer_json_add_array_item_double(wb, NAN); + } + else + buffer_json_add_array_item_double(wb, n); + } + + buffer_json_array_close(wb); + + return i; +} + +static inline void rrdr_dimension_query_points_statistics(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options, bool dview) { + STORAGE_POINT *sp = (dview) ? r->dview : r->dqp; + NETDATA_DOUBLE anomaly_rate_multiplier = (dview) ? RRDR_DVIEW_ANOMALY_COUNT_MULTIPLIER : 1.0; + + if(unlikely(!sp)) + return; + + if(key) + buffer_json_member_add_object(wb, key); + + buffer_json_member_add_array(wb, "min"); + for(size_t c = 0; c < r->d ; c++) { + if (!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + buffer_json_add_array_item_double(wb, sp[c].min); + } + buffer_json_array_close(wb); + + buffer_json_member_add_array(wb, "max"); + for(size_t c = 0; c < r->d ; c++) { + if (!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + buffer_json_add_array_item_double(wb, sp[c].max); + } + buffer_json_array_close(wb); + + if(options & RRDR_OPTION_RETURN_RAW) { + buffer_json_member_add_array(wb, "sum"); + for(size_t c = 0; c < r->d ; c++) { + if (!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + buffer_json_add_array_item_double(wb, sp[c].sum); + } + buffer_json_array_close(wb); + + buffer_json_member_add_array(wb, "cnt"); + for(size_t c = 0; c < r->d ; c++) { + if (!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + buffer_json_add_array_item_uint64(wb, sp[c].count); + } + buffer_json_array_close(wb); + + buffer_json_member_add_array(wb, "arc"); + for(size_t c = 0; c < r->d ; c++) { + if (!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + buffer_json_add_array_item_uint64(wb, storage_point_anomaly_rate(sp[c]) / anomaly_rate_multiplier / 100.0 * sp[c].count); + } + buffer_json_array_close(wb); + } + else { + NETDATA_DOUBLE sum = 0.0; + for(size_t c = 0; c < r->d ; c++) { + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + sum += ABS(sp[c].sum); + } + + buffer_json_member_add_array(wb, "avg"); + for(size_t c = 0; c < r->d ; c++) { + if (!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + buffer_json_add_array_item_double(wb, storage_point_average_value(sp[c])); + } + buffer_json_array_close(wb); + + buffer_json_member_add_array(wb, "arp"); + for(size_t c = 0; c < r->d ; c++) { + if (!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + buffer_json_add_array_item_double(wb, storage_point_anomaly_rate(sp[c]) / anomaly_rate_multiplier); + } + buffer_json_array_close(wb); + + buffer_json_member_add_array(wb, "con"); + for(size_t c = 0; c < r->d ; c++) { + if (!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + NETDATA_DOUBLE con = (sum > 0.0) ? ABS(sp[c].sum) * 100.0 / sum : 0.0; + buffer_json_add_array_item_double(wb, con); + } + buffer_json_array_close(wb); + } + + if(key) + buffer_json_object_close(wb); +} + +void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb) { + QUERY_TARGET *qt = r->internal.qt; + DATASOURCE_FORMAT format = qt->request.format; + RRDR_OPTIONS options = qt->window.options; + + long rows = rrdr_rows(r); + + char kq[2] = "", // key quote + sq[2] = ""; // string quote + + if( options & RRDR_OPTION_GOOGLE_JSON ) { + kq[0] = '\0'; + sq[0] = '\''; + } + else { + kq[0] = '"'; + sq[0] = '"'; + } + + buffer_json_initialize( + wb, kq, sq, 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT); + + buffer_json_member_add_uint64(wb, "api", 1); + buffer_json_member_add_string(wb, "id", qt->id); + buffer_json_member_add_string(wb, "name", qt->id); + buffer_json_member_add_time_t(wb, "view_update_every", r->view.update_every); + buffer_json_member_add_time_t(wb, "update_every", qt->db.minimum_latest_update_every_s); + buffer_json_member_add_time_t(wb, "first_entry", qt->db.first_time_s); + buffer_json_member_add_time_t(wb, "last_entry", qt->db.last_time_s); + buffer_json_member_add_time_t(wb, "after", r->view.after); + buffer_json_member_add_time_t(wb, "before", r->view.before); + buffer_json_member_add_string(wb, "group", time_grouping_tostring(qt->request.time_group_method)); + rrdr_options_to_buffer_json_array(wb, "options", options); + + if(!rrdr_dimension_names(wb, "dimension_names", r, options)) + rows = 0; + + if(!rrdr_dimension_ids(wb, "dimension_ids", r, options)) + rows = 0; + + if (options & RRDR_OPTION_ALL_DIMENSIONS) { + query_target_summary_instances_v1(wb, qt, "full_chart_list"); + query_target_summary_dimensions_v12(wb, qt, "full_dimension_list", false, NULL); + query_target_summary_labels_v12(wb, qt, "full_chart_labels", false, NULL, NULL); + } + + query_target_functions(wb, "functions", r); + + if (!qt->request.st && !jsonwrap_v1_chart_ids(wb, "chart_ids", r, options)) + rows = 0; + + if (qt->instances.chart_label_key_pattern && !query_target_chart_labels_filter_v1(wb, "chart_labels", r, options)) + rows = 0; + + if(!query_target_metrics_latest_values(wb, "latest_values", r, options)) + rows = 0; + + size_t dimensions = rrdr_dimension_view_latest_values(wb, "view_latest_values", r, options); + if(!dimensions) + rows = 0; + + buffer_json_member_add_uint64(wb, "dimensions", dimensions); + buffer_json_member_add_uint64(wb, "points", rows); + buffer_json_member_add_string(wb, "format", rrdr_format_to_string(format)); + + buffer_json_member_add_array(wb, "db_points_per_tier"); + for(size_t tier = 0; tier < storage_tiers ; tier++) + buffer_json_add_array_item_uint64(wb, qt->db.tiers[tier].points); + buffer_json_array_close(wb); + + if(options & RRDR_OPTION_DEBUG) + jsonwrap_query_plan(r, wb); +} + +static void rrdset_rrdcalc_entries_v2(BUFFER *wb, RRDINSTANCE_ACQUIRED *ria) { + RRDSET *st = rrdinstance_acquired_rrdset(ria); + if(st) { + rw_spinlock_read_lock(&st->alerts.spinlock); + if(st->alerts.base) { + buffer_json_member_add_object(wb, "alerts"); + for(RRDCALC *rc = st->alerts.base; rc ;rc = rc->next) { + if(rc->status < RRDCALC_STATUS_CLEAR) + continue; + + buffer_json_member_add_object(wb, string2str(rc->config.name)); + buffer_json_member_add_string(wb, "st", rrdcalc_status2string(rc->status)); + buffer_json_member_add_double(wb, "vl", rc->value); + buffer_json_member_add_string(wb, "un", string2str(rc->config.units)); + buffer_json_object_close(wb); + } + buffer_json_object_close(wb); + } + rw_spinlock_read_unlock(&st->alerts.spinlock); + } +} + +static void query_target_combined_units_v2(BUFFER *wb, QUERY_TARGET *qt, size_t contexts, bool ignore_percentage) { + if(!ignore_percentage && query_target_has_percentage_units(qt)) { + buffer_json_member_add_string(wb, "units", "%"); + } + else if(contexts == 1) { + buffer_json_member_add_string(wb, "units", rrdcontext_acquired_units(qt->contexts.array[0].rca)); + } + else if(contexts > 1) { + DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE); + for(size_t c = 0; c < qt->contexts.used ;c++) + dictionary_set(dict, rrdcontext_acquired_units(qt->contexts.array[c].rca), NULL, 0); + + if(dictionary_entries(dict) == 1) + buffer_json_member_add_string(wb, "units", rrdcontext_acquired_units(qt->contexts.array[0].rca)); + else { + buffer_json_member_add_array(wb, "units"); + const char *s; + dfe_start_read(dict, s) + buffer_json_add_array_item_string(wb, s_dfe.name); + dfe_done(s); + buffer_json_array_close(wb); + } + dictionary_destroy(dict); + } +} + +static void query_target_combined_chart_type(BUFFER *wb, QUERY_TARGET *qt, size_t contexts) { + if(contexts >= 1) + buffer_json_member_add_string(wb, "chart_type", rrdset_type_name(rrdcontext_acquired_chart_type(qt->contexts.array[0].rca))); +} + +static void rrdr_grouped_by_array_v2(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options __maybe_unused) { + QUERY_TARGET *qt = r->internal.qt; + + buffer_json_member_add_array(wb, key); + + // find the deeper group-by + ssize_t g = 0; + for(g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) { + if(qt->request.group_by[g].group_by == RRDR_GROUP_BY_NONE) + break; + } + + if(g > 0) + g--; + + RRDR_GROUP_BY group_by = qt->request.group_by[g].group_by; + + if(group_by & RRDR_GROUP_BY_SELECTED) + buffer_json_add_array_item_string(wb, "selected"); + + else if(group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE) + buffer_json_add_array_item_string(wb, "percentage-of-instance"); + + else { + + if(group_by & RRDR_GROUP_BY_DIMENSION) + buffer_json_add_array_item_string(wb, "dimension"); + + if(group_by & RRDR_GROUP_BY_INSTANCE) + buffer_json_add_array_item_string(wb, "instance"); + + if(group_by & RRDR_GROUP_BY_LABEL) { + BUFFER *b = buffer_create(0, NULL); + for (size_t l = 0; l < qt->group_by[g].used; l++) { + buffer_flush(b); + buffer_fast_strcat(b, "label:", 6); + buffer_strcat(b, qt->group_by[g].label_keys[l]); + buffer_json_add_array_item_string(wb, buffer_tostring(b)); + } + buffer_free(b); + } + + if(group_by & RRDR_GROUP_BY_NODE) + buffer_json_add_array_item_string(wb, "node"); + + if(group_by & RRDR_GROUP_BY_CONTEXT) + buffer_json_add_array_item_string(wb, "context"); + + if(group_by & RRDR_GROUP_BY_UNITS) + buffer_json_add_array_item_string(wb, "units"); + } + + buffer_json_array_close(wb); // group_by_order +} + +static void rrdr_dimension_units_array_v2(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options, bool ignore_percentage) { + if(!r->du) + return; + + bool percentage = !ignore_percentage && query_target_has_percentage_units(r->internal.qt); + + buffer_json_member_add_array(wb, key); + for(size_t c = 0; c < r->d ; c++) { + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + if(percentage) + buffer_json_add_array_item_string(wb, "%"); + else + buffer_json_add_array_item_string(wb, string2str(r->du[c])); + } + buffer_json_array_close(wb); +} + +static void rrdr_dimension_priority_array_v2(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { + if(!r->dp) + return; + + buffer_json_member_add_array(wb, key); + for(size_t c = 0; c < r->d ; c++) { + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + buffer_json_add_array_item_uint64(wb, r->dp[c]); + } + buffer_json_array_close(wb); +} + +static void rrdr_dimension_aggregated_array_v2(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { + if(!r->dgbc) + return; + + buffer_json_member_add_array(wb, key); + for(size_t c = 0; c < r->d ;c++) { + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + buffer_json_add_array_item_uint64(wb, r->dgbc[c]); + } + buffer_json_array_close(wb); +} + +static void query_target_title(BUFFER *wb, QUERY_TARGET *qt, size_t contexts) { + if(contexts == 1) { + buffer_json_member_add_string(wb, "title", rrdcontext_acquired_title(qt->contexts.array[0].rca)); + } + else if(contexts > 1) { + BUFFER *t = buffer_create(0, NULL); + DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE); + + buffer_strcat(t, "Chart for contexts: "); + + size_t added = 0; + for(size_t c = 0; c < qt->contexts.used ;c++) { + bool *set = dictionary_set(dict, rrdcontext_acquired_id(qt->contexts.array[c].rca), NULL, sizeof(*set)); + if(!*set) { + *set = true; + if(added) + buffer_fast_strcat(t, ", ", 2); + + buffer_strcat(t, rrdcontext_acquired_id(qt->contexts.array[c].rca)); + added++; + } + } + buffer_json_member_add_string(wb, "title", buffer_tostring(t)); + dictionary_destroy(dict); + buffer_free(t); + } +} + +static void query_target_detailed_objects_tree(BUFFER *wb, RRDR *r, RRDR_OPTIONS options) { + QUERY_TARGET *qt = r->internal.qt; + buffer_json_member_add_object(wb, "nodes"); + + time_t now_s = now_realtime_sec(); + RRDHOST *last_host = NULL; + RRDCONTEXT_ACQUIRED *last_rca = NULL; + RRDINSTANCE_ACQUIRED *last_ria = NULL; + + size_t h = 0, c = 0, i = 0, m = 0, q = 0; + for(; h < qt->nodes.used ; h++) { + QUERY_NODE *qn = query_node(qt, h); + RRDHOST *host = qn->rrdhost; + + for( ;c < qt->contexts.used ;c++) { + QUERY_CONTEXT *qc = query_context(qt, c); + RRDCONTEXT_ACQUIRED *rca = qc->rca; + if(!rrdcontext_acquired_belongs_to_host(rca, host)) break; + + for( ;i < qt->instances.used ;i++) { + QUERY_INSTANCE *qi = query_instance(qt, i); + RRDINSTANCE_ACQUIRED *ria = qi->ria; + if(!rrdinstance_acquired_belongs_to_context(ria, rca)) break; + + for( ; m < qt->dimensions.used ; m++) { + QUERY_DIMENSION *qd = query_dimension(qt, m); + RRDMETRIC_ACQUIRED *rma = qd->rma; + if(!rrdmetric_acquired_belongs_to_instance(rma, ria)) break; + + QUERY_METRIC *qm = NULL; + bool queried = false; + for( ; q < qt->query.used ;q++) { + QUERY_METRIC *tqm = query_metric(qt, q); + QUERY_DIMENSION *tqd = query_dimension(qt, tqm->link.query_dimension_id); + if(tqd->rma != rma) break; + + queried = tqm->status & RRDR_DIMENSION_QUERIED; + qm = tqm; + } + + if(!queried & !(options & RRDR_OPTION_ALL_DIMENSIONS)) + continue; + + if(host != last_host) { + if(last_host) { + if(last_rca) { + if(last_ria) { + buffer_json_object_close(wb); // dimensions + buffer_json_object_close(wb); // instance + last_ria = NULL; + } + buffer_json_object_close(wb); // instances + buffer_json_object_close(wb); // context + last_rca = NULL; + } + buffer_json_object_close(wb); // contexts + buffer_json_object_close(wb); // host + last_host = NULL; + } + + buffer_json_member_add_object(wb, host->machine_guid); + if(qn->node_id[0]) + buffer_json_member_add_string(wb, "nd", qn->node_id); + buffer_json_member_add_uint64(wb, "ni", qn->slot); + buffer_json_member_add_string(wb, "nm", rrdhost_hostname(host)); + buffer_json_member_add_object(wb, "contexts"); + + last_host = host; + } + + if(rca != last_rca) { + if(last_rca) { + if(last_ria) { + buffer_json_object_close(wb); // dimensions + buffer_json_object_close(wb); // instance + last_ria = NULL; + } + buffer_json_object_close(wb); // instances + buffer_json_object_close(wb); // context + last_rca = NULL; + } + + buffer_json_member_add_object(wb, rrdcontext_acquired_id(rca)); + buffer_json_member_add_object(wb, "instances"); + + last_rca = rca; + } + + if(ria != last_ria) { + if(last_ria) { + buffer_json_object_close(wb); // dimensions + buffer_json_object_close(wb); // instance + last_ria = NULL; + } + + buffer_json_member_add_object(wb, rrdinstance_acquired_id(ria)); + buffer_json_member_add_string(wb, "nm", rrdinstance_acquired_name(ria)); + buffer_json_member_add_time_t(wb, "ue", rrdinstance_acquired_update_every(ria)); + RRDLABELS *labels = rrdinstance_acquired_labels(ria); + if(labels) { + buffer_json_member_add_object(wb, "labels"); + rrdlabels_to_buffer_json_members(labels, wb); + buffer_json_object_close(wb); + } + rrdset_rrdcalc_entries_v2(wb, ria); + buffer_json_member_add_object(wb, "dimensions"); + + last_ria = ria; + } + + buffer_json_member_add_object(wb, rrdmetric_acquired_id(rma)); + { + buffer_json_member_add_string(wb, "nm", rrdmetric_acquired_name(rma)); + buffer_json_member_add_uint64(wb, "qr", queried ? 1 : 0); + time_t first_entry_s = rrdmetric_acquired_first_entry(rma); + time_t last_entry_s = rrdmetric_acquired_last_entry(rma); + buffer_json_member_add_time_t(wb, "fe", first_entry_s); + buffer_json_member_add_time_t(wb, "le", last_entry_s ? last_entry_s : now_s); + + if(qm) { + if(qm->status & RRDR_DIMENSION_GROUPED) { + // buffer_json_member_add_string(wb, "grouped_as_id", string2str(qm->grouped_as.id)); + buffer_json_member_add_string(wb, "as", string2str(qm->grouped_as.name)); + } + + query_target_points_statistics(wb, qt, &qm->query_points); + + if(options & RRDR_OPTION_DEBUG) + jsonwrap_query_metric_plan(wb, qm); + } + } + buffer_json_object_close(wb); // metric + } + } + } + } + + if(last_host) { + if(last_rca) { + if(last_ria) { + buffer_json_object_close(wb); // dimensions + buffer_json_object_close(wb); // instance + last_ria = NULL; + } + buffer_json_object_close(wb); // instances + buffer_json_object_close(wb); // context + last_rca = NULL; + } + buffer_json_object_close(wb); // contexts + buffer_json_object_close(wb); // host + last_host = NULL; + } + buffer_json_object_close(wb); // hosts +} + +void version_hashes_api_v2(BUFFER *wb, struct query_versions *versions) { + buffer_json_member_add_object(wb, "versions"); + buffer_json_member_add_uint64(wb, "routing_hard_hash", 1); + buffer_json_member_add_uint64(wb, "nodes_hard_hash", dictionary_version(rrdhost_root_index)); + buffer_json_member_add_uint64(wb, "contexts_hard_hash", versions->contexts_hard_hash); + buffer_json_member_add_uint64(wb, "contexts_soft_hash", versions->contexts_soft_hash); + buffer_json_member_add_uint64(wb, "alerts_hard_hash", versions->alerts_hard_hash); + buffer_json_member_add_uint64(wb, "alerts_soft_hash", versions->alerts_soft_hash); + buffer_json_object_close(wb); +} + +void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb) { + QUERY_TARGET *qt = r->internal.qt; + RRDR_OPTIONS options = qt->window.options; + + char kq[2] = "\"", // key quote + sq[2] = "\""; // string quote + + if(unlikely(options & RRDR_OPTION_GOOGLE_JSON)) { + kq[0] = '\0'; + sq[0] = '\''; + } + + buffer_json_initialize( + wb, kq, sq, 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT); + buffer_json_member_add_uint64(wb, "api", 2); + + if(options & RRDR_OPTION_DEBUG) { + buffer_json_member_add_string(wb, "id", qt->id); + buffer_json_member_add_object(wb, "request"); + { + buffer_json_member_add_string(wb, "format", rrdr_format_to_string(qt->request.format)); + rrdr_options_to_buffer_json_array(wb, "options", qt->request.options); + + buffer_json_member_add_object(wb, "scope"); + buffer_json_member_add_string(wb, "scope_nodes", qt->request.scope_nodes); + buffer_json_member_add_string(wb, "scope_contexts", qt->request.scope_contexts); + buffer_json_object_close(wb); // scope + + buffer_json_member_add_object(wb, "selectors"); + if (qt->request.host) + buffer_json_member_add_string(wb, "nodes", rrdhost_hostname(qt->request.host)); + else + buffer_json_member_add_string(wb, "nodes", qt->request.nodes); + buffer_json_member_add_string(wb, "contexts", qt->request.contexts); + buffer_json_member_add_string(wb, "instances", qt->request.instances); + buffer_json_member_add_string(wb, "dimensions", qt->request.dimensions); + buffer_json_member_add_string(wb, "labels", qt->request.labels); + buffer_json_member_add_string(wb, "alerts", qt->request.alerts); + buffer_json_object_close(wb); // selectors + + buffer_json_member_add_object(wb, "window"); + buffer_json_member_add_time_t(wb, "after", qt->request.after); + buffer_json_member_add_time_t(wb, "before", qt->request.before); + buffer_json_member_add_uint64(wb, "points", qt->request.points); + if (qt->request.options & RRDR_OPTION_SELECTED_TIER) + buffer_json_member_add_uint64(wb, "tier", qt->request.tier); + else + buffer_json_member_add_string(wb, "tier", NULL); + buffer_json_object_close(wb); // window + + buffer_json_member_add_object(wb, "aggregations"); + { + buffer_json_member_add_object(wb, "time"); + buffer_json_member_add_string(wb, "time_group", time_grouping_tostring(qt->request.time_group_method)); + buffer_json_member_add_string(wb, "time_group_options", qt->request.time_group_options); + if (qt->request.resampling_time > 0) + buffer_json_member_add_time_t(wb, "time_resampling", qt->request.resampling_time); + else + buffer_json_member_add_string(wb, "time_resampling", NULL); + buffer_json_object_close(wb); // time + + buffer_json_member_add_array(wb, "metrics"); + for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) { + if(qt->request.group_by[g].group_by == RRDR_GROUP_BY_NONE) + break; + + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_array(wb, "group_by"); + buffer_json_group_by_to_array(wb, qt->request.group_by[g].group_by); + buffer_json_array_close(wb); + + buffer_json_member_add_array(wb, "group_by_label"); + for (size_t l = 0; l < qt->group_by[g].used; l++) + buffer_json_add_array_item_string(wb, qt->group_by[g].label_keys[l]); + buffer_json_array_close(wb); + + buffer_json_member_add_string( + wb, "aggregation",group_by_aggregate_function_to_string(qt->request.group_by[g].aggregation)); + } + buffer_json_object_close(wb); + } + buffer_json_array_close(wb); // group_by + } + buffer_json_object_close(wb); // aggregations + + buffer_json_member_add_uint64(wb, "timeout", qt->request.timeout_ms); + } + buffer_json_object_close(wb); // request + } + + version_hashes_api_v2(wb, &qt->versions); + + buffer_json_member_add_object(wb, "summary"); + struct summary_total_counts + nodes_totals = { 0 }, + contexts_totals = { 0 }, + instances_totals = { 0 }, + metrics_totals = { 0 }, + label_key_totals = { 0 }, + label_key_value_totals = { 0 }; + { + query_target_summary_nodes_v2(wb, qt, "nodes", &nodes_totals); + r->internal.contexts = query_target_summary_contexts_v2(wb, qt, "contexts", &contexts_totals); + query_target_summary_instances_v2(wb, qt, "instances", &instances_totals); + query_target_summary_dimensions_v12(wb, qt, "dimensions", true, &metrics_totals); + query_target_summary_labels_v12(wb, qt, "labels", true, &label_key_totals, &label_key_value_totals); + query_target_summary_alerts_v2(wb, qt, "alerts"); + } + if(query_target_aggregatable(qt)) { + buffer_json_member_add_object(wb, "globals"); + query_target_points_statistics(wb, qt, &qt->query_points); + buffer_json_object_close(wb); // globals + } + buffer_json_object_close(wb); // summary + + buffer_json_member_add_object(wb, "totals"); + query_target_total_counts(wb, "nodes", &nodes_totals); + query_target_total_counts(wb, "contexts", &contexts_totals); + query_target_total_counts(wb, "instances", &instances_totals); + query_target_total_counts(wb, "dimensions", &metrics_totals); + query_target_total_counts(wb, "label_keys", &label_key_totals); + query_target_total_counts(wb, "label_key_values", &label_key_value_totals); + buffer_json_object_close(wb); // totals + + if(options & RRDR_OPTION_SHOW_DETAILS) { + buffer_json_member_add_object(wb, "detailed"); + query_target_detailed_objects_tree(wb, r, options); + buffer_json_object_close(wb); // detailed + } + + query_target_functions(wb, "functions", r); +} + +//static void annotations_range_for_value_flags(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format __maybe_unused, RRDR_OPTIONS options, RRDR_VALUE_FLAGS flags, const char *type) { +// const size_t dims = r->d, rows = r->rows; +// size_t next_d_idx = 0; +// for(size_t d = 0; d < dims ; d++) { +// if(!rrdr_dimension_should_be_exposed(r->od[d], options)) +// continue; +// +// size_t d_idx = next_d_idx++; +// +// size_t t = 0; +// while(t < rows) { +// +// // find the beginning +// time_t started = 0; +// for(; t < rows ;t++) { +// RRDR_VALUE_FLAGS o = r->o[t * r->d + d]; +// if(o & flags) { +// started = r->t[t]; +// break; +// } +// } +// +// if(started) { +// time_t ended = 0; +// for(; t < rows ;t++) { +// RRDR_VALUE_FLAGS o = r->o[t * r->d + d]; +// if(!(o & flags)) { +// ended = r->t[t]; +// break; +// } +// } +// +// if(!ended) +// ended = r->t[rows - 1]; +// +// buffer_json_add_array_item_object(wb); +// buffer_json_member_add_string(wb, "t", type); +// // buffer_json_member_add_string(wb, "d", string2str(r->dn[d])); +// buffer_json_member_add_uint64(wb, "d", d_idx); +// if(started == ended) { +// if(options & RRDR_OPTION_MILLISECONDS) +// buffer_json_member_add_time_t2ms(wb, "x", started); +// else +// buffer_json_member_add_time_t(wb, "x", started); +// } +// else { +// buffer_json_member_add_array(wb, "x"); +// if(options & RRDR_OPTION_MILLISECONDS) { +// buffer_json_add_array_item_time_t2ms(wb, started); +// buffer_json_add_array_item_time_t2ms(wb, ended); +// } +// else { +// buffer_json_add_array_item_time_t(wb, started); +// buffer_json_add_array_item_time_t(wb, ended); +// } +// buffer_json_array_close(wb); +// } +// buffer_json_object_close(wb); +// } +// } +// } +//} +// +//void rrdr_json_wrapper_annotations(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format __maybe_unused, RRDR_OPTIONS options) { +// buffer_json_member_add_array(wb, "annotations"); +// +// annotations_range_for_value_flags(r, wb, format, options, RRDR_VALUE_EMPTY, "G"); // Gap +// annotations_range_for_value_flags(r, wb, format, options, RRDR_VALUE_RESET, "O"); // Overflow +// annotations_range_for_value_flags(r, wb, format, options, RRDR_VALUE_PARTIAL, "P"); // Partial +// +// buffer_json_array_close(wb); // annotations +//} + +void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb) { + buffer_json_member_add_double(wb, "min", r->view.min); + buffer_json_member_add_double(wb, "max", r->view.max); + + buffer_json_query_timings(wb, "timings", &r->internal.qt->timings); + buffer_json_finalize(wb); +} + +void rrdr_json_wrapper_end2(RRDR *r, BUFFER *wb) { + QUERY_TARGET *qt = r->internal.qt; + DATASOURCE_FORMAT format = qt->request.format; + RRDR_OPTIONS options = qt->window.options; + + buffer_json_member_add_object(wb, "db"); + { + buffer_json_member_add_uint64(wb, "tiers", storage_tiers); + buffer_json_member_add_time_t(wb, "update_every", qt->db.minimum_latest_update_every_s); + buffer_json_member_add_time_t(wb, "first_entry", qt->db.first_time_s); + buffer_json_member_add_time_t(wb, "last_entry", qt->db.last_time_s); + + query_target_combined_units_v2(wb, qt, r->internal.contexts, true); + buffer_json_member_add_object(wb, "dimensions"); + { + rrdr_dimension_ids(wb, "ids", r, options); + rrdr_dimension_units_array_v2(wb, "units", r, options, true); + rrdr_dimension_query_points_statistics(wb, "sts", r, options, false); + } + buffer_json_object_close(wb); // dimensions + + buffer_json_member_add_array(wb, "per_tier"); + for(size_t tier = 0; tier < storage_tiers ; tier++) { + buffer_json_add_array_item_object(wb); + buffer_json_member_add_uint64(wb, "tier", tier); + buffer_json_member_add_uint64(wb, "queries", qt->db.tiers[tier].queries); + buffer_json_member_add_uint64(wb, "points", qt->db.tiers[tier].points); + buffer_json_member_add_time_t(wb, "update_every", qt->db.tiers[tier].update_every); + buffer_json_member_add_time_t(wb, "first_entry", qt->db.tiers[tier].retention.first_time_s); + buffer_json_member_add_time_t(wb, "last_entry", qt->db.tiers[tier].retention.last_time_s); + buffer_json_object_close(wb); + } + buffer_json_array_close(wb); + } + buffer_json_object_close(wb); + + buffer_json_member_add_object(wb, "view"); + { + query_target_title(wb, qt, r->internal.contexts); + buffer_json_member_add_time_t(wb, "update_every", r->view.update_every); + buffer_json_member_add_time_t(wb, "after", r->view.after); + buffer_json_member_add_time_t(wb, "before", r->view.before); + + if(options & RRDR_OPTION_DEBUG) { + buffer_json_member_add_string(wb, "format", rrdr_format_to_string(format)); + rrdr_options_to_buffer_json_array(wb, "options", options); + buffer_json_member_add_string(wb, "time_group", time_grouping_tostring(qt->request.time_group_method)); + } + + if(options & RRDR_OPTION_DEBUG) { + buffer_json_member_add_object(wb, "partial_data_trimming"); + buffer_json_member_add_time_t(wb, "max_update_every", r->partial_data_trimming.max_update_every); + buffer_json_member_add_time_t(wb, "expected_after", r->partial_data_trimming.expected_after); + buffer_json_member_add_time_t(wb, "trimmed_after", r->partial_data_trimming.trimmed_after); + buffer_json_object_close(wb); + } + + if(options & RRDR_OPTION_RETURN_RAW) + buffer_json_member_add_uint64(wb, "points", rrdr_rows(r)); + + query_target_combined_units_v2(wb, qt, r->internal.contexts, false); + query_target_combined_chart_type(wb, qt, r->internal.contexts); + buffer_json_member_add_object(wb, "dimensions"); + { + rrdr_grouped_by_array_v2(wb, "grouped_by", r, options); + rrdr_dimension_ids(wb, "ids", r, options); + rrdr_dimension_names(wb, "names", r, options); + rrdr_dimension_units_array_v2(wb, "units", r, options, false); + rrdr_dimension_priority_array_v2(wb, "priorities", r, options); + rrdr_dimension_aggregated_array_v2(wb, "aggregated", r, options); + rrdr_dimension_query_points_statistics(wb, "sts", r, options, true); + rrdr_json_group_by_labels(wb, "labels", r, options); + } + buffer_json_object_close(wb); // dimensions + buffer_json_member_add_double(wb, "min", r->view.min); + buffer_json_member_add_double(wb, "max", r->view.max); + } + buffer_json_object_close(wb); // view + + buffer_json_agents_v2(wb, &r->internal.qt->timings, 0, false, true); + buffer_json_cloud_timings(wb, "timings", &r->internal.qt->timings); + buffer_json_finalize(wb); +} diff --git a/src/web/api/formatters/json_wrapper.h b/src/web/api/formatters/json_wrapper.h new file mode 100644 index 000000000..a702f3a5c --- /dev/null +++ b/src/web/api/formatters/json_wrapper.h @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_API_FORMATTER_JSON_WRAPPER_H +#define NETDATA_API_FORMATTER_JSON_WRAPPER_H + +#include "rrd2json.h" +#include "web/api/queries/query.h" + +typedef void (*wrapper_begin_t)(RRDR *r, BUFFER *wb); +typedef void (*wrapper_end_t)(RRDR *r, BUFFER *wb); + +void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb); +void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb); + +void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb); +void rrdr_json_wrapper_end2(RRDR *r, BUFFER *wb); + +struct query_versions; +void version_hashes_api_v2(BUFFER *wb, struct query_versions *versions); + +#endif //NETDATA_API_FORMATTER_JSON_WRAPPER_H diff --git a/src/web/api/formatters/rrd2json.c b/src/web/api/formatters/rrd2json.c new file mode 100644 index 000000000..81c9ad5c7 --- /dev/null +++ b/src/web/api/formatters/rrd2json.c @@ -0,0 +1,391 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "web/api/web_api_v1.h" +#include "database/storage_engine.h" + +void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) +{ + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT); + rrdset2json(st, wb, NULL, NULL); + buffer_json_finalize(wb); +} + +const char *rrdr_format_to_string(DATASOURCE_FORMAT format) { + switch(format) { + case DATASOURCE_JSON: + return DATASOURCE_FORMAT_JSON; + + case DATASOURCE_JSON2: + return DATASOURCE_FORMAT_JSON2; + + case DATASOURCE_DATATABLE_JSON: + return DATASOURCE_FORMAT_DATATABLE_JSON; + + case DATASOURCE_DATATABLE_JSONP: + return DATASOURCE_FORMAT_DATATABLE_JSONP; + + case DATASOURCE_JSONP: + return DATASOURCE_FORMAT_JSONP; + + case DATASOURCE_SSV: + return DATASOURCE_FORMAT_SSV; + + case DATASOURCE_CSV: + return DATASOURCE_FORMAT_CSV; + + case DATASOURCE_TSV: + return DATASOURCE_FORMAT_TSV; + + case DATASOURCE_HTML: + return DATASOURCE_FORMAT_HTML; + + case DATASOURCE_JS_ARRAY: + return DATASOURCE_FORMAT_JS_ARRAY; + + case DATASOURCE_SSV_COMMA: + return DATASOURCE_FORMAT_SSV_COMMA; + + default: + return "unknown"; + } +} + +int rrdset2value_api_v1( + RRDSET *st + , BUFFER *wb + , NETDATA_DOUBLE *n + , const char *dimensions + , size_t points + , time_t after + , time_t before + , RRDR_TIME_GROUPING group_method + , const char *group_options + , time_t resampling_time + , uint32_t options + , time_t *db_after + , time_t *db_before + , size_t *db_points_read + , size_t *db_points_per_tier + , size_t *result_points_generated + , int *value_is_null + , NETDATA_DOUBLE *anomaly_rate + , time_t timeout + , size_t tier + , QUERY_SOURCE query_source + , STORAGE_PRIORITY priority +) { + int ret = HTTP_RESP_INTERNAL_SERVER_ERROR; + + ONEWAYALLOC *owa = onewayalloc_create(0); + RRDR *r = rrd2rrdr_legacy( + owa, + st, + points, + after, + before, + group_method, + resampling_time, + options, + dimensions, + group_options, + timeout, + tier, + query_source, + priority); + + if(!r) { + if(value_is_null) *value_is_null = 1; + ret = HTTP_RESP_INTERNAL_SERVER_ERROR; + goto cleanup; + } + + if(db_points_read) + *db_points_read += r->stats.db_points_read; + + if(db_points_per_tier) { + for(size_t t = 0; t < storage_tiers ;t++) + db_points_per_tier[t] += r->internal.qt->db.tiers[t].points; + } + + if(result_points_generated) + *result_points_generated += r->stats.result_points_generated; + + if(rrdr_rows(r) == 0) { + if(db_after) *db_after = 0; + if(db_before) *db_before = 0; + if(value_is_null) *value_is_null = 1; + + ret = HTTP_RESP_BAD_REQUEST; + goto cleanup; + } + + if(wb) { + if (r->view.flags & RRDR_RESULT_FLAG_RELATIVE) + buffer_no_cacheable(wb); + else if (r->view.flags & RRDR_RESULT_FLAG_ABSOLUTE) + buffer_cacheable(wb); + } + + if(db_after) *db_after = r->view.after; + if(db_before) *db_before = r->view.before; + + long i = (!(options & RRDR_OPTION_REVERSED))?(long)rrdr_rows(r) - 1:0; + *n = rrdr2value(r, i, options, value_is_null, anomaly_rate); + ret = HTTP_RESP_OK; + +cleanup: + rrdr_free(owa, r); + onewayalloc_destroy(owa); + return ret; +} + +static inline void buffer_json_member_add_key_only(BUFFER *wb, const char *key) { + buffer_print_json_comma_newline_spacing(wb); + buffer_print_json_key(wb, key); + buffer_fast_strcat(wb, ":", 1); + wb->json.stack[wb->json.depth].count++; +} + +static inline void buffer_json_member_add_string_open(BUFFER *wb, const char *key) { + buffer_json_member_add_key_only(wb, key); + buffer_strcat(wb, wb->json.value_quote); +} + +static inline void buffer_json_member_add_string_close(BUFFER *wb) { + buffer_strcat(wb, wb->json.value_quote); +} + +int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *latest_timestamp) { + wrapper_begin_t wrapper_begin = rrdr_json_wrapper_begin; + wrapper_end_t wrapper_end = rrdr_json_wrapper_end; + + if(qt->request.version == 2) { + wrapper_begin = rrdr_json_wrapper_begin2; + wrapper_end = rrdr_json_wrapper_end2; + } + + RRDR *r = rrd2rrdr(owa, qt); + + if(!r) { + buffer_strcat(wb, "Cannot generate output with these parameters on this chart."); + return HTTP_RESP_INTERNAL_SERVER_ERROR; + } + + if (r->view.flags & RRDR_RESULT_FLAG_CANCEL) { + rrdr_free(owa, r); + return HTTP_RESP_CLIENT_CLOSED_REQUEST; + } + + if(r->view.flags & RRDR_RESULT_FLAG_RELATIVE) + buffer_no_cacheable(wb); + else if(r->view.flags & RRDR_RESULT_FLAG_ABSOLUTE) + buffer_cacheable(wb); + + if(latest_timestamp && rrdr_rows(r) > 0) + *latest_timestamp = r->view.before; + + DATASOURCE_FORMAT format = qt->request.format; + RRDR_OPTIONS options = qt->window.options; + + switch(format) { + case DATASOURCE_SSV: + if(options & RRDR_OPTION_JSON_WRAP) { + wb->content_type = CT_APPLICATION_JSON; + wrapper_begin(r, wb); + buffer_json_member_add_string_open(wb, "result"); + rrdr2ssv(r, wb, options, "", " ", ""); + buffer_json_member_add_string_close(wb); + wrapper_end(r, wb); + } + else { + wb->content_type = CT_TEXT_PLAIN; + rrdr2ssv(r, wb, options, "", " ", ""); + } + break; + + case DATASOURCE_SSV_COMMA: + if(options & RRDR_OPTION_JSON_WRAP) { + wb->content_type = CT_APPLICATION_JSON; + wrapper_begin(r, wb); + buffer_json_member_add_string_open(wb, "result"); + rrdr2ssv(r, wb, options, "", ",", ""); + buffer_json_member_add_string_close(wb); + wrapper_end(r, wb); + } + else { + wb->content_type = CT_TEXT_PLAIN; + rrdr2ssv(r, wb, options, "", ",", ""); + } + break; + + case DATASOURCE_JS_ARRAY: + if(options & RRDR_OPTION_JSON_WRAP) { + wb->content_type = CT_APPLICATION_JSON; + wrapper_begin(r, wb); + buffer_json_member_add_array(wb, "result"); + rrdr2ssv(r, wb, options, "", ",", ""); + buffer_json_array_close(wb); + wrapper_end(r, wb); + } + else { + wb->content_type = CT_APPLICATION_JSON; + rrdr2ssv(r, wb, options, "[", ",", "]"); + } + break; + + case DATASOURCE_CSV: + if(options & RRDR_OPTION_JSON_WRAP) { + wb->content_type = CT_APPLICATION_JSON; + wrapper_begin(r, wb); + buffer_json_member_add_string_open(wb, "result"); + rrdr2csv(r, wb, format, options, "", ",", "\\n", ""); + buffer_json_member_add_string_close(wb); + wrapper_end(r, wb); + } + else { + wb->content_type = CT_TEXT_PLAIN; + rrdr2csv(r, wb, format, options, "", ",", "\r\n", ""); + } + break; + + case DATASOURCE_CSV_MARKDOWN: + if(options & RRDR_OPTION_JSON_WRAP) { + wb->content_type = CT_APPLICATION_JSON; + wrapper_begin(r, wb); + buffer_json_member_add_string_open(wb, "result"); + rrdr2csv(r, wb, format, options, "", "|", "\\n", ""); + buffer_json_member_add_string_close(wb); + wrapper_end(r, wb); + } + else { + wb->content_type = CT_TEXT_PLAIN; + rrdr2csv(r, wb, format, options, "", "|", "\r\n", ""); + } + break; + + case DATASOURCE_CSV_JSON_ARRAY: + wb->content_type = CT_APPLICATION_JSON; + if(options & RRDR_OPTION_JSON_WRAP) { + wrapper_begin(r, wb); + buffer_json_member_add_array(wb, "result"); + rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n"); + buffer_json_array_close(wb); + wrapper_end(r, wb); + } + else { + wb->content_type = CT_APPLICATION_JSON; + buffer_strcat(wb, "[\n"); + rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n"); + buffer_strcat(wb, "\n]"); + } + break; + + case DATASOURCE_TSV: + if(options & RRDR_OPTION_JSON_WRAP) { + wb->content_type = CT_APPLICATION_JSON; + wrapper_begin(r, wb); + buffer_json_member_add_string_open(wb, "result"); + rrdr2csv(r, wb, format, options, "", "\t", "\\n", ""); + buffer_json_member_add_string_close(wb); + wrapper_end(r, wb); + } + else { + wb->content_type = CT_TEXT_PLAIN; + rrdr2csv(r, wb, format, options, "", "\t", "\r\n", ""); + } + break; + + case DATASOURCE_HTML: + if(options & RRDR_OPTION_JSON_WRAP) { + wb->content_type = CT_APPLICATION_JSON; + wrapper_begin(r, wb); + buffer_json_member_add_string_open(wb, "result"); + buffer_strcat(wb, "\\n
\\n\\n"); + rrdr2csv(r, wb, format, options, "\\n", ""); + buffer_strcat(wb, "
", "", "
\\n
\\n\\n"); + buffer_json_member_add_string_close(wb); + wrapper_end(r, wb); + } + else { + wb->content_type = CT_TEXT_HTML; + buffer_strcat(wb, "\n
\n\n"); + rrdr2csv(r, wb, format, options, "\n", ""); + buffer_strcat(wb, "
", "", "
\n
\n\n"); + } + break; + + case DATASOURCE_DATATABLE_JSONP: + wb->content_type = CT_APPLICATION_X_JAVASCRIPT; + + if(options & RRDR_OPTION_JSON_WRAP) { + wrapper_begin(r, wb); + buffer_json_member_add_key_only(wb, "result"); + } + + rrdr2json(r, wb, options, 1); + + if(options & RRDR_OPTION_JSON_WRAP) + wrapper_end(r, wb); + + break; + + case DATASOURCE_DATATABLE_JSON: + wb->content_type = CT_APPLICATION_JSON; + + if(options & RRDR_OPTION_JSON_WRAP) { + wrapper_begin(r, wb); + buffer_json_member_add_key_only(wb, "result"); + } + + rrdr2json(r, wb, options, 1); + + if(options & RRDR_OPTION_JSON_WRAP) + wrapper_end(r, wb); + + break; + + case DATASOURCE_JSONP: + wb->content_type = CT_APPLICATION_X_JAVASCRIPT; + if(options & RRDR_OPTION_JSON_WRAP) { + wrapper_begin(r, wb); + buffer_json_member_add_key_only(wb, "result"); + } + + rrdr2json(r, wb, options, 0); + + if(options & RRDR_OPTION_JSON_WRAP) + wrapper_end(r, wb); + + break; + + case DATASOURCE_JSON: + default: + wb->content_type = CT_APPLICATION_JSON; + + if(options & RRDR_OPTION_JSON_WRAP) { + wrapper_begin(r, wb); + buffer_json_member_add_key_only(wb, "result"); + } + + rrdr2json(r, wb, options, 0); + + if(options & RRDR_OPTION_JSON_WRAP) { + if (options & RRDR_OPTION_RETURN_JWAR) { + buffer_json_member_add_key_only(wb, "anomaly_rates"); + rrdr2json(r, wb, options | RRDR_OPTION_INTERNAL_AR, false); + } + wrapper_end(r, wb); + } + break; + + case DATASOURCE_JSON2: + wb->content_type = CT_APPLICATION_JSON; + wrapper_begin(r, wb); + rrdr2json_v2(r, wb); + wrapper_end(r, wb); + break; + } + + rrdr_free(owa, r); + return HTTP_RESP_OK; +} diff --git a/src/web/api/formatters/rrd2json.h b/src/web/api/formatters/rrd2json.h new file mode 100644 index 000000000..f0c0c39ba --- /dev/null +++ b/src/web/api/formatters/rrd2json.h @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_RRD2JSON_H +#define NETDATA_RRD2JSON_H 1 + +// type of JSON generations +typedef enum { + DATASOURCE_JSON = 0, + DATASOURCE_DATATABLE_JSON = 1, + DATASOURCE_DATATABLE_JSONP = 2, + DATASOURCE_SSV = 3, + DATASOURCE_CSV = 4, + DATASOURCE_JSONP = 5, + DATASOURCE_TSV = 6, + DATASOURCE_HTML = 7, + DATASOURCE_JS_ARRAY = 8, + DATASOURCE_SSV_COMMA = 9, + DATASOURCE_CSV_JSON_ARRAY = 10, + DATASOURCE_CSV_MARKDOWN = 11, + DATASOURCE_JSON2 = 12, +} DATASOURCE_FORMAT; + +#include "web/api/web_api_v1.h" + +#include "web/api/exporters/allmetrics.h" +#include "web/api/queries/rrdr.h" + +#include "web/api/formatters/csv/csv.h" +#include "web/api/formatters/ssv/ssv.h" +#include "web/api/formatters/json/json.h" +#include "web/api/formatters/value/value.h" + +#include "web/api/formatters/rrdset2json.h" +#include "web/api/formatters/charts2json.h" +#include "web/api/formatters/json_wrapper.h" + +#include "web/server/web_client.h" + +#define HOSTNAME_MAX 1024 + +#define DATASOURCE_FORMAT_JSON "json" +#define DATASOURCE_FORMAT_JSON2 "json2" +#define DATASOURCE_FORMAT_DATATABLE_JSON "datatable" +#define DATASOURCE_FORMAT_DATATABLE_JSONP "datasource" +#define DATASOURCE_FORMAT_JSONP "jsonp" +#define DATASOURCE_FORMAT_SSV "ssv" +#define DATASOURCE_FORMAT_CSV "csv" +#define DATASOURCE_FORMAT_TSV "tsv" +#define DATASOURCE_FORMAT_HTML "html" +#define DATASOURCE_FORMAT_JS_ARRAY "array" +#define DATASOURCE_FORMAT_SSV_COMMA "ssvcomma" +#define DATASOURCE_FORMAT_CSV_JSON_ARRAY "csvjsonarray" +#define DATASOURCE_FORMAT_CSV_MARKDOWN "markdown" + +void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb); +const char *rrdr_format_to_string(DATASOURCE_FORMAT format); + +int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, struct query_target *qt, time_t *latest_timestamp); + +void rrdr_json_group_by_labels(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options); + +int rrdset2value_api_v1( + RRDSET *st + , BUFFER *wb + , NETDATA_DOUBLE *n + , const char *dimensions + , size_t points + , time_t after + , time_t before + , RRDR_TIME_GROUPING group_method + , const char *group_options + , time_t resampling_time + , uint32_t options + , time_t *db_after + , time_t *db_before + , size_t *db_points_read + , size_t *db_points_per_tier + , size_t *result_points_generated + , int *value_is_null + , NETDATA_DOUBLE *anomaly_rate + , time_t timeout + , size_t tier + , QUERY_SOURCE query_source + , STORAGE_PRIORITY priority +); + +static inline bool rrdr_dimension_should_be_exposed(RRDR_DIMENSION_FLAGS rrdr_dim_flags, RRDR_OPTIONS options) { + if(unlikely((options & RRDR_OPTION_RETURN_RAW) && (rrdr_dim_flags & RRDR_DIMENSION_QUERIED))) + return true; + + if(unlikely(rrdr_dim_flags & RRDR_DIMENSION_HIDDEN)) return false; + if(unlikely(!(rrdr_dim_flags & RRDR_DIMENSION_QUERIED))) return false; + if(unlikely((options & RRDR_OPTION_NONZERO) && !(rrdr_dim_flags & RRDR_DIMENSION_NONZERO))) return false; + + return true; +} + +#endif /* NETDATA_RRD2JSON_H */ diff --git a/src/web/api/formatters/rrdset2json.c b/src/web/api/formatters/rrdset2json.c new file mode 100644 index 000000000..542178b25 --- /dev/null +++ b/src/web/api/formatters/rrdset2json.c @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rrdset2json.h" + +static int process_label_callback(const char *name, const char *value, RRDLABEL_SRC ls __maybe_unused, void *data) { + BUFFER *wb = data; + buffer_json_member_add_string_or_empty(wb, name, value); + return 1; +} + +void chart_labels2json(RRDSET *st, BUFFER *wb) +{ + if(unlikely(!st->rrdlabels)) + return; + + rrdlabels_walkthrough_read(st->rrdlabels, process_label_callback, wb); +} + +// generate JSON for the /api/v1/chart API call +void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memory_used) +{ + time_t first_entry_t = rrdset_first_entry_s(st); + time_t last_entry_t = rrdset_last_entry_s(st); + char buf[RRD_ID_LENGTH_MAX + 16]; + + buffer_json_member_add_string(wb, "id", rrdset_id(st)); + buffer_json_member_add_string(wb, "name", rrdset_name(st)); + buffer_json_member_add_string(wb, "type", rrdset_parts_type(st)); + buffer_json_member_add_string(wb, "family", rrdset_family(st)); + buffer_json_member_add_string(wb, "context", rrdset_context(st)); + snprintfz(buf, RRD_ID_LENGTH_MAX + 15, "%s (%s)", rrdset_title(st), rrdset_name(st)); + buffer_json_member_add_string(wb, "title", buf); + buffer_json_member_add_int64(wb, "priority", st->priority); + buffer_json_member_add_string(wb, "plugin", rrdset_plugin_name(st)); + buffer_json_member_add_string(wb, "module", rrdset_module_name(st)); + buffer_json_member_add_string(wb, "units", rrdset_units(st)); + + snprintfz(buf, RRD_ID_LENGTH_MAX + 15, "/api/v1/data?chart=%s", rrdset_name(st)); + buffer_json_member_add_string(wb, "data_url", buf); + + buffer_json_member_add_string(wb, "chart_type", rrdset_type_name(st->chart_type)); + buffer_json_member_add_int64(wb, "duration", (int64_t)(last_entry_t - first_entry_t + st->update_every)); + buffer_json_member_add_int64(wb, "first_entry", (int64_t)first_entry_t); + buffer_json_member_add_int64(wb, "last_entry", (int64_t)last_entry_t); + buffer_json_member_add_int64(wb, "update_every", (int64_t)st->update_every); + + unsigned long memory = sizeof(RRDSET); + + size_t dimensions = 0; + buffer_json_member_add_object(wb, "dimensions"); + { + RRDDIM *rd; + rrddim_foreach_read(rd, st) + { + if (rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN) || rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) + continue; + + memory += rrddim_size() + rd->db.memsize; + + buffer_json_member_add_object(wb, rrddim_id(rd)); + buffer_json_member_add_string(wb, "name", rrddim_name(rd)); + buffer_json_object_close(wb); + + dimensions++; + } + rrddim_foreach_done(rd); + } + buffer_json_object_close(wb); + + if(dimensions_count) *dimensions_count += dimensions; + if(memory_used) *memory_used += memory; + + buffer_json_member_add_object(wb, "chart_variables"); + health_api_v1_chart_custom_variables2json(st, wb); + buffer_json_object_close(wb); + + buffer_json_member_add_double(wb, "green", st->green); + buffer_json_member_add_double(wb, "red", st->red); + + { + buffer_json_member_add_object(wb, "alarms"); + RRDCALC *rc; + rw_spinlock_read_lock(&st->alerts.spinlock); + DOUBLE_LINKED_LIST_FOREACH_FORWARD(st->alerts.base, rc, prev, next) + { + { + buffer_json_member_add_object(wb, rrdcalc_name(rc)); + buffer_json_member_add_string_or_empty(wb, "id", rrdcalc_name(rc)); + buffer_json_member_add_string_or_empty(wb, "status", rrdcalc_status2string(rc->status)); + buffer_json_member_add_string_or_empty(wb, "units", rrdcalc_units(rc)); + buffer_json_member_add_int64(wb, "duration", (int64_t)rc->config.update_every); + buffer_json_object_close(wb); + } + } + rw_spinlock_read_unlock(&st->alerts.spinlock); + buffer_json_object_close(wb); + } + + buffer_json_member_add_object(wb, "chart_labels"); + chart_labels2json(st, wb); + buffer_json_object_close(wb); + + buffer_json_member_add_object(wb, "functions"); + chart_functions2json(st, wb); + buffer_json_object_close(wb); +} diff --git a/src/web/api/formatters/rrdset2json.h b/src/web/api/formatters/rrdset2json.h new file mode 100644 index 000000000..8b325c65d --- /dev/null +++ b/src/web/api/formatters/rrdset2json.h @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_API_FORMATTER_RRDSET2JSON_H +#define NETDATA_API_FORMATTER_RRDSET2JSON_H + +#include "rrd2json.h" + +void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memory_used); + +#endif //NETDATA_API_FORMATTER_RRDSET2JSON_H diff --git a/src/web/api/formatters/ssv/README.md b/src/web/api/formatters/ssv/README.md new file mode 100644 index 000000000..b32494014 --- /dev/null +++ b/src/web/api/formatters/ssv/README.md @@ -0,0 +1,63 @@ + + +# SSV formatter + +The SSV formatter sums all dimensions in [results of database queries](/src/web/api/queries/README.md) +to a single value and returns a list of such values showing how it changes through time. + +It supports the following formats: + +| format | content type | description | +|:----:|:----------:|:----------| +| `ssv` | text/plain | a space separated list of values | +| `ssvcomma` | text/plain | a comma separated list of values | +| `array` | application/json | a JSON array | + +The SSV formatter respects the following API `&options=`: + +| option | supported | description | +| :----:|:-------:|:----------| +| `nonzero` | yes | to return only the dimensions that have at least a non-zero value | +| `flip` | yes | to return the numbers older to newer (the default is newer to older) | +| `percent` | yes | to replace all values with their percentage over the row total | +| `abs` | yes | to turn all values positive, before using them | +| `min2max` | yes | to return the delta from the minimum value to the maximum value (across dimensions) | + +## Examples + +Get the average system CPU utilization of the last hour, in 6 values (one every 10 minutes): + +```bash +# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=system.cpu&format=ssv&after=-3600&points=6&group=average' +1.741352 1.6800467 1.769411 1.6761112 1.629862 1.6807968 +``` + +--- + +Get the total mysql bandwidth (in + out) for the last hour, in 6 values (one every 10 minutes): + +Netdata returns bandwidth in `kilobits`. + +```bash +# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=mysql_local.net&format=ssvcomma&after=-3600&points=6&group=sum&options=abs' +72618.7936215,72618.778889,72618.788084,72618.9195918,72618.7760612,72618.6712421 +``` + +--- + +Get the web server max connections for the last hour, in 12 values (one every 5 minutes) +in a JSON array: + +```bash +# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=nginx_local.connections&format=array&after=-3600&points=12&group=max' +[278,258,268,239,259,260,243,266,278,318,264,258] +``` + + diff --git a/src/web/api/formatters/ssv/ssv.c b/src/web/api/formatters/ssv/ssv.c new file mode 100644 index 000000000..2eb26b459 --- /dev/null +++ b/src/web/api/formatters/ssv/ssv.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "ssv.h" + +void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, const char *separator, const char *suffix) { + //netdata_log_info("RRD2SSV(): %s: BEGIN", r->st->id); + long i; + + buffer_strcat(wb, prefix); + long start = 0, end = rrdr_rows(r), step = 1; + if(!(options & RRDR_OPTION_REVERSED)) { + start = rrdr_rows(r) - 1; + end = -1; + step = -1; + } + + // for each line in the array + for(i = start; i != end ;i += step) { + int all_values_are_null = 0; + NETDATA_DOUBLE v = rrdr2value(r, i, options, &all_values_are_null, NULL); + + if(likely(i != start)) { + if(r->view.min > v) r->view.min = v; + if(r->view.max < v) r->view.max = v; + } + else { + r->view.min = v; + r->view.max = v; + } + + if(likely(i != start)) + buffer_strcat(wb, separator); + + if(all_values_are_null) { + if(options & RRDR_OPTION_NULL2ZERO) + buffer_strcat(wb, "0"); + else + buffer_strcat(wb, "null"); + } + else + buffer_print_netdata_double(wb, v); + } + buffer_strcat(wb, suffix); + //netdata_log_info("RRD2SSV(): %s: END", r->st->id); +} diff --git a/src/web/api/formatters/ssv/ssv.h b/src/web/api/formatters/ssv/ssv.h new file mode 100644 index 000000000..f7d4a9548 --- /dev/null +++ b/src/web/api/formatters/ssv/ssv.h @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_API_FORMATTER_SSV_H +#define NETDATA_API_FORMATTER_SSV_H + +#include "../rrd2json.h" + +void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, const char *separator, const char *suffix); + +#endif //NETDATA_API_FORMATTER_SSV_H diff --git a/src/web/api/formatters/value/README.md b/src/web/api/formatters/value/README.md new file mode 100644 index 000000000..8a2df23c6 --- /dev/null +++ b/src/web/api/formatters/value/README.md @@ -0,0 +1,28 @@ + + +# Value formatter + +The Value formatter presents [results of database queries](/src/web/api/queries/README.md) as a single value. + +To calculate the single value to be returned, it sums the values of all dimensions. + +The Value formatter respects the following API `&options=`: + +| option | supported | description | +|:----: |:-------: |:---------- | +| `percent` | yes | to replace all values with their percentage over the row total| +| `abs` | yes | to turn all values positive, before using them | +| `min2max` | yes | to return the delta from the minimum value to the maximum value (across dimensions)| + +The Value formatter is not exposed by the API by itself. +Instead it is used by the [`ssv`](/src/web/api/formatters/ssv/README.md) formatter +and [health monitoring queries](/src/health/README.md). + + diff --git a/src/web/api/formatters/value/value.c b/src/web/api/formatters/value/value.c new file mode 100644 index 000000000..0ec1b1265 --- /dev/null +++ b/src/web/api/formatters/value/value.c @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "value.h" + +inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, NETDATA_DOUBLE *anomaly_rate) { + size_t c; + + NETDATA_DOUBLE *cn = &r->v[ i * r->d ]; + RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ]; + NETDATA_DOUBLE *ar = &r->ar[ i * r->d ]; + + NETDATA_DOUBLE sum = 0, min = NAN, max = NAN, v = NAN; + size_t dims = 0; + + NETDATA_DOUBLE total_anomaly_rate = 0; + + // for each dimension + for (c = 0; c < r->d ; c++) { + if(unlikely(!rrdr_dimension_should_be_exposed(r->od[c], options))) + continue; + + if(unlikely((co[c] & RRDR_VALUE_EMPTY))) + continue; + + NETDATA_DOUBLE n = cn[c]; + + if(unlikely(!dims)) + min = max = n; + + sum += n; + + if (n < min) min = n; + if (n > max) max = n; + + total_anomaly_rate += ar[c]; + + dims++; + } + + if(!dims) { + if(anomaly_rate) + *anomaly_rate = 0; + + if(all_values_are_null) + *all_values_are_null = 1; + + return (options & RRDR_OPTION_NULL2ZERO) ? 0 : NAN; + } + + if(anomaly_rate) + *anomaly_rate = total_anomaly_rate / (NETDATA_DOUBLE)dims; + + if(all_values_are_null) + *all_values_are_null = 0; + + if(options & RRDR_OPTION_DIMS_MIN2MAX) + v = max - min; + else if(options & RRDR_OPTION_DIMS_AVERAGE) + v = sum / (NETDATA_DOUBLE)dims; + else if(options & RRDR_OPTION_DIMS_MIN) + v = min; + else if(options & RRDR_OPTION_DIMS_MAX) + v = max; + else + v = sum; + + if((options & RRDR_OPTION_NULL2ZERO) && (isnan(v) || isinf(v))) + v = 0; + + return v; +} + +QUERY_VALUE rrdmetric2value(RRDHOST *host, + struct rrdcontext_acquired *rca, struct rrdinstance_acquired *ria, struct rrdmetric_acquired *rma, + time_t after, time_t before, + RRDR_OPTIONS options, RRDR_TIME_GROUPING time_group_method, const char *time_group_options, + size_t tier, time_t timeout, QUERY_SOURCE query_source, STORAGE_PRIORITY priority +) { + QUERY_TARGET_REQUEST qtr = { + .version = 1, + .host = host, + .rca = rca, + .ria = ria, + .rma = rma, + .after = after, + .before = before, + .points = 1, + .options = options, + .time_group_method = time_group_method, + .time_group_options = time_group_options, + .tier = tier, + .timeout_ms = timeout, + .query_source = query_source, + .priority = priority, + }; + + ONEWAYALLOC *owa = onewayalloc_create(16 * 1024); + QUERY_TARGET *qt = query_target_create(&qtr); + RRDR *r = rrd2rrdr(owa, qt); + + QUERY_VALUE qv; + + if(!r || rrdr_rows(r) == 0) { + qv = (QUERY_VALUE) { + .value = NAN, + .anomaly_rate = NAN, + .sp = { + .count = 0, + .min = NAN, + .max = NAN, + .sum = NAN, + .anomaly_count = 0, + }, + .duration_ut = (r) ? r->internal.qt->timings.executed_ut - r->internal.qt->timings.received_ut : 0, + }; + } + else { + qv = (QUERY_VALUE) { + .after = r->view.after, + .before = r->view.before, + .points_read = r->stats.db_points_read, + .result_points = r->stats.result_points_generated, + .sp = { + .count = 0, + }, + .duration_ut = r->internal.qt->timings.executed_ut - r->internal.qt->timings.received_ut, + }; + + for(size_t d = 0; d < r->internal.qt->query.used ;d++) { + if(!rrdr_dimension_should_be_exposed(r->internal.qt->query.array[d].status, options)) + continue; + + storage_point_merge_to(qv.sp, r->internal.qt->query.array[d].query_points); + } + + for(size_t t = 0; t < storage_tiers ;t++) + qv.storage_points_per_tier[t] = r->internal.qt->db.tiers[t].points; + + long i = (!(options & RRDR_OPTION_REVERSED))?(long)rrdr_rows(r) - 1:0; + int all_values_are_null = 0; + qv.value = rrdr2value(r, i, options, &all_values_are_null, &qv.anomaly_rate); + if(all_values_are_null) { + qv.value = NAN; + qv.anomaly_rate = NAN; + } + } + + rrdr_free(owa, r); + query_target_release(qt); + onewayalloc_destroy(owa); + + return qv; +} diff --git a/src/web/api/formatters/value/value.h b/src/web/api/formatters/value/value.h new file mode 100644 index 000000000..072ca14f8 --- /dev/null +++ b/src/web/api/formatters/value/value.h @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_API_FORMATTER_VALUE_H +#define NETDATA_API_FORMATTER_VALUE_H + +#include "../rrd2json.h" + +typedef struct storage_value { + NETDATA_DOUBLE value; + NETDATA_DOUBLE anomaly_rate; + time_t after; + time_t before; + size_t points_read; + size_t storage_points_per_tier[RRD_STORAGE_TIERS]; + size_t result_points; + STORAGE_POINT sp; + usec_t duration_ut; +} QUERY_VALUE; + +struct rrdmetric_acquired; +struct rrdinstance_acquired; +struct rrdcontext_acquired; + +QUERY_VALUE rrdmetric2value(RRDHOST *host, + struct rrdcontext_acquired *rca, struct rrdinstance_acquired *ria, struct rrdmetric_acquired *rma, + time_t after, time_t before, + RRDR_OPTIONS options, RRDR_TIME_GROUPING time_group_method, const char *time_group_options, + size_t tier, time_t timeout, QUERY_SOURCE query_source, STORAGE_PRIORITY priority +); + +NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, NETDATA_DOUBLE *anomaly_rate); + +#endif //NETDATA_API_FORMATTER_VALUE_H -- cgit v1.2.3