summaryrefslogtreecommitdiffstats
path: root/src/web/api/formatters
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-05-05 11:19:16 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-05-05 12:07:37 +0000
commitb485aab7e71c1625cfc27e0f92c9509f42378458 (patch)
treeae9abe108601079d1679194de237c9a435ae5b55 /src/web/api/formatters
parentAdding upstream version 1.44.3. (diff)
downloadnetdata-b485aab7e71c1625cfc27e0f92c9509f42378458.tar.xz
netdata-b485aab7e71c1625cfc27e0f92c9509f42378458.zip
Adding upstream version 1.45.3+dfsg.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/web/api/formatters')
-rw-r--r--src/web/api/formatters/README.md82
-rw-r--r--src/web/api/formatters/charts2json.c104
-rw-r--r--src/web/api/formatters/charts2json.h11
-rw-r--r--src/web/api/formatters/csv/README.md148
-rw-r--r--src/web/api/formatters/csv/csv.c109
-rw-r--r--src/web/api/formatters/csv/csv.h12
-rw-r--r--src/web/api/formatters/json/README.md160
-rw-r--r--src/web/api/formatters/json/json.c350
-rw-r--r--src/web/api/formatters/json/json.h11
-rw-r--r--src/web/api/formatters/json_wrapper.c1578
-rw-r--r--src/web/api/formatters/json_wrapper.h21
-rw-r--r--src/web/api/formatters/rrd2json.c391
-rw-r--r--src/web/api/formatters/rrd2json.h98
-rw-r--r--src/web/api/formatters/rrdset2json.c106
-rw-r--r--src/web/api/formatters/rrdset2json.h10
-rw-r--r--src/web/api/formatters/ssv/README.md63
-rw-r--r--src/web/api/formatters/ssv/ssv.c45
-rw-r--r--src/web/api/formatters/ssv/ssv.h10
-rw-r--r--src/web/api/formatters/value/README.md28
-rw-r--r--src/web/api/formatters/value/value.c153
-rw-r--r--src/web/api/formatters/value/value.h33
21 files changed, 3523 insertions, 0 deletions
diff --git a/src/web/api/formatters/README.md b/src/web/api/formatters/README.md
new file mode 100644
index 000000000..df1ae7867
--- /dev/null
+++ b/src/web/api/formatters/README.md
@@ -0,0 +1,82 @@
+<!--
+title: "Query formatting"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/src/web/api/formatters/README.md
+sidebar_label: "Query formatting"
+learn_status: "Published"
+learn_topic_type: "References"
+learn_rel_path: "Developers/Web/Api/Formatters"
+-->
+
+# Query formatting
+
+API data queries need to be formatted before returned to the caller.
+Using API parameters, the caller may define the format he/she wishes to get back.
+
+The following formats are supported:
+
+| format|module|content type|description|
+|:----:|:----:|:----------:|:----------|
+| `array`|[ssv](https://github.com/netdata/netdata/blob/master/src/web/api/formatters/ssv/README.md)|application/json|a JSON array|
+| `csv`|[csv](https://github.com/netdata/netdata/blob/master/src/web/api/formatters/csv/README.md)|text/plain|a text table, comma separated, with a header line (dimension names) and `\r\n` at the end of the lines|
+| `csvjsonarray`|[csv](https://github.com/netdata/netdata/blob/master/src/web/api/formatters/csv/README.md)|application/json|a JSON array, with each row as another array (the first row has the dimension names)|
+| `datasource`|[json](https://github.com/netdata/netdata/blob/master/src/web/api/formatters/json/README.md)|application/json|a Google Visualization Provider `datasource` javascript callback|
+| `datatable`|[json](https://github.com/netdata/netdata/blob/master/src/web/api/formatters/json/README.md)|application/json|a Google `datatable`|
+| `html`|[csv](https://github.com/netdata/netdata/blob/master/src/web/api/formatters/csv/README.md)|text/html|an html table|
+| `json`|[json](https://github.com/netdata/netdata/blob/master/src/web/api/formatters/json/README.md)|application/json|a JSON object|
+| `jsonp`|[json](https://github.com/netdata/netdata/blob/master/src/web/api/formatters/json/README.md)|application/json|a JSONP javascript callback|
+| `markdown`|[csv](https://github.com/netdata/netdata/blob/master/src/web/api/formatters/csv/README.md)|text/plain|a markdown table|
+| `ssv`|[ssv](https://github.com/netdata/netdata/blob/master/src/web/api/formatters/ssv/README.md)|text/plain|a space separated list of values|
+| `ssvcomma`|[ssv](https://github.com/netdata/netdata/blob/master/src/web/api/formatters/ssv/README.md)|text/plain|a comma separated list of values|
+| `tsv`|[csv](https://github.com/netdata/netdata/blob/master/src/web/api/formatters/csv/README.md)|text/plain|a TAB delimited `csv` (MS Excel flavor)|
+
+For examples of each format, check the relative module documentation.
+
+## Metadata with the `jsonwrap` option
+
+All data queries can be encapsulated to JSON object having metadata about the query and the results.
+
+This is done by adding the `options=jsonwrap` to the API URL (if there are other `options` append
+`,jsonwrap` to the existing ones).
+
+This is such an object:
+
+```bash
+# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=system.cpu&after=-3600&points=6&group=average&format=csv&options=nonzero,jsonwrap'
+{
+ "api": 1,
+ "id": "system.cpu",
+ "name": "system.cpu",
+ "view_update_every": 600,
+ "update_every": 1,
+ "first_entry": 1540387074,
+ "last_entry": 1540647070,
+ "before": 1540647000,
+ "after": 1540644000,
+ "dimension_names": ["steal", "softirq", "user", "system", "iowait"],
+ "dimension_ids": ["steal", "softirq", "user", "system", "iowait"],
+ "latest_values": [0, 0.2493766, 1.745636, 0.4987531, 0],
+ "view_latest_values": [0.0158314, 0.0516506, 0.866549, 0.7196127, 0.0050002],
+ "dimensions": 5,
+ "points": 6,
+ "format": "csv",
+ "result": "time,steal,softirq,user,system,iowait\n2018-10-27 13:30:00,0.0158314,0.0516506,0.866549,0.7196127,0.0050002\n2018-10-27 13:20:00,0.0149856,0.0529183,0.8673155,0.7121144,0.0049979\n2018-10-27 13:10:00,0.0137501,0.053315,0.8578097,0.7197613,0.0054209\n2018-10-27 13:00:00,0.0154252,0.0554688,0.899432,0.7200638,0.0067252\n2018-10-27 12:50:00,0.0145866,0.0495922,0.8404341,0.7011141,0.0041688\n2018-10-27 12:40:00,0.0162366,0.0595954,0.8827475,0.7020573,0.0041636\n",
+ "min": 0,
+ "max": 0
+}
+```
+
+## Downloading data query result files
+
+Following the [Google Visualization Provider guidelines](https://developers.google.com/chart/interactive/docs/dev/implementing_data_source),
+Netdata supports parsing `tqx` options.
+
+Using these options, any Netdata data query can instruct the web browser to download
+the result and save it under a given filename.
+
+For example, to download a CSV file with CPU utilization of the last hour,
+[click here](https://registry.my-netdata.io/api/v1/data?chart=system.cpu&after=-3600&format=csv&options=nonzero&tqx=outFileName:system+cpu+utilization+of+the+last_hour.csv).
+
+This is done by appending `&tqx=outFileName:FILENAME` to any data query.
+The output will be in the format given with `&format=`.
+
+
diff --git a/src/web/api/formatters/charts2json.c b/src/web/api/formatters/charts2json.c
new file mode 100644
index 000000000..cab4debae
--- /dev/null
+++ b/src/web/api/formatters/charts2json.c
@@ -0,0 +1,104 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "charts2json.h"
+
+// generate JSON for the /api/v1/charts API call
+
+const char* get_release_channel() {
+ static int use_stable = -1;
+
+ if (use_stable == -1) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s/.environment", netdata_configured_user_config_dir);
+ procfile *ff = procfile_open(filename, "=", PROCFILE_FLAG_NO_ERROR_ON_FILE_IO);
+ if (ff) {
+ procfile_set_quotes(ff, "'\"");
+ ff = procfile_readall(ff);
+ if (ff) {
+ unsigned int i;
+ for (i = 0; i < procfile_lines(ff); i++) {
+ if (!procfile_linewords(ff, i))
+ continue;
+ if (!strcmp(procfile_lineword(ff, i, 0), "RELEASE_CHANNEL")) {
+ if (!strcmp(procfile_lineword(ff, i, 1), "stable"))
+ use_stable = 1;
+ else if (!strcmp(procfile_lineword(ff, i, 1), "nightly"))
+ use_stable = 0;
+ break;
+ }
+ }
+ procfile_close(ff);
+ }
+ }
+ if (use_stable == -1)
+ use_stable = strchr(program_version, '-') ? 0 : 1;
+ }
+ return (use_stable)?"stable":"nightly";
+}
+
+void charts2json(RRDHOST *host, BUFFER *wb) {
+ static char *custom_dashboard_info_js_filename = NULL;
+ size_t c = 0, dimensions = 0, memory = 0, alarms = 0;
+ RRDSET *st;
+
+ time_t now = now_realtime_sec();
+
+ if(unlikely(!custom_dashboard_info_js_filename))
+ custom_dashboard_info_js_filename = config_get(CONFIG_SECTION_WEB, "custom dashboard_info.js", "");
+
+ buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
+
+ buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(host));
+ buffer_json_member_add_string(wb, "version", rrdhost_program_version(host));
+ buffer_json_member_add_string(wb, "release_channel", get_release_channel());
+ buffer_json_member_add_string(wb, "os", rrdhost_os(host));
+ buffer_json_member_add_string(wb, "timezone", rrdhost_timezone(host));
+ buffer_json_member_add_int64(wb, "update_every", host->rrd_update_every);
+ buffer_json_member_add_int64(wb, "history", host->rrd_history_entries);
+ buffer_json_member_add_string(wb, "memory_mode", rrd_memory_mode_name(host->rrd_memory_mode));
+ buffer_json_member_add_string(wb, "custom_info", custom_dashboard_info_js_filename);
+
+ buffer_json_member_add_object(wb, "charts");
+ rrdset_foreach_read(st, host) {
+ if (rrdset_is_available_for_viewers(st)) {
+
+ buffer_json_member_add_object(wb, rrdset_id(st));
+ rrdset2json(st, wb, &dimensions, &memory);
+ buffer_json_object_close(wb);
+ st->last_accessed_time_s = now;
+ c++;
+ }
+ }
+ rrdset_foreach_done(st);
+ buffer_json_object_close(wb);
+
+ RRDCALC *rc;
+ foreach_rrdcalc_in_rrdhost_read(host, rc) {
+ if(rc->rrdset)
+ alarms++;
+ }
+ foreach_rrdcalc_in_rrdhost_done(rc);
+
+ buffer_json_member_add_int64(wb, "charts_count", (int64_t) c);
+ buffer_json_member_add_int64(wb, "dimensions_count", (int64_t) dimensions);
+ buffer_json_member_add_int64(wb, "alarms_count", (int64_t)alarms);
+ buffer_json_member_add_int64(wb, "rrd_memory_bytes", (int64_t)memory);
+ buffer_json_member_add_int64(wb, "hosts_count", (int64_t) rrdhost_hosts_available());
+
+ buffer_json_member_add_array(wb, "hosts");
+ {
+ rrd_rdlock();
+ RRDHOST *h;
+ rrdhost_foreach_read(h) {
+ if(!rrdhost_should_be_removed(h, host, now) /*&& !rrdhost_flag_check(h, RRDHOST_FLAG_ARCHIVED) */) {
+ buffer_json_add_array_item_object(wb);
+ buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(h));
+ buffer_json_object_close(wb);
+ }
+ }
+ rrd_unlock();
+ }
+ buffer_json_array_close(wb);
+
+ buffer_json_finalize(wb);
+}
diff --git a/src/web/api/formatters/charts2json.h b/src/web/api/formatters/charts2json.h
new file mode 100644
index 000000000..7b07af5a0
--- /dev/null
+++ b/src/web/api/formatters/charts2json.h
@@ -0,0 +1,11 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_API_FORMATTER_CHARTS2JSON_H
+#define NETDATA_API_FORMATTER_CHARTS2JSON_H
+
+#include "rrd2json.h"
+
+void charts2json(RRDHOST *host, BUFFER *wb);
+const char* get_release_channel();
+
+#endif //NETDATA_API_FORMATTER_CHARTS2JSON_H
diff --git a/src/web/api/formatters/csv/README.md b/src/web/api/formatters/csv/README.md
new file mode 100644
index 000000000..ee5e3666f
--- /dev/null
+++ b/src/web/api/formatters/csv/README.md
@@ -0,0 +1,148 @@
+<!--
+title: "CSV formatter"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/src/web/api/formatters/csv/README.md
+sidebar_label: "CSV formatter"
+learn_status: "Published"
+learn_topic_type: "References"
+learn_rel_path: "Developers/Web/Api/Formatters"
+-->
+
+# CSV formatter
+
+The CSV formatter presents [results of database queries](https://github.com/netdata/netdata/blob/master/src/web/api/queries/README.md) in the following formats:
+
+| format|content type|description|
+| :----:|:----------:|:----------|
+| `csv`|text/plain|a text table, comma separated, with a header line (dimension names) and `\r\n` at the end of the lines|
+| `csvjsonarray`|application/json|a JSON array, with each row as another array (the first row has the dimension names)|
+| `tsv`|text/plain|like `csv` but TAB is used instead of comma to separate values (MS Excel flavor)|
+| `html`|text/html|an html table|
+| `markdown`|text/plain|markdown table|
+
+In all formats the date and time is the first column.
+
+The CSV formatter respects the following API `&options=`:
+
+| option|supported|description|
+|:----:|:-------:|:----------|
+| `nonzero`|yes|to return only the dimensions that have at least a non-zero value|
+| `flip`|yes|to return the rows older to newer (the default is newer to older)|
+| `seconds`|yes|to return the date and time in unix timestamp|
+| `ms`|yes|to return the date and time in unit timestamp as milliseconds|
+| `percent`|yes|to replace all values with their percentage over the row total|
+| `abs`|yes|to turn all values positive|
+| `null2zero`|yes|to replace gaps with zeros (the default prints the string `null`|
+
+## Examples
+
+Get the system total bandwidth for all physical network interfaces, over the last hour,
+in 6 rows (one for every 10 minutes), in `csv` format:
+
+Netdata always returns bandwidth in `kilobits`.
+
+```bash
+# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=system.net&format=csv&after=-3600&group=sum&points=6&options=abs'
+time,received,sent
+2018-10-26 23:50:00,90214.67847,215137.79762
+2018-10-26 23:40:00,90126.32286,238587.57522
+2018-10-26 23:30:00,86061.22688,213389.23526
+2018-10-26 23:20:00,85590.75164,206129.01608
+2018-10-26 23:10:00,83163.30691,194311.77384
+2018-10-26 23:00:00,85167.29657,197538.07773
+```
+
+---
+
+Get the max RAM used by the SQL server and any cron jobs, over the last hour, in 2 rows (one for every 30
+minutes), in `tsv` format, and format the date and time as unix timestamp:
+
+Netdata always returns memory in `MB`.
+
+```bash
+# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=apps.mem&format=tsv&after=-3600&group=max&points=2&options=nonzero,seconds&dimensions=sql,cron'
+time sql cron
+1540598400 61.95703 0.25
+1540596600 61.95703 0.25
+```
+
+---
+
+Get an HTML table of the last 4 values (4 seconds) of system CPU utilization:
+
+Netdata always returns CPU utilization as `%`.
+
+```bash
+# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=system.cpu&format=html&after=-4&options=nonzero'
+<html>
+<center>
+<table border="0" cellpadding="5" cellspacing="5">
+<tr><td>time</td><td>softirq</td><td>user</td><td>system</td></tr>
+<tr><td>2018-10-27 00:16:07</td><td>0.25</td><td>1</td><td>0.75</td></tr>
+<tr><td>2018-10-27 00:16:06</td><td>0</td><td>1.0025063</td><td>0.5012531</td></tr>
+<tr><td>2018-10-27 00:16:05</td><td>0</td><td>1</td><td>0.75</td></tr>
+<tr><td>2018-10-27 00:16:04</td><td>0</td><td>1.0025063</td><td>0.7518797</td></tr>
+</table>
+</center>
+</html>
+```
+
+This is how it looks when rendered by a web browser:
+
+![image](https://user-images.githubusercontent.com/2662304/47597887-bafbf480-d99c-11e8-864a-d880bb8d2e5b.png)
+
+---
+
+Get a JSON array with the average bandwidth rate of the mysql server, over the last hour, in 6 values
+(one every 10 minutes), and return the date and time in milliseconds:
+
+Netdata always returns bandwidth rates in `kilobits/s`.
+
+```bash
+# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=mysql_local.net&format=csvjsonarray&after=-3600&points=6&group=average&options=abs,ms'
+[
+["time","in","out"],
+[1540599600000,0.7499986,120.2810185],
+[1540599000000,0.7500019,120.2815509],
+[1540598400000,0.7499999,120.2812319],
+[1540597800000,0.7500044,120.2819634],
+[1540597200000,0.7499968,120.2807337],
+[1540596600000,0.7499988,120.2810527]
+]
+```
+
+---
+
+Get the number of processes started per minute, for the last 10 minutes, in `markdown` format:
+
+```bash
+# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=system.forks&format=markdown&after=-600&points=10&group=sum'
+time | started
+:---: |:---:
+2018-10-27 03:52:00| 245.1706149
+2018-10-27 03:51:00| 152.6654636
+2018-10-27 03:50:00| 163.1755789
+2018-10-27 03:49:00| 176.1574766
+2018-10-27 03:48:00| 178.0137076
+2018-10-27 03:47:00| 183.8306543
+2018-10-27 03:46:00| 264.1635621
+2018-10-27 03:45:00| 205.001551
+2018-10-27 03:44:00| 7026.9852167
+2018-10-27 03:43:00| 205.9904794
+```
+
+And this is how it looks when formatted:
+
+| time | started |
+|:--:|:-----:|
+| 2018-10-27 03:52:00 | 245.1706149 |
+| 2018-10-27 03:51:00 | 152.6654636 |
+| 2018-10-27 03:50:00 | 163.1755789 |
+| 2018-10-27 03:49:00 | 176.1574766 |
+| 2018-10-27 03:48:00 | 178.0137076 |
+| 2018-10-27 03:47:00 | 183.8306543 |
+| 2018-10-27 03:46:00 | 264.1635621 |
+| 2018-10-27 03:45:00 | 205.001551 |
+| 2018-10-27 03:44:00 | 7026.9852167 |
+| 2018-10-27 03:43:00 | 205.9904794 |
+
+
diff --git a/src/web/api/formatters/csv/csv.c b/src/web/api/formatters/csv/csv.c
new file mode 100644
index 000000000..d81ddb34e
--- /dev/null
+++ b/src/web/api/formatters/csv/csv.c
@@ -0,0 +1,109 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "libnetdata/libnetdata.h"
+#include "csv.h"
+
+void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines) {
+ //netdata_log_info("RRD2CSV(): %s: BEGIN", r->st->id);
+ long c, i;
+ const long used = (long)r->d;
+
+ // print the csv header
+ for(c = 0, i = 0; c < used ; c++) {
+ if(!rrdr_dimension_should_be_exposed(r->od[c], options))
+ continue;
+
+ if(!i) {
+ buffer_strcat(wb, startline);
+ if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
+ buffer_strcat(wb, "time");
+ if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
+ }
+ buffer_strcat(wb, separator);
+ if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
+ buffer_strcat(wb, string2str(r->dn[c]));
+ if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
+ i++;
+ }
+ buffer_strcat(wb, endline);
+
+ if(format == DATASOURCE_CSV_MARKDOWN) {
+ // print the --- line after header
+ for(c = 0, i = 0; c < used ;c++) {
+ if(!rrdr_dimension_should_be_exposed(r->od[c], options))
+ continue;
+
+ if(!i) {
+ buffer_strcat(wb, startline);
+ if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
+ buffer_strcat(wb, ":---:");
+ if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
+ }
+ buffer_strcat(wb, separator);
+ if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
+ buffer_strcat(wb, ":---:");
+ if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
+ i++;
+ }
+ buffer_strcat(wb, endline);
+ }
+
+ if(!i) {
+ // no dimensions present
+ return;
+ }
+
+ long start = 0, end = rrdr_rows(r), step = 1;
+ if(!(options & RRDR_OPTION_REVERSED)) {
+ start = rrdr_rows(r) - 1;
+ end = -1;
+ step = -1;
+ }
+
+ // for each line in the array
+ for(i = start; i != end ;i += step) {
+ NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
+ RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ];
+
+ buffer_strcat(wb, betweenlines);
+ buffer_strcat(wb, startline);
+
+ time_t now = r->t[i];
+
+ if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) {
+ // print the timestamp of the line
+ buffer_print_netdata_double(wb, (NETDATA_DOUBLE) now);
+ // in ms
+ if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000");
+ }
+ else {
+ // generate the local date time
+ struct tm tmbuf, *tm = localtime_r(&now, &tmbuf);
+ if(!tm) {
+ netdata_log_error("localtime() failed."); continue; }
+ buffer_date(wb, tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
+ }
+
+ // for each dimension
+ for(c = 0; c < used ;c++) {
+ if(!rrdr_dimension_should_be_exposed(r->od[c], options))
+ continue;
+
+ buffer_strcat(wb, separator);
+
+ NETDATA_DOUBLE n = cn[c];
+
+ if(co[c] & RRDR_VALUE_EMPTY) {
+ if(options & RRDR_OPTION_NULL2ZERO)
+ buffer_strcat(wb, "0");
+ else
+ buffer_strcat(wb, "null");
+ }
+ else
+ buffer_print_netdata_double(wb, n);
+ }
+
+ buffer_strcat(wb, endline);
+ }
+ //netdata_log_info("RRD2CSV(): %s: END", r->st->id);
+}
diff --git a/src/web/api/formatters/csv/csv.h b/src/web/api/formatters/csv/csv.h
new file mode 100644
index 000000000..666d4c660
--- /dev/null
+++ b/src/web/api/formatters/csv/csv.h
@@ -0,0 +1,12 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_API_FORMATTER_CSV_H
+#define NETDATA_API_FORMATTER_CSV_H
+
+#include "web/api/queries/rrdr.h"
+
+void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines);
+
+#include "../rrd2json.h"
+
+#endif //NETDATA_API_FORMATTER_CSV_H
diff --git a/src/web/api/formatters/json/README.md b/src/web/api/formatters/json/README.md
new file mode 100644
index 000000000..b0037cb2e
--- /dev/null
+++ b/src/web/api/formatters/json/README.md
@@ -0,0 +1,160 @@
+<!--
+title: "JSON formatter"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/src/web/api/formatters/json/README.md
+sidebar_label: "JSON formatter"
+learn_status: "Published"
+learn_topic_type: "References"
+learn_rel_path: "Developers/Web/Api/Formatters"
+-->
+
+# JSON formatter
+
+The CSV formatter presents [results of database queries](https://github.com/netdata/netdata/blob/master/src/web/api/queries/README.md) in the following formats:
+
+| format | content type | description|
+|:----:|:----------:|:----------|
+| `json` | application/json | return the query result as a json object|
+| `jsonp` | application/json | return the query result as a JSONP javascript callback|
+| `datatable` | application/json | return the query result as a Google `datatable`|
+| `datasource` | application/json | return the query result as a Google Visualization Provider `datasource` javascript callback|
+
+The CSV formatter respects the following API `&options=`:
+
+| option | supported | description|
+|:----:|:-------:|:----------|
+| `google_json` | yes | enable the Google flavor of JSON (using double quotes for strings and `Date()` function for dates|
+| `objectrows` | yes | return each row as an object, instead of an array|
+| `nonzero` | yes | to return only the dimensions that have at least a non-zero value|
+| `flip` | yes | to return the rows older to newer (the default is newer to older)|
+| `seconds` | yes | to return the date and time in unix timestamp|
+| `ms` | yes | to return the date and time in unit timestamp as milliseconds|
+| `percent` | yes | to replace all values with their percentage over the row total|
+| `abs` | yes | to turn all values positive|
+| `null2zero` | yes | to replace gaps with zeros (the default prints the string `null`|
+
+## Examples
+
+To show the differences between each format, in the following examples we query the same
+chart (having just one dimension called `active`), changing only the query `format` and its `options`.
+
+> Using `format=json` and `options=`
+
+```bash
+# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=nginx_local.connections&after=-3600&points=6&group=average&format=json&options='
+{
+ "labels": ["time", "active"],
+ "data":
+ [
+ [ 1540644600, 224.2516667],
+ [ 1540644000, 229.29],
+ [ 1540643400, 222.41],
+ [ 1540642800, 226.6816667],
+ [ 1540642200, 246.4083333],
+ [ 1540641600, 241.0966667]
+ ]
+}
+```
+
+> Using `format=json` and `options=objectrows`
+
+```bash
+# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=nginx_local.connections&after=-3600&points=6&group=average&format=json&options=objectrows'
+{
+ "labels": ["time", "active"],
+ "data":
+ [
+ { "time": 1540644600, "active": 224.2516667},
+ { "time": 1540644000, "active": 229.29},
+ { "time": 1540643400, "active": 222.41},
+ { "time": 1540642800, "active": 226.6816667},
+ { "time": 1540642200, "active": 246.4083333},
+ { "time": 1540641600, "active": 241.0966667}
+ ]
+}
+```
+
+> Using `format=json` and `options=objectrows,google_json`
+
+```bash
+# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=nginx_local.connections&after=-3600&points=6&group=average&formatjson&options=objectrows,google_json'
+{
+ "labels": ["time", "active"],
+ "data":
+ [
+ { "time": new Date(2018,9,27,12,50,0), "active": 224.2516667},
+ { "time": new Date(2018,9,27,12,40,0), "active": 229.29},
+ { "time": new Date(2018,9,27,12,30,0), "active": 222.41},
+ { "time": new Date(2018,9,27,12,20,0), "active": 226.6816667},
+ { "time": new Date(2018,9,27,12,10,0), "active": 246.4083333},
+ { "time": new Date(2018,9,27,12,0,0), "active": 241.0966667}
+ ]
+}
+```
+
+> Using `format=jsonp` and `options=`
+
+```bash
+curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=nginx_local.connections&after=-3600&points=6&group=average&formjsonp&options='
+callback({
+ "labels": ["time", "active"],
+ "data":
+ [
+ [ 1540645200, 235.885],
+ [ 1540644600, 224.2516667],
+ [ 1540644000, 229.29],
+ [ 1540643400, 222.41],
+ [ 1540642800, 226.6816667],
+ [ 1540642200, 246.4083333]
+ ]
+});
+```
+
+> Using `format=datatable` and `options=`
+
+```bash
+curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=nginx_local.connections&after=-3600&points=6&group=average&formdatatable&options='
+{
+ "cols":
+ [
+ {"id":"","label":"time","pattern":"","type":"datetime"},
+ {"id":"","label":"","pattern":"","type":"string","p":{"role":"annotation"}},
+ {"id":"","label":"","pattern":"","type":"string","p":{"role":"annotationText"}},
+ {"id":"","label":"active","pattern":"","type":"number"}
+ ],
+ "rows":
+ [
+ {"c":[{"v":"Date(2018,9,27,13,0,0)"},{"v":null},{"v":null},{"v":235.885}]},
+ {"c":[{"v":"Date(2018,9,27,12,50,0)"},{"v":null},{"v":null},{"v":224.2516667}]},
+ {"c":[{"v":"Date(2018,9,27,12,40,0)"},{"v":null},{"v":null},{"v":229.29}]},
+ {"c":[{"v":"Date(2018,9,27,12,30,0)"},{"v":null},{"v":null},{"v":222.41}]},
+ {"c":[{"v":"Date(2018,9,27,12,20,0)"},{"v":null},{"v":null},{"v":226.6816667}]},
+ {"c":[{"v":"Date(2018,9,27,12,10,0)"},{"v":null},{"v":null},{"v":246.4083333}]}
+ ]
+}
+```
+
+> Using `format=datasource` and `options=`
+
+```bash
+curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=nginx_local.connections&after=-3600&points=6&group=average&format=datasource&options='
+google.visualization.Query.setResponse({version:'0.6',reqId:'0',status:'ok',sig:'1540645368',table:{
+ "cols":
+ [
+ {"id":"","label":"time","pattern":"","type":"datetime"},
+ {"id":"","label":"","pattern":"","type":"string","p":{"role":"annotation"}},
+ {"id":"","label":"","pattern":"","type":"string","p":{"role":"annotationText"}},
+ {"id":"","label":"active","pattern":"","type":"number"}
+ ],
+ "rows":
+ [
+ {"c":[{"v":"Date(2018,9,27,13,0,0)"},{"v":null},{"v":null},{"v":235.885}]},
+ {"c":[{"v":"Date(2018,9,27,12,50,0)"},{"v":null},{"v":null},{"v":224.2516667}]},
+ {"c":[{"v":"Date(2018,9,27,12,40,0)"},{"v":null},{"v":null},{"v":229.29}]},
+ {"c":[{"v":"Date(2018,9,27,12,30,0)"},{"v":null},{"v":null},{"v":222.41}]},
+ {"c":[{"v":"Date(2018,9,27,12,20,0)"},{"v":null},{"v":null},{"v":226.6816667}]},
+ {"c":[{"v":"Date(2018,9,27,12,10,0)"},{"v":null},{"v":null},{"v":246.4083333}]}
+ ]
+}});
+```
+
+
diff --git a/src/web/api/formatters/json/json.c b/src/web/api/formatters/json/json.c
new file mode 100644
index 000000000..7e3f400e9
--- /dev/null
+++ b/src/web/api/formatters/json/json.c
@@ -0,0 +1,350 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "json.h"
+
+#define JSON_DATES_JS 1
+#define JSON_DATES_TIMESTAMP 2
+
+void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
+ //netdata_log_info("RRD2JSON(): %s: BEGIN", r->st->id);
+ int row_annotations = 0, dates, dates_with_new = 0;
+ char kq[2] = "", // key quote
+ sq[2] = "", // string quote
+ pre_label[101] = "", // before each label
+ post_label[101] = "", // after each label
+ pre_date[101] = "", // the beginning of line, to the date
+ post_date[101] = "", // closing the date
+ pre_value[101] = "", // before each value
+ post_value[101] = "", // after each value
+ post_line[101] = "", // at the end of each row
+ normal_annotation[201] = "", // default row annotation
+ overflow_annotation[201] = "", // overflow row annotation
+ data_begin[101] = "", // between labels and values
+ finish[101] = "", // at the end of everything
+ object_rows_time[101] = "";
+
+ if(datatable) {
+ dates = JSON_DATES_JS;
+ if( options & RRDR_OPTION_GOOGLE_JSON ) {
+ kq[0] = '\0';
+ sq[0] = '\'';
+ }
+ else {
+ kq[0] = '"';
+ sq[0] = '"';
+ }
+ row_annotations = 1;
+ snprintfz(pre_date, 100, " {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
+ snprintfz(post_date, 100, "%s}", sq);
+ snprintfz(pre_label, 100, ",\n {%sid%s:%s%s,%slabel%s:%s", kq, kq, sq, sq, kq, kq, sq);
+ snprintfz(post_label, 100, "%s,%spattern%s:%s%s,%stype%s:%snumber%s}", sq, kq, kq, sq, sq, kq, kq, sq, sq);
+ snprintfz(pre_value, 100, ",{%sv%s:", kq, kq);
+ strcpy(post_value, "}");
+ strcpy(post_line, "]}");
+ snprintfz(data_begin, 100, "\n ],\n %srows%s:\n [\n", kq, kq);
+ strcpy(finish, "\n ]\n }");
+
+ snprintfz(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
+ snprintfz(normal_annotation, 200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
+
+ buffer_sprintf(wb, "{\n %scols%s:\n [\n", kq, kq);
+ buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
+ buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
+ buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
+
+ // remove the valueobjects flag
+ // google wants its own keys
+ if(options & RRDR_OPTION_OBJECTSROWS)
+ options &= ~RRDR_OPTION_OBJECTSROWS;
+ }
+ else {
+ kq[0] = '"';
+ sq[0] = '"';
+ if(options & RRDR_OPTION_GOOGLE_JSON) {
+ dates = JSON_DATES_JS;
+ dates_with_new = 1;
+ }
+ else {
+ dates = JSON_DATES_TIMESTAMP;
+ dates_with_new = 0;
+ }
+ if( options & RRDR_OPTION_OBJECTSROWS )
+ strcpy(pre_date, " {");
+ else
+ strcpy(pre_date, " [");
+ strcpy(pre_label, ",\"");
+ strcpy(post_label, "\"");
+ strcpy(pre_value, ",");
+ if( options & RRDR_OPTION_OBJECTSROWS )
+ strcpy(post_line, "}");
+ else
+ strcpy(post_line, "]");
+ snprintfz(data_begin, 100, "],\n %sdata%s:[\n", kq, kq);
+ strcpy(finish, "\n ]\n }");
+
+ buffer_sprintf(wb, "{\n %slabels%s:[", kq, kq);
+ buffer_sprintf(wb, "%stime%s", sq, sq);
+
+ if( options & RRDR_OPTION_OBJECTSROWS )
+ snprintfz(object_rows_time, 100, "%stime%s: ", kq, kq);
+
+ }
+
+ size_t pre_value_len = strlen(pre_value);
+ size_t post_value_len = strlen(post_value);
+ size_t pre_label_len = strlen(pre_label);
+ size_t post_label_len = strlen(post_label);
+ size_t pre_date_len = strlen(pre_date);
+ size_t post_date_len = strlen(post_date);
+ size_t post_line_len = strlen(post_line);
+ size_t normal_annotation_len = strlen(normal_annotation);
+ size_t overflow_annotation_len = strlen(overflow_annotation);
+ size_t object_rows_time_len = strlen(object_rows_time);
+
+ // -------------------------------------------------------------------------
+ // print the JSON header
+
+ long c, i;
+ const long used = (long)r->d;
+
+ // print the header lines
+ for(c = 0, i = 0; c < used ; c++) {
+ if(!rrdr_dimension_should_be_exposed(r->od[c], options))
+ continue;
+
+ buffer_fast_strcat(wb, pre_label, pre_label_len);
+ buffer_strcat(wb, string2str(r->dn[c]));
+ buffer_fast_strcat(wb, post_label, post_label_len);
+ i++;
+ }
+
+ if(!i) {
+ buffer_fast_strcat(wb, pre_label, pre_label_len);
+ buffer_fast_strcat(wb, "no data", 7);
+ buffer_fast_strcat(wb, post_label, post_label_len);
+ }
+ size_t total_number_of_dimensions = i;
+
+ // print the beginning of row data
+ buffer_strcat(wb, data_begin);
+
+ // if all dimensions are hidden, print a null
+ if(!i) {
+ buffer_strcat(wb, finish);
+ return;
+ }
+
+ long start = 0, end = rrdr_rows(r), step = 1;
+ if(!(options & RRDR_OPTION_REVERSED)) {
+ start = rrdr_rows(r) - 1;
+ end = -1;
+ step = -1;
+ }
+
+ // pre-allocate a large enough buffer for us
+ // this does not need to be accurate - it is just a hint to avoid multiple realloc().
+ buffer_need_bytes(wb,
+ ( 20 * rrdr_rows(r)) // timestamp + json overhead
+ + ( (pre_value_len + post_value_len + 4) * total_number_of_dimensions * rrdr_rows(r) ) // number
+ );
+
+ // for each line in the array
+ for(i = start; i != end ;i += step) {
+ NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
+ RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ];
+ NETDATA_DOUBLE *ar = &r->ar[ i * r->d ];
+
+ time_t now = r->t[i];
+
+ if(dates == JSON_DATES_JS) {
+ // generate the local date time
+ struct tm tmbuf, *tm = localtime_r(&now, &tmbuf);
+ if(!tm) {
+ netdata_log_error("localtime_r() failed."); continue; }
+
+ if(likely(i != start)) buffer_fast_strcat(wb, ",\n", 2);
+ buffer_fast_strcat(wb, pre_date, pre_date_len);
+
+ if( options & RRDR_OPTION_OBJECTSROWS )
+ buffer_fast_strcat(wb, object_rows_time, object_rows_time_len);
+
+ if(unlikely(dates_with_new))
+ buffer_fast_strcat(wb, "new ", 4);
+
+ buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
+
+ buffer_fast_strcat(wb, post_date, post_date_len);
+
+ if(unlikely(row_annotations)) {
+ // google supports one annotation per row
+ int annotation_found = 0;
+ for(c = 0; c < used ; c++) {
+ if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;
+
+ if(unlikely(co[c] & RRDR_VALUE_RESET)) {
+ buffer_fast_strcat(wb, overflow_annotation, overflow_annotation_len);
+ annotation_found = 1;
+ break;
+ }
+ }
+ if(likely(!annotation_found))
+ buffer_fast_strcat(wb, normal_annotation, normal_annotation_len);
+ }
+ }
+ else {
+ // print the timestamp of the line
+ if(likely(i != start))
+ buffer_fast_strcat(wb, ",\n", 2);
+
+ buffer_fast_strcat(wb, pre_date, pre_date_len);
+
+ if(unlikely( options & RRDR_OPTION_OBJECTSROWS ))
+ buffer_fast_strcat(wb, object_rows_time, object_rows_time_len);
+
+ buffer_print_netdata_double(wb, (NETDATA_DOUBLE) r->t[i]);
+
+ // in ms
+ if(unlikely(options & RRDR_OPTION_MILLISECONDS))
+ buffer_fast_strcat(wb, "000", 3);
+
+ buffer_fast_strcat(wb, post_date, post_date_len);
+ }
+
+ // for each dimension
+ for(c = 0; c < used ;c++) {
+ if(!rrdr_dimension_should_be_exposed(r->od[c], options))
+ continue;
+
+ NETDATA_DOUBLE n;
+ if(unlikely(options & RRDR_OPTION_INTERNAL_AR))
+ n = ar[c];
+ else
+ n = cn[c];
+
+ buffer_fast_strcat(wb, pre_value, pre_value_len);
+
+ if(unlikely( options & RRDR_OPTION_OBJECTSROWS ))
+ buffer_sprintf(wb, "%s%s%s: ", kq, string2str(r->dn[c]), kq);
+
+ if(co[c] & RRDR_VALUE_EMPTY && !(options & (RRDR_OPTION_INTERNAL_AR))) {
+ if(unlikely(options & RRDR_OPTION_NULL2ZERO))
+ buffer_fast_strcat(wb, "0", 1);
+ else
+ buffer_fast_strcat(wb, "null", 4);
+ }
+ else
+ buffer_print_netdata_double(wb, n);
+
+ buffer_fast_strcat(wb, post_value, post_value_len);
+ }
+
+ buffer_fast_strcat(wb, post_line, post_line_len);
+ }
+
+ buffer_strcat(wb, finish);
+ //netdata_log_info("RRD2JSON(): %s: END", r->st->id);
+}
+
+void rrdr2json_v2(RRDR *r, BUFFER *wb) {
+ QUERY_TARGET *qt = r->internal.qt;
+ RRDR_OPTIONS options = qt->window.options;
+
+ bool send_count = query_target_aggregatable(qt);
+ bool send_hidden = send_count && r->vh && query_has_group_by_aggregation_percentage(qt);
+
+ buffer_json_member_add_object(wb, "result");
+
+ buffer_json_member_add_array(wb, "labels");
+ buffer_json_add_array_item_string(wb, "time");
+ long d, i;
+ const long used = (long)r->d;
+ for(d = 0, i = 0; d < used ; d++) {
+ if(!rrdr_dimension_should_be_exposed(r->od[d], options))
+ continue;
+
+ buffer_json_add_array_item_string(wb, string2str(r->di[d]));
+ i++;
+ }
+ buffer_json_array_close(wb); // labels
+
+ buffer_json_member_add_object(wb, "point");
+ {
+ size_t point_count = 0;
+ buffer_json_member_add_uint64(wb, "value", point_count++);
+ buffer_json_member_add_uint64(wb, "arp", point_count++);
+ buffer_json_member_add_uint64(wb, "pa", point_count++);
+ if (send_count)
+ buffer_json_member_add_uint64(wb, "count", point_count++);
+ if (send_hidden)
+ buffer_json_member_add_uint64(wb, "hidden", point_count++);
+ }
+ buffer_json_object_close(wb); // point
+
+ buffer_json_member_add_array(wb, "data");
+ if(i) {
+ long start = 0, end = rrdr_rows(r), step = 1;
+ if (!(options & RRDR_OPTION_REVERSED)) {
+ start = rrdr_rows(r) - 1;
+ end = -1;
+ step = -1;
+ }
+
+ // for each line in the array
+ for (i = start; i != end; i += step) {
+ NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
+ NETDATA_DOUBLE *ch = send_hidden ? &r->vh[i * r->d ] : NULL;
+ RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ];
+ NETDATA_DOUBLE *ar = &r->ar[ i * r->d ];
+ uint32_t *gbc = &r->gbc [ i * r->d ];
+ time_t now = r->t[i];
+
+ buffer_json_add_array_item_array(wb); // row
+
+ if (options & RRDR_OPTION_MILLISECONDS)
+ buffer_json_add_array_item_time_ms(wb, now); // the time
+ else
+ buffer_json_add_array_item_time_t(wb, now); // the time
+
+ for (d = 0; d < used; d++) {
+ if (!rrdr_dimension_should_be_exposed(r->od[d], options))
+ continue;
+
+ RRDR_VALUE_FLAGS o = co[d];
+
+ buffer_json_add_array_item_array(wb); // point
+
+ // add the value
+ NETDATA_DOUBLE n = cn[d];
+
+ if(o & RRDR_VALUE_EMPTY) {
+ if (unlikely(options & RRDR_OPTION_NULL2ZERO))
+ buffer_json_add_array_item_double(wb, 0);
+ else
+ buffer_json_add_array_item_double(wb, NAN);
+ }
+ else
+ buffer_json_add_array_item_double(wb, n);
+
+ // add the anomaly
+ buffer_json_add_array_item_double(wb, ar[d]);
+
+ // add the point annotations
+ buffer_json_add_array_item_uint64(wb, o);
+
+ // add the count
+ if(send_count)
+ buffer_json_add_array_item_uint64(wb, gbc[d]);
+ if(send_hidden)
+ buffer_json_add_array_item_double(wb, ch[d]);
+
+ buffer_json_array_close(wb); // point
+ }
+
+ buffer_json_array_close(wb); // row
+ }
+ }
+
+ buffer_json_array_close(wb); // data
+
+ buffer_json_object_close(wb); // annotations
+}
diff --git a/src/web/api/formatters/json/json.h b/src/web/api/formatters/json/json.h
new file mode 100644
index 000000000..d1ab4f901
--- /dev/null
+++ b/src/web/api/formatters/json/json.h
@@ -0,0 +1,11 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_API_FORMATTER_JSON_H
+#define NETDATA_API_FORMATTER_JSON_H
+
+#include "../rrd2json.h"
+
+void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable);
+void rrdr2json_v2(RRDR *r, BUFFER *wb);
+
+#endif //NETDATA_API_FORMATTER_JSON_H
diff --git a/src/web/api/formatters/json_wrapper.c b/src/web/api/formatters/json_wrapper.c
new file mode 100644
index 000000000..fca5a0b83
--- /dev/null
+++ b/src/web/api/formatters/json_wrapper.c
@@ -0,0 +1,1578 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "json_wrapper.h"
+
+static void jsonwrap_query_metric_plan(BUFFER *wb, QUERY_METRIC *qm) {
+ buffer_json_member_add_array(wb, "plans");
+ for (size_t p = 0; p < qm->plan.used; p++) {
+ QUERY_PLAN_ENTRY *qp = &qm->plan.array[p];
+
+ buffer_json_add_array_item_object(wb);
+ buffer_json_member_add_uint64(wb, "tr", qp->tier);
+ buffer_json_member_add_time_t(wb, "af", qp->after);
+ buffer_json_member_add_time_t(wb, "bf", qp->before);
+ buffer_json_object_close(wb);
+ }
+ buffer_json_array_close(wb);
+
+ buffer_json_member_add_array(wb, "tiers");
+ for (size_t tier = 0; tier < storage_tiers; tier++) {
+ buffer_json_add_array_item_object(wb);
+ buffer_json_member_add_uint64(wb, "tr", tier);
+ buffer_json_member_add_time_t(wb, "fe", qm->tiers[tier].db_first_time_s);
+ buffer_json_member_add_time_t(wb, "le", qm->tiers[tier].db_last_time_s);
+ buffer_json_member_add_int64(wb, "wg", qm->tiers[tier].weight);
+ buffer_json_object_close(wb);
+ }
+ buffer_json_array_close(wb);
+}
+
+void jsonwrap_query_plan(RRDR *r, BUFFER *wb) {
+ QUERY_TARGET *qt = r->internal.qt;
+
+ buffer_json_member_add_object(wb, "query_plan");
+ for(size_t m = 0; m < qt->query.used; m++) {
+ QUERY_METRIC *qm = query_metric(qt, m);
+ buffer_json_member_add_object(wb, query_metric_id(qt, qm));
+ jsonwrap_query_metric_plan(wb, qm);
+ buffer_json_object_close(wb);
+ }
+ buffer_json_object_close(wb);
+}
+
+static inline size_t rrdr_dimension_names(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
+ const size_t dimensions = r->d;
+ size_t c, i;
+
+ buffer_json_member_add_array(wb, key);
+ for(c = 0, i = 0; c < dimensions ; c++) {
+ if(!rrdr_dimension_should_be_exposed(r->od[c], options))
+ continue;
+
+ buffer_json_add_array_item_string(wb, string2str(r->dn[c]));
+ i++;
+ }
+ buffer_json_array_close(wb);
+
+ return i;
+}
+
+static inline size_t rrdr_dimension_ids(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
+ const size_t dimensions = r->d;
+ size_t c, i;
+
+ buffer_json_member_add_array(wb, key);
+ for(c = 0, i = 0; c < dimensions ; c++) {
+ if(!rrdr_dimension_should_be_exposed(r->od[c], options))
+ continue;
+
+ buffer_json_add_array_item_string(wb, string2str(r->di[c]));
+ i++;
+ }
+ buffer_json_array_close(wb);
+
+ return i;
+}
+
+static inline long jsonwrap_v1_chart_ids(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
+ QUERY_TARGET *qt = r->internal.qt;
+ const long query_used = qt->query.used;
+ long c, i;
+
+ buffer_json_member_add_array(wb, key);
+ for (c = 0, i = 0; c < query_used; c++) {
+ if(!rrdr_dimension_should_be_exposed(r->od[c], options))
+ continue;
+
+ QUERY_METRIC *qm = query_metric(qt, c);
+ QUERY_INSTANCE *qi = query_instance(qt, qm->link.query_instance_id);
+ buffer_json_add_array_item_string(wb, rrdinstance_acquired_id(qi->ria));
+ i++;
+ }
+ buffer_json_array_close(wb);
+
+ return i;
+}
+
+struct summary_total_counts {
+ size_t selected;
+ size_t excluded;
+ size_t queried;
+ size_t failed;
+};
+
+static inline void aggregate_into_summary_totals(struct summary_total_counts *totals, QUERY_METRICS_COUNTS *metrics) {
+ if(unlikely(!totals || !metrics))
+ return;
+
+ if(metrics->selected) {
+ totals->selected++;
+
+ if(metrics->queried)
+ totals->queried++;
+
+ else if(metrics->failed)
+ totals->failed++;
+ }
+ else
+ totals->excluded++;
+}
+
+static inline void query_target_total_counts(BUFFER *wb, const char *key, struct summary_total_counts *totals) {
+ if(!totals->selected && !totals->queried && !totals->failed && !totals->excluded)
+ return;
+
+ buffer_json_member_add_object(wb, key);
+
+ if(totals->selected)
+ buffer_json_member_add_uint64(wb, "sl", totals->selected);
+
+ if(totals->excluded)
+ buffer_json_member_add_uint64(wb, "ex", totals->excluded);
+
+ if(totals->queried)
+ buffer_json_member_add_uint64(wb, "qr", totals->queried);
+
+ if(totals->failed)
+ buffer_json_member_add_uint64(wb, "fl", totals->failed);
+
+ buffer_json_object_close(wb);
+}
+
+static inline void query_target_metric_counts(BUFFER *wb, QUERY_METRICS_COUNTS *metrics) {
+ if(!metrics->selected && !metrics->queried && !metrics->failed && !metrics->excluded)
+ return;
+
+ buffer_json_member_add_object(wb, "ds");
+
+ if(metrics->selected)
+ buffer_json_member_add_uint64(wb, "sl", metrics->selected);
+
+ if(metrics->excluded)
+ buffer_json_member_add_uint64(wb, "ex", metrics->excluded);
+
+ if(metrics->queried)
+ buffer_json_member_add_uint64(wb, "qr", metrics->queried);
+
+ if(metrics->failed)
+ buffer_json_member_add_uint64(wb, "fl", metrics->failed);
+
+ buffer_json_object_close(wb);
+}
+
+static inline void query_target_instance_counts(BUFFER *wb, QUERY_INSTANCES_COUNTS *instances) {
+ if(!instances->selected && !instances->queried && !instances->failed && !instances->excluded)
+ return;
+
+ buffer_json_member_add_object(wb, "is");
+
+ if(instances->selected)
+ buffer_json_member_add_uint64(wb, "sl", instances->selected);
+
+ if(instances->excluded)
+ buffer_json_member_add_uint64(wb, "ex", instances->excluded);
+
+ if(instances->queried)
+ buffer_json_member_add_uint64(wb, "qr", instances->queried);
+
+ if(instances->failed)
+ buffer_json_member_add_uint64(wb, "fl", instances->failed);
+
+ buffer_json_object_close(wb);
+}
+
+static inline void query_target_alerts_counts(BUFFER *wb, QUERY_ALERTS_COUNTS *alerts, const char *name, bool array) {
+ if(!alerts->clear && !alerts->other && !alerts->critical && !alerts->warning)
+ return;
+
+ if(array)
+ buffer_json_add_array_item_object(wb);
+ else
+ buffer_json_member_add_object(wb, "al");
+
+ if(name)
+ buffer_json_member_add_string(wb, "nm", name);
+
+ if(alerts->clear)
+ buffer_json_member_add_uint64(wb, "cl", alerts->clear);
+
+ if(alerts->warning)
+ buffer_json_member_add_uint64(wb, "wr", alerts->warning);
+
+ if(alerts->critical)
+ buffer_json_member_add_uint64(wb, "cr", alerts->critical);
+
+ if(alerts->other)
+ buffer_json_member_add_uint64(wb, "ot", alerts->other);
+
+ buffer_json_object_close(wb);
+}
+
+static inline void query_target_points_statistics(BUFFER *wb, QUERY_TARGET *qt, STORAGE_POINT *sp) {
+ if(!sp->count)
+ return;
+
+ buffer_json_member_add_object(wb, "sts");
+
+ buffer_json_member_add_double(wb, "min", sp->min);
+ buffer_json_member_add_double(wb, "max", sp->max);
+
+ if(query_target_aggregatable(qt)) {
+ buffer_json_member_add_uint64(wb, "cnt", sp->count);
+
+ if(sp->sum != 0.0) {
+ buffer_json_member_add_double(wb, "sum", sp->sum);
+ buffer_json_member_add_double(wb, "vol", sp->sum * (NETDATA_DOUBLE) query_view_update_every(qt));
+ }
+
+ if(sp->anomaly_count != 0)
+ buffer_json_member_add_uint64(wb, "arc", sp->anomaly_count);
+ }
+ else {
+ NETDATA_DOUBLE avg = (sp->count) ? sp->sum / (NETDATA_DOUBLE)sp->count : 0.0;
+ if(avg != 0.0)
+ buffer_json_member_add_double(wb, "avg", avg);
+
+ NETDATA_DOUBLE arp = storage_point_anomaly_rate(*sp);
+ if(arp != 0.0)
+ buffer_json_member_add_double(wb, "arp", arp);
+
+ NETDATA_DOUBLE con = (qt->query_points.sum > 0.0) ? sp->sum * 100.0 / qt->query_points.sum : 0.0;
+ if(con != 0.0)
+ buffer_json_member_add_double(wb, "con", con);
+ }
+ buffer_json_object_close(wb);
+}
+
+static void query_target_summary_nodes_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key, struct summary_total_counts *totals) {
+ buffer_json_member_add_array(wb, key);
+ for (size_t c = 0; c < qt->nodes.used; c++) {
+ QUERY_NODE *qn = query_node(qt, c);
+ RRDHOST *host = qn->rrdhost;
+ buffer_json_add_array_item_object(wb);
+ buffer_json_node_add_v2(wb, host, qn->slot, qn->duration_ut, true);
+ query_target_instance_counts(wb, &qn->instances);
+ query_target_metric_counts(wb, &qn->metrics);
+ query_target_alerts_counts(wb, &qn->alerts, NULL, false);
+ query_target_points_statistics(wb, qt, &qn->query_points);
+ buffer_json_object_close(wb);
+
+ aggregate_into_summary_totals(totals, &qn->metrics);
+ }
+ buffer_json_array_close(wb);
+}
+
+static size_t query_target_summary_contexts_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key, struct summary_total_counts *totals) {
+ buffer_json_member_add_array(wb, key);
+ DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
+
+ struct {
+ STORAGE_POINT query_points;
+ QUERY_INSTANCES_COUNTS instances;
+ QUERY_METRICS_COUNTS metrics;
+ QUERY_ALERTS_COUNTS alerts;
+ } *z;
+
+ for (long c = 0; c < (long) qt->contexts.used; c++) {
+ QUERY_CONTEXT *qc = query_context(qt, c);
+
+ z = dictionary_set(dict, rrdcontext_acquired_id(qc->rca), NULL, sizeof(*z));
+
+ z->instances.selected += qc->instances.selected;
+ z->instances.excluded += qc->instances.selected;
+ z->instances.queried += qc->instances.queried;
+ z->instances.failed += qc->instances.failed;
+
+ z->metrics.selected += qc->metrics.selected;
+ z->metrics.excluded += qc->metrics.excluded;
+ z->metrics.queried += qc->metrics.queried;
+ z->metrics.failed += qc->metrics.failed;
+
+ z->alerts.clear += qc->alerts.clear;
+ z->alerts.warning += qc->alerts.warning;
+ z->alerts.critical += qc->alerts.critical;
+
+ storage_point_merge_to(z->query_points, qc->query_points);
+ }
+
+ size_t unique_contexts = dictionary_entries(dict);
+ dfe_start_read(dict, z) {
+ buffer_json_add_array_item_object(wb);
+ buffer_json_member_add_string(wb, "id", z_dfe.name);
+ query_target_instance_counts(wb, &z->instances);
+ query_target_metric_counts(wb, &z->metrics);
+ query_target_alerts_counts(wb, &z->alerts, NULL, false);
+ query_target_points_statistics(wb, qt, &z->query_points);
+ buffer_json_object_close(wb);
+
+ aggregate_into_summary_totals(totals, &z->metrics);
+ }
+ dfe_done(z);
+ buffer_json_array_close(wb);
+ dictionary_destroy(dict);
+
+ return unique_contexts;
+}
+
+static void query_target_summary_instances_v1(BUFFER *wb, QUERY_TARGET *qt, const char *key) {
+ char name[RRD_ID_LENGTH_MAX * 2 + 2];
+
+ buffer_json_member_add_array(wb, key);
+ DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
+ for (long c = 0; c < (long) qt->instances.used; c++) {
+ QUERY_INSTANCE *qi = query_instance(qt, c);
+
+ snprintfz(name, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s",
+ rrdinstance_acquired_id(qi->ria),
+ rrdinstance_acquired_name(qi->ria));
+
+ bool *set = dictionary_set(dict, name, NULL, sizeof(*set));
+ if (!*set) {
+ *set = true;
+ buffer_json_add_array_item_array(wb);
+ buffer_json_add_array_item_string(wb, rrdinstance_acquired_id(qi->ria));
+ buffer_json_add_array_item_string(wb, rrdinstance_acquired_name(qi->ria));
+ buffer_json_array_close(wb);
+ }
+ }
+ dictionary_destroy(dict);
+ buffer_json_array_close(wb);
+}
+
+static void query_target_summary_instances_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key, struct summary_total_counts *totals) {
+ buffer_json_member_add_array(wb, key);
+ for (long c = 0; c < (long) qt->instances.used; c++) {
+ QUERY_INSTANCE *qi = query_instance(qt, c);
+// QUERY_HOST *qh = query_host(qt, qi->query_host_id);
+
+ buffer_json_add_array_item_object(wb);
+ buffer_json_member_add_string(wb, "id", rrdinstance_acquired_id(qi->ria));
+
+ if(!rrdinstance_acquired_id_and_name_are_same(qi->ria))
+ buffer_json_member_add_string(wb, "nm", rrdinstance_acquired_name(qi->ria));
+
+ buffer_json_member_add_uint64(wb, "ni", qi->query_host_id);
+// buffer_json_member_add_string(wb, "id", string2str(qi->id_fqdn));
+// buffer_json_member_add_string(wb, "nm", string2str(qi->name_fqdn));
+// buffer_json_member_add_string(wb, "lc", rrdinstance_acquired_name(qi->ria));
+// buffer_json_member_add_string(wb, "mg", qh->host->machine_guid);
+// if(qh->node_id[0])
+// buffer_json_member_add_string(wb, "nd", qh->node_id);
+ query_target_metric_counts(wb, &qi->metrics);
+ query_target_alerts_counts(wb, &qi->alerts, NULL, false);
+ query_target_points_statistics(wb, qt, &qi->query_points);
+ buffer_json_object_close(wb);
+
+ aggregate_into_summary_totals(totals, &qi->metrics);
+ }
+ buffer_json_array_close(wb);
+}
+
+struct dimensions_sorted_walkthrough_data {
+ BUFFER *wb;
+ struct summary_total_counts *totals;
+ QUERY_TARGET *qt;
+};
+
+struct dimensions_sorted_entry {
+ const char *id;
+ const char *name;
+ STORAGE_POINT query_points;
+ QUERY_METRICS_COUNTS metrics;
+ uint32_t priority;
+};
+
+static int dimensions_sorted_walktrhough_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) {
+ struct dimensions_sorted_walkthrough_data *sdwd = data;
+ BUFFER *wb = sdwd->wb;
+ struct summary_total_counts *totals = sdwd->totals;
+ QUERY_TARGET *qt = sdwd->qt;
+ struct dimensions_sorted_entry *z = value;
+
+ buffer_json_add_array_item_object(wb);
+ buffer_json_member_add_string(wb, "id", z->id);
+ if (z->id != z->name && z->name)
+ buffer_json_member_add_string(wb, "nm", z->name);
+
+ query_target_metric_counts(wb, &z->metrics);
+ query_target_points_statistics(wb, qt, &z->query_points);
+ buffer_json_member_add_uint64(wb, "pri", z->priority);
+ buffer_json_object_close(wb);
+
+ aggregate_into_summary_totals(totals, &z->metrics);
+
+ return 1;
+}
+
+int dimensions_sorted_compar(const DICTIONARY_ITEM **item1, const DICTIONARY_ITEM **item2) {
+ struct dimensions_sorted_entry *z1 = dictionary_acquired_item_value(*item1);
+ struct dimensions_sorted_entry *z2 = dictionary_acquired_item_value(*item2);
+
+ if(z1->priority == z2->priority)
+ return strcmp(dictionary_acquired_item_name(*item1), dictionary_acquired_item_name(*item2));
+ else if(z1->priority < z2->priority)
+ return -1;
+ else
+ return 1;
+}
+
+static void query_target_summary_dimensions_v12(BUFFER *wb, QUERY_TARGET *qt, const char *key, bool v2, struct summary_total_counts *totals) {
+ char buf[RRD_ID_LENGTH_MAX * 2 + 2];
+
+ buffer_json_member_add_array(wb, key);
+ DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
+ struct dimensions_sorted_entry *z;
+ size_t q = 0;
+ for (long c = 0; c < (long) qt->dimensions.used; c++) {
+ QUERY_DIMENSION * qd = query_dimension(qt, c);
+ RRDMETRIC_ACQUIRED *rma = qd->rma;
+
+ QUERY_METRIC *qm = NULL;
+ for( ; q < qt->query.used ;q++) {
+ QUERY_METRIC *tqm = query_metric(qt, q);
+ QUERY_DIMENSION *tqd = query_dimension(qt, tqm->link.query_dimension_id);
+ if(tqd->rma != rma) break;
+ qm = tqm;
+ }
+
+ const char *k, *id, *name;
+
+ if(v2) {
+ k = rrdmetric_acquired_name(rma);
+ id = k;
+ name = k;
+ }
+ else {
+ snprintfz(buf, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s",
+ rrdmetric_acquired_id(rma),
+ rrdmetric_acquired_name(rma));
+ k = buf;
+ id = rrdmetric_acquired_id(rma);
+ name = rrdmetric_acquired_name(rma);
+ }
+
+ z = dictionary_set(dict, k, NULL, sizeof(*z));
+ if(!z->id) {
+ z->id = id;
+ z->name = name;
+ z->priority = qd->priority;
+ }
+ else {
+ if(qd->priority < z->priority)
+ z->priority = qd->priority;
+ }
+
+ if(qm) {
+ z->metrics.selected += (qm->status & RRDR_DIMENSION_SELECTED) ? 1 : 0;
+ z->metrics.failed += (qm->status & RRDR_DIMENSION_FAILED) ? 1 : 0;
+
+ if(qm->status & RRDR_DIMENSION_QUERIED) {
+ z->metrics.queried++;
+ storage_point_merge_to(z->query_points, qm->query_points);
+ }
+ }
+ else
+ z->metrics.excluded++;
+ }
+
+ if(v2) {
+ struct dimensions_sorted_walkthrough_data t = {
+ .wb = wb,
+ .totals = totals,
+ .qt = qt,
+ };
+ dictionary_sorted_walkthrough_rw(dict, DICTIONARY_LOCK_READ, dimensions_sorted_walktrhough_cb,
+ &t, dimensions_sorted_compar);
+ }
+ else {
+ // v1
+ dfe_start_read(dict, z) {
+ buffer_json_add_array_item_array(wb);
+ buffer_json_add_array_item_string(wb, z->id);
+ buffer_json_add_array_item_string(wb, z->name);
+ buffer_json_array_close(wb);
+ }
+ dfe_done(z);
+ }
+ dictionary_destroy(dict);
+ buffer_json_array_close(wb);
+}
+
+struct rrdlabels_formatting_v2 {
+ DICTIONARY *keys;
+ QUERY_INSTANCE *qi;
+ bool v2;
+};
+
+struct rrdlabels_keys_dict_entry {
+ const char *name;
+ DICTIONARY *values;
+ STORAGE_POINT query_points;
+ QUERY_METRICS_COUNTS metrics;
+};
+
+struct rrdlabels_key_value_dict_entry {
+ const char *key;
+ const char *value;
+ STORAGE_POINT query_points;
+ QUERY_METRICS_COUNTS metrics;
+};
+
+static int rrdlabels_formatting_v2(const char *name, const char *value, RRDLABEL_SRC ls __maybe_unused, void *data) {
+ struct rrdlabels_formatting_v2 *t = data;
+
+ struct rrdlabels_keys_dict_entry *d = dictionary_set(t->keys, name, NULL, sizeof(*d));
+ if(!d->values) {
+ d->name = name;
+ d->values = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
+ }
+
+ char n[RRD_ID_LENGTH_MAX * 2 + 2];
+ snprintfz(n, RRD_ID_LENGTH_MAX * 2, "%s:%s", name, value);
+
+ struct rrdlabels_key_value_dict_entry *z = dictionary_set(d->values, n, NULL, sizeof(*z));
+ if(!z->key) {
+ z->key = name;
+ z->value = value;
+ }
+
+ if(t->v2) {
+ QUERY_INSTANCE *qi = t->qi;
+
+ z->metrics.selected += qi->metrics.selected;
+ z->metrics.excluded += qi->metrics.excluded;
+ z->metrics.queried += qi->metrics.queried;
+ z->metrics.failed += qi->metrics.failed;
+
+ d->metrics.selected += qi->metrics.selected;
+ d->metrics.excluded += qi->metrics.excluded;
+ d->metrics.queried += qi->metrics.queried;
+ d->metrics.failed += qi->metrics.failed;
+
+ storage_point_merge_to(z->query_points, qi->query_points);
+ storage_point_merge_to(d->query_points, qi->query_points);
+ }
+
+ return 1;
+}
+
+static void query_target_summary_labels_v12(BUFFER *wb, QUERY_TARGET *qt, const char *key, bool v2, struct summary_total_counts *key_totals, struct summary_total_counts *value_totals) {
+ buffer_json_member_add_array(wb, key);
+ struct rrdlabels_formatting_v2 t = {
+ .keys = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE),
+ .v2 = v2,
+ };
+ for (long c = 0; c < (long) qt->instances.used; c++) {
+ QUERY_INSTANCE *qi = query_instance(qt, c);
+ RRDINSTANCE_ACQUIRED *ria = qi->ria;
+ t.qi = qi;
+ rrdlabels_walkthrough_read(rrdinstance_acquired_labels(ria), rrdlabels_formatting_v2, &t);
+ }
+ struct rrdlabels_keys_dict_entry *d;
+ dfe_start_read(t.keys, d) {
+ if(v2) {
+ buffer_json_add_array_item_object(wb);
+ buffer_json_member_add_string(wb, "id", d_dfe.name);
+ query_target_metric_counts(wb, &d->metrics);
+ query_target_points_statistics(wb, qt, &d->query_points);
+ aggregate_into_summary_totals(key_totals, &d->metrics);
+ buffer_json_member_add_array(wb, "vl");
+ }
+ struct rrdlabels_key_value_dict_entry *z;
+ dfe_start_read(d->values, z){
+ if (v2) {
+ buffer_json_add_array_item_object(wb);
+ buffer_json_member_add_string(wb, "id", z->value);
+ query_target_metric_counts(wb, &z->metrics);
+ query_target_points_statistics(wb, qt, &z->query_points);
+ buffer_json_object_close(wb);
+ aggregate_into_summary_totals(value_totals, &z->metrics);
+ } else {
+ buffer_json_add_array_item_array(wb);
+ buffer_json_add_array_item_string(wb, z->key);
+ buffer_json_add_array_item_string(wb, z->value);
+ buffer_json_array_close(wb);
+ }
+ }
+ dfe_done(z);
+ dictionary_destroy(d->values);
+ if(v2) {
+ buffer_json_array_close(wb);
+ buffer_json_object_close(wb);
+ }
+ }
+ dfe_done(d);
+ dictionary_destroy(t.keys);
+ buffer_json_array_close(wb);
+}
+
+static void query_target_summary_alerts_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key) {
+ buffer_json_member_add_array(wb, key);
+ QUERY_ALERTS_COUNTS *z;
+
+ DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
+ for (long c = 0; c < (long) qt->instances.used; c++) {
+ QUERY_INSTANCE *qi = query_instance(qt, c);
+ RRDSET *st = rrdinstance_acquired_rrdset(qi->ria);
+ if (st) {
+ rw_spinlock_read_lock(&st->alerts.spinlock);
+ if (st->alerts.base) {
+ for (RRDCALC *rc = st->alerts.base; rc; rc = rc->next) {
+ z = dictionary_set(dict, string2str(rc->config.name), NULL, sizeof(*z));
+
+ switch(rc->status) {
+ case RRDCALC_STATUS_CLEAR:
+ z->clear++;
+ break;
+
+ case RRDCALC_STATUS_WARNING:
+ z->warning++;
+ break;
+
+ case RRDCALC_STATUS_CRITICAL:
+ z->critical++;
+ break;
+
+ default:
+ case RRDCALC_STATUS_UNINITIALIZED:
+ case RRDCALC_STATUS_UNDEFINED:
+ case RRDCALC_STATUS_REMOVED:
+ z->other++;
+ break;
+ }
+ }
+ }
+ rw_spinlock_read_unlock(&st->alerts.spinlock);
+ }
+ }
+ dfe_start_read(dict, z)
+ query_target_alerts_counts(wb, z, z_dfe.name, true);
+ dfe_done(z);
+ dictionary_destroy(dict);
+ buffer_json_array_close(wb); // alerts
+}
+
+static inline void query_target_functions(BUFFER *wb, const char *key, RRDR *r) {
+ QUERY_TARGET *qt = r->internal.qt;
+ const long query_used = qt->query.used;
+
+ DICTIONARY *funcs = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
+ RRDINSTANCE_ACQUIRED *ria = NULL;
+ for (long c = 0; c < query_used ; c++) {
+ QUERY_METRIC *qm = query_metric(qt, c);
+ QUERY_INSTANCE *qi = query_instance(qt, qm->link.query_instance_id);
+ if(qi->ria == ria)
+ continue;
+
+ ria = qi->ria;
+ chart_functions_to_dict(rrdinstance_acquired_functions(ria), funcs, NULL, 0);
+ }
+
+ buffer_json_member_add_array(wb, key);
+ void *t; (void)t;
+ dfe_start_read(funcs, t)
+ buffer_json_add_array_item_string(wb, t_dfe.name);
+ dfe_done(t);
+ dictionary_destroy(funcs);
+ buffer_json_array_close(wb);
+}
+
+static inline long query_target_chart_labels_filter_v1(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
+ QUERY_TARGET *qt = r->internal.qt;
+ const long query_used = qt->query.used;
+ long c, i = 0;
+
+ buffer_json_member_add_object(wb, key);
+
+ SIMPLE_PATTERN *pattern = qt->instances.chart_label_key_pattern;
+ char *label_key = NULL;
+ while (pattern && (label_key = simple_pattern_iterate(&pattern))) {
+ buffer_json_member_add_array(wb, label_key);
+
+ for (c = 0, i = 0; c < query_used; c++) {
+ if(!rrdr_dimension_should_be_exposed(r->od[c], options))
+ continue;
+
+ QUERY_METRIC *qm = query_metric(qt, c);
+ QUERY_INSTANCE *qi = query_instance(qt, qm->link.query_instance_id);
+ rrdlabels_value_to_buffer_array_item_or_null(rrdinstance_acquired_labels(qi->ria), wb, label_key);
+ i++;
+ }
+ buffer_json_array_close(wb);
+ }
+
+ buffer_json_object_close(wb);
+
+ return i;
+}
+
+static inline long query_target_metrics_latest_values(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
+ QUERY_TARGET *qt = r->internal.qt;
+ const long query_used = qt->query.used;
+ long c, i;
+
+ buffer_json_member_add_array(wb, key);
+
+ for(c = 0, i = 0; c < query_used ;c++) {
+ if(!rrdr_dimension_should_be_exposed(r->od[c], options))
+ continue;
+
+ QUERY_METRIC *qm = query_metric(qt, c);
+ QUERY_DIMENSION *qd = query_dimension(qt, qm->link.query_dimension_id);
+ buffer_json_add_array_item_double(wb, rrdmetric_acquired_last_stored_value(qd->rma));
+ i++;
+ }
+
+ buffer_json_array_close(wb);
+
+ return i;
+}
+
+static inline size_t rrdr_dimension_view_latest_values(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
+ buffer_json_member_add_array(wb, key);
+
+ size_t c, i;
+ for(c = 0, i = 0; c < r->d ; c++) {
+ if(!rrdr_dimension_should_be_exposed(r->od[c], options))
+ continue;
+
+ i++;
+
+ NETDATA_DOUBLE *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ];
+ RRDR_VALUE_FLAGS *co = &r->o[ (rrdr_rows(r) - 1) * r->d ];
+ NETDATA_DOUBLE n = cn[c];
+
+ if(co[c] & RRDR_VALUE_EMPTY) {
+ if(options & RRDR_OPTION_NULL2ZERO)
+ buffer_json_add_array_item_double(wb, 0.0);
+ else
+ buffer_json_add_array_item_double(wb, NAN);
+ }
+ else
+ buffer_json_add_array_item_double(wb, n);
+ }
+
+ buffer_json_array_close(wb);
+
+ return i;
+}
+
+static inline void rrdr_dimension_query_points_statistics(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options, bool dview) {
+ STORAGE_POINT *sp = (dview) ? r->dview : r->dqp;
+ NETDATA_DOUBLE anomaly_rate_multiplier = (dview) ? RRDR_DVIEW_ANOMALY_COUNT_MULTIPLIER : 1.0;
+
+ if(unlikely(!sp))
+ return;
+
+ if(key)
+ buffer_json_member_add_object(wb, key);
+
+ buffer_json_member_add_array(wb, "min");
+ for(size_t c = 0; c < r->d ; c++) {
+ if (!rrdr_dimension_should_be_exposed(r->od[c], options))
+ continue;
+
+ buffer_json_add_array_item_double(wb, sp[c].min);
+ }
+ buffer_json_array_close(wb);
+
+ buffer_json_member_add_array(wb, "max");
+ for(size_t c = 0; c < r->d ; c++) {
+ if (!rrdr_dimension_should_be_exposed(r->od[c], options))
+ continue;
+
+ buffer_json_add_array_item_double(wb, sp[c].max);
+ }
+ buffer_json_array_close(wb);
+
+ if(options & RRDR_OPTION_RETURN_RAW) {
+ buffer_json_member_add_array(wb, "sum");
+ for(size_t c = 0; c < r->d ; c++) {
+ if (!rrdr_dimension_should_be_exposed(r->od[c], options))
+ continue;
+
+ buffer_json_add_array_item_double(wb, sp[c].sum);
+ }
+ buffer_json_array_close(wb);
+
+ buffer_json_member_add_array(wb, "cnt");
+ for(size_t c = 0; c < r->d ; c++) {
+ if (!rrdr_dimension_should_be_exposed(r->od[c], options))
+ continue;
+
+ buffer_json_add_array_item_uint64(wb, sp[c].count);
+ }
+ buffer_json_array_close(wb);
+
+ buffer_json_member_add_array(wb, "arc");
+ for(size_t c = 0; c < r->d ; c++) {
+ if (!rrdr_dimension_should_be_exposed(r->od[c], options))
+ continue;
+
+ buffer_json_add_array_item_uint64(wb, storage_point_anomaly_rate(sp[c]) / anomaly_rate_multiplier / 100.0 * sp[c].count);
+ }
+ buffer_json_array_close(wb);
+ }
+ else {
+ NETDATA_DOUBLE sum = 0.0;
+ for(size_t c = 0; c < r->d ; c++) {
+ if(!rrdr_dimension_should_be_exposed(r->od[c], options))
+ continue;
+
+ sum += ABS(sp[c].sum);
+ }
+
+ buffer_json_member_add_array(wb, "avg");
+ for(size_t c = 0; c < r->d ; c++) {
+ if (!rrdr_dimension_should_be_exposed(r->od[c], options))
+ continue;
+
+ buffer_json_add_array_item_double(wb, storage_point_average_value(sp[c]));
+ }
+ buffer_json_array_close(wb);
+
+ buffer_json_member_add_array(wb, "arp");
+ for(size_t c = 0; c < r->d ; c++) {
+ if (!rrdr_dimension_should_be_exposed(r->od[c], options))
+ continue;
+
+ buffer_json_add_array_item_double(wb, storage_point_anomaly_rate(sp[c]) / anomaly_rate_multiplier);
+ }
+ buffer_json_array_close(wb);
+
+ buffer_json_member_add_array(wb, "con");
+ for(size_t c = 0; c < r->d ; c++) {
+ if (!rrdr_dimension_should_be_exposed(r->od[c], options))
+ continue;
+
+ NETDATA_DOUBLE con = (sum > 0.0) ? ABS(sp[c].sum) * 100.0 / sum : 0.0;
+ buffer_json_add_array_item_double(wb, con);
+ }
+ buffer_json_array_close(wb);
+ }
+
+ if(key)
+ buffer_json_object_close(wb);
+}
+
+void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb) {
+ QUERY_TARGET *qt = r->internal.qt;
+ DATASOURCE_FORMAT format = qt->request.format;
+ RRDR_OPTIONS options = qt->window.options;
+
+ long rows = rrdr_rows(r);
+
+ char kq[2] = "", // key quote
+ sq[2] = ""; // string quote
+
+ if( options & RRDR_OPTION_GOOGLE_JSON ) {
+ kq[0] = '\0';
+ sq[0] = '\'';
+ }
+ else {
+ kq[0] = '"';
+ sq[0] = '"';
+ }
+
+ buffer_json_initialize(
+ wb, kq, sq, 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
+
+ buffer_json_member_add_uint64(wb, "api", 1);
+ buffer_json_member_add_string(wb, "id", qt->id);
+ buffer_json_member_add_string(wb, "name", qt->id);
+ buffer_json_member_add_time_t(wb, "view_update_every", r->view.update_every);
+ buffer_json_member_add_time_t(wb, "update_every", qt->db.minimum_latest_update_every_s);
+ buffer_json_member_add_time_t(wb, "first_entry", qt->db.first_time_s);
+ buffer_json_member_add_time_t(wb, "last_entry", qt->db.last_time_s);
+ buffer_json_member_add_time_t(wb, "after", r->view.after);
+ buffer_json_member_add_time_t(wb, "before", r->view.before);
+ buffer_json_member_add_string(wb, "group", time_grouping_tostring(qt->request.time_group_method));
+ rrdr_options_to_buffer_json_array(wb, "options", options);
+
+ if(!rrdr_dimension_names(wb, "dimension_names", r, options))
+ rows = 0;
+
+ if(!rrdr_dimension_ids(wb, "dimension_ids", r, options))
+ rows = 0;
+
+ if (options & RRDR_OPTION_ALL_DIMENSIONS) {
+ query_target_summary_instances_v1(wb, qt, "full_chart_list");
+ query_target_summary_dimensions_v12(wb, qt, "full_dimension_list", false, NULL);
+ query_target_summary_labels_v12(wb, qt, "full_chart_labels", false, NULL, NULL);
+ }
+
+ query_target_functions(wb, "functions", r);
+
+ if (!qt->request.st && !jsonwrap_v1_chart_ids(wb, "chart_ids", r, options))
+ rows = 0;
+
+ if (qt->instances.chart_label_key_pattern && !query_target_chart_labels_filter_v1(wb, "chart_labels", r, options))
+ rows = 0;
+
+ if(!query_target_metrics_latest_values(wb, "latest_values", r, options))
+ rows = 0;
+
+ size_t dimensions = rrdr_dimension_view_latest_values(wb, "view_latest_values", r, options);
+ if(!dimensions)
+ rows = 0;
+
+ buffer_json_member_add_uint64(wb, "dimensions", dimensions);
+ buffer_json_member_add_uint64(wb, "points", rows);
+ buffer_json_member_add_string(wb, "format", rrdr_format_to_string(format));
+
+ buffer_json_member_add_array(wb, "db_points_per_tier");
+ for(size_t tier = 0; tier < storage_tiers ; tier++)
+ buffer_json_add_array_item_uint64(wb, qt->db.tiers[tier].points);
+ buffer_json_array_close(wb);
+
+ if(options & RRDR_OPTION_DEBUG)
+ jsonwrap_query_plan(r, wb);
+}
+
+static void rrdset_rrdcalc_entries_v2(BUFFER *wb, RRDINSTANCE_ACQUIRED *ria) {
+ RRDSET *st = rrdinstance_acquired_rrdset(ria);
+ if(st) {
+ rw_spinlock_read_lock(&st->alerts.spinlock);
+ if(st->alerts.base) {
+ buffer_json_member_add_object(wb, "alerts");
+ for(RRDCALC *rc = st->alerts.base; rc ;rc = rc->next) {
+ if(rc->status < RRDCALC_STATUS_CLEAR)
+ continue;
+
+ buffer_json_member_add_object(wb, string2str(rc->config.name));
+ buffer_json_member_add_string(wb, "st", rrdcalc_status2string(rc->status));
+ buffer_json_member_add_double(wb, "vl", rc->value);
+ buffer_json_member_add_string(wb, "un", string2str(rc->config.units));
+ buffer_json_object_close(wb);
+ }
+ buffer_json_object_close(wb);
+ }
+ rw_spinlock_read_unlock(&st->alerts.spinlock);
+ }
+}
+
+static void query_target_combined_units_v2(BUFFER *wb, QUERY_TARGET *qt, size_t contexts, bool ignore_percentage) {
+ if(!ignore_percentage && query_target_has_percentage_units(qt)) {
+ buffer_json_member_add_string(wb, "units", "%");
+ }
+ else if(contexts == 1) {
+ buffer_json_member_add_string(wb, "units", rrdcontext_acquired_units(qt->contexts.array[0].rca));
+ }
+ else if(contexts > 1) {
+ DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
+ for(size_t c = 0; c < qt->contexts.used ;c++)
+ dictionary_set(dict, rrdcontext_acquired_units(qt->contexts.array[c].rca), NULL, 0);
+
+ if(dictionary_entries(dict) == 1)
+ buffer_json_member_add_string(wb, "units", rrdcontext_acquired_units(qt->contexts.array[0].rca));
+ else {
+ buffer_json_member_add_array(wb, "units");
+ const char *s;
+ dfe_start_read(dict, s)
+ buffer_json_add_array_item_string(wb, s_dfe.name);
+ dfe_done(s);
+ buffer_json_array_close(wb);
+ }
+ dictionary_destroy(dict);
+ }
+}
+
+static void query_target_combined_chart_type(BUFFER *wb, QUERY_TARGET *qt, size_t contexts) {
+ if(contexts >= 1)
+ buffer_json_member_add_string(wb, "chart_type", rrdset_type_name(rrdcontext_acquired_chart_type(qt->contexts.array[0].rca)));
+}
+
+static void rrdr_grouped_by_array_v2(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options __maybe_unused) {
+ QUERY_TARGET *qt = r->internal.qt;
+
+ buffer_json_member_add_array(wb, key);
+
+ // find the deeper group-by
+ ssize_t g = 0;
+ for(g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) {
+ if(qt->request.group_by[g].group_by == RRDR_GROUP_BY_NONE)
+ break;
+ }
+
+ if(g > 0)
+ g--;
+
+ RRDR_GROUP_BY group_by = qt->request.group_by[g].group_by;
+
+ if(group_by & RRDR_GROUP_BY_SELECTED)
+ buffer_json_add_array_item_string(wb, "selected");
+
+ else if(group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE)
+ buffer_json_add_array_item_string(wb, "percentage-of-instance");
+
+ else {
+
+ if(group_by & RRDR_GROUP_BY_DIMENSION)
+ buffer_json_add_array_item_string(wb, "dimension");
+
+ if(group_by & RRDR_GROUP_BY_INSTANCE)
+ buffer_json_add_array_item_string(wb, "instance");
+
+ if(group_by & RRDR_GROUP_BY_LABEL) {
+ BUFFER *b = buffer_create(0, NULL);
+ for (size_t l = 0; l < qt->group_by[g].used; l++) {
+ buffer_flush(b);
+ buffer_fast_strcat(b, "label:", 6);
+ buffer_strcat(b, qt->group_by[g].label_keys[l]);
+ buffer_json_add_array_item_string(wb, buffer_tostring(b));
+ }
+ buffer_free(b);
+ }
+
+ if(group_by & RRDR_GROUP_BY_NODE)
+ buffer_json_add_array_item_string(wb, "node");
+
+ if(group_by & RRDR_GROUP_BY_CONTEXT)
+ buffer_json_add_array_item_string(wb, "context");
+
+ if(group_by & RRDR_GROUP_BY_UNITS)
+ buffer_json_add_array_item_string(wb, "units");
+ }
+
+ buffer_json_array_close(wb); // group_by_order
+}
+
+static void rrdr_dimension_units_array_v2(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options, bool ignore_percentage) {
+ if(!r->du)
+ return;
+
+ bool percentage = !ignore_percentage && query_target_has_percentage_units(r->internal.qt);
+
+ buffer_json_member_add_array(wb, key);
+ for(size_t c = 0; c < r->d ; c++) {
+ if(!rrdr_dimension_should_be_exposed(r->od[c], options))
+ continue;
+
+ if(percentage)
+ buffer_json_add_array_item_string(wb, "%");
+ else
+ buffer_json_add_array_item_string(wb, string2str(r->du[c]));
+ }
+ buffer_json_array_close(wb);
+}
+
+static void rrdr_dimension_priority_array_v2(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
+ if(!r->dp)
+ return;
+
+ buffer_json_member_add_array(wb, key);
+ for(size_t c = 0; c < r->d ; c++) {
+ if(!rrdr_dimension_should_be_exposed(r->od[c], options))
+ continue;
+
+ buffer_json_add_array_item_uint64(wb, r->dp[c]);
+ }
+ buffer_json_array_close(wb);
+}
+
+static void rrdr_dimension_aggregated_array_v2(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
+ if(!r->dgbc)
+ return;
+
+ buffer_json_member_add_array(wb, key);
+ for(size_t c = 0; c < r->d ;c++) {
+ if(!rrdr_dimension_should_be_exposed(r->od[c], options))
+ continue;
+
+ buffer_json_add_array_item_uint64(wb, r->dgbc[c]);
+ }
+ buffer_json_array_close(wb);
+}
+
+static void query_target_title(BUFFER *wb, QUERY_TARGET *qt, size_t contexts) {
+ if(contexts == 1) {
+ buffer_json_member_add_string(wb, "title", rrdcontext_acquired_title(qt->contexts.array[0].rca));
+ }
+ else if(contexts > 1) {
+ BUFFER *t = buffer_create(0, NULL);
+ DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
+
+ buffer_strcat(t, "Chart for contexts: ");
+
+ size_t added = 0;
+ for(size_t c = 0; c < qt->contexts.used ;c++) {
+ bool *set = dictionary_set(dict, rrdcontext_acquired_id(qt->contexts.array[c].rca), NULL, sizeof(*set));
+ if(!*set) {
+ *set = true;
+ if(added)
+ buffer_fast_strcat(t, ", ", 2);
+
+ buffer_strcat(t, rrdcontext_acquired_id(qt->contexts.array[c].rca));
+ added++;
+ }
+ }
+ buffer_json_member_add_string(wb, "title", buffer_tostring(t));
+ dictionary_destroy(dict);
+ buffer_free(t);
+ }
+}
+
+static void query_target_detailed_objects_tree(BUFFER *wb, RRDR *r, RRDR_OPTIONS options) {
+ QUERY_TARGET *qt = r->internal.qt;
+ buffer_json_member_add_object(wb, "nodes");
+
+ time_t now_s = now_realtime_sec();
+ RRDHOST *last_host = NULL;
+ RRDCONTEXT_ACQUIRED *last_rca = NULL;
+ RRDINSTANCE_ACQUIRED *last_ria = NULL;
+
+ size_t h = 0, c = 0, i = 0, m = 0, q = 0;
+ for(; h < qt->nodes.used ; h++) {
+ QUERY_NODE *qn = query_node(qt, h);
+ RRDHOST *host = qn->rrdhost;
+
+ for( ;c < qt->contexts.used ;c++) {
+ QUERY_CONTEXT *qc = query_context(qt, c);
+ RRDCONTEXT_ACQUIRED *rca = qc->rca;
+ if(!rrdcontext_acquired_belongs_to_host(rca, host)) break;
+
+ for( ;i < qt->instances.used ;i++) {
+ QUERY_INSTANCE *qi = query_instance(qt, i);
+ RRDINSTANCE_ACQUIRED *ria = qi->ria;
+ if(!rrdinstance_acquired_belongs_to_context(ria, rca)) break;
+
+ for( ; m < qt->dimensions.used ; m++) {
+ QUERY_DIMENSION *qd = query_dimension(qt, m);
+ RRDMETRIC_ACQUIRED *rma = qd->rma;
+ if(!rrdmetric_acquired_belongs_to_instance(rma, ria)) break;
+
+ QUERY_METRIC *qm = NULL;
+ bool queried = false;
+ for( ; q < qt->query.used ;q++) {
+ QUERY_METRIC *tqm = query_metric(qt, q);
+ QUERY_DIMENSION *tqd = query_dimension(qt, tqm->link.query_dimension_id);
+ if(tqd->rma != rma) break;
+
+ queried = tqm->status & RRDR_DIMENSION_QUERIED;
+ qm = tqm;
+ }
+
+ if(!queried & !(options & RRDR_OPTION_ALL_DIMENSIONS))
+ continue;
+
+ if(host != last_host) {
+ if(last_host) {
+ if(last_rca) {
+ if(last_ria) {
+ buffer_json_object_close(wb); // dimensions
+ buffer_json_object_close(wb); // instance
+ last_ria = NULL;
+ }
+ buffer_json_object_close(wb); // instances
+ buffer_json_object_close(wb); // context
+ last_rca = NULL;
+ }
+ buffer_json_object_close(wb); // contexts
+ buffer_json_object_close(wb); // host
+ last_host = NULL;
+ }
+
+ buffer_json_member_add_object(wb, host->machine_guid);
+ if(qn->node_id[0])
+ buffer_json_member_add_string(wb, "nd", qn->node_id);
+ buffer_json_member_add_uint64(wb, "ni", qn->slot);
+ buffer_json_member_add_string(wb, "nm", rrdhost_hostname(host));
+ buffer_json_member_add_object(wb, "contexts");
+
+ last_host = host;
+ }
+
+ if(rca != last_rca) {
+ if(last_rca) {
+ if(last_ria) {
+ buffer_json_object_close(wb); // dimensions
+ buffer_json_object_close(wb); // instance
+ last_ria = NULL;
+ }
+ buffer_json_object_close(wb); // instances
+ buffer_json_object_close(wb); // context
+ last_rca = NULL;
+ }
+
+ buffer_json_member_add_object(wb, rrdcontext_acquired_id(rca));
+ buffer_json_member_add_object(wb, "instances");
+
+ last_rca = rca;
+ }
+
+ if(ria != last_ria) {
+ if(last_ria) {
+ buffer_json_object_close(wb); // dimensions
+ buffer_json_object_close(wb); // instance
+ last_ria = NULL;
+ }
+
+ buffer_json_member_add_object(wb, rrdinstance_acquired_id(ria));
+ buffer_json_member_add_string(wb, "nm", rrdinstance_acquired_name(ria));
+ buffer_json_member_add_time_t(wb, "ue", rrdinstance_acquired_update_every(ria));
+ RRDLABELS *labels = rrdinstance_acquired_labels(ria);
+ if(labels) {
+ buffer_json_member_add_object(wb, "labels");
+ rrdlabels_to_buffer_json_members(labels, wb);
+ buffer_json_object_close(wb);
+ }
+ rrdset_rrdcalc_entries_v2(wb, ria);
+ buffer_json_member_add_object(wb, "dimensions");
+
+ last_ria = ria;
+ }
+
+ buffer_json_member_add_object(wb, rrdmetric_acquired_id(rma));
+ {
+ buffer_json_member_add_string(wb, "nm", rrdmetric_acquired_name(rma));
+ buffer_json_member_add_uint64(wb, "qr", queried ? 1 : 0);
+ time_t first_entry_s = rrdmetric_acquired_first_entry(rma);
+ time_t last_entry_s = rrdmetric_acquired_last_entry(rma);
+ buffer_json_member_add_time_t(wb, "fe", first_entry_s);
+ buffer_json_member_add_time_t(wb, "le", last_entry_s ? last_entry_s : now_s);
+
+ if(qm) {
+ if(qm->status & RRDR_DIMENSION_GROUPED) {
+ // buffer_json_member_add_string(wb, "grouped_as_id", string2str(qm->grouped_as.id));
+ buffer_json_member_add_string(wb, "as", string2str(qm->grouped_as.name));
+ }
+
+ query_target_points_statistics(wb, qt, &qm->query_points);
+
+ if(options & RRDR_OPTION_DEBUG)
+ jsonwrap_query_metric_plan(wb, qm);
+ }
+ }
+ buffer_json_object_close(wb); // metric
+ }
+ }
+ }
+ }
+
+ if(last_host) {
+ if(last_rca) {
+ if(last_ria) {
+ buffer_json_object_close(wb); // dimensions
+ buffer_json_object_close(wb); // instance
+ last_ria = NULL;
+ }
+ buffer_json_object_close(wb); // instances
+ buffer_json_object_close(wb); // context
+ last_rca = NULL;
+ }
+ buffer_json_object_close(wb); // contexts
+ buffer_json_object_close(wb); // host
+ last_host = NULL;
+ }
+ buffer_json_object_close(wb); // hosts
+}
+
+void version_hashes_api_v2(BUFFER *wb, struct query_versions *versions) {
+ buffer_json_member_add_object(wb, "versions");
+ buffer_json_member_add_uint64(wb, "routing_hard_hash", 1);
+ buffer_json_member_add_uint64(wb, "nodes_hard_hash", dictionary_version(rrdhost_root_index));
+ buffer_json_member_add_uint64(wb, "contexts_hard_hash", versions->contexts_hard_hash);
+ buffer_json_member_add_uint64(wb, "contexts_soft_hash", versions->contexts_soft_hash);
+ buffer_json_member_add_uint64(wb, "alerts_hard_hash", versions->alerts_hard_hash);
+ buffer_json_member_add_uint64(wb, "alerts_soft_hash", versions->alerts_soft_hash);
+ buffer_json_object_close(wb);
+}
+
+void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb) {
+ QUERY_TARGET *qt = r->internal.qt;
+ RRDR_OPTIONS options = qt->window.options;
+
+ char kq[2] = "\"", // key quote
+ sq[2] = "\""; // string quote
+
+ if(unlikely(options & RRDR_OPTION_GOOGLE_JSON)) {
+ kq[0] = '\0';
+ sq[0] = '\'';
+ }
+
+ buffer_json_initialize(
+ wb, kq, sq, 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
+ buffer_json_member_add_uint64(wb, "api", 2);
+
+ if(options & RRDR_OPTION_DEBUG) {
+ buffer_json_member_add_string(wb, "id", qt->id);
+ buffer_json_member_add_object(wb, "request");
+ {
+ buffer_json_member_add_string(wb, "format", rrdr_format_to_string(qt->request.format));
+ rrdr_options_to_buffer_json_array(wb, "options", qt->request.options);
+
+ buffer_json_member_add_object(wb, "scope");
+ buffer_json_member_add_string(wb, "scope_nodes", qt->request.scope_nodes);
+ buffer_json_member_add_string(wb, "scope_contexts", qt->request.scope_contexts);
+ buffer_json_object_close(wb); // scope
+
+ buffer_json_member_add_object(wb, "selectors");
+ if (qt->request.host)
+ buffer_json_member_add_string(wb, "nodes", rrdhost_hostname(qt->request.host));
+ else
+ buffer_json_member_add_string(wb, "nodes", qt->request.nodes);
+ buffer_json_member_add_string(wb, "contexts", qt->request.contexts);
+ buffer_json_member_add_string(wb, "instances", qt->request.instances);
+ buffer_json_member_add_string(wb, "dimensions", qt->request.dimensions);
+ buffer_json_member_add_string(wb, "labels", qt->request.labels);
+ buffer_json_member_add_string(wb, "alerts", qt->request.alerts);
+ buffer_json_object_close(wb); // selectors
+
+ buffer_json_member_add_object(wb, "window");
+ buffer_json_member_add_time_t(wb, "after", qt->request.after);
+ buffer_json_member_add_time_t(wb, "before", qt->request.before);
+ buffer_json_member_add_uint64(wb, "points", qt->request.points);
+ if (qt->request.options & RRDR_OPTION_SELECTED_TIER)
+ buffer_json_member_add_uint64(wb, "tier", qt->request.tier);
+ else
+ buffer_json_member_add_string(wb, "tier", NULL);
+ buffer_json_object_close(wb); // window
+
+ buffer_json_member_add_object(wb, "aggregations");
+ {
+ buffer_json_member_add_object(wb, "time");
+ buffer_json_member_add_string(wb, "time_group", time_grouping_tostring(qt->request.time_group_method));
+ buffer_json_member_add_string(wb, "time_group_options", qt->request.time_group_options);
+ if (qt->request.resampling_time > 0)
+ buffer_json_member_add_time_t(wb, "time_resampling", qt->request.resampling_time);
+ else
+ buffer_json_member_add_string(wb, "time_resampling", NULL);
+ buffer_json_object_close(wb); // time
+
+ buffer_json_member_add_array(wb, "metrics");
+ for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) {
+ if(qt->request.group_by[g].group_by == RRDR_GROUP_BY_NONE)
+ break;
+
+ buffer_json_add_array_item_object(wb);
+ {
+ buffer_json_member_add_array(wb, "group_by");
+ buffer_json_group_by_to_array(wb, qt->request.group_by[g].group_by);
+ buffer_json_array_close(wb);
+
+ buffer_json_member_add_array(wb, "group_by_label");
+ for (size_t l = 0; l < qt->group_by[g].used; l++)
+ buffer_json_add_array_item_string(wb, qt->group_by[g].label_keys[l]);
+ buffer_json_array_close(wb);
+
+ buffer_json_member_add_string(
+ wb, "aggregation",group_by_aggregate_function_to_string(qt->request.group_by[g].aggregation));
+ }
+ buffer_json_object_close(wb);
+ }
+ buffer_json_array_close(wb); // group_by
+ }
+ buffer_json_object_close(wb); // aggregations
+
+ buffer_json_member_add_uint64(wb, "timeout", qt->request.timeout_ms);
+ }
+ buffer_json_object_close(wb); // request
+ }
+
+ version_hashes_api_v2(wb, &qt->versions);
+
+ buffer_json_member_add_object(wb, "summary");
+ struct summary_total_counts
+ nodes_totals = { 0 },
+ contexts_totals = { 0 },
+ instances_totals = { 0 },
+ metrics_totals = { 0 },
+ label_key_totals = { 0 },
+ label_key_value_totals = { 0 };
+ {
+ query_target_summary_nodes_v2(wb, qt, "nodes", &nodes_totals);
+ r->internal.contexts = query_target_summary_contexts_v2(wb, qt, "contexts", &contexts_totals);
+ query_target_summary_instances_v2(wb, qt, "instances", &instances_totals);
+ query_target_summary_dimensions_v12(wb, qt, "dimensions", true, &metrics_totals);
+ query_target_summary_labels_v12(wb, qt, "labels", true, &label_key_totals, &label_key_value_totals);
+ query_target_summary_alerts_v2(wb, qt, "alerts");
+ }
+ if(query_target_aggregatable(qt)) {
+ buffer_json_member_add_object(wb, "globals");
+ query_target_points_statistics(wb, qt, &qt->query_points);
+ buffer_json_object_close(wb); // globals
+ }
+ buffer_json_object_close(wb); // summary
+
+ buffer_json_member_add_object(wb, "totals");
+ query_target_total_counts(wb, "nodes", &nodes_totals);
+ query_target_total_counts(wb, "contexts", &contexts_totals);
+ query_target_total_counts(wb, "instances", &instances_totals);
+ query_target_total_counts(wb, "dimensions", &metrics_totals);
+ query_target_total_counts(wb, "label_keys", &label_key_totals);
+ query_target_total_counts(wb, "label_key_values", &label_key_value_totals);
+ buffer_json_object_close(wb); // totals
+
+ if(options & RRDR_OPTION_SHOW_DETAILS) {
+ buffer_json_member_add_object(wb, "detailed");
+ query_target_detailed_objects_tree(wb, r, options);
+ buffer_json_object_close(wb); // detailed
+ }
+
+ query_target_functions(wb, "functions", r);
+}
+
+//static void annotations_range_for_value_flags(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format __maybe_unused, RRDR_OPTIONS options, RRDR_VALUE_FLAGS flags, const char *type) {
+// const size_t dims = r->d, rows = r->rows;
+// size_t next_d_idx = 0;
+// for(size_t d = 0; d < dims ; d++) {
+// if(!rrdr_dimension_should_be_exposed(r->od[d], options))
+// continue;
+//
+// size_t d_idx = next_d_idx++;
+//
+// size_t t = 0;
+// while(t < rows) {
+//
+// // find the beginning
+// time_t started = 0;
+// for(; t < rows ;t++) {
+// RRDR_VALUE_FLAGS o = r->o[t * r->d + d];
+// if(o & flags) {
+// started = r->t[t];
+// break;
+// }
+// }
+//
+// if(started) {
+// time_t ended = 0;
+// for(; t < rows ;t++) {
+// RRDR_VALUE_FLAGS o = r->o[t * r->d + d];
+// if(!(o & flags)) {
+// ended = r->t[t];
+// break;
+// }
+// }
+//
+// if(!ended)
+// ended = r->t[rows - 1];
+//
+// buffer_json_add_array_item_object(wb);
+// buffer_json_member_add_string(wb, "t", type);
+// // buffer_json_member_add_string(wb, "d", string2str(r->dn[d]));
+// buffer_json_member_add_uint64(wb, "d", d_idx);
+// if(started == ended) {
+// if(options & RRDR_OPTION_MILLISECONDS)
+// buffer_json_member_add_time_t2ms(wb, "x", started);
+// else
+// buffer_json_member_add_time_t(wb, "x", started);
+// }
+// else {
+// buffer_json_member_add_array(wb, "x");
+// if(options & RRDR_OPTION_MILLISECONDS) {
+// buffer_json_add_array_item_time_t2ms(wb, started);
+// buffer_json_add_array_item_time_t2ms(wb, ended);
+// }
+// else {
+// buffer_json_add_array_item_time_t(wb, started);
+// buffer_json_add_array_item_time_t(wb, ended);
+// }
+// buffer_json_array_close(wb);
+// }
+// buffer_json_object_close(wb);
+// }
+// }
+// }
+//}
+//
+//void rrdr_json_wrapper_annotations(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format __maybe_unused, RRDR_OPTIONS options) {
+// buffer_json_member_add_array(wb, "annotations");
+//
+// annotations_range_for_value_flags(r, wb, format, options, RRDR_VALUE_EMPTY, "G"); // Gap
+// annotations_range_for_value_flags(r, wb, format, options, RRDR_VALUE_RESET, "O"); // Overflow
+// annotations_range_for_value_flags(r, wb, format, options, RRDR_VALUE_PARTIAL, "P"); // Partial
+//
+// buffer_json_array_close(wb); // annotations
+//}
+
+void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb) {
+ buffer_json_member_add_double(wb, "min", r->view.min);
+ buffer_json_member_add_double(wb, "max", r->view.max);
+
+ buffer_json_query_timings(wb, "timings", &r->internal.qt->timings);
+ buffer_json_finalize(wb);
+}
+
+void rrdr_json_wrapper_end2(RRDR *r, BUFFER *wb) {
+ QUERY_TARGET *qt = r->internal.qt;
+ DATASOURCE_FORMAT format = qt->request.format;
+ RRDR_OPTIONS options = qt->window.options;
+
+ buffer_json_member_add_object(wb, "db");
+ {
+ buffer_json_member_add_uint64(wb, "tiers", storage_tiers);
+ buffer_json_member_add_time_t(wb, "update_every", qt->db.minimum_latest_update_every_s);
+ buffer_json_member_add_time_t(wb, "first_entry", qt->db.first_time_s);
+ buffer_json_member_add_time_t(wb, "last_entry", qt->db.last_time_s);
+
+ query_target_combined_units_v2(wb, qt, r->internal.contexts, true);
+ buffer_json_member_add_object(wb, "dimensions");
+ {
+ rrdr_dimension_ids(wb, "ids", r, options);
+ rrdr_dimension_units_array_v2(wb, "units", r, options, true);
+ rrdr_dimension_query_points_statistics(wb, "sts", r, options, false);
+ }
+ buffer_json_object_close(wb); // dimensions
+
+ buffer_json_member_add_array(wb, "per_tier");
+ for(size_t tier = 0; tier < storage_tiers ; tier++) {
+ buffer_json_add_array_item_object(wb);
+ buffer_json_member_add_uint64(wb, "tier", tier);
+ buffer_json_member_add_uint64(wb, "queries", qt->db.tiers[tier].queries);
+ buffer_json_member_add_uint64(wb, "points", qt->db.tiers[tier].points);
+ buffer_json_member_add_time_t(wb, "update_every", qt->db.tiers[tier].update_every);
+ buffer_json_member_add_time_t(wb, "first_entry", qt->db.tiers[tier].retention.first_time_s);
+ buffer_json_member_add_time_t(wb, "last_entry", qt->db.tiers[tier].retention.last_time_s);
+ buffer_json_object_close(wb);
+ }
+ buffer_json_array_close(wb);
+ }
+ buffer_json_object_close(wb);
+
+ buffer_json_member_add_object(wb, "view");
+ {
+ query_target_title(wb, qt, r->internal.contexts);
+ buffer_json_member_add_time_t(wb, "update_every", r->view.update_every);
+ buffer_json_member_add_time_t(wb, "after", r->view.after);
+ buffer_json_member_add_time_t(wb, "before", r->view.before);
+
+ if(options & RRDR_OPTION_DEBUG) {
+ buffer_json_member_add_string(wb, "format", rrdr_format_to_string(format));
+ rrdr_options_to_buffer_json_array(wb, "options", options);
+ buffer_json_member_add_string(wb, "time_group", time_grouping_tostring(qt->request.time_group_method));
+ }
+
+ if(options & RRDR_OPTION_DEBUG) {
+ buffer_json_member_add_object(wb, "partial_data_trimming");
+ buffer_json_member_add_time_t(wb, "max_update_every", r->partial_data_trimming.max_update_every);
+ buffer_json_member_add_time_t(wb, "expected_after", r->partial_data_trimming.expected_after);
+ buffer_json_member_add_time_t(wb, "trimmed_after", r->partial_data_trimming.trimmed_after);
+ buffer_json_object_close(wb);
+ }
+
+ if(options & RRDR_OPTION_RETURN_RAW)
+ buffer_json_member_add_uint64(wb, "points", rrdr_rows(r));
+
+ query_target_combined_units_v2(wb, qt, r->internal.contexts, false);
+ query_target_combined_chart_type(wb, qt, r->internal.contexts);
+ buffer_json_member_add_object(wb, "dimensions");
+ {
+ rrdr_grouped_by_array_v2(wb, "grouped_by", r, options);
+ rrdr_dimension_ids(wb, "ids", r, options);
+ rrdr_dimension_names(wb, "names", r, options);
+ rrdr_dimension_units_array_v2(wb, "units", r, options, false);
+ rrdr_dimension_priority_array_v2(wb, "priorities", r, options);
+ rrdr_dimension_aggregated_array_v2(wb, "aggregated", r, options);
+ rrdr_dimension_query_points_statistics(wb, "sts", r, options, true);
+ rrdr_json_group_by_labels(wb, "labels", r, options);
+ }
+ buffer_json_object_close(wb); // dimensions
+ buffer_json_member_add_double(wb, "min", r->view.min);
+ buffer_json_member_add_double(wb, "max", r->view.max);
+ }
+ buffer_json_object_close(wb); // view
+
+ buffer_json_agents_v2(wb, &r->internal.qt->timings, 0, false, true);
+ buffer_json_cloud_timings(wb, "timings", &r->internal.qt->timings);
+ buffer_json_finalize(wb);
+}
diff --git a/src/web/api/formatters/json_wrapper.h b/src/web/api/formatters/json_wrapper.h
new file mode 100644
index 000000000..a702f3a5c
--- /dev/null
+++ b/src/web/api/formatters/json_wrapper.h
@@ -0,0 +1,21 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_API_FORMATTER_JSON_WRAPPER_H
+#define NETDATA_API_FORMATTER_JSON_WRAPPER_H
+
+#include "rrd2json.h"
+#include "web/api/queries/query.h"
+
+typedef void (*wrapper_begin_t)(RRDR *r, BUFFER *wb);
+typedef void (*wrapper_end_t)(RRDR *r, BUFFER *wb);
+
+void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb);
+void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb);
+
+void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb);
+void rrdr_json_wrapper_end2(RRDR *r, BUFFER *wb);
+
+struct query_versions;
+void version_hashes_api_v2(BUFFER *wb, struct query_versions *versions);
+
+#endif //NETDATA_API_FORMATTER_JSON_WRAPPER_H
diff --git a/src/web/api/formatters/rrd2json.c b/src/web/api/formatters/rrd2json.c
new file mode 100644
index 000000000..81c9ad5c7
--- /dev/null
+++ b/src/web/api/formatters/rrd2json.c
@@ -0,0 +1,391 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "web/api/web_api_v1.h"
+#include "database/storage_engine.h"
+
+void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb)
+{
+ buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
+ rrdset2json(st, wb, NULL, NULL);
+ buffer_json_finalize(wb);
+}
+
+const char *rrdr_format_to_string(DATASOURCE_FORMAT format) {
+ switch(format) {
+ case DATASOURCE_JSON:
+ return DATASOURCE_FORMAT_JSON;
+
+ case DATASOURCE_JSON2:
+ return DATASOURCE_FORMAT_JSON2;
+
+ case DATASOURCE_DATATABLE_JSON:
+ return DATASOURCE_FORMAT_DATATABLE_JSON;
+
+ case DATASOURCE_DATATABLE_JSONP:
+ return DATASOURCE_FORMAT_DATATABLE_JSONP;
+
+ case DATASOURCE_JSONP:
+ return DATASOURCE_FORMAT_JSONP;
+
+ case DATASOURCE_SSV:
+ return DATASOURCE_FORMAT_SSV;
+
+ case DATASOURCE_CSV:
+ return DATASOURCE_FORMAT_CSV;
+
+ case DATASOURCE_TSV:
+ return DATASOURCE_FORMAT_TSV;
+
+ case DATASOURCE_HTML:
+ return DATASOURCE_FORMAT_HTML;
+
+ case DATASOURCE_JS_ARRAY:
+ return DATASOURCE_FORMAT_JS_ARRAY;
+
+ case DATASOURCE_SSV_COMMA:
+ return DATASOURCE_FORMAT_SSV_COMMA;
+
+ default:
+ return "unknown";
+ }
+}
+
+int rrdset2value_api_v1(
+ RRDSET *st
+ , BUFFER *wb
+ , NETDATA_DOUBLE *n
+ , const char *dimensions
+ , size_t points
+ , time_t after
+ , time_t before
+ , RRDR_TIME_GROUPING group_method
+ , const char *group_options
+ , time_t resampling_time
+ , uint32_t options
+ , time_t *db_after
+ , time_t *db_before
+ , size_t *db_points_read
+ , size_t *db_points_per_tier
+ , size_t *result_points_generated
+ , int *value_is_null
+ , NETDATA_DOUBLE *anomaly_rate
+ , time_t timeout
+ , size_t tier
+ , QUERY_SOURCE query_source
+ , STORAGE_PRIORITY priority
+) {
+ int ret = HTTP_RESP_INTERNAL_SERVER_ERROR;
+
+ ONEWAYALLOC *owa = onewayalloc_create(0);
+ RRDR *r = rrd2rrdr_legacy(
+ owa,
+ st,
+ points,
+ after,
+ before,
+ group_method,
+ resampling_time,
+ options,
+ dimensions,
+ group_options,
+ timeout,
+ tier,
+ query_source,
+ priority);
+
+ if(!r) {
+ if(value_is_null) *value_is_null = 1;
+ ret = HTTP_RESP_INTERNAL_SERVER_ERROR;
+ goto cleanup;
+ }
+
+ if(db_points_read)
+ *db_points_read += r->stats.db_points_read;
+
+ if(db_points_per_tier) {
+ for(size_t t = 0; t < storage_tiers ;t++)
+ db_points_per_tier[t] += r->internal.qt->db.tiers[t].points;
+ }
+
+ if(result_points_generated)
+ *result_points_generated += r->stats.result_points_generated;
+
+ if(rrdr_rows(r) == 0) {
+ if(db_after) *db_after = 0;
+ if(db_before) *db_before = 0;
+ if(value_is_null) *value_is_null = 1;
+
+ ret = HTTP_RESP_BAD_REQUEST;
+ goto cleanup;
+ }
+
+ if(wb) {
+ if (r->view.flags & RRDR_RESULT_FLAG_RELATIVE)
+ buffer_no_cacheable(wb);
+ else if (r->view.flags & RRDR_RESULT_FLAG_ABSOLUTE)
+ buffer_cacheable(wb);
+ }
+
+ if(db_after) *db_after = r->view.after;
+ if(db_before) *db_before = r->view.before;
+
+ long i = (!(options & RRDR_OPTION_REVERSED))?(long)rrdr_rows(r) - 1:0;
+ *n = rrdr2value(r, i, options, value_is_null, anomaly_rate);
+ ret = HTTP_RESP_OK;
+
+cleanup:
+ rrdr_free(owa, r);
+ onewayalloc_destroy(owa);
+ return ret;
+}
+
+static inline void buffer_json_member_add_key_only(BUFFER *wb, const char *key) {
+ buffer_print_json_comma_newline_spacing(wb);
+ buffer_print_json_key(wb, key);
+ buffer_fast_strcat(wb, ":", 1);
+ wb->json.stack[wb->json.depth].count++;
+}
+
+static inline void buffer_json_member_add_string_open(BUFFER *wb, const char *key) {
+ buffer_json_member_add_key_only(wb, key);
+ buffer_strcat(wb, wb->json.value_quote);
+}
+
+static inline void buffer_json_member_add_string_close(BUFFER *wb) {
+ buffer_strcat(wb, wb->json.value_quote);
+}
+
+int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *latest_timestamp) {
+ wrapper_begin_t wrapper_begin = rrdr_json_wrapper_begin;
+ wrapper_end_t wrapper_end = rrdr_json_wrapper_end;
+
+ if(qt->request.version == 2) {
+ wrapper_begin = rrdr_json_wrapper_begin2;
+ wrapper_end = rrdr_json_wrapper_end2;
+ }
+
+ RRDR *r = rrd2rrdr(owa, qt);
+
+ if(!r) {
+ buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");
+ return HTTP_RESP_INTERNAL_SERVER_ERROR;
+ }
+
+ if (r->view.flags & RRDR_RESULT_FLAG_CANCEL) {
+ rrdr_free(owa, r);
+ return HTTP_RESP_CLIENT_CLOSED_REQUEST;
+ }
+
+ if(r->view.flags & RRDR_RESULT_FLAG_RELATIVE)
+ buffer_no_cacheable(wb);
+ else if(r->view.flags & RRDR_RESULT_FLAG_ABSOLUTE)
+ buffer_cacheable(wb);
+
+ if(latest_timestamp && rrdr_rows(r) > 0)
+ *latest_timestamp = r->view.before;
+
+ DATASOURCE_FORMAT format = qt->request.format;
+ RRDR_OPTIONS options = qt->window.options;
+
+ switch(format) {
+ case DATASOURCE_SSV:
+ if(options & RRDR_OPTION_JSON_WRAP) {
+ wb->content_type = CT_APPLICATION_JSON;
+ wrapper_begin(r, wb);
+ buffer_json_member_add_string_open(wb, "result");
+ rrdr2ssv(r, wb, options, "", " ", "");
+ buffer_json_member_add_string_close(wb);
+ wrapper_end(r, wb);
+ }
+ else {
+ wb->content_type = CT_TEXT_PLAIN;
+ rrdr2ssv(r, wb, options, "", " ", "");
+ }
+ break;
+
+ case DATASOURCE_SSV_COMMA:
+ if(options & RRDR_OPTION_JSON_WRAP) {
+ wb->content_type = CT_APPLICATION_JSON;
+ wrapper_begin(r, wb);
+ buffer_json_member_add_string_open(wb, "result");
+ rrdr2ssv(r, wb, options, "", ",", "");
+ buffer_json_member_add_string_close(wb);
+ wrapper_end(r, wb);
+ }
+ else {
+ wb->content_type = CT_TEXT_PLAIN;
+ rrdr2ssv(r, wb, options, "", ",", "");
+ }
+ break;
+
+ case DATASOURCE_JS_ARRAY:
+ if(options & RRDR_OPTION_JSON_WRAP) {
+ wb->content_type = CT_APPLICATION_JSON;
+ wrapper_begin(r, wb);
+ buffer_json_member_add_array(wb, "result");
+ rrdr2ssv(r, wb, options, "", ",", "");
+ buffer_json_array_close(wb);
+ wrapper_end(r, wb);
+ }
+ else {
+ wb->content_type = CT_APPLICATION_JSON;
+ rrdr2ssv(r, wb, options, "[", ",", "]");
+ }
+ break;
+
+ case DATASOURCE_CSV:
+ if(options & RRDR_OPTION_JSON_WRAP) {
+ wb->content_type = CT_APPLICATION_JSON;
+ wrapper_begin(r, wb);
+ buffer_json_member_add_string_open(wb, "result");
+ rrdr2csv(r, wb, format, options, "", ",", "\\n", "");
+ buffer_json_member_add_string_close(wb);
+ wrapper_end(r, wb);
+ }
+ else {
+ wb->content_type = CT_TEXT_PLAIN;
+ rrdr2csv(r, wb, format, options, "", ",", "\r\n", "");
+ }
+ break;
+
+ case DATASOURCE_CSV_MARKDOWN:
+ if(options & RRDR_OPTION_JSON_WRAP) {
+ wb->content_type = CT_APPLICATION_JSON;
+ wrapper_begin(r, wb);
+ buffer_json_member_add_string_open(wb, "result");
+ rrdr2csv(r, wb, format, options, "", "|", "\\n", "");
+ buffer_json_member_add_string_close(wb);
+ wrapper_end(r, wb);
+ }
+ else {
+ wb->content_type = CT_TEXT_PLAIN;
+ rrdr2csv(r, wb, format, options, "", "|", "\r\n", "");
+ }
+ break;
+
+ case DATASOURCE_CSV_JSON_ARRAY:
+ wb->content_type = CT_APPLICATION_JSON;
+ if(options & RRDR_OPTION_JSON_WRAP) {
+ wrapper_begin(r, wb);
+ buffer_json_member_add_array(wb, "result");
+ rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n");
+ buffer_json_array_close(wb);
+ wrapper_end(r, wb);
+ }
+ else {
+ wb->content_type = CT_APPLICATION_JSON;
+ buffer_strcat(wb, "[\n");
+ rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n");
+ buffer_strcat(wb, "\n]");
+ }
+ break;
+
+ case DATASOURCE_TSV:
+ if(options & RRDR_OPTION_JSON_WRAP) {
+ wb->content_type = CT_APPLICATION_JSON;
+ wrapper_begin(r, wb);
+ buffer_json_member_add_string_open(wb, "result");
+ rrdr2csv(r, wb, format, options, "", "\t", "\\n", "");
+ buffer_json_member_add_string_close(wb);
+ wrapper_end(r, wb);
+ }
+ else {
+ wb->content_type = CT_TEXT_PLAIN;
+ rrdr2csv(r, wb, format, options, "", "\t", "\r\n", "");
+ }
+ break;
+
+ case DATASOURCE_HTML:
+ if(options & RRDR_OPTION_JSON_WRAP) {
+ wb->content_type = CT_APPLICATION_JSON;
+ wrapper_begin(r, wb);
+ buffer_json_member_add_string_open(wb, "result");
+ buffer_strcat(wb, "<html>\\n<center>\\n<table border=\\\"0\\\" cellpadding=\\\"5\\\" cellspacing=\\\"5\\\">\\n");
+ rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\\n", "");
+ buffer_strcat(wb, "</table>\\n</center>\\n</html>\\n");
+ buffer_json_member_add_string_close(wb);
+ wrapper_end(r, wb);
+ }
+ else {
+ wb->content_type = CT_TEXT_HTML;
+ buffer_strcat(wb, "<html>\n<center>\n<table border=\"0\" cellpadding=\"5\" cellspacing=\"5\">\n");
+ rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\n", "");
+ buffer_strcat(wb, "</table>\n</center>\n</html>\n");
+ }
+ break;
+
+ case DATASOURCE_DATATABLE_JSONP:
+ wb->content_type = CT_APPLICATION_X_JAVASCRIPT;
+
+ if(options & RRDR_OPTION_JSON_WRAP) {
+ wrapper_begin(r, wb);
+ buffer_json_member_add_key_only(wb, "result");
+ }
+
+ rrdr2json(r, wb, options, 1);
+
+ if(options & RRDR_OPTION_JSON_WRAP)
+ wrapper_end(r, wb);
+
+ break;
+
+ case DATASOURCE_DATATABLE_JSON:
+ wb->content_type = CT_APPLICATION_JSON;
+
+ if(options & RRDR_OPTION_JSON_WRAP) {
+ wrapper_begin(r, wb);
+ buffer_json_member_add_key_only(wb, "result");
+ }
+
+ rrdr2json(r, wb, options, 1);
+
+ if(options & RRDR_OPTION_JSON_WRAP)
+ wrapper_end(r, wb);
+
+ break;
+
+ case DATASOURCE_JSONP:
+ wb->content_type = CT_APPLICATION_X_JAVASCRIPT;
+ if(options & RRDR_OPTION_JSON_WRAP) {
+ wrapper_begin(r, wb);
+ buffer_json_member_add_key_only(wb, "result");
+ }
+
+ rrdr2json(r, wb, options, 0);
+
+ if(options & RRDR_OPTION_JSON_WRAP)
+ wrapper_end(r, wb);
+
+ break;
+
+ case DATASOURCE_JSON:
+ default:
+ wb->content_type = CT_APPLICATION_JSON;
+
+ if(options & RRDR_OPTION_JSON_WRAP) {
+ wrapper_begin(r, wb);
+ buffer_json_member_add_key_only(wb, "result");
+ }
+
+ rrdr2json(r, wb, options, 0);
+
+ if(options & RRDR_OPTION_JSON_WRAP) {
+ if (options & RRDR_OPTION_RETURN_JWAR) {
+ buffer_json_member_add_key_only(wb, "anomaly_rates");
+ rrdr2json(r, wb, options | RRDR_OPTION_INTERNAL_AR, false);
+ }
+ wrapper_end(r, wb);
+ }
+ break;
+
+ case DATASOURCE_JSON2:
+ wb->content_type = CT_APPLICATION_JSON;
+ wrapper_begin(r, wb);
+ rrdr2json_v2(r, wb);
+ wrapper_end(r, wb);
+ break;
+ }
+
+ rrdr_free(owa, r);
+ return HTTP_RESP_OK;
+}
diff --git a/src/web/api/formatters/rrd2json.h b/src/web/api/formatters/rrd2json.h
new file mode 100644
index 000000000..f0c0c39ba
--- /dev/null
+++ b/src/web/api/formatters/rrd2json.h
@@ -0,0 +1,98 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_RRD2JSON_H
+#define NETDATA_RRD2JSON_H 1
+
+// type of JSON generations
+typedef enum {
+ DATASOURCE_JSON = 0,
+ DATASOURCE_DATATABLE_JSON = 1,
+ DATASOURCE_DATATABLE_JSONP = 2,
+ DATASOURCE_SSV = 3,
+ DATASOURCE_CSV = 4,
+ DATASOURCE_JSONP = 5,
+ DATASOURCE_TSV = 6,
+ DATASOURCE_HTML = 7,
+ DATASOURCE_JS_ARRAY = 8,
+ DATASOURCE_SSV_COMMA = 9,
+ DATASOURCE_CSV_JSON_ARRAY = 10,
+ DATASOURCE_CSV_MARKDOWN = 11,
+ DATASOURCE_JSON2 = 12,
+} DATASOURCE_FORMAT;
+
+#include "web/api/web_api_v1.h"
+
+#include "web/api/exporters/allmetrics.h"
+#include "web/api/queries/rrdr.h"
+
+#include "web/api/formatters/csv/csv.h"
+#include "web/api/formatters/ssv/ssv.h"
+#include "web/api/formatters/json/json.h"
+#include "web/api/formatters/value/value.h"
+
+#include "web/api/formatters/rrdset2json.h"
+#include "web/api/formatters/charts2json.h"
+#include "web/api/formatters/json_wrapper.h"
+
+#include "web/server/web_client.h"
+
+#define HOSTNAME_MAX 1024
+
+#define DATASOURCE_FORMAT_JSON "json"
+#define DATASOURCE_FORMAT_JSON2 "json2"
+#define DATASOURCE_FORMAT_DATATABLE_JSON "datatable"
+#define DATASOURCE_FORMAT_DATATABLE_JSONP "datasource"
+#define DATASOURCE_FORMAT_JSONP "jsonp"
+#define DATASOURCE_FORMAT_SSV "ssv"
+#define DATASOURCE_FORMAT_CSV "csv"
+#define DATASOURCE_FORMAT_TSV "tsv"
+#define DATASOURCE_FORMAT_HTML "html"
+#define DATASOURCE_FORMAT_JS_ARRAY "array"
+#define DATASOURCE_FORMAT_SSV_COMMA "ssvcomma"
+#define DATASOURCE_FORMAT_CSV_JSON_ARRAY "csvjsonarray"
+#define DATASOURCE_FORMAT_CSV_MARKDOWN "markdown"
+
+void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb);
+const char *rrdr_format_to_string(DATASOURCE_FORMAT format);
+
+int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, struct query_target *qt, time_t *latest_timestamp);
+
+void rrdr_json_group_by_labels(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options);
+
+int rrdset2value_api_v1(
+ RRDSET *st
+ , BUFFER *wb
+ , NETDATA_DOUBLE *n
+ , const char *dimensions
+ , size_t points
+ , time_t after
+ , time_t before
+ , RRDR_TIME_GROUPING group_method
+ , const char *group_options
+ , time_t resampling_time
+ , uint32_t options
+ , time_t *db_after
+ , time_t *db_before
+ , size_t *db_points_read
+ , size_t *db_points_per_tier
+ , size_t *result_points_generated
+ , int *value_is_null
+ , NETDATA_DOUBLE *anomaly_rate
+ , time_t timeout
+ , size_t tier
+ , QUERY_SOURCE query_source
+ , STORAGE_PRIORITY priority
+);
+
+static inline bool rrdr_dimension_should_be_exposed(RRDR_DIMENSION_FLAGS rrdr_dim_flags, RRDR_OPTIONS options) {
+ if(unlikely((options & RRDR_OPTION_RETURN_RAW) && (rrdr_dim_flags & RRDR_DIMENSION_QUERIED)))
+ return true;
+
+ if(unlikely(rrdr_dim_flags & RRDR_DIMENSION_HIDDEN)) return false;
+ if(unlikely(!(rrdr_dim_flags & RRDR_DIMENSION_QUERIED))) return false;
+ if(unlikely((options & RRDR_OPTION_NONZERO) && !(rrdr_dim_flags & RRDR_DIMENSION_NONZERO))) return false;
+
+ return true;
+}
+
+#endif /* NETDATA_RRD2JSON_H */
diff --git a/src/web/api/formatters/rrdset2json.c b/src/web/api/formatters/rrdset2json.c
new file mode 100644
index 000000000..542178b25
--- /dev/null
+++ b/src/web/api/formatters/rrdset2json.c
@@ -0,0 +1,106 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "rrdset2json.h"
+
+static int process_label_callback(const char *name, const char *value, RRDLABEL_SRC ls __maybe_unused, void *data) {
+ BUFFER *wb = data;
+ buffer_json_member_add_string_or_empty(wb, name, value);
+ return 1;
+}
+
+void chart_labels2json(RRDSET *st, BUFFER *wb)
+{
+ if(unlikely(!st->rrdlabels))
+ return;
+
+ rrdlabels_walkthrough_read(st->rrdlabels, process_label_callback, wb);
+}
+
+// generate JSON for the /api/v1/chart API call
+void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memory_used)
+{
+ time_t first_entry_t = rrdset_first_entry_s(st);
+ time_t last_entry_t = rrdset_last_entry_s(st);
+ char buf[RRD_ID_LENGTH_MAX + 16];
+
+ buffer_json_member_add_string(wb, "id", rrdset_id(st));
+ buffer_json_member_add_string(wb, "name", rrdset_name(st));
+ buffer_json_member_add_string(wb, "type", rrdset_parts_type(st));
+ buffer_json_member_add_string(wb, "family", rrdset_family(st));
+ buffer_json_member_add_string(wb, "context", rrdset_context(st));
+ snprintfz(buf, RRD_ID_LENGTH_MAX + 15, "%s (%s)", rrdset_title(st), rrdset_name(st));
+ buffer_json_member_add_string(wb, "title", buf);
+ buffer_json_member_add_int64(wb, "priority", st->priority);
+ buffer_json_member_add_string(wb, "plugin", rrdset_plugin_name(st));
+ buffer_json_member_add_string(wb, "module", rrdset_module_name(st));
+ buffer_json_member_add_string(wb, "units", rrdset_units(st));
+
+ snprintfz(buf, RRD_ID_LENGTH_MAX + 15, "/api/v1/data?chart=%s", rrdset_name(st));
+ buffer_json_member_add_string(wb, "data_url", buf);
+
+ buffer_json_member_add_string(wb, "chart_type", rrdset_type_name(st->chart_type));
+ buffer_json_member_add_int64(wb, "duration", (int64_t)(last_entry_t - first_entry_t + st->update_every));
+ buffer_json_member_add_int64(wb, "first_entry", (int64_t)first_entry_t);
+ buffer_json_member_add_int64(wb, "last_entry", (int64_t)last_entry_t);
+ buffer_json_member_add_int64(wb, "update_every", (int64_t)st->update_every);
+
+ unsigned long memory = sizeof(RRDSET);
+
+ size_t dimensions = 0;
+ buffer_json_member_add_object(wb, "dimensions");
+ {
+ RRDDIM *rd;
+ rrddim_foreach_read(rd, st)
+ {
+ if (rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN) || rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE))
+ continue;
+
+ memory += rrddim_size() + rd->db.memsize;
+
+ buffer_json_member_add_object(wb, rrddim_id(rd));
+ buffer_json_member_add_string(wb, "name", rrddim_name(rd));
+ buffer_json_object_close(wb);
+
+ dimensions++;
+ }
+ rrddim_foreach_done(rd);
+ }
+ buffer_json_object_close(wb);
+
+ if(dimensions_count) *dimensions_count += dimensions;
+ if(memory_used) *memory_used += memory;
+
+ buffer_json_member_add_object(wb, "chart_variables");
+ health_api_v1_chart_custom_variables2json(st, wb);
+ buffer_json_object_close(wb);
+
+ buffer_json_member_add_double(wb, "green", st->green);
+ buffer_json_member_add_double(wb, "red", st->red);
+
+ {
+ buffer_json_member_add_object(wb, "alarms");
+ RRDCALC *rc;
+ rw_spinlock_read_lock(&st->alerts.spinlock);
+ DOUBLE_LINKED_LIST_FOREACH_FORWARD(st->alerts.base, rc, prev, next)
+ {
+ {
+ buffer_json_member_add_object(wb, rrdcalc_name(rc));
+ buffer_json_member_add_string_or_empty(wb, "id", rrdcalc_name(rc));
+ buffer_json_member_add_string_or_empty(wb, "status", rrdcalc_status2string(rc->status));
+ buffer_json_member_add_string_or_empty(wb, "units", rrdcalc_units(rc));
+ buffer_json_member_add_int64(wb, "duration", (int64_t)rc->config.update_every);
+ buffer_json_object_close(wb);
+ }
+ }
+ rw_spinlock_read_unlock(&st->alerts.spinlock);
+ buffer_json_object_close(wb);
+ }
+
+ buffer_json_member_add_object(wb, "chart_labels");
+ chart_labels2json(st, wb);
+ buffer_json_object_close(wb);
+
+ buffer_json_member_add_object(wb, "functions");
+ chart_functions2json(st, wb);
+ buffer_json_object_close(wb);
+}
diff --git a/src/web/api/formatters/rrdset2json.h b/src/web/api/formatters/rrdset2json.h
new file mode 100644
index 000000000..8b325c65d
--- /dev/null
+++ b/src/web/api/formatters/rrdset2json.h
@@ -0,0 +1,10 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_API_FORMATTER_RRDSET2JSON_H
+#define NETDATA_API_FORMATTER_RRDSET2JSON_H
+
+#include "rrd2json.h"
+
+void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memory_used);
+
+#endif //NETDATA_API_FORMATTER_RRDSET2JSON_H
diff --git a/src/web/api/formatters/ssv/README.md b/src/web/api/formatters/ssv/README.md
new file mode 100644
index 000000000..2e9dd3886
--- /dev/null
+++ b/src/web/api/formatters/ssv/README.md
@@ -0,0 +1,63 @@
+<!--
+title: "SSV formatter"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/src/web/api/formatters/ssv/README.md
+sidebar_label: "SSV formatter"
+learn_status: "Published"
+learn_topic_type: "References"
+learn_rel_path: "Developers/Web/Api/Formatters"
+-->
+
+# SSV formatter
+
+The SSV formatter sums all dimensions in [results of database queries](https://github.com/netdata/netdata/blob/master/src/web/api/queries/README.md)
+to a single value and returns a list of such values showing how it changes through time.
+
+It supports the following formats:
+
+| format | content type | description |
+|:----:|:----------:|:----------|
+| `ssv` | text/plain | a space separated list of values |
+| `ssvcomma` | text/plain | a comma separated list of values |
+| `array` | application/json | a JSON array |
+
+The SSV formatter respects the following API `&options=`:
+
+| option | supported | description |
+| :----:|:-------:|:----------|
+| `nonzero` | yes | to return only the dimensions that have at least a non-zero value |
+| `flip` | yes | to return the numbers older to newer (the default is newer to older) |
+| `percent` | yes | to replace all values with their percentage over the row total |
+| `abs` | yes | to turn all values positive, before using them |
+| `min2max` | yes | to return the delta from the minimum value to the maximum value (across dimensions) |
+
+## Examples
+
+Get the average system CPU utilization of the last hour, in 6 values (one every 10 minutes):
+
+```bash
+# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=system.cpu&format=ssv&after=-3600&points=6&group=average'
+1.741352 1.6800467 1.769411 1.6761112 1.629862 1.6807968
+```
+
+---
+
+Get the total mysql bandwidth (in + out) for the last hour, in 6 values (one every 10 minutes):
+
+Netdata returns bandwidth in `kilobits`.
+
+```bash
+# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=mysql_local.net&format=ssvcomma&after=-3600&points=6&group=sum&options=abs'
+72618.7936215,72618.778889,72618.788084,72618.9195918,72618.7760612,72618.6712421
+```
+
+---
+
+Get the web server max connections for the last hour, in 12 values (one every 5 minutes)
+in a JSON array:
+
+```bash
+# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=nginx_local.connections&format=array&after=-3600&points=12&group=max'
+[278,258,268,239,259,260,243,266,278,318,264,258]
+```
+
+
diff --git a/src/web/api/formatters/ssv/ssv.c b/src/web/api/formatters/ssv/ssv.c
new file mode 100644
index 000000000..2eb26b459
--- /dev/null
+++ b/src/web/api/formatters/ssv/ssv.c
@@ -0,0 +1,45 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "ssv.h"
+
+void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, const char *separator, const char *suffix) {
+ //netdata_log_info("RRD2SSV(): %s: BEGIN", r->st->id);
+ long i;
+
+ buffer_strcat(wb, prefix);
+ long start = 0, end = rrdr_rows(r), step = 1;
+ if(!(options & RRDR_OPTION_REVERSED)) {
+ start = rrdr_rows(r) - 1;
+ end = -1;
+ step = -1;
+ }
+
+ // for each line in the array
+ for(i = start; i != end ;i += step) {
+ int all_values_are_null = 0;
+ NETDATA_DOUBLE v = rrdr2value(r, i, options, &all_values_are_null, NULL);
+
+ if(likely(i != start)) {
+ if(r->view.min > v) r->view.min = v;
+ if(r->view.max < v) r->view.max = v;
+ }
+ else {
+ r->view.min = v;
+ r->view.max = v;
+ }
+
+ if(likely(i != start))
+ buffer_strcat(wb, separator);
+
+ if(all_values_are_null) {
+ if(options & RRDR_OPTION_NULL2ZERO)
+ buffer_strcat(wb, "0");
+ else
+ buffer_strcat(wb, "null");
+ }
+ else
+ buffer_print_netdata_double(wb, v);
+ }
+ buffer_strcat(wb, suffix);
+ //netdata_log_info("RRD2SSV(): %s: END", r->st->id);
+}
diff --git a/src/web/api/formatters/ssv/ssv.h b/src/web/api/formatters/ssv/ssv.h
new file mode 100644
index 000000000..f7d4a9548
--- /dev/null
+++ b/src/web/api/formatters/ssv/ssv.h
@@ -0,0 +1,10 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_API_FORMATTER_SSV_H
+#define NETDATA_API_FORMATTER_SSV_H
+
+#include "../rrd2json.h"
+
+void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, const char *separator, const char *suffix);
+
+#endif //NETDATA_API_FORMATTER_SSV_H
diff --git a/src/web/api/formatters/value/README.md b/src/web/api/formatters/value/README.md
new file mode 100644
index 000000000..3599a836e
--- /dev/null
+++ b/src/web/api/formatters/value/README.md
@@ -0,0 +1,28 @@
+<!--
+title: "Value formatter"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/src/web/api/formatters/value/README.md
+sidebar_label: "Value formatter"
+learn_status: "Published"
+learn_topic_type: "References"
+learn_rel_path: "Developers/Web/Api/Formatters"
+-->
+
+# Value formatter
+
+The Value formatter presents [results of database queries](https://github.com/netdata/netdata/blob/master/src/web/api/queries/README.md) as a single value.
+
+To calculate the single value to be returned, it sums the values of all dimensions.
+
+The Value formatter respects the following API `&options=`:
+
+| option | supported | description |
+|:----: |:-------: |:---------- |
+| `percent` | yes | to replace all values with their percentage over the row total|
+| `abs` | yes | to turn all values positive, before using them |
+| `min2max` | yes | to return the delta from the minimum value to the maximum value (across dimensions)|
+
+The Value formatter is not exposed by the API by itself.
+Instead it is used by the [`ssv`](https://github.com/netdata/netdata/blob/master/src/web/api/formatters/ssv/README.md) formatter
+and [health monitoring queries](https://github.com/netdata/netdata/blob/master/src/health/README.md).
+
+
diff --git a/src/web/api/formatters/value/value.c b/src/web/api/formatters/value/value.c
new file mode 100644
index 000000000..0ec1b1265
--- /dev/null
+++ b/src/web/api/formatters/value/value.c
@@ -0,0 +1,153 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "value.h"
+
+inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, NETDATA_DOUBLE *anomaly_rate) {
+ size_t c;
+
+ NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
+ RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ];
+ NETDATA_DOUBLE *ar = &r->ar[ i * r->d ];
+
+ NETDATA_DOUBLE sum = 0, min = NAN, max = NAN, v = NAN;
+ size_t dims = 0;
+
+ NETDATA_DOUBLE total_anomaly_rate = 0;
+
+ // for each dimension
+ for (c = 0; c < r->d ; c++) {
+ if(unlikely(!rrdr_dimension_should_be_exposed(r->od[c], options)))
+ continue;
+
+ if(unlikely((co[c] & RRDR_VALUE_EMPTY)))
+ continue;
+
+ NETDATA_DOUBLE n = cn[c];
+
+ if(unlikely(!dims))
+ min = max = n;
+
+ sum += n;
+
+ if (n < min) min = n;
+ if (n > max) max = n;
+
+ total_anomaly_rate += ar[c];
+
+ dims++;
+ }
+
+ if(!dims) {
+ if(anomaly_rate)
+ *anomaly_rate = 0;
+
+ if(all_values_are_null)
+ *all_values_are_null = 1;
+
+ return (options & RRDR_OPTION_NULL2ZERO) ? 0 : NAN;
+ }
+
+ if(anomaly_rate)
+ *anomaly_rate = total_anomaly_rate / (NETDATA_DOUBLE)dims;
+
+ if(all_values_are_null)
+ *all_values_are_null = 0;
+
+ if(options & RRDR_OPTION_DIMS_MIN2MAX)
+ v = max - min;
+ else if(options & RRDR_OPTION_DIMS_AVERAGE)
+ v = sum / (NETDATA_DOUBLE)dims;
+ else if(options & RRDR_OPTION_DIMS_MIN)
+ v = min;
+ else if(options & RRDR_OPTION_DIMS_MAX)
+ v = max;
+ else
+ v = sum;
+
+ if((options & RRDR_OPTION_NULL2ZERO) && (isnan(v) || isinf(v)))
+ v = 0;
+
+ return v;
+}
+
+QUERY_VALUE rrdmetric2value(RRDHOST *host,
+ struct rrdcontext_acquired *rca, struct rrdinstance_acquired *ria, struct rrdmetric_acquired *rma,
+ time_t after, time_t before,
+ RRDR_OPTIONS options, RRDR_TIME_GROUPING time_group_method, const char *time_group_options,
+ size_t tier, time_t timeout, QUERY_SOURCE query_source, STORAGE_PRIORITY priority
+) {
+ QUERY_TARGET_REQUEST qtr = {
+ .version = 1,
+ .host = host,
+ .rca = rca,
+ .ria = ria,
+ .rma = rma,
+ .after = after,
+ .before = before,
+ .points = 1,
+ .options = options,
+ .time_group_method = time_group_method,
+ .time_group_options = time_group_options,
+ .tier = tier,
+ .timeout_ms = timeout,
+ .query_source = query_source,
+ .priority = priority,
+ };
+
+ ONEWAYALLOC *owa = onewayalloc_create(16 * 1024);
+ QUERY_TARGET *qt = query_target_create(&qtr);
+ RRDR *r = rrd2rrdr(owa, qt);
+
+ QUERY_VALUE qv;
+
+ if(!r || rrdr_rows(r) == 0) {
+ qv = (QUERY_VALUE) {
+ .value = NAN,
+ .anomaly_rate = NAN,
+ .sp = {
+ .count = 0,
+ .min = NAN,
+ .max = NAN,
+ .sum = NAN,
+ .anomaly_count = 0,
+ },
+ .duration_ut = (r) ? r->internal.qt->timings.executed_ut - r->internal.qt->timings.received_ut : 0,
+ };
+ }
+ else {
+ qv = (QUERY_VALUE) {
+ .after = r->view.after,
+ .before = r->view.before,
+ .points_read = r->stats.db_points_read,
+ .result_points = r->stats.result_points_generated,
+ .sp = {
+ .count = 0,
+ },
+ .duration_ut = r->internal.qt->timings.executed_ut - r->internal.qt->timings.received_ut,
+ };
+
+ for(size_t d = 0; d < r->internal.qt->query.used ;d++) {
+ if(!rrdr_dimension_should_be_exposed(r->internal.qt->query.array[d].status, options))
+ continue;
+
+ storage_point_merge_to(qv.sp, r->internal.qt->query.array[d].query_points);
+ }
+
+ for(size_t t = 0; t < storage_tiers ;t++)
+ qv.storage_points_per_tier[t] = r->internal.qt->db.tiers[t].points;
+
+ long i = (!(options & RRDR_OPTION_REVERSED))?(long)rrdr_rows(r) - 1:0;
+ int all_values_are_null = 0;
+ qv.value = rrdr2value(r, i, options, &all_values_are_null, &qv.anomaly_rate);
+ if(all_values_are_null) {
+ qv.value = NAN;
+ qv.anomaly_rate = NAN;
+ }
+ }
+
+ rrdr_free(owa, r);
+ query_target_release(qt);
+ onewayalloc_destroy(owa);
+
+ return qv;
+}
diff --git a/src/web/api/formatters/value/value.h b/src/web/api/formatters/value/value.h
new file mode 100644
index 000000000..072ca14f8
--- /dev/null
+++ b/src/web/api/formatters/value/value.h
@@ -0,0 +1,33 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_API_FORMATTER_VALUE_H
+#define NETDATA_API_FORMATTER_VALUE_H
+
+#include "../rrd2json.h"
+
+typedef struct storage_value {
+ NETDATA_DOUBLE value;
+ NETDATA_DOUBLE anomaly_rate;
+ time_t after;
+ time_t before;
+ size_t points_read;
+ size_t storage_points_per_tier[RRD_STORAGE_TIERS];
+ size_t result_points;
+ STORAGE_POINT sp;
+ usec_t duration_ut;
+} QUERY_VALUE;
+
+struct rrdmetric_acquired;
+struct rrdinstance_acquired;
+struct rrdcontext_acquired;
+
+QUERY_VALUE rrdmetric2value(RRDHOST *host,
+ struct rrdcontext_acquired *rca, struct rrdinstance_acquired *ria, struct rrdmetric_acquired *rma,
+ time_t after, time_t before,
+ RRDR_OPTIONS options, RRDR_TIME_GROUPING time_group_method, const char *time_group_options,
+ size_t tier, time_t timeout, QUERY_SOURCE query_source, STORAGE_PRIORITY priority
+);
+
+NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, NETDATA_DOUBLE *anomaly_rate);
+
+#endif //NETDATA_API_FORMATTER_VALUE_H