diff options
Diffstat (limited to 'web/api/formatters')
26 files changed, 2254 insertions, 0 deletions
diff --git a/web/api/formatters/Makefile.am b/web/api/formatters/Makefile.am new file mode 100644 index 0000000..11f239c --- /dev/null +++ b/web/api/formatters/Makefile.am @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +SUBDIRS = \ + csv \ + json \ + ssv \ + value \ + $(NULL) + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/web/api/formatters/README.md b/web/api/formatters/README.md new file mode 100644 index 0000000..1fd2b30 --- /dev/null +++ b/web/api/formatters/README.md @@ -0,0 +1,78 @@ +<!-- +title: "Query formatting" +custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/formatters/README.md +--> + +# Query formatting + +API data queries need to be formatted before returned to the caller. +Using API parameters, the caller may define the format he/she wishes to get back. + +The following formats are supported: + +| format|module|content type|description| +|:----:|:----:|:----------:|:----------| +| `array`|[ssv](/web/api/formatters/ssv/README.md)|application/json|a JSON array| +| `csv`|[csv](/web/api/formatters/csv/README.md)|text/plain|a text table, comma separated, with a header line (dimension names) and `\r\n` at the end of the lines| +| `csvjsonarray`|[csv](/web/api/formatters/csv/README.md)|application/json|a JSON array, with each row as another array (the first row has the dimension names)| +| `datasource`|[json](/web/api/formatters/json/README.md)|application/json|a Google Visualization Provider `datasource` javascript callback| +| `datatable`|[json](/web/api/formatters/json/README.md)|application/json|a Google `datatable`| +| `html`|[csv](/web/api/formatters/csv/README.md)|text/html|an html table| +| `json`|[json](/web/api/formatters/json/README.md)|application/json|a JSON object| +| `jsonp`|[json](/web/api/formatters/json/README.md)|application/json|a JSONP javascript callback| +| `markdown`|[csv](/web/api/formatters/csv/README.md)|text/plain|a markdown table| +| `ssv`|[ssv](/web/api/formatters/ssv/README.md)|text/plain|a space separated list of values| +| `ssvcomma`|[ssv](/web/api/formatters/ssv/README.md)|text/plain|a comma separated list of values| +| `tsv`|[csv](/web/api/formatters/csv/README.md)|text/plain|a TAB delimited `csv` (MS Excel flavor)| + +For examples of each format, check the relative module documentation. + +## Metadata with the `jsonwrap` option + +All data queries can be encapsulated to JSON object having metadata about the query and the results. + +This is done by adding the `options=jsonwrap` to the API URL (if there are other `options` append +`,jsonwrap` to the existing ones). + +This is such an object: + +```bash +# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=system.cpu&after=-3600&points=6&group=average&format=csv&options=nonzero,jsonwrap' +{ + "api": 1, + "id": "system.cpu", + "name": "system.cpu", + "view_update_every": 600, + "update_every": 1, + "first_entry": 1540387074, + "last_entry": 1540647070, + "before": 1540647000, + "after": 1540644000, + "dimension_names": ["steal", "softirq", "user", "system", "iowait"], + "dimension_ids": ["steal", "softirq", "user", "system", "iowait"], + "latest_values": [0, 0.2493766, 1.745636, 0.4987531, 0], + "view_latest_values": [0.0158314, 0.0516506, 0.866549, 0.7196127, 0.0050002], + "dimensions": 5, + "points": 6, + "format": "csv", + "result": "time,steal,softirq,user,system,iowait\n2018-10-27 13:30:00,0.0158314,0.0516506,0.866549,0.7196127,0.0050002\n2018-10-27 13:20:00,0.0149856,0.0529183,0.8673155,0.7121144,0.0049979\n2018-10-27 13:10:00,0.0137501,0.053315,0.8578097,0.7197613,0.0054209\n2018-10-27 13:00:00,0.0154252,0.0554688,0.899432,0.7200638,0.0067252\n2018-10-27 12:50:00,0.0145866,0.0495922,0.8404341,0.7011141,0.0041688\n2018-10-27 12:40:00,0.0162366,0.0595954,0.8827475,0.7020573,0.0041636\n", + "min": 0, + "max": 0 +} +``` + +## Downloading data query result files + +Following the [Google Visualization Provider guidelines](https://developers.google.com/chart/interactive/docs/dev/implementing_data_source), +Netdata supports parsing `tqx` options. + +Using these options, any Netdata data query can instruct the web browser to download +the result and save it under a given filename. + +For example, to download a CSV file with CPU utilization of the last hour, +[click here](https://registry.my-netdata.io/api/v1/data?chart=system.cpu&after=-3600&format=csv&options=nonzero&tqx=outFileName:system+cpu+utilization+of+the+last_hour.csv). + +This is done by appending `&tqx=outFileName:FILENAME` to any data query. +The output will be in the format given with `&format=`. + +[](<>) diff --git a/web/api/formatters/charts2json.c b/web/api/formatters/charts2json.c new file mode 100644 index 0000000..856ffb5 --- /dev/null +++ b/web/api/formatters/charts2json.c @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "charts2json.h" + +// generate JSON for the /api/v1/charts API call + +const char* get_release_channel() { + static int use_stable = -1; + + if (use_stable == -1) { + char filename[FILENAME_MAX + 1]; + snprintfz(filename, FILENAME_MAX, "%s/.environment", netdata_configured_user_config_dir); + procfile *ff = procfile_open(filename, "=", PROCFILE_FLAG_DEFAULT); + if(!ff) { + use_stable=1; + } else { + procfile_set_quotes(ff, "'\""); + ff = procfile_readall(ff); + if(!ff) { + use_stable=1; + } else { + unsigned int i; + for(i = 0; i < procfile_lines(ff); i++) { + if (!procfile_linewords(ff, i)) continue; + + if (!strcmp(procfile_lineword(ff, i, 0), "RELEASE_CHANNEL") && !strcmp(procfile_lineword(ff, i, 1), "stable")) { + use_stable = 1; + break; + } + } + procfile_close(ff); + if (use_stable == -1) use_stable = 0; + } + } + } + return (use_stable)?"stable":"nightly"; +} + +void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived) { + static char *custom_dashboard_info_js_filename = NULL; + size_t c, dimensions = 0, memory = 0, alarms = 0; + RRDSET *st; + + time_t now = now_realtime_sec(); + + if(unlikely(!custom_dashboard_info_js_filename)) + custom_dashboard_info_js_filename = config_get(CONFIG_SECTION_WEB, "custom dashboard_info.js", ""); + + buffer_sprintf(wb, "{\n" + "\t\"hostname\": \"%s\"" + ",\n\t\"version\": \"%s\"" + ",\n\t\"release_channel\": \"%s\"" + ",\n\t\"os\": \"%s\"" + ",\n\t\"timezone\": \"%s\"" + ",\n\t\"update_every\": %d" + ",\n\t\"history\": %ld" + ",\n\t\"memory_mode\": \"%s\"" + ",\n\t\"custom_info\": \"%s\"" + ",\n\t\"charts\": {" + , host->hostname + , host->program_version + , get_release_channel() + , host->os + , host->timezone + , host->rrd_update_every + , host->rrd_history_entries + , rrd_memory_mode_name(host->rrd_memory_mode) + , custom_dashboard_info_js_filename + ); + + c = 0; + rrdhost_rdlock(host); + rrdset_foreach_read(st, host) { + if ((!show_archived && rrdset_is_available_for_viewers(st)) || (show_archived && rrdset_is_archived(st))) { + if(c) buffer_strcat(wb, ","); + buffer_strcat(wb, "\n\t\t\""); + buffer_strcat(wb, st->id); + buffer_strcat(wb, "\": "); + rrdset2json(st, wb, &dimensions, &memory, skip_volatile); + + c++; + st->last_accessed_time = now; + } + } + + RRDCALC *rc; + for(rc = host->alarms; rc ; rc = rc->next) { + if(rc->rrdset) + alarms++; + } + rrdhost_unlock(host); + + buffer_sprintf(wb + , "\n\t}" + ",\n\t\"charts_count\": %zu" + ",\n\t\"dimensions_count\": %zu" + ",\n\t\"alarms_count\": %zu" + ",\n\t\"rrd_memory_bytes\": %zu" + ",\n\t\"hosts_count\": %zu" + ",\n\t\"hosts\": [" + , c + , dimensions + , alarms + , memory + , rrd_hosts_available + ); + + if(unlikely(rrd_hosts_available > 1)) { + rrd_rdlock(); + + size_t found = 0; + RRDHOST *h; + rrdhost_foreach_read(h) { + if(!rrdhost_should_be_removed(h, host, now) && !rrdhost_flag_check(h, RRDHOST_FLAG_ARCHIVED)) { + buffer_sprintf(wb + , "%s\n\t\t{" + "\n\t\t\t\"hostname\": \"%s\"" + "\n\t\t}" + , (found > 0) ? "," : "" + , h->hostname + ); + + found++; + } + } + + rrd_unlock(); + } + else { + buffer_sprintf(wb + , "\n\t\t{" + "\n\t\t\t\"hostname\": \"%s\"" + "\n\t\t}" + , host->hostname + ); + } + + buffer_sprintf(wb, "\n\t]\n}\n"); +} + +// generate collectors list for the api/v1/info call + +struct collector { + char *plugin; + char *module; +}; + +struct array_printer { + int c; + BUFFER *wb; +}; + +int print_collector(void *entry, void *data) { + struct array_printer *ap = (struct array_printer *)data; + BUFFER *wb = ap->wb; + struct collector *col=(struct collector *) entry; + if(ap->c) buffer_strcat(wb, ","); + buffer_strcat(wb, "\n\t\t{\n\t\t\t\"plugin\": \""); + buffer_strcat(wb, col->plugin); + buffer_strcat(wb, "\",\n\t\t\t\"module\": \""); + buffer_strcat(wb, col->module); + buffer_strcat(wb, "\"\n\t\t}"); + (ap->c)++; + return 0; +} + +void chartcollectors2json(RRDHOST *host, BUFFER *wb) { + DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + RRDSET *st; + char name[500]; + + time_t now = now_realtime_sec(); + rrdhost_rdlock(host); + rrdset_foreach_read(st, host) { + if (rrdset_is_available_for_viewers(st)) { + struct collector col = { + .plugin = st->plugin_name ? st->plugin_name : "", + .module = st->module_name ? st->module_name : "" + }; + sprintf(name, "%s:%s", col.plugin, col.module); + dictionary_set(dict, name, &col, sizeof(struct collector)); + st->last_accessed_time = now; + } + } + rrdhost_unlock(host); + struct array_printer ap = { + .c = 0, + .wb = wb + }; + dictionary_get_all(dict, print_collector, &ap); + dictionary_destroy(dict); +} diff --git a/web/api/formatters/charts2json.h b/web/api/formatters/charts2json.h new file mode 100644 index 0000000..2d8cce3 --- /dev/null +++ b/web/api/formatters/charts2json.h @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_API_FORMATTER_CHARTS2JSON_H +#define NETDATA_API_FORMATTER_CHARTS2JSON_H + +#include "rrd2json.h" + +extern void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived); +extern void chartcollectors2json(RRDHOST *host, BUFFER *wb); +extern const char* get_release_channel(); + +#endif //NETDATA_API_FORMATTER_CHARTS2JSON_H diff --git a/web/api/formatters/csv/Makefile.am b/web/api/formatters/csv/Makefile.am new file mode 100644 index 0000000..161784b --- /dev/null +++ b/web/api/formatters/csv/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/web/api/formatters/csv/README.md b/web/api/formatters/csv/README.md new file mode 100644 index 0000000..2a859e2 --- /dev/null +++ b/web/api/formatters/csv/README.md @@ -0,0 +1,144 @@ +<!-- +title: "CSV formatter" +custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/formatters/csv/README.md +--> + +# CSV formatter + +The CSV formatter presents [results of database queries](/web/api/queries/README.md) in the following formats: + +| format|content type|description| +| :----:|:----------:|:----------| +| `csv`|text/plain|a text table, comma separated, with a header line (dimension names) and `\r\n` at the end of the lines| +| `csvjsonarray`|application/json|a JSON array, with each row as another array (the first row has the dimension names)| +| `tsv`|text/plain|like `csv` but TAB is used instead of comma to separate values (MS Excel flavor)| +| `html`|text/html|an html table| +| `markdown`|text/plain|markdown table| + +In all formats the date and time is the first column. + +The CSV formatter respects the following API `&options=`: + +| option|supported|description| +|:----:|:-------:|:----------| +| `nonzero`|yes|to return only the dimensions that have at least a non-zero value| +| `flip`|yes|to return the rows older to newer (the default is newer to older)| +| `seconds`|yes|to return the date and time in unix timestamp| +| `ms`|yes|to return the date and time in unit timestamp as milliseconds| +| `percent`|yes|to replace all values with their percentage over the row total| +| `abs`|yes|to turn all values positive| +| `null2zero`|yes|to replace gaps with zeros (the default prints the string `null`| + +## Examples + +Get the system total bandwidth for all physical network interfaces, over the last hour, +in 6 rows (one for every 10 minutes), in `csv` format: + +Netdata always returns bandwidth in `kilobits`. + +```bash +# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=system.net&format=csv&after=-3600&group=sum&points=6&options=abs' +time,received,sent +2018-10-26 23:50:00,90214.67847,215137.79762 +2018-10-26 23:40:00,90126.32286,238587.57522 +2018-10-26 23:30:00,86061.22688,213389.23526 +2018-10-26 23:20:00,85590.75164,206129.01608 +2018-10-26 23:10:00,83163.30691,194311.77384 +2018-10-26 23:00:00,85167.29657,197538.07773 +``` + +--- + +Get the max RAM used by the SQL server and any cron jobs, over the last hour, in 2 rows (one for every 30 +minutes), in `tsv` format, and format the date and time as unix timestamp: + +Netdata always returns memory in `MB`. + +```bash +# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=apps.mem&format=tsv&after=-3600&group=max&points=2&options=nonzero,seconds&dimensions=sql,cron' +time sql cron +1540598400 61.95703 0.25 +1540596600 61.95703 0.25 +``` + +--- + +Get an HTML table of the last 4 values (4 seconds) of system CPU utilization: + +Netdata always returns CPU utilization as `%`. + +```bash +# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=system.cpu&format=html&after=-4&options=nonzero' +<html> +<center> +<table border="0" cellpadding="5" cellspacing="5"> +<tr><td>time</td><td>softirq</td><td>user</td><td>system</td></tr> +<tr><td>2018-10-27 00:16:07</td><td>0.25</td><td>1</td><td>0.75</td></tr> +<tr><td>2018-10-27 00:16:06</td><td>0</td><td>1.0025063</td><td>0.5012531</td></tr> +<tr><td>2018-10-27 00:16:05</td><td>0</td><td>1</td><td>0.75</td></tr> +<tr><td>2018-10-27 00:16:04</td><td>0</td><td>1.0025063</td><td>0.7518797</td></tr> +</table> +</center> +</html> +``` + +This is how it looks when rendered by a web browser: + + + +--- + +Get a JSON array with the average bandwidth rate of the mysql server, over the last hour, in 6 values +(one every 10 minutes), and return the date and time in milliseconds: + +Netdata always returns bandwidth rates in `kilobits/s`. + +```bash +# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=mysql_local.net&format=csvjsonarray&after=-3600&points=6&group=average&options=abs,ms' +[ +["time","in","out"], +[1540599600000,0.7499986,120.2810185], +[1540599000000,0.7500019,120.2815509], +[1540598400000,0.7499999,120.2812319], +[1540597800000,0.7500044,120.2819634], +[1540597200000,0.7499968,120.2807337], +[1540596600000,0.7499988,120.2810527] +] +``` + +--- + +Get the number of processes started per minute, for the last 10 minutes, in `markdown` format: + +```bash +# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=system.forks&format=markdown&after=-600&points=10&group=sum' +time | started +:---: |:---: +2018-10-27 03:52:00| 245.1706149 +2018-10-27 03:51:00| 152.6654636 +2018-10-27 03:50:00| 163.1755789 +2018-10-27 03:49:00| 176.1574766 +2018-10-27 03:48:00| 178.0137076 +2018-10-27 03:47:00| 183.8306543 +2018-10-27 03:46:00| 264.1635621 +2018-10-27 03:45:00| 205.001551 +2018-10-27 03:44:00| 7026.9852167 +2018-10-27 03:43:00| 205.9904794 +``` + +And this is how it looks when formatted: + +| time | started | +|:--:|:-----:| +| 2018-10-27 03:52:00 | 245.1706149 | +| 2018-10-27 03:51:00 | 152.6654636 | +| 2018-10-27 03:50:00 | 163.1755789 | +| 2018-10-27 03:49:00 | 176.1574766 | +| 2018-10-27 03:48:00 | 178.0137076 | +| 2018-10-27 03:47:00 | 183.8306543 | +| 2018-10-27 03:46:00 | 264.1635621 | +| 2018-10-27 03:45:00 | 205.001551 | +| 2018-10-27 03:44:00 | 7026.9852167 | +| 2018-10-27 03:43:00 | 205.9904794 | + +[](<>) diff --git a/web/api/formatters/csv/csv.c b/web/api/formatters/csv/csv.c new file mode 100644 index 0000000..da0a6b5 --- /dev/null +++ b/web/api/formatters/csv/csv.c @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "libnetdata/libnetdata.h" +#include "csv.h" + +void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines, RRDDIM *temp_rd) { + rrdset_check_rdlock(r->st); + + //info("RRD2CSV(): %s: BEGIN", r->st->id); + long c, i; + RRDDIM *d; + + // print the csv header + for(c = 0, i = 0, d = temp_rd?temp_rd:r->st->dimensions; d && c < r->d ;c++, d = d->next) { + if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; + + if(!i) { + buffer_strcat(wb, startline); + if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); + buffer_strcat(wb, "time"); + if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); + } + buffer_strcat(wb, separator); + if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); + buffer_strcat(wb, d->name); + if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); + i++; + } + buffer_strcat(wb, endline); + + if(format == DATASOURCE_CSV_MARKDOWN) { + // print the --- line after header + for(c = 0, i = 0, d = temp_rd?temp_rd:r->st->dimensions; d && c < r->d ;c++, d = d->next) { + if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; + + if(!i) { + buffer_strcat(wb, startline); + if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); + buffer_strcat(wb, ":---:"); + if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); + } + buffer_strcat(wb, separator); + if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); + buffer_strcat(wb, ":---:"); + if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); + i++; + } + buffer_strcat(wb, endline); + } + + if(!i) { + // no dimensions present + return; + } + + long start = 0, end = rrdr_rows(r), step = 1; + if(!(options & RRDR_OPTION_REVERSED)) { + start = rrdr_rows(r) - 1; + end = -1; + step = -1; + } + + // for each line in the array + calculated_number total = 1; + for(i = start; i != end ;i += step) { + calculated_number *cn = &r->v[ i * r->d ]; + RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ]; + + buffer_strcat(wb, betweenlines); + buffer_strcat(wb, startline); + + time_t now = r->t[i]; + + if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) { + // print the timestamp of the line + buffer_rrd_value(wb, (calculated_number)now); + // in ms + if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000"); + } + else { + // generate the local date time + struct tm tmbuf, *tm = localtime_r(&now, &tmbuf); + if(!tm) { error("localtime() failed."); continue; } + buffer_date(wb, tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec); + } + + int set_min_max = 0; + if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { + total = 0; + for(c = 0, d = temp_rd?temp_rd:r->st->dimensions; d && c < r->d ;c++, d = d->next) { + calculated_number n = cn[c]; + + if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) + n = -n; + + total += n; + } + // prevent a division by zero + if(total == 0) total = 1; + set_min_max = 1; + } + + // for each dimension + for(c = 0, d = temp_rd?temp_rd:r->st->dimensions; d && c < r->d ;c++, d = d->next) { + if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; + + buffer_strcat(wb, separator); + + calculated_number n = cn[c]; + + if(co[c] & RRDR_VALUE_EMPTY) { + if(options & RRDR_OPTION_NULL2ZERO) + buffer_strcat(wb, "0"); + else + buffer_strcat(wb, "null"); + } + else { + if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) + n = -n; + + if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { + n = n * 100 / total; + + if(unlikely(set_min_max)) { + r->min = r->max = n; + set_min_max = 0; + } + + if(n < r->min) r->min = n; + if(n > r->max) r->max = n; + } + + buffer_rrd_value(wb, n); + } + } + + buffer_strcat(wb, endline); + } + //info("RRD2CSV(): %s: END", r->st->id); +} diff --git a/web/api/formatters/csv/csv.h b/web/api/formatters/csv/csv.h new file mode 100644 index 0000000..cf6020d --- /dev/null +++ b/web/api/formatters/csv/csv.h @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_API_FORMATTER_CSV_H +#define NETDATA_API_FORMATTER_CSV_H + +#include "web/api/queries/rrdr.h" + +extern void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines, RRDDIM *temp_rd); + +#include "../rrd2json.h" + +#endif //NETDATA_API_FORMATTER_CSV_H diff --git a/web/api/formatters/json/Makefile.am b/web/api/formatters/json/Makefile.am new file mode 100644 index 0000000..161784b --- /dev/null +++ b/web/api/formatters/json/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/web/api/formatters/json/README.md b/web/api/formatters/json/README.md new file mode 100644 index 0000000..685a3f2 --- /dev/null +++ b/web/api/formatters/json/README.md @@ -0,0 +1,156 @@ +<!-- +title: "JSON formatter" +custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/formatters/json/README.md +--> + +# JSON formatter + +The CSV formatter presents [results of database queries](/web/api/queries/README.md) in the following formats: + +| format | content type | description| +|:----:|:----------:|:----------| +| `json` | application/json | return the query result as a json object| +| `jsonp` | application/json | return the query result as a JSONP javascript callback| +| `datatable` | application/json | return the query result as a Google `datatable`| +| `datasource` | application/json | return the query result as a Google Visualization Provider `datasource` javascript callback| + +The CSV formatter respects the following API `&options=`: + +| option | supported | description| +|:----:|:-------:|:----------| +| `google_json` | yes | enable the Google flavor of JSON (using double quotes for strings and `Date()` function for dates| +| `objectrows` | yes | return each row as an object, instead of an array| +| `nonzero` | yes | to return only the dimensions that have at least a non-zero value| +| `flip` | yes | to return the rows older to newer (the default is newer to older)| +| `seconds` | yes | to return the date and time in unix timestamp| +| `ms` | yes | to return the date and time in unit timestamp as milliseconds| +| `percent` | yes | to replace all values with their percentage over the row total| +| `abs` | yes | to turn all values positive| +| `null2zero` | yes | to replace gaps with zeros (the default prints the string `null`| + +## Examples + +To show the differences between each format, in the following examples we query the same +chart (having just one dimension called `active`), changing only the query `format` and its `options`. + +> Using `format=json` and `options=` + +```bash +# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=nginx_local.connections&after=-3600&points=6&group=average&format=json&options=' +{ + "labels": ["time", "active"], + "data": + [ + [ 1540644600, 224.2516667], + [ 1540644000, 229.29], + [ 1540643400, 222.41], + [ 1540642800, 226.6816667], + [ 1540642200, 246.4083333], + [ 1540641600, 241.0966667] + ] +} +``` + +> Using `format=json` and `options=objectrows` + +```bash +# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=nginx_local.connections&after=-3600&points=6&group=average&format=json&options=objectrows' +{ + "labels": ["time", "active"], + "data": + [ + { "time": 1540644600, "active": 224.2516667}, + { "time": 1540644000, "active": 229.29}, + { "time": 1540643400, "active": 222.41}, + { "time": 1540642800, "active": 226.6816667}, + { "time": 1540642200, "active": 246.4083333}, + { "time": 1540641600, "active": 241.0966667} + ] +} +``` + +> Using `format=json` and `options=objectrows,google_json` + +```bash +# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=nginx_local.connections&after=-3600&points=6&group=average&formatjson&options=objectrows,google_json' +{ + "labels": ["time", "active"], + "data": + [ + { "time": new Date(2018,9,27,12,50,0), "active": 224.2516667}, + { "time": new Date(2018,9,27,12,40,0), "active": 229.29}, + { "time": new Date(2018,9,27,12,30,0), "active": 222.41}, + { "time": new Date(2018,9,27,12,20,0), "active": 226.6816667}, + { "time": new Date(2018,9,27,12,10,0), "active": 246.4083333}, + { "time": new Date(2018,9,27,12,0,0), "active": 241.0966667} + ] +} +``` + +> Using `format=jsonp` and `options=` + +```bash +curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=nginx_local.connections&after=-3600&points=6&group=average&formjsonp&options=' +callback({ + "labels": ["time", "active"], + "data": + [ + [ 1540645200, 235.885], + [ 1540644600, 224.2516667], + [ 1540644000, 229.29], + [ 1540643400, 222.41], + [ 1540642800, 226.6816667], + [ 1540642200, 246.4083333] + ] +}); +``` + +> Using `format=datatable` and `options=` + +```bash +curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=nginx_local.connections&after=-3600&points=6&group=average&formdatatable&options=' +{ + "cols": + [ + {"id":"","label":"time","pattern":"","type":"datetime"}, + {"id":"","label":"","pattern":"","type":"string","p":{"role":"annotation"}}, + {"id":"","label":"","pattern":"","type":"string","p":{"role":"annotationText"}}, + {"id":"","label":"active","pattern":"","type":"number"} + ], + "rows": + [ + {"c":[{"v":"Date(2018,9,27,13,0,0)"},{"v":null},{"v":null},{"v":235.885}]}, + {"c":[{"v":"Date(2018,9,27,12,50,0)"},{"v":null},{"v":null},{"v":224.2516667}]}, + {"c":[{"v":"Date(2018,9,27,12,40,0)"},{"v":null},{"v":null},{"v":229.29}]}, + {"c":[{"v":"Date(2018,9,27,12,30,0)"},{"v":null},{"v":null},{"v":222.41}]}, + {"c":[{"v":"Date(2018,9,27,12,20,0)"},{"v":null},{"v":null},{"v":226.6816667}]}, + {"c":[{"v":"Date(2018,9,27,12,10,0)"},{"v":null},{"v":null},{"v":246.4083333}]} + ] +} +``` + +> Using `format=datasource` and `options=` + +```bash +curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=nginx_local.connections&after=-3600&points=6&group=average&format=datasource&options=' +google.visualization.Query.setResponse({version:'0.6',reqId:'0',status:'ok',sig:'1540645368',table:{ + "cols": + [ + {"id":"","label":"time","pattern":"","type":"datetime"}, + {"id":"","label":"","pattern":"","type":"string","p":{"role":"annotation"}}, + {"id":"","label":"","pattern":"","type":"string","p":{"role":"annotationText"}}, + {"id":"","label":"active","pattern":"","type":"number"} + ], + "rows": + [ + {"c":[{"v":"Date(2018,9,27,13,0,0)"},{"v":null},{"v":null},{"v":235.885}]}, + {"c":[{"v":"Date(2018,9,27,12,50,0)"},{"v":null},{"v":null},{"v":224.2516667}]}, + {"c":[{"v":"Date(2018,9,27,12,40,0)"},{"v":null},{"v":null},{"v":229.29}]}, + {"c":[{"v":"Date(2018,9,27,12,30,0)"},{"v":null},{"v":null},{"v":222.41}]}, + {"c":[{"v":"Date(2018,9,27,12,20,0)"},{"v":null},{"v":null},{"v":226.6816667}]}, + {"c":[{"v":"Date(2018,9,27,12,10,0)"},{"v":null},{"v":null},{"v":246.4083333}]} + ] +}}); +``` + +[](<>) diff --git a/web/api/formatters/json/json.c b/web/api/formatters/json/json.c new file mode 100644 index 0000000..f28eb57 --- /dev/null +++ b/web/api/formatters/json/json.c @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "json.h" + +#define JSON_DATES_JS 1 +#define JSON_DATES_TIMESTAMP 2 + +void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, RRDDIM *temp_rd) { + rrdset_check_rdlock(r->st); + + //info("RRD2JSON(): %s: BEGIN", r->st->id); + int row_annotations = 0, dates, dates_with_new = 0; + char kq[2] = "", // key quote + sq[2] = "", // string quote + pre_label[101] = "", // before each label + post_label[101] = "", // after each label + pre_date[101] = "", // the beginning of line, to the date + post_date[101] = "", // closing the date + pre_value[101] = "", // before each value + post_value[101] = "", // after each value + post_line[101] = "", // at the end of each row + normal_annotation[201] = "", // default row annotation + overflow_annotation[201] = "", // overflow row annotation + data_begin[101] = "", // between labels and values + finish[101] = ""; // at the end of everything + + if(datatable) { + dates = JSON_DATES_JS; + if( options & RRDR_OPTION_GOOGLE_JSON ) { + kq[0] = '\0'; + sq[0] = '\''; + } + else { + kq[0] = '"'; + sq[0] = '"'; + } + row_annotations = 1; + snprintfz(pre_date, 100, " {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq); + snprintfz(post_date, 100, "%s}", sq); + snprintfz(pre_label, 100, ",\n {%sid%s:%s%s,%slabel%s:%s", kq, kq, sq, sq, kq, kq, sq); + snprintfz(post_label, 100, "%s,%spattern%s:%s%s,%stype%s:%snumber%s}", sq, kq, kq, sq, sq, kq, kq, sq, sq); + snprintfz(pre_value, 100, ",{%sv%s:", kq, kq); + strcpy(post_value, "}"); + strcpy(post_line, "]}"); + snprintfz(data_begin, 100, "\n ],\n %srows%s:\n [\n", kq, kq); + strcpy(finish, "\n ]\n}"); + + snprintfz(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq); + snprintfz(normal_annotation, 200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq); + + buffer_sprintf(wb, "{\n %scols%s:\n [\n", kq, kq); + buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq); + buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq); + buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq); + + // remove the valueobjects flag + // google wants its own keys + if(options & RRDR_OPTION_OBJECTSROWS) + options &= ~RRDR_OPTION_OBJECTSROWS; + } + else { + kq[0] = '"'; + sq[0] = '"'; + if(options & RRDR_OPTION_GOOGLE_JSON) { + dates = JSON_DATES_JS; + dates_with_new = 1; + } + else { + dates = JSON_DATES_TIMESTAMP; + dates_with_new = 0; + } + if( options & RRDR_OPTION_OBJECTSROWS ) + strcpy(pre_date, " { "); + else + strcpy(pre_date, " [ "); + strcpy(pre_label, ", \""); + strcpy(post_label, "\""); + strcpy(pre_value, ", "); + if( options & RRDR_OPTION_OBJECTSROWS ) + strcpy(post_line, "}"); + else + strcpy(post_line, "]"); + snprintfz(data_begin, 100, "],\n %sdata%s:\n [\n", kq, kq); + strcpy(finish, "\n ]\n}"); + + buffer_sprintf(wb, "{\n %slabels%s: [", kq, kq); + buffer_sprintf(wb, "%stime%s", sq, sq); + } + + // ------------------------------------------------------------------------- + // print the JSON header + + long c, i; + RRDDIM *rd; + + // print the header lines + for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { + if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; + + buffer_strcat(wb, pre_label); + buffer_strcat(wb, rd->name); +// buffer_strcat(wb, "."); +// buffer_strcat(wb, rd->rrdset->name); + buffer_strcat(wb, post_label); + i++; + } + if(!i) { + buffer_strcat(wb, pre_label); + buffer_strcat(wb, "no data"); + buffer_strcat(wb, post_label); + } + + // print the begin of row data + buffer_strcat(wb, data_begin); + + // if all dimensions are hidden, print a null + if(!i) { + buffer_strcat(wb, finish); + return; + } + + long start = 0, end = rrdr_rows(r), step = 1; + if(!(options & RRDR_OPTION_REVERSED)) { + start = rrdr_rows(r) - 1; + end = -1; + step = -1; + } + + // for each line in the array + calculated_number total = 1; + for(i = start; i != end ;i += step) { + calculated_number *cn = &r->v[ i * r->d ]; + RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ]; + + time_t now = r->t[i]; + + if(dates == JSON_DATES_JS) { + // generate the local date time + struct tm tmbuf, *tm = localtime_r(&now, &tmbuf); + if(!tm) { error("localtime_r() failed."); continue; } + + if(likely(i != start)) buffer_strcat(wb, ",\n"); + buffer_strcat(wb, pre_date); + + if( options & RRDR_OPTION_OBJECTSROWS ) + buffer_sprintf(wb, "%stime%s: ", kq, kq); + + if(dates_with_new) + buffer_strcat(wb, "new "); + + buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec); + + buffer_strcat(wb, post_date); + + if(row_annotations) { + // google supports one annotation per row + int annotation_found = 0; + for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd ;c++, rd = rd->next) { + if(unlikely(!(r->od[c] & RRDR_DIMENSION_SELECTED))) continue; + + if(co[c] & RRDR_VALUE_RESET) { + buffer_strcat(wb, overflow_annotation); + annotation_found = 1; + break; + } + } + if(!annotation_found) + buffer_strcat(wb, normal_annotation); + } + } + else { + // print the timestamp of the line + if(likely(i != start)) buffer_strcat(wb, ",\n"); + buffer_strcat(wb, pre_date); + + if( options & RRDR_OPTION_OBJECTSROWS ) + buffer_sprintf(wb, "%stime%s: ", kq, kq); + + buffer_rrd_value(wb, (calculated_number)r->t[i]); + // in ms + if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000"); + + buffer_strcat(wb, post_date); + } + + int set_min_max = 0; + if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { + total = 0; + for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { + calculated_number n = cn[c]; + + if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) + n = -n; + + total += n; + } + // prevent a division by zero + if(total == 0) total = 1; + set_min_max = 1; + } + + // for each dimension + for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { + if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; + + calculated_number n = cn[c]; + + buffer_strcat(wb, pre_value); + + if( options & RRDR_OPTION_OBJECTSROWS ) + buffer_sprintf(wb, "%s%s%s: ", kq, rd->name, kq); + + if(co[c] & RRDR_VALUE_EMPTY) { + if(options & RRDR_OPTION_NULL2ZERO) + buffer_strcat(wb, "0"); + else + buffer_strcat(wb, "null"); + } + else { + if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) + n = -n; + + if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { + n = n * 100 / total; + + if(unlikely(set_min_max)) { + r->min = r->max = n; + set_min_max = 0; + } + + if(n < r->min) r->min = n; + if(n > r->max) r->max = n; + } + + buffer_rrd_value(wb, n); + } + + buffer_strcat(wb, post_value); + } + + buffer_strcat(wb, post_line); + } + + buffer_strcat(wb, finish); + //info("RRD2JSON(): %s: END", r->st->id); +} diff --git a/web/api/formatters/json/json.h b/web/api/formatters/json/json.h new file mode 100644 index 0000000..6d73a3f --- /dev/null +++ b/web/api/formatters/json/json.h @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_API_FORMATTER_JSON_H +#define NETDATA_API_FORMATTER_JSON_H + +#include "../rrd2json.h" + +extern void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, RRDDIM *temp_rd); + +#endif //NETDATA_API_FORMATTER_JSON_H diff --git a/web/api/formatters/json_wrapper.c b/web/api/formatters/json_wrapper.c new file mode 100644 index 0000000..cf4f109 --- /dev/null +++ b/web/api/formatters/json_wrapper.c @@ -0,0 +1,295 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "json_wrapper.h" + +void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value, RRDDIM *temp_rd, char *chart_label_key) { + rrdset_check_rdlock(r->st); + + long rows = rrdr_rows(r); + long c, i; + RRDDIM *rd; + + //info("JSONWRAPPER(): %s: BEGIN", r->st->id); + char kq[2] = "", // key quote + sq[2] = ""; // string quote + + if( options & RRDR_OPTION_GOOGLE_JSON ) { + kq[0] = '\0'; + sq[0] = '\''; + } + else { + kq[0] = '"'; + sq[0] = '"'; + } + + rrdset_rdlock(r->st); + buffer_sprintf(wb, "{\n" + " %sapi%s: 1,\n" + " %sid%s: %s%s%s,\n" + " %sname%s: %s%s%s,\n" + " %sview_update_every%s: %d,\n" + " %supdate_every%s: %d,\n" + " %sfirst_entry%s: %u,\n" + " %slast_entry%s: %u,\n" + " %sbefore%s: %u,\n" + " %safter%s: %u,\n" + " %sdimension_names%s: [" + , kq, kq + , kq, kq, sq, temp_rd?r->st->context:r->st->id, sq + , kq, kq, sq, temp_rd?r->st->context:r->st->name, sq + , kq, kq, r->update_every + , kq, kq, r->st->update_every + , kq, kq, (uint32_t)rrdset_first_entry_t_nolock(r->st) + , kq, kq, (uint32_t)rrdset_last_entry_t_nolock(r->st) + , kq, kq, (uint32_t)r->before + , kq, kq, (uint32_t)r->after + , kq, kq); + rrdset_unlock(r->st); + + for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { + if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; + + if(i) buffer_strcat(wb, ", "); + buffer_strcat(wb, sq); + buffer_strcat(wb, rd->name); + buffer_strcat(wb, sq); + i++; + } + if(!i) { +#ifdef NETDATA_INTERNAL_CHECKS + error("RRDR is empty for %s (RRDR has %d dimensions, options is 0x%08x)", r->st->id, r->d, options); +#endif + rows = 0; + buffer_strcat(wb, sq); + buffer_strcat(wb, "no data"); + buffer_strcat(wb, sq); + } + + buffer_sprintf(wb, "],\n" + " %sdimension_ids%s: [" + , kq, kq); + + for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { + if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; + + if(i) buffer_strcat(wb, ", "); + buffer_strcat(wb, sq); + buffer_strcat(wb, rd->id); + buffer_strcat(wb, sq); + i++; + } + if(!i) { + rows = 0; + buffer_strcat(wb, sq); + buffer_strcat(wb, "no data"); + buffer_strcat(wb, sq); + } + buffer_strcat(wb, "],\n"); + + // Composite charts + if (temp_rd) { + buffer_sprintf( + wb, + " %schart_ids%s: [", + kq, kq); + + for (c = 0, i = 0, rd = temp_rd ; rd && c < r->d; c++, rd = rd->next) { + if (unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) + continue; + if (unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) + continue; + + if (i) + buffer_strcat(wb, ", "); + buffer_strcat(wb, sq); + buffer_strcat(wb, rd->rrdset->name); + buffer_strcat(wb, sq); + i++; + } + if (!i) { + rows = 0; + buffer_strcat(wb, sq); + buffer_strcat(wb, "no data"); + buffer_strcat(wb, sq); + } + buffer_strcat(wb, "],\n"); + if (chart_label_key) { + buffer_sprintf(wb, " %schart_labels%s: { ", kq, kq); + + SIMPLE_PATTERN *pattern = simple_pattern_create(chart_label_key, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT); + SIMPLE_PATTERN *original_pattern = pattern; + char *label_key = NULL; + int keys = 0; + while (pattern && (label_key = simple_pattern_iterate(&pattern))) { + uint32_t key_hash = simple_hash(label_key); + struct label *current_label; + + if (keys) + buffer_strcat(wb, ", "); + buffer_sprintf(wb, "%s%s%s : [", kq, label_key, kq); + keys++; + + for (c = 0, i = 0, rd = temp_rd; rd && c < r->d; c++, rd = rd->next) { + if (unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) + continue; + if (unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) + continue; + if (i) + buffer_strcat(wb, ", "); + + current_label = rrdset_lookup_label_key(rd->rrdset, label_key, key_hash); + if (current_label) { + buffer_strcat(wb, sq); + buffer_strcat(wb, current_label->value); + buffer_strcat(wb, sq); + } else + buffer_strcat(wb, "null"); + i++; + } + if (!i) { + rows = 0; + buffer_strcat(wb, sq); + buffer_strcat(wb, "no data"); + buffer_strcat(wb, sq); + } + buffer_strcat(wb, "]"); + } + buffer_strcat(wb, "},\n"); + simple_pattern_free(original_pattern); + } + } + + buffer_sprintf(wb, " %slatest_values%s: [" + , kq, kq); + + for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { + if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; + + if(i) buffer_strcat(wb, ", "); + i++; + + calculated_number value = rd->last_stored_value; + if (NAN == value) + buffer_strcat(wb, "null"); + else + buffer_rrd_value(wb, value); + /* + storage_number n = rd->values[rrdset_last_slot(r->st)]; + + if(!does_storage_number_exist(n)) + buffer_strcat(wb, "null"); + else + buffer_rrd_value(wb, unpack_storage_number(n)); + */ + } + if(!i) { + rows = 0; + buffer_strcat(wb, "null"); + } + + buffer_sprintf(wb, "],\n" + " %sview_latest_values%s: [" + , kq, kq); + + i = 0; + if(rows) { + calculated_number total = 1; + + if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { + total = 0; + for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { + calculated_number *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ]; + calculated_number n = cn[c]; + + if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) + n = -n; + + total += n; + } + // prevent a division by zero + if(total == 0) total = 1; + } + + for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { + if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; + + if(i) buffer_strcat(wb, ", "); + i++; + + calculated_number *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ]; + RRDR_VALUE_FLAGS *co = &r->o[ (rrdr_rows(r) - 1) * r->d ]; + calculated_number n = cn[c]; + + if(co[c] & RRDR_VALUE_EMPTY) { + if(options & RRDR_OPTION_NULL2ZERO) + buffer_strcat(wb, "0"); + else + buffer_strcat(wb, "null"); + } + else { + if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) + n = -n; + + if(unlikely(options & RRDR_OPTION_PERCENTAGE)) + n = n * 100 / total; + + buffer_rrd_value(wb, n); + } + } + } + if(!i) { + rows = 0; + buffer_strcat(wb, "null"); + } + + buffer_sprintf(wb, "],\n" + " %sdimensions%s: %ld,\n" + " %spoints%s: %ld,\n" + " %sformat%s: %s" + , kq, kq, i + , kq, kq, rows + , kq, kq, sq + ); + + rrdr_buffer_print_format(wb, format); + + if((options & RRDR_OPTION_CUSTOM_VARS) && (options & RRDR_OPTION_JSON_WRAP)) { + buffer_sprintf(wb, "%s,\n %schart_variables%s: ", sq, kq, kq); + health_api_v1_chart_custom_variables2json(r->st, wb); + } + else + buffer_sprintf(wb, "%s", sq); + + buffer_sprintf(wb, ",\n %sresult%s: ", kq, kq); + + if(string_value) buffer_strcat(wb, sq); + //info("JSONWRAPPER(): %s: END", r->st->id); +} + +void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value) { + (void)format; + + char kq[2] = "", // key quote + sq[2] = ""; // string quote + + if( options & RRDR_OPTION_GOOGLE_JSON ) { + kq[0] = '\0'; + sq[0] = '\''; + } + else { + kq[0] = '"'; + sq[0] = '"'; + } + + if(string_value) buffer_strcat(wb, sq); + + buffer_sprintf(wb, ",\n %smin%s: ", kq, kq); + buffer_rrd_value(wb, r->min); + buffer_sprintf(wb, ",\n %smax%s: ", kq, kq); + buffer_rrd_value(wb, r->max); + buffer_strcat(wb, "\n}\n"); +} diff --git a/web/api/formatters/json_wrapper.h b/web/api/formatters/json_wrapper.h new file mode 100644 index 0000000..d48d5d1 --- /dev/null +++ b/web/api/formatters/json_wrapper.h @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_API_FORMATTER_JSON_WRAPPER_H +#define NETDATA_API_FORMATTER_JSON_WRAPPER_H + +#include "rrd2json.h" + +extern void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value, RRDDIM *temp_rd, char *chart_key); +extern void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value); + +#endif //NETDATA_API_FORMATTER_JSON_WRAPPER_H diff --git a/web/api/formatters/rrd2json.c b/web/api/formatters/rrd2json.c new file mode 100644 index 0000000..d8e2480 --- /dev/null +++ b/web/api/formatters/rrd2json.c @@ -0,0 +1,401 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "web/api/web_api_v1.h" + +static inline void free_temp_rrddim(RRDDIM *temp_rd) +{ + if (unlikely(!temp_rd)) + return; + + RRDDIM *t; + while (temp_rd) { + t = temp_rd->next; + freez((char *)temp_rd->id); + freez((char *)temp_rd->name); +#ifdef ENABLE_DBENGINE + if (temp_rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) + freez(temp_rd->state->metric_uuid); +#endif + freez(temp_rd->state); + freez(temp_rd); + temp_rd = t; + } +} + +void free_context_param_list(struct context_param **param_list) +{ + if (unlikely(!param_list || !*param_list)) + return; + + free_temp_rrddim(((*param_list)->rd)); + freez((*param_list)); + *param_list = NULL; +} + +void rebuild_context_param_list(struct context_param *context_param_list, time_t after_requested) +{ + RRDDIM *temp_rd = context_param_list->rd; + RRDDIM *new_rd_list = NULL, *t; + while (temp_rd) { + t = temp_rd->next; + if (rrdset_last_entry_t(temp_rd->rrdset) >= after_requested) { + temp_rd->next = new_rd_list; + new_rd_list = temp_rd; + } else { + freez((char *)temp_rd->id); + freez((char *)temp_rd->name); +#ifdef ENABLE_DBENGINE + if (temp_rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) + freez(temp_rd->state->metric_uuid); +#endif + freez(temp_rd->state); + freez(temp_rd); + } + temp_rd = t; + } + context_param_list->rd = new_rd_list; +}; + +void build_context_param_list(struct context_param **param_list, RRDSET *st) +{ + if (unlikely(!param_list || !st)) + return; + + if (unlikely(!(*param_list))) { + *param_list = mallocz(sizeof(struct context_param)); + (*param_list)->first_entry_t = LONG_MAX; + (*param_list)->last_entry_t = 0; + (*param_list)->rd = NULL; + } + + RRDDIM *rd1; + st->last_accessed_time = now_realtime_sec(); + rrdset_rdlock(st); + + (*param_list)->first_entry_t = MIN((*param_list)->first_entry_t, rrdset_first_entry_t_nolock(st)); + (*param_list)->last_entry_t = MAX((*param_list)->last_entry_t, rrdset_last_entry_t_nolock(st)); + + rrddim_foreach_read(rd1, st) { + RRDDIM *rd = mallocz(rd1->memsize); + memcpy(rd, rd1, rd1->memsize); + rd->id = strdupz(rd1->id); + rd->name = strdupz(rd1->name); + rd->state = mallocz(sizeof(*rd->state)); + memcpy(rd->state, rd1->state, sizeof(*rd->state)); + memcpy(&rd->state->collect_ops, &rd1->state->collect_ops, sizeof(struct rrddim_collect_ops)); + memcpy(&rd->state->query_ops, &rd1->state->query_ops, sizeof(struct rrddim_query_ops)); +#ifdef ENABLE_DBENGINE + if (rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { + rd->state->metric_uuid = mallocz(sizeof(uuid_t)); + uuid_copy(*rd->state->metric_uuid, *rd1->state->metric_uuid); + } +#endif + rd->next = (*param_list)->rd; + (*param_list)->rd = rd; + } + + rrdset_unlock(st); +} + +void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) { + rrdset2json(st, wb, NULL, NULL, 0); +} + +void rrdr_buffer_print_format(BUFFER *wb, uint32_t format) { + switch(format) { + case DATASOURCE_JSON: + buffer_strcat(wb, DATASOURCE_FORMAT_JSON); + break; + + case DATASOURCE_DATATABLE_JSON: + buffer_strcat(wb, DATASOURCE_FORMAT_DATATABLE_JSON); + break; + + case DATASOURCE_DATATABLE_JSONP: + buffer_strcat(wb, DATASOURCE_FORMAT_DATATABLE_JSONP); + break; + + case DATASOURCE_JSONP: + buffer_strcat(wb, DATASOURCE_FORMAT_JSONP); + break; + + case DATASOURCE_SSV: + buffer_strcat(wb, DATASOURCE_FORMAT_SSV); + break; + + case DATASOURCE_CSV: + buffer_strcat(wb, DATASOURCE_FORMAT_CSV); + break; + + case DATASOURCE_TSV: + buffer_strcat(wb, DATASOURCE_FORMAT_TSV); + break; + + case DATASOURCE_HTML: + buffer_strcat(wb, DATASOURCE_FORMAT_HTML); + break; + + case DATASOURCE_JS_ARRAY: + buffer_strcat(wb, DATASOURCE_FORMAT_JS_ARRAY); + break; + + case DATASOURCE_SSV_COMMA: + buffer_strcat(wb, DATASOURCE_FORMAT_SSV_COMMA); + break; + + default: + buffer_strcat(wb, "unknown"); + break; + } +} + +int rrdset2value_api_v1( + RRDSET *st + , BUFFER *wb + , calculated_number *n + , const char *dimensions + , long points + , long long after + , long long before + , int group_method + , long group_time + , uint32_t options + , time_t *db_after + , time_t *db_before + , int *value_is_null +) { + + RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions, NULL); + + if(!r) { + if(value_is_null) *value_is_null = 1; + return HTTP_RESP_INTERNAL_SERVER_ERROR; + } + + if(rrdr_rows(r) == 0) { + rrdr_free(r); + + if(db_after) *db_after = 0; + if(db_before) *db_before = 0; + if(value_is_null) *value_is_null = 1; + + return HTTP_RESP_BAD_REQUEST; + } + + if(wb) { + if (r->result_options & RRDR_RESULT_OPTION_RELATIVE) + buffer_no_cacheable(wb); + else if (r->result_options & RRDR_RESULT_OPTION_ABSOLUTE) + buffer_cacheable(wb); + } + + if(db_after) *db_after = r->after; + if(db_before) *db_before = r->before; + + long i = (!(options & RRDR_OPTION_REVERSED))?rrdr_rows(r) - 1:0; + *n = rrdr2value(r, i, options, value_is_null); + + rrdr_free(r); + return HTTP_RESP_OK; +} + +int rrdset2anything_api_v1( + RRDSET *st + , BUFFER *wb + , BUFFER *dimensions + , uint32_t format + , long points + , long long after + , long long before + , int group_method + , long group_time + , uint32_t options + , time_t *latest_timestamp + , struct context_param *context_param_list + , char *chart_label_key +) { + time_t last_accessed_time = now_realtime_sec(); + st->last_accessed_time = last_accessed_time; + + + RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions?buffer_tostring(dimensions):NULL, context_param_list); + if(!r) { + buffer_strcat(wb, "Cannot generate output with these parameters on this chart."); + return HTTP_RESP_INTERNAL_SERVER_ERROR; + } + + RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL; + + if(r->result_options & RRDR_RESULT_OPTION_RELATIVE) + buffer_no_cacheable(wb); + else if(r->result_options & RRDR_RESULT_OPTION_ABSOLUTE) + buffer_cacheable(wb); + + if(latest_timestamp && rrdr_rows(r) > 0) + *latest_timestamp = r->before; + + switch(format) { + case DATASOURCE_SSV: + if(options & RRDR_OPTION_JSON_WRAP) { + wb->contenttype = CT_APPLICATION_JSON; + rrdr_json_wrapper_begin(r, wb, format, options, 1, temp_rd, chart_label_key); + rrdr2ssv(r, wb, options, "", " ", ""); + rrdr_json_wrapper_end(r, wb, format, options, 1); + } + else { + wb->contenttype = CT_TEXT_PLAIN; + rrdr2ssv(r, wb, options, "", " ", ""); + } + break; + + case DATASOURCE_SSV_COMMA: + if(options & RRDR_OPTION_JSON_WRAP) { + wb->contenttype = CT_APPLICATION_JSON; + rrdr_json_wrapper_begin(r, wb, format, options, 1, temp_rd, chart_label_key); + rrdr2ssv(r, wb, options, "", ",", ""); + rrdr_json_wrapper_end(r, wb, format, options, 1); + } + else { + wb->contenttype = CT_TEXT_PLAIN; + rrdr2ssv(r, wb, options, "", ",", ""); + } + break; + + case DATASOURCE_JS_ARRAY: + if(options & RRDR_OPTION_JSON_WRAP) { + wb->contenttype = CT_APPLICATION_JSON; + rrdr_json_wrapper_begin(r, wb, format, options, 0, temp_rd, chart_label_key); + rrdr2ssv(r, wb, options, "[", ",", "]"); + rrdr_json_wrapper_end(r, wb, format, options, 0); + } + else { + wb->contenttype = CT_APPLICATION_JSON; + rrdr2ssv(r, wb, options, "[", ",", "]"); + } + break; + + case DATASOURCE_CSV: + if(options & RRDR_OPTION_JSON_WRAP) { + wb->contenttype = CT_APPLICATION_JSON; + rrdr_json_wrapper_begin(r, wb, format, options, 1, temp_rd, chart_label_key); + rrdr2csv(r, wb, format, options, "", ",", "\\n", "", temp_rd); + rrdr_json_wrapper_end(r, wb, format, options, 1); + } + else { + wb->contenttype = CT_TEXT_PLAIN; + rrdr2csv(r, wb, format, options, "", ",", "\r\n", "", temp_rd); + } + break; + + case DATASOURCE_CSV_MARKDOWN: + if(options & RRDR_OPTION_JSON_WRAP) { + wb->contenttype = CT_APPLICATION_JSON; + rrdr_json_wrapper_begin(r, wb, format, options, 1, temp_rd, chart_label_key); + rrdr2csv(r, wb, format, options, "", "|", "\\n", "", temp_rd); + rrdr_json_wrapper_end(r, wb, format, options, 1); + } + else { + wb->contenttype = CT_TEXT_PLAIN; + rrdr2csv(r, wb, format, options, "", "|", "\r\n", "", temp_rd); + } + break; + + case DATASOURCE_CSV_JSON_ARRAY: + wb->contenttype = CT_APPLICATION_JSON; + if(options & RRDR_OPTION_JSON_WRAP) { + rrdr_json_wrapper_begin(r, wb, format, options, 0, temp_rd, chart_label_key); + buffer_strcat(wb, "[\n"); + rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n", temp_rd); + buffer_strcat(wb, "\n]"); + rrdr_json_wrapper_end(r, wb, format, options, 0); + } + else { + wb->contenttype = CT_APPLICATION_JSON; + buffer_strcat(wb, "[\n"); + rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n", temp_rd); + buffer_strcat(wb, "\n]"); + } + break; + + case DATASOURCE_TSV: + if(options & RRDR_OPTION_JSON_WRAP) { + wb->contenttype = CT_APPLICATION_JSON; + rrdr_json_wrapper_begin(r, wb, format, options, 1, temp_rd, chart_label_key); + rrdr2csv(r, wb, format, options, "", "\t", "\\n", "", temp_rd); + rrdr_json_wrapper_end(r, wb, format, options, 1); + } + else { + wb->contenttype = CT_TEXT_PLAIN; + rrdr2csv(r, wb, format, options, "", "\t", "\r\n", "", temp_rd); + } + break; + + case DATASOURCE_HTML: + if(options & RRDR_OPTION_JSON_WRAP) { + wb->contenttype = CT_APPLICATION_JSON; + rrdr_json_wrapper_begin(r, wb, format, options, 1, temp_rd, chart_label_key); + buffer_strcat(wb, "<html>\\n<center>\\n<table border=\\\"0\\\" cellpadding=\\\"5\\\" cellspacing=\\\"5\\\">\\n"); + rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\\n", "", temp_rd); + buffer_strcat(wb, "</table>\\n</center>\\n</html>\\n"); + rrdr_json_wrapper_end(r, wb, format, options, 1); + } + else { + wb->contenttype = CT_TEXT_HTML; + buffer_strcat(wb, "<html>\n<center>\n<table border=\"0\" cellpadding=\"5\" cellspacing=\"5\">\n"); + rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\n", "", temp_rd); + buffer_strcat(wb, "</table>\n</center>\n</html>\n"); + } + break; + + case DATASOURCE_DATATABLE_JSONP: + wb->contenttype = CT_APPLICATION_X_JAVASCRIPT; + + if(options & RRDR_OPTION_JSON_WRAP) + rrdr_json_wrapper_begin(r, wb, format, options, 0, temp_rd, chart_label_key); + + rrdr2json(r, wb, options, 1, temp_rd); + + if(options & RRDR_OPTION_JSON_WRAP) + rrdr_json_wrapper_end(r, wb, format, options, 0); + break; + + case DATASOURCE_DATATABLE_JSON: + wb->contenttype = CT_APPLICATION_JSON; + + if(options & RRDR_OPTION_JSON_WRAP) + rrdr_json_wrapper_begin(r, wb, format, options, 0, temp_rd, chart_label_key); + + rrdr2json(r, wb, options, 1, temp_rd); + + if(options & RRDR_OPTION_JSON_WRAP) + rrdr_json_wrapper_end(r, wb, format, options, 0); + break; + + case DATASOURCE_JSONP: + wb->contenttype = CT_APPLICATION_X_JAVASCRIPT; + if(options & RRDR_OPTION_JSON_WRAP) + rrdr_json_wrapper_begin(r, wb, format, options, 0, temp_rd, chart_label_key); + + rrdr2json(r, wb, options, 0, temp_rd); + + if(options & RRDR_OPTION_JSON_WRAP) + rrdr_json_wrapper_end(r, wb, format, options, 0); + break; + + case DATASOURCE_JSON: + default: + wb->contenttype = CT_APPLICATION_JSON; + + if(options & RRDR_OPTION_JSON_WRAP) + rrdr_json_wrapper_begin(r, wb, format, options, 0, temp_rd, chart_label_key); + + rrdr2json(r, wb, options, 0, temp_rd); + + if(options & RRDR_OPTION_JSON_WRAP) + rrdr_json_wrapper_end(r, wb, format, options, 0); + break; + } + + rrdr_free(r); + return HTTP_RESP_OK; +} diff --git a/web/api/formatters/rrd2json.h b/web/api/formatters/rrd2json.h new file mode 100644 index 0000000..3dc5989 --- /dev/null +++ b/web/api/formatters/rrd2json.h @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_RRD2JSON_H +#define NETDATA_RRD2JSON_H 1 + +#include "web/api/web_api_v1.h" +#include "web/api/exporters/allmetrics.h" +#include "web/api/queries/rrdr.h" + +#include "web/api/formatters/csv/csv.h" +#include "web/api/formatters/ssv/ssv.h" +#include "web/api/formatters/json/json.h" +#include "web/api/formatters/value/value.h" + +#include "web/api/formatters/rrdset2json.h" +#include "web/api/formatters/charts2json.h" +#include "web/api/formatters/json_wrapper.h" + +#include "web/server/web_client.h" + +#define HOSTNAME_MAX 1024 + +#define API_RELATIVE_TIME_MAX (3 * 365 * 86400) + +// type of JSON generations +#define DATASOURCE_INVALID (-1) +#define DATASOURCE_JSON 0 +#define DATASOURCE_DATATABLE_JSON 1 +#define DATASOURCE_DATATABLE_JSONP 2 +#define DATASOURCE_SSV 3 +#define DATASOURCE_CSV 4 +#define DATASOURCE_JSONP 5 +#define DATASOURCE_TSV 6 +#define DATASOURCE_HTML 7 +#define DATASOURCE_JS_ARRAY 8 +#define DATASOURCE_SSV_COMMA 9 +#define DATASOURCE_CSV_JSON_ARRAY 10 +#define DATASOURCE_CSV_MARKDOWN 11 + +#define DATASOURCE_FORMAT_JSON "json" +#define DATASOURCE_FORMAT_DATATABLE_JSON "datatable" +#define DATASOURCE_FORMAT_DATATABLE_JSONP "datasource" +#define DATASOURCE_FORMAT_JSONP "jsonp" +#define DATASOURCE_FORMAT_SSV "ssv" +#define DATASOURCE_FORMAT_CSV "csv" +#define DATASOURCE_FORMAT_TSV "tsv" +#define DATASOURCE_FORMAT_HTML "html" +#define DATASOURCE_FORMAT_JS_ARRAY "array" +#define DATASOURCE_FORMAT_SSV_COMMA "ssvcomma" +#define DATASOURCE_FORMAT_CSV_JSON_ARRAY "csvjsonarray" +#define DATASOURCE_FORMAT_CSV_MARKDOWN "markdown" + +extern void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb); +extern void rrdr_buffer_print_format(BUFFER *wb, uint32_t format); + +extern int rrdset2anything_api_v1( + RRDSET *st + , BUFFER *wb + , BUFFER *dimensions + , uint32_t format + , long points + , long long after + , long long before + , int group_method + , long group_time + , uint32_t options + , time_t *latest_timestamp + , struct context_param *context_param_list + , char *chart_label_key +); + +extern int rrdset2value_api_v1( + RRDSET *st + , BUFFER *wb + , calculated_number *n + , const char *dimensions + , long points + , long long after + , long long before + , int group_method + , long group_time + , uint32_t options + , time_t *db_after + , time_t *db_before + , int *value_is_null +); + +extern void build_context_param_list(struct context_param **param_list, RRDSET *st); +extern void rebuild_context_param_list(struct context_param *context_param_list, time_t after_requested); +extern void free_context_param_list(struct context_param **param_list); + +#endif /* NETDATA_RRD2JSON_H */ diff --git a/web/api/formatters/rrdset2json.c b/web/api/formatters/rrdset2json.c new file mode 100644 index 0000000..5482881 --- /dev/null +++ b/web/api/formatters/rrdset2json.c @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rrdset2json.h" + +void chart_labels2json(RRDSET *st, BUFFER *wb, size_t indentation) +{ + char tabs[11]; + struct label_index *labels = &st->state->labels; + + if (indentation > 10) + indentation = 10; + + tabs[0] = '\0'; + while (indentation) { + strcat(tabs, "\t"); + indentation--; + } + + int count = 0; + netdata_rwlock_rdlock(&labels->labels_rwlock); + for (struct label *label = labels->head; label; label = label->next) { + if(count > 0) buffer_strcat(wb, ",\n"); + buffer_strcat(wb, tabs); + + char value[CONFIG_MAX_VALUE * 2 + 1]; + sanitize_json_string(value, label->value, CONFIG_MAX_VALUE * 2); + buffer_sprintf(wb, "\"%s\": \"%s\"", label->key, value); + + count++; + } + buffer_strcat(wb, "\n"); + netdata_rwlock_unlock(&labels->labels_rwlock); +} + +// generate JSON for the /api/v1/chart API call + +void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memory_used, int skip_volatile) { + rrdset_rdlock(st); + + time_t first_entry_t = rrdset_first_entry_t_nolock(st); + time_t last_entry_t = rrdset_last_entry_t_nolock(st); + + buffer_sprintf(wb, + "\t\t{\n" + "\t\t\t\"id\": \"%s\",\n" + "\t\t\t\"name\": \"%s\",\n" + "\t\t\t\"type\": \"%s\",\n" + "\t\t\t\"family\": \"%s\",\n" + "\t\t\t\"context\": \"%s\",\n" + "\t\t\t\"title\": \"%s (%s)\",\n" + "\t\t\t\"priority\": %ld,\n" + "\t\t\t\"plugin\": \"%s\",\n" + "\t\t\t\"module\": \"%s\",\n" + "\t\t\t\"enabled\": %s,\n" + "\t\t\t\"units\": \"%s\",\n" + "\t\t\t\"data_url\": \"/api/v1/data?chart=%s\",\n" + "\t\t\t\"chart_type\": \"%s\",\n" + , st->id + , st->name + , st->type + , st->family + , st->context + , st->title, st->name + , st->priority + , st->plugin_name?st->plugin_name:"" + , st->module_name?st->module_name:"" + , rrdset_flag_check(st, RRDSET_FLAG_ENABLED)?"true":"false" + , st->units + , st->name + , rrdset_type_name(st->chart_type) + ); + + if (likely(!skip_volatile)) + buffer_sprintf(wb, + "\t\t\t\"duration\": %ld,\n" + , last_entry_t - first_entry_t + st->update_every//st->entries * st->update_every + ); + + buffer_sprintf(wb, + "\t\t\t\"first_entry\": %ld,\n" + , first_entry_t //rrdset_first_entry_t(st) + ); + + if (likely(!skip_volatile)) + buffer_sprintf(wb, + "\t\t\t\"last_entry\": %ld,\n" + , last_entry_t//rrdset_last_entry_t(st) + ); + + buffer_sprintf(wb, + "\t\t\t\"update_every\": %d,\n" + "\t\t\t\"dimensions\": {\n" + , st->update_every + ); + + unsigned long memory = st->memsize; + + size_t dimensions = 0; + RRDDIM *rd; + rrddim_foreach_read(rd, st) { + if(rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN) || rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) continue; + + memory += rd->memsize; + + if (dimensions) + buffer_strcat(wb, ",\n\t\t\t\t\""); + else + buffer_strcat(wb, "\t\t\t\t\""); + buffer_strcat_jsonescape(wb, rd->id); + buffer_strcat(wb, "\": { \"name\": \""); + buffer_strcat_jsonescape(wb, rd->name); + buffer_strcat(wb, "\" }"); + + dimensions++; + } + + if(dimensions_count) *dimensions_count += dimensions; + if(memory_used) *memory_used += memory; + + buffer_sprintf(wb, "\n\t\t\t},\n\t\t\t\"chart_variables\": "); + health_api_v1_chart_custom_variables2json(st, wb); + + buffer_strcat(wb, ",\n\t\t\t\"green\": "); + buffer_rrd_value(wb, st->green); + buffer_strcat(wb, ",\n\t\t\t\"red\": "); + buffer_rrd_value(wb, st->red); + + if (likely(!skip_volatile)) { + buffer_strcat(wb, ",\n\t\t\t\"alarms\": {\n"); + size_t alarms = 0; + RRDCALC *rc; + for (rc = st->alarms; rc; rc = rc->rrdset_next) { + buffer_sprintf( + wb, + "%s" + "\t\t\t\t\"%s\": {\n" + "\t\t\t\t\t\"id\": %u,\n" + "\t\t\t\t\t\"status\": \"%s\",\n" + "\t\t\t\t\t\"units\": \"%s\",\n" + "\t\t\t\t\t\"update_every\": %d\n" + "\t\t\t\t}", + (alarms) ? ",\n" : "", rc->name, rc->id, rrdcalc_status2string(rc->status), rc->units, + rc->update_every); + + alarms++; + } + buffer_sprintf(wb, + "\n\t\t\t}" + ); + } + buffer_strcat(wb, ",\n\t\t\t\"chart_labels\": {\n"); + chart_labels2json(st, wb, 2); + buffer_strcat(wb, "\t\t\t}\n"); + + + buffer_sprintf(wb, + "\n\t\t}" + ); + + rrdset_unlock(st); +} diff --git a/web/api/formatters/rrdset2json.h b/web/api/formatters/rrdset2json.h new file mode 100644 index 0000000..697c846 --- /dev/null +++ b/web/api/formatters/rrdset2json.h @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_API_FORMATTER_RRDSET2JSON_H +#define NETDATA_API_FORMATTER_RRDSET2JSON_H + +#include "rrd2json.h" + +extern void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memory_used, int skip_volatile); + +#endif //NETDATA_API_FORMATTER_RRDSET2JSON_H diff --git a/web/api/formatters/ssv/Makefile.am b/web/api/formatters/ssv/Makefile.am new file mode 100644 index 0000000..161784b --- /dev/null +++ b/web/api/formatters/ssv/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/web/api/formatters/ssv/README.md b/web/api/formatters/ssv/README.md new file mode 100644 index 0000000..d439949 --- /dev/null +++ b/web/api/formatters/ssv/README.md @@ -0,0 +1,59 @@ +<!-- +title: "SSV formatter" +custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/formatters/ssv/README.md +--> + +# SSV formatter + +The SSV formatter sums all dimensions in [results of database queries](/web/api/queries/README.md) +to a single value and returns a list of such values showing how it changes through time. + +It supports the following formats: + +| format | content type | description | +|:----:|:----------:|:----------| +| `ssv` | text/plain | a space separated list of values | +| `ssvcomma` | text/plain | a comma separated list of values | +| `array` | application/json | a JSON array | + +The SSV formatter respects the following API `&options=`: + +| option | supported | description | +| :----:|:-------:|:----------| +| `nonzero` | yes | to return only the dimensions that have at least a non-zero value | +| `flip` | yes | to return the numbers older to newer (the default is newer to older) | +| `percent` | yes | to replace all values with their percentage over the row total | +| `abs` | yes | to turn all values positive, before using them | +| `min2max` | yes | to return the delta from the minimum value to the maximum value (across dimensions) | + +## Examples + +Get the average system CPU utilization of the last hour, in 6 values (one every 10 minutes): + +```bash +# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=system.cpu&format=ssv&after=-3600&points=6&group=average' +1.741352 1.6800467 1.769411 1.6761112 1.629862 1.6807968 +``` + +--- + +Get the total mysql bandwidth (in + out) for the last hour, in 6 values (one every 10 minutes): + +Netdata returns bandwidth in `kilobits`. + +```bash +# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=mysql_local.net&format=ssvcomma&after=-3600&points=6&group=sum&options=abs' +72618.7936215,72618.778889,72618.788084,72618.9195918,72618.7760612,72618.6712421 +``` + +--- + +Get the web server max connections for the last hour, in 12 values (one every 5 minutes) +in a JSON array: + +```bash +# curl -Ss 'https://registry.my-netdata.io/api/v1/data?chart=nginx_local.connections&format=array&after=-3600&points=12&group=max' +[278,258,268,239,259,260,243,266,278,318,264,258] +``` + +[](<>) diff --git a/web/api/formatters/ssv/ssv.c b/web/api/formatters/ssv/ssv.c new file mode 100644 index 0000000..eeba028 --- /dev/null +++ b/web/api/formatters/ssv/ssv.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "ssv.h" + +void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, const char *separator, const char *suffix) { + //info("RRD2SSV(): %s: BEGIN", r->st->id); + long i; + + buffer_strcat(wb, prefix); + long start = 0, end = rrdr_rows(r), step = 1; + if(!(options & RRDR_OPTION_REVERSED)) { + start = rrdr_rows(r) - 1; + end = -1; + step = -1; + } + + // for each line in the array + for(i = start; i != end ;i += step) { + int all_values_are_null = 0; + calculated_number v = rrdr2value(r, i, options, &all_values_are_null); + + if(likely(i != start)) { + if(r->min > v) r->min = v; + if(r->max < v) r->max = v; + } + else { + r->min = v; + r->max = v; + } + + if(likely(i != start)) + buffer_strcat(wb, separator); + + if(all_values_are_null) { + if(options & RRDR_OPTION_NULL2ZERO) + buffer_strcat(wb, "0"); + else + buffer_strcat(wb, "null"); + } + else + buffer_rrd_value(wb, v); + } + buffer_strcat(wb, suffix); + //info("RRD2SSV(): %s: END", r->st->id); +} diff --git a/web/api/formatters/ssv/ssv.h b/web/api/formatters/ssv/ssv.h new file mode 100644 index 0000000..6963dcf --- /dev/null +++ b/web/api/formatters/ssv/ssv.h @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_API_FORMATTER_SSV_H +#define NETDATA_API_FORMATTER_SSV_H + +#include "../rrd2json.h" + +extern void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, const char *separator, const char *suffix); + +#endif //NETDATA_API_FORMATTER_SSV_H diff --git a/web/api/formatters/value/Makefile.am b/web/api/formatters/value/Makefile.am new file mode 100644 index 0000000..161784b --- /dev/null +++ b/web/api/formatters/value/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/web/api/formatters/value/README.md b/web/api/formatters/value/README.md new file mode 100644 index 0000000..21c9370 --- /dev/null +++ b/web/api/formatters/value/README.md @@ -0,0 +1,24 @@ +<!-- +title: "Value formatter" +custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/formatters/value/README.md +--> + +# Value formatter + +The Value formatter presents [results of database queries](/web/api/queries/README.md) as a single value. + +To calculate the single value to be returned, it sums the values of all dimensions. + +The Value formatter respects the following API `&options=`: + +| option | supported | description | +|:----: |:-------: |:---------- | +| `percent` | yes | to replace all values with their percentage over the row total| +| `abs` | yes | to turn all values positive, before using them | +| `min2max` | yes | to return the delta from the minimum value to the maximum value (across dimensions)| + +The Value formatter is not exposed by the API by itself. +Instead it is used by the [`ssv`](/web/api/formatters/ssv/README.md) formatter +and [health monitoring queries](/health/README.md). + +[](<>) diff --git a/web/api/formatters/value/value.c b/web/api/formatters/value/value.c new file mode 100644 index 0000000..aea6c16 --- /dev/null +++ b/web/api/formatters/value/value.c @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "value.h" + + +inline calculated_number rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null) { + rrdset_check_rdlock(r->st); + + long c; + RRDDIM *d; + + calculated_number *cn = &r->v[ i * r->d ]; + RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ]; + + calculated_number sum = 0, min = 0, max = 0, v; + int all_null = 1, init = 1; + + calculated_number total = 1; + int set_min_max = 0; + if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { + total = 0; + for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) { + calculated_number n = cn[c]; + + if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) + n = -n; + + total += n; + } + // prevent a division by zero + if(total == 0) total = 1; + set_min_max = 1; + } + + // for each dimension + for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) { + if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; + + calculated_number n = cn[c]; + + if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) + n = -n; + + if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { + n = n * 100 / total; + + if(unlikely(set_min_max)) { + r->min = r->max = n; + set_min_max = 0; + } + + if(n < r->min) r->min = n; + if(n > r->max) r->max = n; + } + + if(unlikely(init)) { + if(n > 0) { + min = 0; + max = n; + } + else { + min = n; + max = 0; + } + init = 0; + } + + if(likely(!(co[c] & RRDR_VALUE_EMPTY))) { + all_null = 0; + sum += n; + } + + if(n < min) min = n; + if(n > max) max = n; + } + + if(unlikely(all_null)) { + if(likely(all_values_are_null)) + *all_values_are_null = 1; + return 0; + } + else { + if(likely(all_values_are_null)) + *all_values_are_null = 0; + } + + if(options & RRDR_OPTION_MIN2MAX) + v = max - min; + else + v = sum; + + return v; +} diff --git a/web/api/formatters/value/value.h b/web/api/formatters/value/value.h new file mode 100644 index 0000000..d9e981f --- /dev/null +++ b/web/api/formatters/value/value.h @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_API_FORMATTER_VALUE_H +#define NETDATA_API_FORMATTER_VALUE_H + +#include "../rrd2json.h" + +extern calculated_number rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null); + +#endif //NETDATA_API_FORMATTER_VALUE_H |