From 6cf8f2d5174a53f582e61d715edbb88d6e3367cc Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 14 Jun 2023 21:20:33 +0200 Subject: Adding upstream version 1.40.0. Signed-off-by: Daniel Baumann --- web/api/formatters/json/json.c | 26 ++++++++----- web/api/formatters/rrd2json.h | 2 +- web/api/netdata-swagger.yaml | 11 +++++- web/api/queries/query.c | 87 +++++++++++++++++++++++------------------- web/api/queries/query.h | 1 + web/api/web_api_v1.c | 2 +- 6 files changed, 78 insertions(+), 51 deletions(-) (limited to 'web/api') diff --git a/web/api/formatters/json/json.c b/web/api/formatters/json/json.c index d5b8c7570..3a7a23ba1 100644 --- a/web/api/formatters/json/json.c +++ b/web/api/formatters/json/json.c @@ -244,12 +244,12 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) { //info("RRD2JSON(): %s: END", r->st->id); } - void rrdr2json_v2(RRDR *r, BUFFER *wb) { QUERY_TARGET *qt = r->internal.qt; RRDR_OPTIONS options = qt->window.options; - bool expose_gbc = query_target_aggregatable(qt); + bool send_count = query_target_aggregatable(qt); + bool send_hidden = send_count && r->vh && query_has_group_by_aggregation_percentage(qt); buffer_json_member_add_object(wb, "result"); @@ -267,12 +267,17 @@ void rrdr2json_v2(RRDR *r, BUFFER *wb) { buffer_json_array_close(wb); // labels buffer_json_member_add_object(wb, "point"); - buffer_json_member_add_uint64(wb, "value", 0); - buffer_json_member_add_uint64(wb, "arp", 1); - buffer_json_member_add_uint64(wb, "pa", 2); - if(expose_gbc) - buffer_json_member_add_uint64(wb, "count", 3); - buffer_json_object_close(wb); + { + size_t point_count = 0; + buffer_json_member_add_uint64(wb, "value", point_count++); + buffer_json_member_add_uint64(wb, "arp", point_count++); + buffer_json_member_add_uint64(wb, "pa", point_count++); + if (send_count) + buffer_json_member_add_uint64(wb, "count", point_count++); + if (send_hidden) + buffer_json_member_add_uint64(wb, "hidden", point_count++); + } + buffer_json_object_close(wb); // point buffer_json_member_add_array(wb, "data"); if(i) { @@ -286,6 +291,7 @@ void rrdr2json_v2(RRDR *r, BUFFER *wb) { // for each line in the array for (i = start; i != end; i += step) { NETDATA_DOUBLE *cn = &r->v[ i * r->d ]; + NETDATA_DOUBLE *ch = send_hidden ? &r->vh[i * r->d ] : NULL; RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ]; NETDATA_DOUBLE *ar = &r->ar[ i * r->d ]; uint32_t *gbc = &r->gbc [ i * r->d ]; @@ -325,8 +331,10 @@ void rrdr2json_v2(RRDR *r, BUFFER *wb) { buffer_json_add_array_item_uint64(wb, o); // add the count - if(expose_gbc) + if(send_count) buffer_json_add_array_item_uint64(wb, gbc[d]); + if(send_hidden) + buffer_json_add_array_item_double(wb, ch[d]); buffer_json_array_close(wb); // point } diff --git a/web/api/formatters/rrd2json.h b/web/api/formatters/rrd2json.h index def26c754..ca3a41aae 100644 --- a/web/api/formatters/rrd2json.h +++ b/web/api/formatters/rrd2json.h @@ -87,7 +87,7 @@ int rrdset2value_api_v1( ); static inline bool rrdr_dimension_should_be_exposed(RRDR_DIMENSION_FLAGS rrdr_dim_flags, RRDR_OPTIONS options) { - if(unlikely(options & RRDR_OPTION_RETURN_RAW)) + if(unlikely((options & RRDR_OPTION_RETURN_RAW) && (rrdr_dim_flags & RRDR_DIMENSION_QUERIED))) return true; if(unlikely(rrdr_dim_flags & RRDR_DIMENSION_HIDDEN)) return false; diff --git a/web/api/netdata-swagger.yaml b/web/api/netdata-swagger.yaml index c25f0b719..b050f3407 100644 --- a/web/api/netdata-swagger.yaml +++ b/web/api/netdata-swagger.yaml @@ -241,6 +241,7 @@ paths: A comma separated list of the groupings required. All possible values can be combined together, except `selected`. If `selected` is given in the list, all others are ignored. The order they are placed in the list is currently ignored. + This parameter is also accepted as `group_by[0]` and `group_by[1]` when multiple grouping passes are required. required: false schema: type: array @@ -261,6 +262,7 @@ paths: in: query description: | A comma separated list of the label keys to group by their values. The order of the labels in the list is respected. + This parameter is also accepted as `group_by_label[0]` and `group_by_label[1]` when multiple grouping passes are required. required: false schema: type: string @@ -271,6 +273,7 @@ paths: description: | The aggregation function to apply when grouping metrics together. When option `raw` is given, `average` and `avg` behave like `sum` and the caller is expected to calculate the average. + This parameter is also accepted as `aggregation[0]` and `aggregation[1]` when multiple grouping passes are required. required: false schema: type: string @@ -280,6 +283,7 @@ paths: - avg - average - sum + - percentage default: average - $ref: '#/components/parameters/scopeNodes' - $ref: '#/components/parameters/scopeContexts' @@ -2741,8 +2745,13 @@ components: type: integer count: description: | - The number of metrics aggregated into this point. This exists only when the option `raw` is given to the query. + The number of metrics aggregated into this point. + This exists only when the option `raw` is given to the query and the final aggregation point is NOT `percentage`. type: integer + hidden: + description: | + The sum of the non-selected dimensions aggregated for this group item point. + This exists only when the option `raw` is given to the query and the final aggregation method is `percentage`. data: type: array items: diff --git a/web/api/queries/query.c b/web/api/queries/query.c index 3770d4770..a0347f6fe 100644 --- a/web/api/queries/query.c +++ b/web/api/queries/query.c @@ -883,6 +883,9 @@ RRDR_GROUP_BY_FUNCTION group_by_aggregate_function_parse(const char *s) { if(strcmp(s, "sum") == 0) return RRDR_GROUP_BY_FUNCTION_SUM; + if(strcmp(s, "percentage") == 0) + return RRDR_GROUP_BY_FUNCTION_PERCENTAGE; + return RRDR_GROUP_BY_FUNCTION_AVERAGE; } @@ -900,6 +903,9 @@ const char *group_by_aggregate_function_to_string(RRDR_GROUP_BY_FUNCTION group_b case RRDR_GROUP_BY_FUNCTION_SUM: return "sum"; + + case RRDR_GROUP_BY_FUNCTION_PERCENTAGE: + return "percentage"; } } @@ -2555,9 +2561,9 @@ static void rrd2rrdr_set_timestamps(RRDR *r) { before_wanted, r->t[points_wanted - 1]); } -static void query_group_by_make_dimension_key(BUFFER *key, RRDR_GROUP_BY group_by, size_t group_by_id, QUERY_TARGET *qt, QUERY_NODE *qn, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi, QUERY_DIMENSION *qd __maybe_unused, QUERY_METRIC *qm, bool query_has_percentage_of_instance) { +static void query_group_by_make_dimension_key(BUFFER *key, RRDR_GROUP_BY group_by, size_t group_by_id, QUERY_TARGET *qt, QUERY_NODE *qn, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi, QUERY_DIMENSION *qd __maybe_unused, QUERY_METRIC *qm, bool query_has_percentage_of_group) { buffer_flush(key); - if(unlikely(!query_has_percentage_of_instance && qm->status & RRDR_DIMENSION_HIDDEN)) { + if(unlikely(!query_has_percentage_of_group && qm->status & RRDR_DIMENSION_HIDDEN)) { buffer_strcat(key, "__hidden_dimensions__"); } else if(unlikely(group_by & RRDR_GROUP_BY_SELECTED)) { @@ -2599,9 +2605,9 @@ static void query_group_by_make_dimension_key(BUFFER *key, RRDR_GROUP_BY group_b } } -static void query_group_by_make_dimension_id(BUFFER *key, RRDR_GROUP_BY group_by, size_t group_by_id, QUERY_TARGET *qt, QUERY_NODE *qn, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi, QUERY_DIMENSION *qd __maybe_unused, QUERY_METRIC *qm, bool query_has_percentage_of_instance) { +static void query_group_by_make_dimension_id(BUFFER *key, RRDR_GROUP_BY group_by, size_t group_by_id, QUERY_TARGET *qt, QUERY_NODE *qn, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi, QUERY_DIMENSION *qd __maybe_unused, QUERY_METRIC *qm, bool query_has_percentage_of_group) { buffer_flush(key); - if(unlikely(!query_has_percentage_of_instance && qm->status & RRDR_DIMENSION_HIDDEN)) { + if(unlikely(!query_has_percentage_of_group && qm->status & RRDR_DIMENSION_HIDDEN)) { buffer_strcat(key, "__hidden_dimensions__"); } else if(unlikely(group_by & RRDR_GROUP_BY_SELECTED)) { @@ -2654,9 +2660,9 @@ static void query_group_by_make_dimension_id(BUFFER *key, RRDR_GROUP_BY group_by } } -static void query_group_by_make_dimension_name(BUFFER *key, RRDR_GROUP_BY group_by, size_t group_by_id, QUERY_TARGET *qt, QUERY_NODE *qn, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi, QUERY_DIMENSION *qd __maybe_unused, QUERY_METRIC *qm, bool query_has_percentage_of_instance) { +static void query_group_by_make_dimension_name(BUFFER *key, RRDR_GROUP_BY group_by, size_t group_by_id, QUERY_TARGET *qt, QUERY_NODE *qn, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi, QUERY_DIMENSION *qd __maybe_unused, QUERY_METRIC *qm, bool query_has_percentage_of_group) { buffer_flush(key); - if(unlikely(!query_has_percentage_of_instance && qm->status & RRDR_DIMENSION_HIDDEN)) { + if(unlikely(!query_has_percentage_of_group && qm->status & RRDR_DIMENSION_HIDDEN)) { buffer_strcat(key, "__hidden_dimensions__"); } else if(unlikely(group_by & RRDR_GROUP_BY_SELECTED)) { @@ -2758,16 +2764,16 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) { } // make sure there are valid group-by methods - bool query_has_percentage_of_instance = false; - for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES - 1 ;g++) { + for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) { if(!(qt->request.group_by[g].group_by & SUPPORTED_GROUP_BY_METHODS)) qt->request.group_by[g].group_by = (g == 0) ? RRDR_GROUP_BY_DIMENSION : RRDR_GROUP_BY_NONE; - - if(qt->request.group_by[g].group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE) - query_has_percentage_of_instance = true; } - // merge all group-by options to upper levels + bool query_has_percentage_of_group = query_target_has_percentage_of_group(qt); + + // merge all group-by options to upper levels, + // so that the top level has all the groupings of the inner levels, + // and each subsequent level has all the groupings of its inner levels. for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES - 1 ;g++) { if(qt->request.group_by[g].group_by == RRDR_GROUP_BY_NONE) continue; @@ -2815,6 +2821,7 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) { for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) { RRDR_GROUP_BY group_by = qt->request.group_by[g].group_by; + RRDR_GROUP_BY_FUNCTION aggregation_method = qt->request.group_by[g].aggregation; if(group_by == RRDR_GROUP_BY_NONE) break; @@ -2855,7 +2862,7 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) { // -------------------------------------------------------------------- // generate the group by key - query_group_by_make_dimension_key(key, group_by, g, qt, qn, qc, qi, qd, qm, query_has_percentage_of_instance); + query_group_by_make_dimension_key(key, group_by, g, qt, qn, qc, qi, qd, qm, query_has_percentage_of_group); // lookup the key in the dictionary @@ -2869,13 +2876,13 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) { // ---------------------------------------------------------------- // generate the dimension id - query_group_by_make_dimension_id(key, group_by, g, qt, qn, qc, qi, qd, qm, query_has_percentage_of_instance); + query_group_by_make_dimension_id(key, group_by, g, qt, qn, qc, qi, qd, qm, query_has_percentage_of_group); entries[pos].id = string_strdupz(buffer_tostring(key)); // ---------------------------------------------------------------- // generate the dimension name - query_group_by_make_dimension_name(key, group_by, g, qt, qn, qc, qi, qd, qm, query_has_percentage_of_instance); + query_group_by_make_dimension_name(key, group_by, g, qt, qn, qc, qi, qd, qm, query_has_percentage_of_group); entries[pos].name = string_strdupz(buffer_tostring(key)); // add the rest of the info @@ -2914,9 +2921,9 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) { // the query target adds to it the non-zero flag qm->status |= RRDR_DIMENSION_GROUPED; - if(query_has_percentage_of_instance) - // when the query has percentage of instance - // there will be no hidden dimensions in the final query + if(query_has_percentage_of_group) + // when the query has percentage of group + // there will be no hidden dimensions in the final query, // so we have to remove the hidden flag from all dimensions entries[pos].od |= qm->status & ~RRDR_DIMENSION_HIDDEN; else @@ -2934,12 +2941,10 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) { qt->id, qt->window.after, qt->window.before, added, qt->window.points); goto cleanup; } - - bool hidden_dimension_on_percentage_of_instance = hidden_dimensions && (group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE); - - // prevent double cleanup in case of error + // prevent double free at cleanup in case of error added = 0; + // link this RRDR if(!last_r) first_r = last_r = r; else @@ -2954,7 +2959,7 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) { r->gbc = onewayalloc_callocz(owa, r->n * r->d, sizeof(*r->gbc)); r->dqp = onewayalloc_callocz(owa, r->d, sizeof(STORAGE_POINT)); - if(hidden_dimension_on_percentage_of_instance) + if(hidden_dimensions && ((group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE) || (aggregation_method == RRDR_GROUP_BY_FUNCTION_PERCENTAGE))) // this is where we are going to group the hidden dimensions r->vh = onewayalloc_mallocz(owa, r->n * r->d * sizeof(*r->vh)); @@ -2987,7 +2992,7 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) { // initialize partial trimming r->partial_data_trimming.max_update_every = update_every_max; r->partial_data_trimming.expected_after = - (!(qt->window.options & RRDR_OPTION_RETURN_RAW) && + (!query_target_aggregatable(qt) && qt->window.before >= qt->window.now - update_every_max) ? qt->window.before - update_every_max : qt->window.before; @@ -3006,7 +3011,7 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) { co[d] = RRDR_VALUE_EMPTY; if(vh) - *vh = NAN; + vh[d] = NAN; } } } @@ -3073,9 +3078,9 @@ static void rrd2rrdr_group_by_add_metric(RRDR *r_dst, size_t d_dst, RRDR *r_tmp, internal_fatal(!r_dst->dqp, "QUERY: group-by destination is not properly prepared (missing dqp array)"); internal_fatal(!r_dst->gbc, "QUERY: group-by destination is not properly prepared (missing gbc array)"); - bool hidden_dimension_on_percentage_of_instance = (r_tmp->od[d_tmp] & RRDR_DIMENSION_HIDDEN) && r_dst->vh; + bool hidden_dimension_on_percentage_of_group = (r_tmp->od[d_tmp] & RRDR_DIMENSION_HIDDEN) && r_dst->vh; - if(!hidden_dimension_on_percentage_of_instance) { + if(!hidden_dimension_on_percentage_of_group) { r_dst->od[d_dst] |= r_tmp->od[d_tmp]; storage_point_merge_to(r_dst->dqp[d_dst], *query_points); } @@ -3092,7 +3097,7 @@ static void rrd2rrdr_group_by_add_metric(RRDR *r_dst, size_t d_dst, RRDR *r_tmp, continue; size_t idx_dst = i * r_dst->d + d_dst; - NETDATA_DOUBLE *cn = (hidden_dimension_on_percentage_of_instance) ? &r_dst->vh[ idx_dst ] : &r_dst->v[ idx_dst ]; + NETDATA_DOUBLE *cn = (hidden_dimension_on_percentage_of_group) ? &r_dst->vh[ idx_dst ] : &r_dst->v[ idx_dst ]; RRDR_VALUE_FLAGS *co = &r_dst->o[ idx_dst ]; NETDATA_DOUBLE *ar = &r_dst->ar[ idx_dst ]; uint32_t *gbc = &r_dst->gbc[ idx_dst ]; @@ -3101,6 +3106,7 @@ static void rrd2rrdr_group_by_add_metric(RRDR *r_dst, size_t d_dst, RRDR *r_tmp, default: case RRDR_GROUP_BY_FUNCTION_AVERAGE: case RRDR_GROUP_BY_FUNCTION_SUM: + case RRDR_GROUP_BY_FUNCTION_PERCENTAGE: if(isnan(*cn)) *cn = n_tmp; else @@ -3118,7 +3124,7 @@ static void rrd2rrdr_group_by_add_metric(RRDR *r_dst, size_t d_dst, RRDR *r_tmp, break; } - if(!hidden_dimension_on_percentage_of_instance) { + if(!hidden_dimension_on_percentage_of_group) { *co &= ~RRDR_VALUE_EMPTY; *co |= (o_tmp & (RRDR_VALUE_RESET | RRDR_VALUE_PARTIAL)); *ar += ar_tmp; @@ -3161,10 +3167,13 @@ static void rrdr2rrdr_group_by_partial_trimming(RRDR *r) { } } -static void rrdr2rrdr_group_by_calculate_percentage_of_instance(RRDR *r) { +static void rrdr2rrdr_group_by_calculate_percentage_of_group(RRDR *r) { if(!r->vh) return; + if(query_target_aggregatable(r->internal.qt) && query_has_group_by_aggregation_percentage(r->internal.qt)) + return; + for(size_t i = 0; i < r->n ;i++) { NETDATA_DOUBLE *cn = &r->v[ i * r->d ]; NETDATA_DOUBLE *ch = &r->vh[ i * r->d ]; @@ -3185,7 +3194,10 @@ static void rrdr2rrdr_group_by_calculate_percentage_of_instance(RRDR *r) { } } -static void rrd2rrdr_convert_to_percentage(RRDR *r) { +static void rrd2rrdr_convert_values_to_percentage_of_total(RRDR *r) { + if(!(r->internal.qt->window.options & RRDR_OPTION_PERCENTAGE) || query_target_aggregatable(r->internal.qt)) + return; + size_t global_min_max_values = 0; NETDATA_DOUBLE global_min = NAN, global_max = NAN; @@ -3279,19 +3291,17 @@ static void rrd2rrdr_convert_to_percentage(RRDR *r) { static RRDR *rrd2rrdr_group_by_finalize(RRDR *r_tmp) { QUERY_TARGET *qt = r_tmp->internal.qt; - RRDR_OPTIONS options = qt->window.options; if(!r_tmp->group_by.r) { // v1 query - if(options & RRDR_OPTION_PERCENTAGE) - rrd2rrdr_convert_to_percentage(r_tmp); + rrd2rrdr_convert_values_to_percentage_of_total(r_tmp); return r_tmp; } // v2 query // do the additional passes on RRDRs RRDR *last_r = r_tmp->group_by.r; - rrdr2rrdr_group_by_calculate_percentage_of_instance(last_r); + rrdr2rrdr_group_by_calculate_percentage_of_group(last_r); RRDR *r = last_r->group_by.r; size_t pass = 0; @@ -3302,7 +3312,7 @@ static RRDR *rrd2rrdr_group_by_finalize(RRDR *r_tmp) { qt->request.group_by[pass].aggregation, &last_r->dqp[d], pass); } - rrdr2rrdr_group_by_calculate_percentage_of_instance(r); + rrdr2rrdr_group_by_calculate_percentage_of_group(r); last_r = r; r = last_r->group_by.r; @@ -3324,7 +3334,7 @@ static RRDR *rrd2rrdr_group_by_finalize(RRDR *r_tmp) { if(qt->request.group_by[g].group_by != RRDR_GROUP_BY_NONE) aggregation = qt->request.group_by[g].aggregation; - if(!(options & RRDR_OPTION_RETURN_RAW) && r->partial_data_trimming.expected_after < qt->window.before) + if(!query_target_aggregatable(qt) && r->partial_data_trimming.expected_after < qt->window.before) rrdr2rrdr_group_by_partial_trimming(r); // apply averaging, remove RRDR_VALUE_EMPTY, find the non-zero dimensions, min and max @@ -3416,8 +3426,7 @@ static RRDR *rrd2rrdr_group_by_finalize(RRDR *r_tmp) { qt->window.options &= ~RRDR_OPTION_NONZERO; } - if(options & RRDR_OPTION_PERCENTAGE && !(options & RRDR_OPTION_RETURN_RAW)) - rrd2rrdr_convert_to_percentage(r); + rrd2rrdr_convert_values_to_percentage_of_total(r); // update query instance counts in query host and query context { diff --git a/web/api/queries/query.h b/web/api/queries/query.h index e6fdcfbe4..5eabb6c03 100644 --- a/web/api/queries/query.h +++ b/web/api/queries/query.h @@ -85,6 +85,7 @@ typedef enum rrdr_group_by_function { RRDR_GROUP_BY_FUNCTION_MIN, RRDR_GROUP_BY_FUNCTION_MAX, RRDR_GROUP_BY_FUNCTION_SUM, + RRDR_GROUP_BY_FUNCTION_PERCENTAGE, } RRDR_GROUP_BY_FUNCTION; RRDR_GROUP_BY_FUNCTION group_by_aggregate_function_parse(const char *s); diff --git a/web/api/web_api_v1.c b/web/api/web_api_v1.c index 6e23549d4..637329696 100644 --- a/web/api/web_api_v1.c +++ b/web/api/web_api_v1.c @@ -354,7 +354,7 @@ inline int web_client_api_request_v1_alarm_log(RRDHOST *host, struct web_client buffer_flush(w->response.data); w->response.data->content_type = CT_APPLICATION_JSON; - health_alarm_log2json(host, w->response.data, after, chart); + sql_health_alarm_log2json(host, w->response.data, after, chart); return HTTP_RESP_OK; } -- cgit v1.2.3