Merging upstream version 1.45.3+dfsg.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 0 insertions, 1652 deletions

diff --git a/health/health.c b/health/health.c
deleted file mode 100644
index c1a11167c..000000000
--- a/health/health.c
+++ /dev/null
@@ -1,1652 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#include "health.h"
-
-#define WORKER_HEALTH_JOB_RRD_LOCK              0
-#define WORKER_HEALTH_JOB_HOST_LOCK             1
-#define WORKER_HEALTH_JOB_DB_QUERY              2
-#define WORKER_HEALTH_JOB_CALC_EVAL             3
-#define WORKER_HEALTH_JOB_WARNING_EVAL          4
-#define WORKER_HEALTH_JOB_CRITICAL_EVAL         5
-#define WORKER_HEALTH_JOB_ALARM_LOG_ENTRY       6
-#define WORKER_HEALTH_JOB_ALARM_LOG_PROCESS     7
-#define WORKER_HEALTH_JOB_DELAYED_INIT_RRDSET   8
-#define WORKER_HEALTH_JOB_DELAYED_INIT_RRDDIM   9
-
-#if WORKER_UTILIZATION_MAX_JOB_TYPES < 10
-#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 10
-#endif
-
-unsigned int default_health_enabled = 1;
-char *silencers_filename;
-SIMPLE_PATTERN *conf_enabled_alarms = NULL;
-DICTIONARY *health_rrdvars;
-
-void health_entry_flags_to_json_array(BUFFER *wb, const char *key, HEALTH_ENTRY_FLAGS flags) {
-    buffer_json_member_add_array(wb, key);
-
-    if(flags & HEALTH_ENTRY_FLAG_PROCESSED)
-        buffer_json_add_array_item_string(wb, "PROCESSED");
-    if(flags & HEALTH_ENTRY_FLAG_UPDATED)
-        buffer_json_add_array_item_string(wb, "UPDATED");
-    if(flags & HEALTH_ENTRY_FLAG_EXEC_RUN)
-        buffer_json_add_array_item_string(wb, "EXEC_RUN");
-    if(flags & HEALTH_ENTRY_FLAG_EXEC_FAILED)
-        buffer_json_add_array_item_string(wb, "EXEC_FAILED");
-    if(flags & HEALTH_ENTRY_FLAG_SILENCED)
-        buffer_json_add_array_item_string(wb, "SILENCED");
-    if(flags & HEALTH_ENTRY_RUN_ONCE)
-        buffer_json_add_array_item_string(wb, "RUN_ONCE");
-    if(flags & HEALTH_ENTRY_FLAG_EXEC_IN_PROGRESS)
-        buffer_json_add_array_item_string(wb, "EXEC_IN_PROGRESS");
-    if(flags & HEALTH_ENTRY_FLAG_IS_REPEATING)
-        buffer_json_add_array_item_string(wb, "RECURRING");
-    if(flags & HEALTH_ENTRY_FLAG_SAVED)
-        buffer_json_add_array_item_string(wb, "SAVED");
-    if(flags & HEALTH_ENTRY_FLAG_ACLK_QUEUED)
-        buffer_json_add_array_item_string(wb, "ACLK_QUEUED");
-    if(flags & HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION)
-        buffer_json_add_array_item_string(wb, "NO_CLEAR_NOTIFICATION");
-
-    buffer_json_array_close(wb);
-}
-
-static bool prepare_command(BUFFER *wb,
-                            const char *exec,
-                            const char *recipient,
-                            const char *registry_hostname,
-                            uint32_t unique_id,
-                            uint32_t alarm_id,
-                            uint32_t alarm_event_id,
-                            uint32_t when,
-                            const char *alert_name,
-                            const char *alert_chart_name,
-                            const char *new_status,
-                            const char *old_status,
-                            NETDATA_DOUBLE new_value,
-                            NETDATA_DOUBLE old_value,
-                            const char *alert_source,
-                            uint32_t duration,
-                            uint32_t non_clear_duration,
-                            const char *alert_units,
-                            const char *alert_info,
-                            const char *new_value_string,
-                            const char *old_value_string,
-                            const char *source,
-                            const char *error_msg,
-                            int n_warn,
-                            int n_crit,
-                            const char *warn_alarms,
-                            const char *crit_alarms,
-                            const char *classification,
-                            const char *edit_command,
-                            const char *machine_guid,
-                            uuid_t *transition_id,
-                            const char *summary,
-                            const char *context,
-                            const char *component,
-                            const char *type
-) {
-    char buf[8192];
-    size_t n = sizeof(buf) - 1;
-
-    buffer_strcat(wb, "exec");
-
-    if (!sanitize_command_argument_string(buf, exec, n))
-        return false;
-    buffer_sprintf(wb, " '%s'", buf);
-
-    if (!sanitize_command_argument_string(buf, recipient, n))
-        return false;
-    buffer_sprintf(wb, " '%s'", buf);
-
-    if (!sanitize_command_argument_string(buf, registry_hostname, n))
-        return false;
-    buffer_sprintf(wb, " '%s'", buf);
-
-    buffer_sprintf(wb, " '%u'", unique_id);
-
-    buffer_sprintf(wb, " '%u'", alarm_id);
-
-    buffer_sprintf(wb, " '%u'", alarm_event_id);
-
-    buffer_sprintf(wb, " '%u'", when);
-
-    if (!sanitize_command_argument_string(buf, alert_name, n))
-        return false;
-    buffer_sprintf(wb, " '%s'", buf);
-
-    if (!sanitize_command_argument_string(buf, alert_chart_name, n))
-        return false;
-    buffer_sprintf(wb, " '%s'", buf);
-
-    if (!sanitize_command_argument_string(buf, new_status, n))
-        return false;
-    buffer_sprintf(wb, " '%s'", buf);
-
-    if (!sanitize_command_argument_string(buf, old_status, n))
-        return false;
-    buffer_sprintf(wb, " '%s'", buf);
-
-    buffer_sprintf(wb, " '" NETDATA_DOUBLE_FORMAT_ZERO "'", new_value);
-
-    buffer_sprintf(wb, " '" NETDATA_DOUBLE_FORMAT_ZERO "'", old_value);
-
-    if (!sanitize_command_argument_string(buf, alert_source, n))
-        return false;
-    buffer_sprintf(wb, " '%s'", buf);
-
-    buffer_sprintf(wb, " '%u'", duration);
-
-    buffer_sprintf(wb, " '%u'", non_clear_duration);
-
-    if (!sanitize_command_argument_string(buf, alert_units, n))
-        return false;
-    buffer_sprintf(wb, " '%s'", buf);
-
-    if (!sanitize_command_argument_string(buf, alert_info, n))
-        return false;
-    buffer_sprintf(wb, " '%s'", buf);
-
-    if (!sanitize_command_argument_string(buf, new_value_string, n))
-        return false;
-    buffer_sprintf(wb, " '%s'", buf);
-
-    if (!sanitize_command_argument_string(buf, old_value_string, n))
-        return false;
-    buffer_sprintf(wb, " '%s'", buf);
-
-    if (!sanitize_command_argument_string(buf, source, n))
-        return false;
-    buffer_sprintf(wb, " '%s'", buf);
-
-    if (!sanitize_command_argument_string(buf, error_msg, n))
-        return false;
-    buffer_sprintf(wb, " '%s'", buf);
-
-    buffer_sprintf(wb, " '%d'", n_warn);
-
-    buffer_sprintf(wb, " '%d'", n_crit);
-
-    if (!sanitize_command_argument_string(buf, warn_alarms, n))
-        return false;
-    buffer_sprintf(wb, " '%s'", buf);
-
-    if (!sanitize_command_argument_string(buf, crit_alarms, n))
-        return false;
-    buffer_sprintf(wb, " '%s'", buf);
-
-    if (!sanitize_command_argument_string(buf, classification, n))
-        return false;
-    buffer_sprintf(wb, " '%s'", buf);
-
-    if (!sanitize_command_argument_string(buf, edit_command, n))
-        return false;
-    buffer_sprintf(wb, " '%s'", buf);
-
-    if (!sanitize_command_argument_string(buf, machine_guid, n))
-        return false;
-    buffer_sprintf(wb, " '%s'", buf);
-
-    char tr_id[UUID_STR_LEN];
-    uuid_unparse_lower(*transition_id, tr_id);
-    if (!sanitize_command_argument_string(buf, tr_id, n))
-        return false;
-    buffer_sprintf(wb, " '%s'", buf);
-
-    if (!sanitize_command_argument_string(buf, summary, n))
-        return false;
-    buffer_sprintf(wb, " '%s'", buf);
-
-    if (!sanitize_command_argument_string(buf, context, n))
-        return false;
-    buffer_sprintf(wb, " '%s'", buf);
-
-    if (!sanitize_command_argument_string(buf, component, n))
-        return false;
-    buffer_sprintf(wb, " '%s'", buf);
-
-    if (!sanitize_command_argument_string(buf, type, n))
-        return false;
-    buffer_sprintf(wb, " '%s'", buf);
-
-    return true;
-}
-
-// the queue of executed alarm notifications that haven't been waited for yet
-static struct {
-    ALARM_ENTRY *head; // oldest
-    ALARM_ENTRY *tail; // latest
-} alarm_notifications_in_progress = {NULL, NULL};
-
-typedef struct active_alerts {
-    char *name;
-    time_t last_status_change;
-    RRDCALC_STATUS status;
-} active_alerts_t;
-
-static inline void enqueue_alarm_notify_in_progress(ALARM_ENTRY *ae)
-{
-    ae->prev_in_progress = NULL;
-    ae->next_in_progress = NULL;
-
-    if (NULL != alarm_notifications_in_progress.tail) {
-        ae->prev_in_progress = alarm_notifications_in_progress.tail;
-        alarm_notifications_in_progress.tail->next_in_progress = ae;
-    }
-    if (NULL == alarm_notifications_in_progress.head) {
-        alarm_notifications_in_progress.head = ae;
-    }
-    alarm_notifications_in_progress.tail = ae;
-
-}
-
-static inline void unlink_alarm_notify_in_progress(ALARM_ENTRY *ae)
-{
-    struct alarm_entry *prev = ae->prev_in_progress;
-    struct alarm_entry *next = ae->next_in_progress;
-
-    if (NULL != prev) {
-        prev->next_in_progress = next;
-    }
-    if (NULL != next) {
-        next->prev_in_progress = prev;
-    }
-    if (ae == alarm_notifications_in_progress.head) {
-        alarm_notifications_in_progress.head = next;
-    }
-    if (ae == alarm_notifications_in_progress.tail) {
-        alarm_notifications_in_progress.tail = prev;
-    }
-}
-// ----------------------------------------------------------------------------
-// health initialization
-
-/**
- * User Config directory
- *
- * Get the config directory for health and return it.
- *
- * @return a pointer to the user config directory
- */
-inline char *health_user_config_dir(void) {
-    char buffer[FILENAME_MAX + 1];
-    snprintfz(buffer, FILENAME_MAX, "%s/health.d", netdata_configured_user_config_dir);
-    return config_get(CONFIG_SECTION_DIRECTORIES, "health config", buffer);
-}
-
-/**
- * Stock Config Directory
- *
- * Get the Stock config directory and return it.
- *
- * @return a pointer to the stock config directory.
- */
-inline char *health_stock_config_dir(void) {
-    char buffer[FILENAME_MAX + 1];
-    snprintfz(buffer, FILENAME_MAX, "%s/health.d", netdata_configured_stock_config_dir);
-    return config_get(CONFIG_SECTION_DIRECTORIES, "stock health config", buffer);
-}
-
-/**
- * Silencers init
- *
- * Function used to initialize the silencer structure.
- */
-static void health_silencers_init(void) {
-    FILE *fd = fopen(silencers_filename, "r");
-    if (fd) {
-        fseek(fd, 0 , SEEK_END);
-        off_t length = (off_t) ftell(fd);
-        fseek(fd, 0 , SEEK_SET);
-
-        if (length > 0 && length < HEALTH_SILENCERS_MAX_FILE_LEN) {
-            char *str = mallocz((length+1)* sizeof(char));
-            if(str) {
-                size_t copied;
-                copied = fread(str, sizeof(char), length, fd);
-                if (copied == (length* sizeof(char))) {
-                    str[length] = 0x00;
-                    json_parse(str, NULL, health_silencers_json_read_callback);
-                    netdata_log_info("Parsed health silencers file %s", silencers_filename);
-                } else {
-                    netdata_log_error("Cannot read the data from health silencers file %s", silencers_filename);
-                }
-                freez(str);
-            }
-        } else {
-            netdata_log_error("Health silencers file %s has the size %" PRId64 " that is out of range[ 1 , %d ]. Aborting read.",
-                              silencers_filename,
-                              (int64_t)length,
-                              HEALTH_SILENCERS_MAX_FILE_LEN);
-        }
-        fclose(fd);
-    } else {
-        netdata_log_info("Cannot open the file %s, so Netdata will work with the default health configuration.",
-                         silencers_filename);
-    }
-}
-
-/**
- * Health Init
- *
- * Initialize the health thread.
- */
-void health_init(void) {
-    netdata_log_debug(D_HEALTH, "Health configuration initializing");
-
-    if(!(default_health_enabled = (unsigned int)config_get_boolean(CONFIG_SECTION_HEALTH, "enabled", default_health_enabled))) {
-        netdata_log_debug(D_HEALTH, "Health is disabled.");
-        return;
-    }
-
-    health_silencers_init();
-}
-
-// ----------------------------------------------------------------------------
-// re-load health configuration
-
-/**
- * Reload host
- *
- * Reload configuration for a specific host.
- *
- * @param host the structure of the host that the function will reload the configuration.
- */
-static void health_reload_host(RRDHOST *host) {
-    if(unlikely(!host->health.health_enabled) && !rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH))
-        return;
-
-    nd_log(NDLS_DAEMON, NDLP_DEBUG,
-           "[%s]: Reloading health.",
-           rrdhost_hostname(host));
-
-    char *user_path = health_user_config_dir();
-    char *stock_path = health_stock_config_dir();
-
-    // free all running alarms
-    rrdcalc_delete_all(host);
-    rrdcalctemplate_delete_all(host);
-
-    // invalidate all previous entries in the alarm log
-    rw_spinlock_read_lock(&host->health_log.spinlock);
-    ALARM_ENTRY *t;
-    for(t = host->health_log.alarms ; t ; t = t->next) {
-        if(t->new_status != RRDCALC_STATUS_REMOVED)
-            t->flags |= HEALTH_ENTRY_FLAG_UPDATED;
-    }
-    rw_spinlock_read_unlock(&host->health_log.spinlock);
-
-    // reset all thresholds to all charts
-    RRDSET *st;
-    rrdset_foreach_read(st, host) {
-        st->green = NAN;
-        st->red = NAN;
-    }
-    rrdset_foreach_done(st);
-
-    // load the new alarms
-    health_readdir(host, user_path, stock_path, NULL);
-
-    //Discard alarms with labels that do not apply to host
-    rrdcalc_delete_alerts_not_matching_host_labels_from_this_host(host);
-
-    // link the loaded alarms to their charts
-    rrdset_foreach_write(st, host) {
-        rrdcalc_link_matching_alerts_to_rrdset(st);
-        rrdcalctemplate_link_matching_templates_to_rrdset(st);
-    }
-    rrdset_foreach_done(st);
-
-#ifdef ENABLE_ACLK
-    if (netdata_cloud_enabled) {
-        struct aclk_sync_cfg_t *wc = host->aclk_config;
-        if (likely(wc)) {
-            wc->alert_queue_removed = SEND_REMOVED_AFTER_HEALTH_LOOPS;
-        }
-    }
-#endif
-}
-
-/**
- * Reload
- *
- * Reload the host configuration for all hosts.
- */
-void health_reload(void) {
-    sql_refresh_hashes();
-
-    RRDHOST *host;
-    dfe_start_reentrant(rrdhost_root_index, host){
-        health_reload_host(host);
-    }
-    dfe_done(host);
-}
-
-// ----------------------------------------------------------------------------
-// health main thread and friends
-
-static inline RRDCALC_STATUS rrdcalc_value2status(NETDATA_DOUBLE n) {
-    if(isnan(n) || isinf(n)) return RRDCALC_STATUS_UNDEFINED;
-    if(n) return RRDCALC_STATUS_RAISED;
-    return RRDCALC_STATUS_CLEAR;
-}
-
-#define ACTIVE_ALARMS_LIST_EXAMINE 500
-#define ACTIVE_ALARMS_LIST 15
-
-static inline int compare_active_alerts(const void * a, const void * b) {
-    active_alerts_t *active_alerts_a = (active_alerts_t *)a;
-    active_alerts_t *active_alerts_b = (active_alerts_t *)b;
-
-    return (int) ( active_alerts_b->last_status_change - active_alerts_a->last_status_change );
-}
-
-static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
-    ae->flags |= HEALTH_ENTRY_FLAG_PROCESSED;
-
-    if(unlikely(ae->new_status < RRDCALC_STATUS_CLEAR)) {
-        // do not send notifications for internal statuses
-        netdata_log_debug(D_HEALTH, "Health not sending notification for alarm '%s.%s' status %s (internal statuses)", ae_chart_id(ae), ae_name(ae), rrdcalc_status2string(ae->new_status));
-        goto done;
-    }
-
-    if(unlikely(ae->new_status <= RRDCALC_STATUS_CLEAR && (ae->flags & HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION))) {
-        // do not send notifications for disabled statuses
-
-        nd_log(NDLS_DAEMON, NDLP_DEBUG,
-               "[%s]: Health not sending notification for alarm '%s.%s' status %s (it has no-clear-notification enabled)",
-               rrdhost_hostname(host), ae_chart_id(ae), ae_name(ae), rrdcalc_status2string(ae->new_status));
-
-        // mark it as run, so that we will send the same alarm if it happens again
-        goto done;
-    }
-
-    // find the previous notification for the same alarm
-    // which we have run the exec script
-    // exception: alarms with HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION set
-    RRDCALC_STATUS last_executed_status = -3;
-    if(likely(!(ae->flags & HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION))) {
-        int ret = sql_health_get_last_executed_event(host, ae, &last_executed_status);
-
-        if (likely(ret == 1)) {
-            // we have executed this alarm notification in the past
-            if(last_executed_status == ae->new_status && !(ae->flags & HEALTH_ENTRY_FLAG_IS_REPEATING)) {
-                // don't send the notification for the same status again
-                nd_log(NDLS_DAEMON, NDLP_DEBUG,
-                       "[%s]: Health not sending again notification for alarm '%s.%s' status %s",
-                       rrdhost_hostname(host), ae_chart_id(ae), ae_name(ae),
-                       rrdcalc_status2string(ae->new_status));
-                goto done;
-            }
-        }
-        else {
-            // we have not executed this alarm notification in the past
-            // so, don't send CLEAR notifications
-            if(unlikely(ae->new_status == RRDCALC_STATUS_CLEAR)) {
-                if((!(ae->flags & HEALTH_ENTRY_RUN_ONCE)) || (ae->flags & HEALTH_ENTRY_RUN_ONCE && ae->old_status < RRDCALC_STATUS_RAISED) ) {
-                    netdata_log_debug(D_HEALTH, "Health not sending notification for first initialization of alarm '%s.%s' status %s"
-                    , ae_chart_id(ae), ae_name(ae), rrdcalc_status2string(ae->new_status));
-                    goto done;
-                }
-            }
-        }
-    }
-
-    // Check if alarm notifications are silenced
-    if (ae->flags & HEALTH_ENTRY_FLAG_SILENCED) {
-        nd_log(NDLS_DAEMON, NDLP_DEBUG,
-               "[%s]: Health not sending notification for alarm '%s.%s' status %s "
-               "(command API has disabled notifications)",
-               rrdhost_hostname(host), ae_chart_id(ae), ae_name(ae), rrdcalc_status2string(ae->new_status));
-        goto done;
-    }
-
-    nd_log(NDLS_DAEMON, NDLP_DEBUG,
-           "[%s]: Sending notification for alarm '%s.%s' status %s.",
-           rrdhost_hostname(host), ae_chart_id(ae), ae_name(ae), rrdcalc_status2string(ae->new_status));
-
-    const char *exec      = (ae->exec)      ? ae_exec(ae)      : string2str(host->health.health_default_exec);
-    const char *recipient = (ae->recipient) ? ae_recipient(ae) : string2str(host->health.health_default_recipient);
-
-    int n_warn=0, n_crit=0;
-    RRDCALC *rc;
-    EVAL_EXPRESSION *expr=NULL;
-    BUFFER *warn_alarms, *crit_alarms;
-    active_alerts_t *active_alerts = callocz(ACTIVE_ALARMS_LIST_EXAMINE, sizeof(active_alerts_t));
-
-    warn_alarms = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE, &netdata_buffers_statistics.buffers_health);
-    crit_alarms = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE, &netdata_buffers_statistics.buffers_health);
-
-    foreach_rrdcalc_in_rrdhost_read(host, rc) {
-        if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec))
-            continue;
-
-        if(unlikely((n_warn + n_crit) >= ACTIVE_ALARMS_LIST_EXAMINE))
-            break;
-
-        if (unlikely(rc->status == RRDCALC_STATUS_WARNING)) {
-            if (likely(ae->alarm_id != rc->id) || likely(ae->alarm_event_id != rc->next_event_id - 1)) {
-                active_alerts[n_warn+n_crit].name = (char *)rrdcalc_name(rc);
-                active_alerts[n_warn+n_crit].last_status_change = rc->last_status_change;
-                active_alerts[n_warn+n_crit].status = rc->status;
-                n_warn++;
-            } else if (ae->alarm_id == rc->id)
-                expr = rc->warning;
-        } else if (unlikely(rc->status == RRDCALC_STATUS_CRITICAL)) {
-            if (likely(ae->alarm_id != rc->id) || likely(ae->alarm_event_id != rc->next_event_id - 1)) {
-                active_alerts[n_warn+n_crit].name = (char *)rrdcalc_name(rc);
-                active_alerts[n_warn+n_crit].last_status_change = rc->last_status_change;
-                active_alerts[n_warn+n_crit].status = rc->status;
-                n_crit++;
-            } else if (ae->alarm_id == rc->id)
-                expr = rc->critical;
-        } else if (unlikely(rc->status == RRDCALC_STATUS_CLEAR)) {
-            if (ae->alarm_id == rc->id)
-                expr = rc->warning;
-        }
-    }
-    foreach_rrdcalc_in_rrdhost_done(rc);
-
-    if (n_warn+n_crit>1)
-        qsort (active_alerts, n_warn+n_crit, sizeof(active_alerts_t), compare_active_alerts);
-
-    int count_w = 0, count_c = 0;
-    while (count_w + count_c < n_warn + n_crit && count_w + count_c < ACTIVE_ALARMS_LIST) {
-        if (active_alerts[count_w+count_c].status == RRDCALC_STATUS_WARNING) {
-            if (count_w)
-                buffer_strcat(warn_alarms, ",");
-            buffer_strcat(warn_alarms, active_alerts[count_w+count_c].name);
-            buffer_strcat(warn_alarms, "=");
-            buffer_snprintf(warn_alarms, 11, "%"PRId64"", (int64_t)active_alerts[count_w+count_c].last_status_change);
-            count_w++;
-        }
-        else if (active_alerts[count_w+count_c].status == RRDCALC_STATUS_CRITICAL) {
-            if (count_c)
-                buffer_strcat(crit_alarms, ",");
-            buffer_strcat(crit_alarms, active_alerts[count_w+count_c].name);
-            buffer_strcat(crit_alarms, "=");
-            buffer_snprintf(crit_alarms, 11, "%"PRId64"", (int64_t)active_alerts[count_w+count_c].last_status_change);
-            count_c++;
-        }
-    }
-
-    char *edit_command = ae->source ? health_edit_command_from_source(ae_source(ae)) : strdupz("UNKNOWN=0=UNKNOWN");
-
-    BUFFER *wb = buffer_create(8192, &netdata_buffers_statistics.buffers_health);
-    bool ok = prepare_command(wb,
-                              exec,
-                              recipient,
-                              rrdhost_registry_hostname(host),
-                              ae->unique_id,
-                              ae->alarm_id,
-                              ae->alarm_event_id,
-                              (unsigned long)ae->when,
-                              ae_name(ae),
-                              ae->chart?ae_chart_id(ae):"NOCHART",
-                              rrdcalc_status2string(ae->new_status),
-                              rrdcalc_status2string(ae->old_status),
-                              ae->new_value,
-                              ae->old_value,
-                              ae->source?ae_source(ae):"UNKNOWN",
-                              (uint32_t)ae->duration,
-                              (ae->flags & HEALTH_ENTRY_FLAG_IS_REPEATING && ae->new_status >= RRDCALC_STATUS_WARNING) ? (uint32_t)ae->duration : (uint32_t)ae->non_clear_duration,
-                              ae_units(ae),
-                              ae_info(ae),
-                              ae_new_value_string(ae),
-                              ae_old_value_string(ae),
-                              (expr && expr->source)?expr->source:"NOSOURCE",
-                              (expr && expr->error_msg)?buffer_tostring(expr->error_msg):"NOERRMSG",
-                              n_warn,
-                              n_crit,
-                              buffer_tostring(warn_alarms),
-                              buffer_tostring(crit_alarms),
-                              ae->classification?ae_classification(ae):"Unknown",
-                              edit_command,
-                              host->machine_guid,
-                              &ae->transition_id,
-                              host->health.use_summary_for_notifications && ae->summary?ae_summary(ae):ae_name(ae),
-                              string2str(ae->chart_context),
-                              string2str(ae->component),
-                              string2str(ae->type)
-                              );
-
-    const char *command_to_run = buffer_tostring(wb);
-    if (ok) {
-        ae->flags |= HEALTH_ENTRY_FLAG_EXEC_RUN;
-        ae->exec_run_timestamp = now_realtime_sec(); /* will be updated by real time after spawning */
-
-        netdata_log_debug(D_HEALTH, "executing command '%s'", command_to_run);
-        ae->flags |= HEALTH_ENTRY_FLAG_EXEC_IN_PROGRESS;
-        ae->exec_spawn_serial = spawn_enq_cmd(command_to_run);
-        enqueue_alarm_notify_in_progress(ae);
-        health_alarm_log_save(host, ae);
-    } else {
-        netdata_log_error("Failed to format command arguments");
-    }
-
-    buffer_free(wb);
-    freez(edit_command);
-    buffer_free(warn_alarms);
-    buffer_free(crit_alarms);
-    freez(active_alerts);
-
-    return; //health_alarm_wait_for_execution
-done:
-    health_alarm_log_save(host, ae);
-}
-
-static inline void health_alarm_wait_for_execution(ALARM_ENTRY *ae) {
-    if (!(ae->flags & HEALTH_ENTRY_FLAG_EXEC_IN_PROGRESS))
-        return;
-
-    spawn_wait_cmd(ae->exec_spawn_serial, &ae->exec_code, &ae->exec_run_timestamp);
-    netdata_log_debug(D_HEALTH, "done executing command - returned with code %d", ae->exec_code);
-    ae->flags &= ~HEALTH_ENTRY_FLAG_EXEC_IN_PROGRESS;
-
-    if(ae->exec_code != 0)
-        ae->flags |= HEALTH_ENTRY_FLAG_EXEC_FAILED;
-
-    unlink_alarm_notify_in_progress(ae);
-}
-
-static inline void health_process_notifications(RRDHOST *host, ALARM_ENTRY *ae) {
-    netdata_log_debug(D_HEALTH, "Health alarm '%s.%s' = " NETDATA_DOUBLE_FORMAT_AUTO " - changed status from %s to %s",
-         ae->chart?ae_chart_id(ae):"NOCHART", ae_name(ae),
-         ae->new_value,
-         rrdcalc_status2string(ae->old_status),
-         rrdcalc_status2string(ae->new_status)
-    );
-
-    health_alarm_execute(host, ae);
-}
-
-static inline void health_alarm_log_process(RRDHOST *host) {
-    uint32_t first_waiting = (host->health_log.alarms)?host->health_log.alarms->unique_id:0;
-    time_t now = now_realtime_sec();
-
-    rw_spinlock_read_lock(&host->health_log.spinlock);
-
-    ALARM_ENTRY *ae;
-    for(ae = host->health_log.alarms; ae && ae->unique_id >= host->health_last_processed_id; ae = ae->next) {
-        if(unlikely(
-                    !(ae->flags & HEALTH_ENTRY_FLAG_PROCESSED) &&
-                    !(ae->flags & HEALTH_ENTRY_FLAG_UPDATED)
-                    )) {
-            if(unlikely(ae->unique_id < first_waiting))
-                first_waiting = ae->unique_id;
-
-            if(likely(now >= ae->delay_up_to_timestamp))
-                health_process_notifications(host, ae);
-        }
-    }
-
-    rw_spinlock_read_unlock(&host->health_log.spinlock);
-
-    // remember this for the next iteration
-    host->health_last_processed_id = first_waiting;
-
-    //delete those that are updated, no in progress execution, and is not repeating
-    rw_spinlock_write_lock(&host->health_log.spinlock);
-
-    ALARM_ENTRY *prev = NULL, *next = NULL;
-    for(ae = host->health_log.alarms; ae ; ae = next) {
-        next = ae->next; // set it here, for the next iteration
-
-        if((likely(!(ae->flags & HEALTH_ENTRY_FLAG_IS_REPEATING)) &&
-           (ae->flags & HEALTH_ENTRY_FLAG_UPDATED) &&
-           (ae->flags & HEALTH_ENTRY_FLAG_SAVED) &&
-           !(ae->flags & HEALTH_ENTRY_FLAG_EXEC_IN_PROGRESS))
-            ||
-           ((ae->new_status == RRDCALC_STATUS_REMOVED) &&
-           (ae->flags & HEALTH_ENTRY_FLAG_SAVED) &&
-           (ae->when + 86400 < now_realtime_sec())))
-            {
-
-            if(host->health_log.alarms == ae) {
-                host->health_log.alarms = next;
-                // prev is also NULL here
-            }
-            else {
-                prev->next = next;
-                // prev should not be touched here - we need it for the next iteration
-                // because we may have to also remove the next item
-            }
-
-            health_alarm_log_free_one_nochecks_nounlink(ae);
-        }
-        else
-            prev = ae;
-    }
-
-    rw_spinlock_write_unlock(&host->health_log.spinlock);
-}
-
-static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run) {
-    if(unlikely(!rc->rrdset)) {
-        netdata_log_debug(D_HEALTH, "Health not running alarm '%s.%s'. It is not linked to a chart.", rrdcalc_chart_name(rc), rrdcalc_name(rc));
-        return 0;
-    }
-
-    if(unlikely(rc->next_update > now)) {
-        if (unlikely(*next_run > rc->next_update)) {
-            // update the next_run time of the main loop
-            // to run this alarm precisely the time required
-            *next_run = rc->next_update;
-        }
-
-        netdata_log_debug(D_HEALTH, "Health not examining alarm '%s.%s' yet (will do in %d secs).", rrdcalc_chart_name(rc), rrdcalc_name(rc), (int) (rc->next_update - now));
-        return 0;
-    }
-
-    if(unlikely(!rc->update_every)) {
-        netdata_log_debug(D_HEALTH, "Health not running alarm '%s.%s'. It does not have an update frequency", rrdcalc_chart_name(rc), rrdcalc_name(rc));
-        return 0;
-    }
-
-    if(unlikely(rrdset_flag_check(rc->rrdset, RRDSET_FLAG_OBSOLETE))) {
-        netdata_log_debug(D_HEALTH, "Health not running alarm '%s.%s'. The chart has been marked as obsolete", rrdcalc_chart_name(rc), rrdcalc_name(rc));
-        return 0;
-    }
-
-    if(unlikely(!rc->rrdset->last_collected_time.tv_sec || rc->rrdset->counter_done < 2)) {
-        netdata_log_debug(D_HEALTH, "Health not running alarm '%s.%s'. Chart is not fully collected yet.", rrdcalc_chart_name(rc), rrdcalc_name(rc));
-        return 0;
-    }
-
-    int update_every = rc->rrdset->update_every;
-    time_t first = rrdset_first_entry_s(rc->rrdset);
-    time_t last = rrdset_last_entry_s(rc->rrdset);
-
-    if(unlikely(now + update_every < first /* || now - update_every > last */)) {
-        netdata_log_debug(D_HEALTH
-              , "Health not examining alarm '%s.%s' yet (wanted time is out of bounds - we need %lu but got %lu - %lu)."
-              , rrdcalc_chart_name(rc), rrdcalc_name(rc), (unsigned long) now, (unsigned long) first
-              , (unsigned long) last);
-        return 0;
-    }
-
-    if(RRDCALC_HAS_DB_LOOKUP(rc)) {
-        time_t needed = now + rc->before + rc->after;
-
-        if(needed + update_every < first || needed - update_every > last) {
-            netdata_log_debug(D_HEALTH
-                  , "Health not examining alarm '%s.%s' yet (not enough data yet - we need %lu but got %lu - %lu)."
-                  , rrdcalc_chart_name(rc), rrdcalc_name(rc), (unsigned long) needed, (unsigned long) first
-                  , (unsigned long) last);
-            return 0;
-        }
-    }
-
-    return 1;
-}
-
-static inline int check_if_resumed_from_suspension(void) {
-    static usec_t last_realtime = 0, last_monotonic = 0;
-    usec_t realtime = now_realtime_usec(), monotonic = now_monotonic_usec();
-    int ret = 0;
-
-    // detect if monotonic and realtime have twice the difference
-    // in which case we assume the system was just waken from hibernation
-
-    if(last_realtime && last_monotonic && realtime - last_realtime > 2 * (monotonic - last_monotonic))
-        ret = 1;
-
-    last_realtime = realtime;
-    last_monotonic = monotonic;
-
-    return ret;
-}
-
-static void health_main_cleanup(void *ptr) {
-    worker_unregister();
-
-    struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
-    static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
-    netdata_log_info("cleaning up...");
-    static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
-
-    nd_log(NDLS_DAEMON, NDLP_DEBUG,
-           "Health thread ended.");
-}
-
-static void initialize_health(RRDHOST *host)
-{
-    if(!host->health.health_enabled ||
-        rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH) ||
-        !service_running(SERVICE_HEALTH))
-        return;
-
-    rrdhost_flag_set(host, RRDHOST_FLAG_INITIALIZED_HEALTH);
-
-    nd_log(NDLS_DAEMON, NDLP_DEBUG,
-           "[%s]: Initializing health.",
-           rrdhost_hostname(host));
-
-    host->health.health_default_warn_repeat_every = config_get_duration(CONFIG_SECTION_HEALTH, "default repeat warning", "never");
-    host->health.health_default_crit_repeat_every = config_get_duration(CONFIG_SECTION_HEALTH, "default repeat critical", "never");
-
-    host->health_log.next_log_id = 1;
-    host->health_log.next_alarm_id = 1;
-    host->health_log.max = 1000;
-    host->health_log.next_log_id = (uint32_t)now_realtime_sec();
-    host->health_log.next_alarm_id = 0;
-
-    long n = config_get_number(CONFIG_SECTION_HEALTH, "in memory max health log entries", host->health_log.max);
-    if(n < 10) {
-        nd_log(NDLS_DAEMON, NDLP_WARNING,
-               "Host '%s': health configuration has invalid max log entries %ld. "
-               "Using default %u",
-               rrdhost_hostname(host), n, host->health_log.max);
-
-        config_set_number(CONFIG_SECTION_HEALTH, "in memory max health log entries", (long)host->health_log.max);
-    }
-    else
-        host->health_log.max = (unsigned int)n;
-
-    uint32_t m = config_get_number(CONFIG_SECTION_HEALTH, "health log history", HEALTH_LOG_DEFAULT_HISTORY);
-    if (m < HEALTH_LOG_MINIMUM_HISTORY) {
-        nd_log(NDLS_DAEMON, NDLP_WARNING,
-               "Host '%s': health configuration has invalid health log history %u. "
-               "Using minimum %d",
-               rrdhost_hostname(host), m, HEALTH_LOG_MINIMUM_HISTORY);
-
-        config_set_number(CONFIG_SECTION_HEALTH, "health log history", HEALTH_LOG_MINIMUM_HISTORY);
-        m = HEALTH_LOG_MINIMUM_HISTORY;
-    }
-
-    //default health log history is 5 days and not less than a day
-    if (host->health_log.health_log_history) {
-        if (host->health_log.health_log_history < HEALTH_LOG_MINIMUM_HISTORY)
-            host->health_log.health_log_history = HEALTH_LOG_MINIMUM_HISTORY;
-    } else
-        host->health_log.health_log_history = m;
-
-    nd_log(NDLS_DAEMON, NDLP_DEBUG,
-           "[%s]: Health log history is set to %u seconds (%u days)",
-           rrdhost_hostname(host), host->health_log.health_log_history, host->health_log.health_log_history / 86400);
-
-    conf_enabled_alarms = simple_pattern_create(config_get(CONFIG_SECTION_HEALTH, "enabled alarms", "*"), NULL,
-                                                SIMPLE_PATTERN_EXACT, true);
-
-    rw_spinlock_init(&host->health_log.spinlock);
-
-    char filename[FILENAME_MAX + 1];
-
-    snprintfz(filename, FILENAME_MAX, "%s/alarm-notify.sh", netdata_configured_primary_plugins_dir);
-    host->health.health_default_exec = string_strdupz(config_get(CONFIG_SECTION_HEALTH, "script to execute on alarm", filename));
-    host->health.health_default_recipient = string_strdupz("root");
-    host->health.use_summary_for_notifications = config_get_boolean(CONFIG_SECTION_HEALTH, "use summary for notifications", CONFIG_BOOLEAN_YES);
-
-    sql_health_alarm_log_load(host);
-
-    // ------------------------------------------------------------------------
-    // load health configuration
-
-    health_readdir(host, health_user_config_dir(), health_stock_config_dir(), NULL);
-
-    // link the loaded alarms to their charts
-    RRDSET *st;
-    rrdset_foreach_reentrant(st, host) {
-        rrdcalc_link_matching_alerts_to_rrdset(st);
-        rrdcalctemplate_link_matching_templates_to_rrdset(st);
-    }
-    rrdset_foreach_done(st);
-
-    //Discard alarms with labels that do not apply to host
-    rrdcalc_delete_alerts_not_matching_host_labels_from_this_host(host);
-}
-
-static void health_sleep(time_t next_run, unsigned int loop __maybe_unused) {
-    time_t now = now_realtime_sec();
-    if(now < next_run) {
-        worker_is_idle();
-        netdata_log_debug(D_HEALTH, "Health monitoring iteration no %u done. Next iteration in %d secs", loop, (int) (next_run - now));
-        while (now < next_run && service_running(SERVICE_HEALTH)) {
-            sleep_usec(USEC_PER_SEC);
-            now = now_realtime_sec();
-        }
-    }
-    else {
-        netdata_log_debug(D_HEALTH, "Health monitoring iteration no %u done. Next iteration now", loop);
-    }
-}
-
-static SILENCE_TYPE check_silenced(RRDCALC *rc, const char *host)
-{
-    SILENCER *s;
-
-    for (s = silencers->silencers; s!=NULL; s=s->next){
-        if (
-                (!s->alarms_pattern || (rc->name && s->alarms_pattern && simple_pattern_matches_string(s->alarms_pattern, rc->name))) &&
-                (!s->contexts_pattern || (rc->rrdset && rc->rrdset->context && s->contexts_pattern && simple_pattern_matches_string(s->contexts_pattern, rc->rrdset->context))) &&
-                (!s->hosts_pattern || (host && s->hosts_pattern && simple_pattern_matches(s->hosts_pattern, host))) &&
-                (!s->charts_pattern || (rc->chart && s->charts_pattern && simple_pattern_matches_string(s->charts_pattern, rc->chart)))
-                ) {
-            netdata_log_debug(D_HEALTH, "Alarm matches command API silence entry %s:%s:%s:%s", s->alarms,s->charts, s->contexts, s->hosts);
-            if (unlikely(silencers->stype == STYPE_NONE)) {
-                netdata_log_debug(D_HEALTH, "Alarm %s matched a silence entry, but no SILENCE or DISABLE command was issued via the command API. The match has no effect.", rrdcalc_name(rc));
-            } else {
-                netdata_log_debug(D_HEALTH, "Alarm %s via the command API - name:%s context:%s chart:%s host:%s"
-                        , (silencers->stype == STYPE_DISABLE_ALARMS)?"Disabled":"Silenced"
-                        , rrdcalc_name(rc)
-                        , (rc->rrdset)?rrdset_context(rc->rrdset):""
-                        , rrdcalc_chart_name(rc)
-                        , host
-                        );
-            }
-            return silencers->stype;
-        }
-    }
-    return STYPE_NONE;
-}
-
-/**
- * Update Disabled Silenced
- *
- * Update the variable rrdcalc_flags of the structure RRDCALC according with the values of the host structure
- *
- * @param host structure that contains information about the host monitored.
- * @param rc structure with information about the alarm
- *
- * @return It returns 1 case rrdcalc_flags is DISABLED or 0 otherwise
- */
-static int update_disabled_silenced(RRDHOST *host, RRDCALC *rc) {
-    uint32_t rrdcalc_flags_old = rc->run_flags;
-    // Clear the flags
-    rc->run_flags &= ~(RRDCALC_FLAG_DISABLED | RRDCALC_FLAG_SILENCED);
-    if (unlikely(silencers->all_alarms)) {
-        if (silencers->stype == STYPE_DISABLE_ALARMS) rc->run_flags |= RRDCALC_FLAG_DISABLED;
-        else if (silencers->stype == STYPE_SILENCE_NOTIFICATIONS) rc->run_flags |= RRDCALC_FLAG_SILENCED;
-    } else {
-        SILENCE_TYPE st = check_silenced(rc, rrdhost_hostname(host));
-        if (st == STYPE_DISABLE_ALARMS) rc->run_flags |= RRDCALC_FLAG_DISABLED;
-        else if (st == STYPE_SILENCE_NOTIFICATIONS) rc->run_flags |= RRDCALC_FLAG_SILENCED;
-    }
-
-    if (rrdcalc_flags_old != rc->run_flags) {
-        netdata_log_info(
-            "Alarm silencing changed for host '%s' alarm '%s': Disabled %s->%s Silenced %s->%s",
-            rrdhost_hostname(host),
-            rrdcalc_name(rc),
-            (rrdcalc_flags_old & RRDCALC_FLAG_DISABLED) ? "true" : "false",
-            (rc->run_flags & RRDCALC_FLAG_DISABLED) ? "true" : "false",
-            (rrdcalc_flags_old & RRDCALC_FLAG_SILENCED) ? "true" : "false",
-            (rc->run_flags & RRDCALC_FLAG_SILENCED) ? "true" : "false");
-    }
-    if (rc->run_flags & RRDCALC_FLAG_DISABLED)
-        return 1;
-    else
-        return 0;
-}
-
-static void sql_health_postpone_queue_removed(RRDHOST *host __maybe_unused) {
-#ifdef ENABLE_ACLK
-    if (netdata_cloud_enabled) {
-        struct aclk_sync_cfg_t *wc = host->aclk_config;
-        if (unlikely(!wc)) {
-            return;
-        }
-
-        if (wc->alert_queue_removed >= 1) {
-            wc->alert_queue_removed+=6;
-        }
-    }
-#endif
-}
-
-static void health_execute_delayed_initializations(RRDHOST *host) {
-    RRDSET *st;
-    bool must_postpone = false;
-
-    if (!rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION)) return;
-    rrdhost_flag_clear(host, RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION);
-
-    rrdset_foreach_reentrant(st, host) {
-        if(!rrdset_flag_check(st, RRDSET_FLAG_PENDING_HEALTH_INITIALIZATION)) continue;
-        rrdset_flag_clear(st, RRDSET_FLAG_PENDING_HEALTH_INITIALIZATION);
-
-        worker_is_busy(WORKER_HEALTH_JOB_DELAYED_INIT_RRDSET);
-
-        rrdcalc_link_matching_alerts_to_rrdset(st);
-        rrdcalctemplate_link_matching_templates_to_rrdset(st);
-
-        RRDDIM *rd;
-        rrddim_foreach_read(rd, st) {
-            if(!rrddim_flag_check(rd, RRDDIM_FLAG_PENDING_HEALTH_INITIALIZATION)) continue;
-            rrddim_flag_clear(rd, RRDDIM_FLAG_PENDING_HEALTH_INITIALIZATION);
-
-            worker_is_busy(WORKER_HEALTH_JOB_DELAYED_INIT_RRDDIM);
-
-            RRDCALCTEMPLATE *rt;
-            foreach_rrdcalctemplate_read(host, rt) {
-                if(!rt->foreach_dimension_pattern)
-                    continue;
-
-                if(rrdcalctemplate_check_rrdset_conditions(rt, st, host)) {
-                    rrdcalctemplate_check_rrddim_conditions_and_link(rt, st, rd, host);
-                }
-            }
-            foreach_rrdcalctemplate_done(rt);
-
-            if (health_variable_check(health_rrdvars, st, rd) || rrdset_flag_check(st, RRDSET_FLAG_HAS_RRDCALC_LINKED))
-                rrdvar_store_for_chart(host, st);
-        }
-        rrddim_foreach_done(rd);
-        must_postpone = true;
-    }
-    rrdset_foreach_done(st);
-    if (must_postpone)
-        sql_health_postpone_queue_removed(host);
-}
-
-/**
- * Health Main
- *
- * The main thread of the health system. In this function all the alarms will be processed.
- *
- * @param ptr is a pointer to the netdata_static_thread structure.
- *
- * @return It always returns NULL
- */
-
-void *health_main(void *ptr) {
-    worker_register("HEALTH");
-    worker_register_job_name(WORKER_HEALTH_JOB_RRD_LOCK, "rrd lock");
-    worker_register_job_name(WORKER_HEALTH_JOB_HOST_LOCK, "host lock");
-    worker_register_job_name(WORKER_HEALTH_JOB_DB_QUERY, "db lookup");
-    worker_register_job_name(WORKER_HEALTH_JOB_CALC_EVAL, "calc eval");
-    worker_register_job_name(WORKER_HEALTH_JOB_WARNING_EVAL, "warning eval");
-    worker_register_job_name(WORKER_HEALTH_JOB_CRITICAL_EVAL, "critical eval");
-    worker_register_job_name(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY, "alarm log entry");
-    worker_register_job_name(WORKER_HEALTH_JOB_ALARM_LOG_PROCESS, "alarm log process");
-    worker_register_job_name(WORKER_HEALTH_JOB_DELAYED_INIT_RRDSET, "rrdset init");
-    worker_register_job_name(WORKER_HEALTH_JOB_DELAYED_INIT_RRDDIM, "rrddim init");
-
-    netdata_thread_cleanup_push(health_main_cleanup, ptr);
-
-    int min_run_every = (int)config_get_number(CONFIG_SECTION_HEALTH, "run at least every seconds", 10);
-    if(min_run_every < 1) min_run_every = 1;
-
-    time_t hibernation_delay  = config_get_number(CONFIG_SECTION_HEALTH, "postpone alarms during hibernation for seconds", 60);
-
-    bool health_running_logged = false;
-
-    rrdcalc_delete_alerts_not_matching_host_labels_from_all_hosts();
-
-    unsigned int loop = 0;
-
-    while(service_running(SERVICE_HEALTH)) {
-        loop++;
-        netdata_log_debug(D_HEALTH, "Health monitoring iteration no %u started", loop);
-
-        time_t now = now_realtime_sec();
-        int runnable = 0, apply_hibernation_delay = 0;
-        time_t next_run = now + min_run_every;
-        RRDCALC *rc;
-        RRDHOST *host;
-
-        if (unlikely(check_if_resumed_from_suspension())) {
-            apply_hibernation_delay = 1;
-
-            nd_log(NDLS_DAEMON, NDLP_NOTICE,
-                       "Postponing alarm checks for %"PRId64" seconds, "
-                       "because it seems that the system was just resumed from suspension.",
-                       (int64_t)hibernation_delay);
-        }
-
-        if (unlikely(silencers->all_alarms && silencers->stype == STYPE_DISABLE_ALARMS)) {
-            static int logged=0;
-            if (!logged) {
-                nd_log(NDLS_DAEMON, NDLP_DEBUG,
-                       "Skipping health checks, because all alarms are disabled via a %s command.",
-                       HEALTH_CMDAPI_CMD_DISABLEALL);
-                logged = 1;
-            }
-        }
-
-        worker_is_busy(WORKER_HEALTH_JOB_RRD_LOCK);
-        dfe_start_reentrant(rrdhost_root_index, host) {
-
-            if(unlikely(!service_running(SERVICE_HEALTH)))
-                break;
-
-            if (unlikely(!host->health.health_enabled))
-                continue;
-
-            if (unlikely(!rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH)))
-                initialize_health(host);
-
-            health_execute_delayed_initializations(host);
-
-            rrdcalc_delete_alerts_not_matching_host_labels_from_this_host(host);
-
-            if (unlikely(apply_hibernation_delay)) {
-                nd_log(NDLS_DAEMON, NDLP_DEBUG,
-                           "[%s]: Postponing health checks for %"PRId64" seconds.",
-                           rrdhost_hostname(host),
-                           (int64_t)hibernation_delay);
-
-                host->health.health_delay_up_to = now + hibernation_delay;
-            }
-
-            if (unlikely(host->health.health_delay_up_to)) {
-                if (unlikely(now < host->health.health_delay_up_to)) {
-                    continue;
-                }
-
-                nd_log(NDLS_DAEMON, NDLP_DEBUG,
-                       "[%s]: Resuming health checks after delay.",
-                       rrdhost_hostname(host));
-
-                host->health.health_delay_up_to = 0;
-            }
-
-            // wait until cleanup of obsolete charts on children is complete
-            if (host != localhost) {
-                if (unlikely(host->trigger_chart_obsoletion_check == 1)) {
-
-                    nd_log(NDLS_DAEMON, NDLP_DEBUG,
-                           "[%s]: Waiting for chart obsoletion check.",
-                           rrdhost_hostname(host));
-
-                    continue;
-                }
-            }
-
-            if (!health_running_logged) {
-                nd_log(NDLS_DAEMON, NDLP_DEBUG,
-                       "[%s]: Health is running.",
-                       rrdhost_hostname(host));
-
-                health_running_logged = true;
-            }
-
-            worker_is_busy(WORKER_HEALTH_JOB_HOST_LOCK);
-
-            // the first loop is to lookup values from the db
-            foreach_rrdcalc_in_rrdhost_read(host, rc) {
-
-                if(unlikely(!service_running(SERVICE_HEALTH)))
-                    break;
-
-                rrdcalc_update_info_using_rrdset_labels(rc);
-
-                if (update_disabled_silenced(host, rc))
-                    continue;
-
-                // create an alert removed event if the chart is obsolete and
-                // has stopped being collected for 60 seconds
-                if (unlikely(rc->rrdset && rc->status != RRDCALC_STATUS_REMOVED &&
-                             rrdset_flag_check(rc->rrdset, RRDSET_FLAG_OBSOLETE) &&
-                             now > (rc->rrdset->last_collected_time.tv_sec + 60))) {
-                    if (!rrdcalc_isrepeating(rc)) {
-                        worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
-                        time_t now = now_realtime_sec();
-
-                        ALARM_ENTRY *ae = health_create_alarm_entry(
-                                                                    host,
-                                                                    rc->id,
-                                                                    rc->next_event_id++,
-                                                                    rc->config_hash_id,
-                                                                    now,
-                                                                    rc->name,
-                                                                    rc->rrdset->id,
-                                                                    rc->rrdset->context,
-                                                                    rc->rrdset->name,
-                                                                    rc->classification,
-                                                                    rc->component,
-                                                                    rc->type,
-                                                                    rc->exec,
-                                                                    rc->recipient,
-                                                                    now - rc->last_status_change,
-                                                                    rc->value,
-                                                                    NAN,
-                                                                    rc->status,
-                                                                    RRDCALC_STATUS_REMOVED,
-                                                                    rc->source,
-                                                                    rc->units,
-                                                                    rc->summary,
-                                                                    rc->info,
-                                                                    0,
-                                                                    rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0);
-
-                        if (ae) {
-                            health_log_alert(host, ae);
-                            health_alarm_log_add_entry(host, ae);
-                            rc->old_status = rc->status;
-                            rc->status = RRDCALC_STATUS_REMOVED;
-                            rc->last_status_change = now;
-                            rc->last_status_change_value = rc->value;
-                            rc->last_updated = now;
-                            rc->value = NAN;
-                            rc->ae = ae;
-
-#ifdef ENABLE_ACLK
-                            if (netdata_cloud_enabled)
-                                sql_queue_alarm_to_aclk(host, ae, true);
-#endif
-                        }
-                    }
-                }
-
-                if (unlikely(!rrdcalc_isrunnable(rc, now, &next_run))) {
-                    if (unlikely(rc->run_flags & RRDCALC_FLAG_RUNNABLE))
-                        rc->run_flags &= ~RRDCALC_FLAG_RUNNABLE;
-                    continue;
-                }
-
-                runnable++;
-                rc->old_value = rc->value;
-                rc->run_flags |= RRDCALC_FLAG_RUNNABLE;
-
-                // ------------------------------------------------------------
-                // if there is database lookup, do it
-
-                if (unlikely(RRDCALC_HAS_DB_LOOKUP(rc))) {
-                    worker_is_busy(WORKER_HEALTH_JOB_DB_QUERY);
-
-                    /* time_t old_db_timestamp = rc->db_before; */
-                    int value_is_null = 0;
-
-                    int ret = rrdset2value_api_v1(rc->rrdset, NULL, &rc->value, rrdcalc_dimensions(rc), 1,
-                                                  rc->after, rc->before, rc->group, NULL,
-                                                  0, rc->options | RRDR_OPTION_SELECTED_TIER,
-                                                  &rc->db_after,&rc->db_before,
-                                                  NULL, NULL, NULL,
-                                                  &value_is_null, NULL, 0, 0,
-                                                  QUERY_SOURCE_HEALTH, STORAGE_PRIORITY_LOW);
-
-                    if (unlikely(ret != 200)) {
-                        // database lookup failed
-                        rc->value = NAN;
-                        rc->run_flags |= RRDCALC_FLAG_DB_ERROR;
-
-                        netdata_log_debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup returned error %d",
-                              rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), ret
-                              );
-                    } else
-                        rc->run_flags &= ~RRDCALC_FLAG_DB_ERROR;
-
-                    if (unlikely(value_is_null)) {
-                        // collected value is null
-                        rc->value = NAN;
-                        rc->run_flags |= RRDCALC_FLAG_DB_NAN;
-
-                        netdata_log_debug(D_HEALTH,
-                              "Health on host '%s', alarm '%s.%s': database lookup returned empty value (possibly value is not collected yet)",
-                              rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc)
-                              );
-                    } else
-                        rc->run_flags &= ~RRDCALC_FLAG_DB_NAN;
-
-                    netdata_log_debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup gave value " NETDATA_DOUBLE_FORMAT,
-                          rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), rc->value
-                          );
-                }
-
-                // ------------------------------------------------------------
-                // if there is calculation expression, run it
-
-                if (unlikely(rc->calculation)) {
-                    worker_is_busy(WORKER_HEALTH_JOB_CALC_EVAL);
-
-                    if (unlikely(!expression_evaluate(rc->calculation))) {
-                        // calculation failed
-                        rc->value = NAN;
-                        rc->run_flags |= RRDCALC_FLAG_CALC_ERROR;
-
-                        netdata_log_debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': expression '%s' failed: %s",
-                              rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),
-                              rc->calculation->parsed_as, buffer_tostring(rc->calculation->error_msg)
-                              );
-                    } else {
-                        rc->run_flags &= ~RRDCALC_FLAG_CALC_ERROR;
-
-                        netdata_log_debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': expression '%s' gave value "
-                              NETDATA_DOUBLE_FORMAT
-                              ": %s (source: %s)", rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),
-                              rc->calculation->parsed_as, rc->calculation->result,
-                              buffer_tostring(rc->calculation->error_msg), rrdcalc_source(rc)
-                              );
-
-                        rc->value = rc->calculation->result;
-                    }
-                }
-            }
-            foreach_rrdcalc_in_rrdhost_done(rc);
-
-            if (unlikely(runnable && service_running(SERVICE_HEALTH))) {
-                foreach_rrdcalc_in_rrdhost_read(host, rc) {
-                    if(unlikely(!service_running(SERVICE_HEALTH)))
-                        break;
-
-                    if (unlikely(!(rc->run_flags & RRDCALC_FLAG_RUNNABLE)))
-                        continue;
-
-                    if (rc->run_flags & RRDCALC_FLAG_DISABLED) {
-                        continue;
-                    }
-                    RRDCALC_STATUS warning_status = RRDCALC_STATUS_UNDEFINED;
-                    RRDCALC_STATUS critical_status = RRDCALC_STATUS_UNDEFINED;
-
-                    // --------------------------------------------------------
-                    // check the warning expression
-
-                    if (likely(rc->warning)) {
-                        worker_is_busy(WORKER_HEALTH_JOB_WARNING_EVAL);
-
-                        if (unlikely(!expression_evaluate(rc->warning))) {
-                            // calculation failed
-                            rc->run_flags |= RRDCALC_FLAG_WARN_ERROR;
-
-                            netdata_log_debug(D_HEALTH,
-                                  "Health on host '%s', alarm '%s.%s': warning expression failed with error: %s",
-                                  rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),
-                                  buffer_tostring(rc->warning->error_msg)
-                                  );
-                        } else {
-                            rc->run_flags &= ~RRDCALC_FLAG_WARN_ERROR;
-                            netdata_log_debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': warning expression gave value "
-                                  NETDATA_DOUBLE_FORMAT
-                                  ": %s (source: %s)", rrdhost_hostname(host), rrdcalc_chart_name(rc),
-                                  rrdcalc_name(rc), rc->warning->result, buffer_tostring(rc->warning->error_msg), rrdcalc_source(rc)
-                                  );
-                            warning_status = rrdcalc_value2status(rc->warning->result);
-                        }
-                    }
-
-                    // --------------------------------------------------------
-                    // check the critical expression
-
-                    if (likely(rc->critical)) {
-                        worker_is_busy(WORKER_HEALTH_JOB_CRITICAL_EVAL);
-
-                        if (unlikely(!expression_evaluate(rc->critical))) {
-                            // calculation failed
-                            rc->run_flags |= RRDCALC_FLAG_CRIT_ERROR;
-
-                            netdata_log_debug(D_HEALTH,
-                                  "Health on host '%s', alarm '%s.%s': critical expression failed with error: %s",
-                                  rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),
-                                  buffer_tostring(rc->critical->error_msg)
-                                  );
-                        } else {
-                            rc->run_flags &= ~RRDCALC_FLAG_CRIT_ERROR;
-                            netdata_log_debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': critical expression gave value "
-                                  NETDATA_DOUBLE_FORMAT
-                                  ": %s (source: %s)", rrdhost_hostname(host), rrdcalc_chart_name(rc),
-                                  rrdcalc_name(rc), rc->critical->result, buffer_tostring(rc->critical->error_msg),
-                                  rrdcalc_source(rc)
-                                  );
-                            critical_status = rrdcalc_value2status(rc->critical->result);
-                        }
-                    }
-
-                    // --------------------------------------------------------
-                    // decide the final alarm status
-
-                    RRDCALC_STATUS status = RRDCALC_STATUS_UNDEFINED;
-
-                    switch (warning_status) {
-                        case RRDCALC_STATUS_CLEAR:
-                            status = RRDCALC_STATUS_CLEAR;
-                            break;
-
-                        case RRDCALC_STATUS_RAISED:
-                            status = RRDCALC_STATUS_WARNING;
-                            break;
-
-                        default:
-                            break;
-                    }
-
-                    switch (critical_status) {
-                        case RRDCALC_STATUS_CLEAR:
-                            if (status == RRDCALC_STATUS_UNDEFINED)
-                                status = RRDCALC_STATUS_CLEAR;
-                            break;
-
-                        case RRDCALC_STATUS_RAISED:
-                            status = RRDCALC_STATUS_CRITICAL;
-                            break;
-
-                        default:
-                            break;
-                    }
-
-                    // --------------------------------------------------------
-                    // check if the new status and the old differ
-
-                    if (status != rc->status) {
-
-                        worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
-                        int delay = 0;
-
-                        // apply trigger hysteresis
-
-                        if (now > rc->delay_up_to_timestamp) {
-                            rc->delay_up_current = rc->delay_up_duration;
-                            rc->delay_down_current = rc->delay_down_duration;
-                            rc->delay_last = 0;
-                            rc->delay_up_to_timestamp = 0;
-                        } else {
-                            rc->delay_up_current = (int) (rc->delay_up_current * rc->delay_multiplier);
-                            if (rc->delay_up_current > rc->delay_max_duration)
-                                rc->delay_up_current = rc->delay_max_duration;
-
-                            rc->delay_down_current = (int) (rc->delay_down_current * rc->delay_multiplier);
-                            if (rc->delay_down_current > rc->delay_max_duration)
-                                rc->delay_down_current = rc->delay_max_duration;
-                        }
-
-                        if (status > rc->status)
-                            delay = rc->delay_up_current;
-                        else
-                            delay = rc->delay_down_current;
-
-                        // COMMENTED: because we do need to send raising alarms
-                        // if(now + delay < rc->delay_up_to_timestamp)
-                        //      delay = (int)(rc->delay_up_to_timestamp - now);
-
-                        rc->delay_last = delay;
-                        rc->delay_up_to_timestamp = now + delay;
-
-                        ALARM_ENTRY *ae = health_create_alarm_entry(
-                                                                    host,
-                                                                    rc->id,
-                                                                    rc->next_event_id++,
-                                                                    rc->config_hash_id,
-                                                                    now,
-                                                                    rc->name,
-                                                                    rc->rrdset->id,
-                                                                    rc->rrdset->context,
-                                                                    rc->rrdset->name,
-                                                                    rc->classification,
-                                                                    rc->component,
-                                                                    rc->type,
-                                                                    rc->exec,
-                                                                    rc->recipient,
-                                                                    now - rc->last_status_change,
-                                                                    rc->old_value,
-                                                                    rc->value,
-                                                                    rc->status,
-                                                                    status,
-                                                                    rc->source,
-                                                                    rc->units,
-                                                                    rc->summary,
-                                                                    rc->info,
-                                                                    rc->delay_last,
-                                                                    (
-                                                                     ((rc->options & RRDCALC_OPTION_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) |
-                                                                     ((rc->run_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) |
-                                                                     (rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0)
-                                                                     )
-                                                                    );
-
-                        health_log_alert(host, ae);
-                        health_alarm_log_add_entry(host, ae);
-
-                        nd_log(NDLS_DAEMON, NDLP_DEBUG,
-                               "[%s]: Alert event for [%s.%s], value [%s], status [%s].",
-                               rrdhost_hostname(host), ae_chart_id(ae), ae_name(ae), ae_new_value_string(ae),
-                               rrdcalc_status2string(ae->new_status));
-
-                        rc->last_status_change_value = rc->value;
-                        rc->last_status_change = now;
-                        rc->old_status = rc->status;
-                        rc->status = status;
-                        rc->ae = ae;
-
-                        if(unlikely(rrdcalc_isrepeating(rc))) {
-                            rc->last_repeat = now;
-                            if (rc->status == RRDCALC_STATUS_CLEAR)
-                                rc->run_flags |= RRDCALC_FLAG_RUN_ONCE;
-                        }
-                    }
-
-                    rc->last_updated = now;
-                    rc->next_update = now + rc->update_every;
-
-                    if (next_run > rc->next_update)
-                        next_run = rc->next_update;
-                }
-                foreach_rrdcalc_in_rrdhost_done(rc);
-
-                // process repeating alarms
-                foreach_rrdcalc_in_rrdhost_read(host, rc) {
-                    if(unlikely(!service_running(SERVICE_HEALTH)))
-                        break;
-
-                    int repeat_every = 0;
-                    if(unlikely(rrdcalc_isrepeating(rc) && rc->delay_up_to_timestamp <= now)) {
-                        if(unlikely(rc->status == RRDCALC_STATUS_WARNING)) {
-                            rc->run_flags &= ~RRDCALC_FLAG_RUN_ONCE;
-                            repeat_every = rc->warn_repeat_every;
-                        } else if(unlikely(rc->status == RRDCALC_STATUS_CRITICAL)) {
-                            rc->run_flags &= ~RRDCALC_FLAG_RUN_ONCE;
-                            repeat_every = rc->crit_repeat_every;
-                        } else if(unlikely(rc->status == RRDCALC_STATUS_CLEAR)) {
-                            if(!(rc->run_flags & RRDCALC_FLAG_RUN_ONCE)) {
-                                if(rc->old_status == RRDCALC_STATUS_CRITICAL) {
-                                    repeat_every = 1;
-                                } else if (rc->old_status == RRDCALC_STATUS_WARNING) {
-                                    repeat_every = 1;
-                                }
-                            }
-                        }
-                    } else {
-                        continue;
-                    }
-
-                    if(unlikely(repeat_every > 0 && (rc->last_repeat + repeat_every) <= now)) {
-                        worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
-                        rc->last_repeat = now;
-                        if (likely(rc->times_repeat < UINT32_MAX)) rc->times_repeat++;
-                        ALARM_ENTRY *ae = health_create_alarm_entry(
-                                                                    host,
-                                                                    rc->id,
-                                                                    rc->next_event_id++,
-                                                                    rc->config_hash_id,
-                                                                    now,
-                                                                    rc->name,
-                                                                    rc->rrdset->id,
-                                                                    rc->rrdset->context,
-                                                                    rc->rrdset->name,
-                                                                    rc->classification,
-                                                                    rc->component,
-                                                                    rc->type,
-                                                                    rc->exec,
-                                                                    rc->recipient,
-                                                                    now - rc->last_status_change,
-                                                                    rc->old_value,
-                                                                    rc->value,
-                                                                    rc->old_status,
-                                                                    rc->status,
-                                                                    rc->source,
-                                                                    rc->units,
-                                                                    rc->summary,
-                                                                    rc->info,
-                                                                    rc->delay_last,
-                                                                    (
-                                                                     ((rc->options & RRDCALC_OPTION_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) |
-                                                                     ((rc->run_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) |
-                                                                     (rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0)
-                                                                     )
-                                                                    );
-
-                        health_log_alert(host, ae);
-                        ae->last_repeat = rc->last_repeat;
-                        if (!(rc->run_flags & RRDCALC_FLAG_RUN_ONCE) && rc->status == RRDCALC_STATUS_CLEAR) {
-                            ae->flags |= HEALTH_ENTRY_RUN_ONCE;
-                        }
-                        rc->run_flags |= RRDCALC_FLAG_RUN_ONCE;
-                        health_process_notifications(host, ae);
-                        netdata_log_debug(D_HEALTH, "Notification sent for the repeating alarm %u.", ae->alarm_id);
-                        health_alarm_wait_for_execution(ae);
-                        health_alarm_log_free_one_nochecks_nounlink(ae);
-                    }
-                }
-                foreach_rrdcalc_in_rrdhost_done(rc);
-            }
-
-            if (unlikely(!service_running(SERVICE_HEALTH)))
-                break;
-
-            // execute notifications
-            // and cleanup
-            worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_PROCESS);
-            health_alarm_log_process(host);
-
-            if (unlikely(!service_running(SERVICE_HEALTH))) {
-                // wait for all notifications to finish before allowing health to be cleaned up
-                ALARM_ENTRY *ae;
-                while (NULL != (ae = alarm_notifications_in_progress.head)) {
-                    if(unlikely(!service_running(SERVICE_HEALTH)))
-                        break;
-
-                    health_alarm_wait_for_execution(ae);
-                }
-                break;
-            }
-#ifdef ENABLE_ACLK
-            if (netdata_cloud_enabled) {
-                struct aclk_sync_cfg_t *wc = host->aclk_config;
-                if (unlikely(!wc))
-                    continue;
-
-                if (wc->alert_queue_removed == 1) {
-                    sql_queue_removed_alerts_to_aclk(host);
-                } else if (wc->alert_queue_removed > 1) {
-                    wc->alert_queue_removed--;
-                }
-
-                if (wc->alert_checkpoint_req == 1) {
-                    aclk_push_alarm_checkpoint(host);
-                } else if (wc->alert_checkpoint_req > 1) {
-                    wc->alert_checkpoint_req--;
-                }
-            }
-#endif
-        }
-        dfe_done(host);
-
-        // wait for all notifications to finish before allowing health to be cleaned up
-        ALARM_ENTRY *ae;
-        while (NULL != (ae = alarm_notifications_in_progress.head)) {
-            if(unlikely(!service_running(SERVICE_HEALTH)))
-                break;
-
-            health_alarm_wait_for_execution(ae);
-        }
-
-        if(unlikely(!service_running(SERVICE_HEALTH)))
-            break;
-
-        health_sleep(next_run, loop);
-
-    } // forever
-
-    netdata_thread_cleanup_pop(1);
-    return NULL;
-}