path: root/src/health/health_event_loop.c

// SPDX-License-Identifier: GPL-3.0-or-later

#include "health.h"
#include "health_internals.h"

#define WORKER_HEALTH_JOB_RRD_LOCK              0
#define WORKER_HEALTH_JOB_HOST_LOCK             1
#define WORKER_HEALTH_JOB_DB_QUERY              2
#define WORKER_HEALTH_JOB_CALC_EVAL             3
#define WORKER_HEALTH_JOB_WARNING_EVAL          4
#define WORKER_HEALTH_JOB_CRITICAL_EVAL         5
#define WORKER_HEALTH_JOB_ALARM_LOG_ENTRY       6
#define WORKER_HEALTH_JOB_ALARM_LOG_PROCESS     7
#define WORKER_HEALTH_JOB_DELAYED_INIT_RRDSET   8
#define WORKER_HEALTH_JOB_DELAYED_INIT_RRDDIM   9

#if WORKER_UTILIZATION_MAX_JOB_TYPES < 10
#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 10
#endif

// ----------------------------------------------------------------------------
// health main thread and friends

static inline RRDCALC_STATUS rrdcalc_value2status(NETDATA_DOUBLE n) {
    if(isnan(n) || isinf(n)) return RRDCALC_STATUS_UNDEFINED;
    if(n) return RRDCALC_STATUS_RAISED;
    return RRDCALC_STATUS_CLEAR;
}

static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run) {
    if(unlikely(!rc->rrdset)) {
        netdata_log_debug(D_HEALTH, "Health not running alarm '%s.%s'. It is not linked to a chart.", rrdcalc_chart_name(rc), rrdcalc_name(rc));
        return 0;
    }

    if(unlikely(rc->next_update > now)) {
        if (unlikely(*next_run > rc->next_update)) {
            // update the next_run time of the main loop
            // to run this alarm precisely the time required
            *next_run = rc->next_update;
        }

        netdata_log_debug(D_HEALTH, "Health not examining alarm '%s.%s' yet (will do in %d secs).", rrdcalc_chart_name(rc), rrdcalc_name(rc), (int) (rc->next_update - now));
        return 0;
    }

    if(unlikely(!rc->config.update_every)) {
        netdata_log_debug(D_HEALTH, "Health not running alarm '%s.%s'. It does not have an update frequency", rrdcalc_chart_name(rc), rrdcalc_name(rc));
        return 0;
    }

    if(unlikely(rrdset_flag_check(rc->rrdset, RRDSET_FLAG_OBSOLETE))) {
        netdata_log_debug(D_HEALTH, "Health not running alarm '%s.%s'. The chart has been marked as obsolete", rrdcalc_chart_name(rc), rrdcalc_name(rc));
        return 0;
    }

    if(unlikely(!rc->rrdset->last_collected_time.tv_sec || rc->rrdset->counter_done < 2)) {
        netdata_log_debug(D_HEALTH, "Health not running alarm '%s.%s'. Chart is not fully collected yet.", rrdcalc_chart_name(rc), rrdcalc_name(rc));
        return 0;
    }

    int update_every = rc->rrdset->update_every;
    time_t first = rrdset_first_entry_s(rc->rrdset);
    time_t last = rrdset_last_entry_s(rc->rrdset);

    if(unlikely(now + update_every < first /* || now - update_every > last */)) {
        netdata_log_debug(D_HEALTH
                          , "Health not examining alarm '%s.%s' yet (wanted time is out of bounds - we need %lu but got %lu - %lu)."
                          , rrdcalc_chart_name(rc), rrdcalc_name(rc), (unsigned long) now, (unsigned long) first
                          , (unsigned long) last);
        return 0;
    }

    if(RRDCALC_HAS_DB_LOOKUP(rc)) {
        time_t needed = now + rc->config.before + rc->config.after;

        if(needed + update_every < first || needed - update_every > last) {
            netdata_log_debug(D_HEALTH,
                "Health not examining alarm '%s.%s' yet (not enough data yet - we need %lu but got %lu - %lu).",
                rrdcalc_chart_name(rc),
                rrdcalc_name(rc),
                (unsigned long) needed,
                (unsigned long) first,
                (unsigned long) last);
            return 0;
        }
    }

    return 1;
}

static void health_sleep(time_t next_run, unsigned int loop __maybe_unused) {
    time_t now = now_realtime_sec();
    if(now < next_run) {
        worker_is_idle();
        netdata_log_debug(D_HEALTH, "Health monitoring iteration no %u done. Next iteration in %d secs", loop, (int) (next_run - now));
        while (now < next_run && service_running(SERVICE_HEALTH)) {
            sleep_usec(USEC_PER_SEC);
            now = now_realtime_sec();
        }
    }
    else {
        netdata_log_debug(D_HEALTH, "Health monitoring iteration no %u done. Next iteration now", loop);
    }
}

static void health_execute_delayed_initializations(RRDHOST *host) {
    health_plugin_init();

    RRDSET *st;

    if (!rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION)) return;
    rrdhost_flag_clear(host, RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION);

    rrdset_foreach_reentrant(st, host) {
        if(!rrdset_flag_check(st, RRDSET_FLAG_PENDING_HEALTH_INITIALIZATION)) continue;
        rrdset_flag_clear(st, RRDSET_FLAG_PENDING_HEALTH_INITIALIZATION);

        worker_is_busy(WORKER_HEALTH_JOB_DELAYED_INIT_RRDSET);
        health_prototype_alerts_for_rrdset_incrementally(st);
    }
    rrdset_foreach_done(st);
}

static void health_initialize_rrdhost(RRDHOST *host) {
    health_plugin_init();

    if(!host->health.health_enabled ||
        rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH) ||
        !service_running(SERVICE_HEALTH))
        return;

    rrdhost_flag_set(host, RRDHOST_FLAG_INITIALIZED_HEALTH);

    host->health_log.max = health_globals.config.health_log_entries_max;
    host->health_log.health_log_retention_s = health_globals.config.health_log_retention_s;
    host->health.health_default_exec = string_dup(health_globals.config.default_exec);
    host->health.health_default_recipient = string_dup(health_globals.config.default_recipient);
    host->health.use_summary_for_notifications = health_globals.config.use_summary_for_notifications;

    host->health_log.next_log_id = get_uint32_id();
    host->health_log.next_alarm_id = 0;

    rw_spinlock_init(&host->health_log.spinlock);
    sql_health_alarm_log_load(host);
    health_apply_prototypes_to_host(host);
}

static inline int check_if_resumed_from_suspension(void) {
    static usec_t last_realtime = 0, last_monotonic = 0;
    usec_t realtime = now_realtime_usec(), monotonic = now_monotonic_usec();
    int ret = 0;

    // detect if monotonic and realtime have twice the difference
    // in which case we assume the system was just waken from hibernation

    if(last_realtime && last_monotonic && realtime - last_realtime > 2 * (monotonic - last_monotonic))
        ret = 1;

    last_realtime = realtime;
    last_monotonic = monotonic;

    return ret;
}

static void do_eval_expression(
    RRDCALC *rc,
    EVAL_EXPRESSION *expression,
    const char *expression_type __maybe_unused,
    size_t job_type,
    RRDCALC_FLAGS error_type,
    RRDCALC_STATUS *calc_status,
    NETDATA_DOUBLE *result)
{
    if (!expression || (!calc_status && !result))
        return;

    worker_is_busy(job_type);

    if (unlikely(!expression_evaluate(expression))) {
        // calculation failed
        rc->run_flags |= error_type;
        if (result)
            *result = NAN;

        netdata_log_debug(D_HEALTH,
                          "Health on host '%s', alarm '%s.%s': %s expression failed with error: %s",
                          rrdhost_hostname(rc->rrdset->rrdhost), rrdcalc_chart_name(rc), rrdcalc_name(rc), expression_type,
                          expression_error_msg(expression)
        );
        return;
    }
    rc->run_flags &= ~error_type;
    netdata_log_debug(D_HEALTH,
                      "Health on host '%s', alarm '%s.%s': %s expression gave value "
                      NETDATA_DOUBLE_FORMAT ": %s (source: %s)",
                      rrdhost_hostname(rc->rrdset->rrdhost),
                      rrdcalc_chart_name(rc),
                      rrdcalc_name(rc),
                      expression_type,
                      expression_result(expression),
                      expression_error_msg(expression),
                      rrdcalc_source(rc));
    if (calc_status)
        *calc_status = rrdcalc_value2status(expression_result(expression));
    else
        *result = expression_result(expression);
}

static void health_event_loop(void) {
    bool health_running_logged = false;

    unsigned int loop = 0;

    while(service_running(SERVICE_HEALTH)) {
        loop++;
        netdata_log_debug(D_HEALTH, "Health monitoring iteration no %u started", loop);

        time_t now = now_realtime_sec();
        int runnable = 0, apply_hibernation_delay = 0;
        time_t next_run = now + health_globals.config.run_at_least_every_seconds;
        RRDCALC *rc;
        RRDHOST *host;

        if (unlikely(check_if_resumed_from_suspension())) {
            apply_hibernation_delay = 1;

            nd_log(NDLS_DAEMON, NDLP_NOTICE,
                   "Postponing alarm checks for %"PRId32" seconds, "
                   "because it seems that the system was just resumed from suspension.",
                   (int32_t)health_globals.config.postpone_alarms_during_hibernation_for_seconds);
            schedule_node_state_update(localhost, 0);
        }

        if (unlikely(silencers->all_alarms && silencers->stype == STYPE_DISABLE_ALARMS)) {
            static int logged=0;
            if (!logged) {
                nd_log(NDLS_DAEMON, NDLP_DEBUG,
                       "Skipping health checks, because all alarms are disabled via API command.");
                logged = 1;
            }
        }

        worker_is_busy(WORKER_HEALTH_JOB_RRD_LOCK);
        dfe_start_reentrant(rrdhost_root_index, host) {

            if(unlikely(!service_running(SERVICE_HEALTH)))
                break;

            if (unlikely(!host->health.health_enabled))
                continue;

            if (unlikely(!rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH)))
                health_initialize_rrdhost(host);

            health_execute_delayed_initializations(host);

            if (unlikely(apply_hibernation_delay)) {
                nd_log(NDLS_DAEMON, NDLP_DEBUG,
                       "[%s]: Postponing health checks for %"PRId32" seconds.",
                       rrdhost_hostname(host),
                       health_globals.config.postpone_alarms_during_hibernation_for_seconds);

                host->health.health_delay_up_to =
                    now + health_globals.config.postpone_alarms_during_hibernation_for_seconds;
            }

            if (unlikely(host->health.health_delay_up_to)) {
                if (unlikely(now < host->health.health_delay_up_to)) {
                    continue;
                }

                nd_log(NDLS_DAEMON, NDLP_DEBUG,
                       "[%s]: Resuming health checks after delay.",
                       rrdhost_hostname(host));

                host->health.health_delay_up_to = 0;
            }

            // wait until cleanup of obsolete charts on children is complete
            if (host != localhost) {
                if (unlikely(host->trigger_chart_obsoletion_check == 1)) {

                    nd_log(NDLS_DAEMON, NDLP_DEBUG,
                           "[%s]: Waiting for chart obsoletion check.",
                           rrdhost_hostname(host));

                    continue;
                }
            }

            if (!health_running_logged) {
                nd_log(NDLS_DAEMON, NDLP_DEBUG,
                       "[%s]: Health is running.",
                       rrdhost_hostname(host));

                health_running_logged = true;
            }

            worker_is_busy(WORKER_HEALTH_JOB_HOST_LOCK);
            {
                struct aclk_sync_cfg_t *wc = host->aclk_config;
                if (wc && wc->send_snapshot == 2)
                    continue;
            }

            // the first loop is to lookup values from the db
            foreach_rrdcalc_in_rrdhost_read(host, rc) {

                if(unlikely(!service_running(SERVICE_HEALTH)))
                    break;

                rrdcalc_update_info_using_rrdset_labels(rc);

                if (health_silencers_update_disabled_silenced(host, rc))
                    continue;

                // create an alert removed event if the chart is obsolete and
                // has stopped being collected for 60 seconds
                if (unlikely(rc->rrdset && rc->status != RRDCALC_STATUS_REMOVED &&
                             rrdset_flag_check(rc->rrdset, RRDSET_FLAG_OBSOLETE) &&
                             now > (rc->rrdset->last_collected_time.tv_sec + 60))) {

                    if (!rrdcalc_isrepeating(rc)) {
                        worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
                        time_t now_tmp = now_realtime_sec();

                        ALARM_ENTRY *ae =
                            health_create_alarm_entry(
                                host,
                                rc,
                                now_tmp,
                                now_tmp - rc->last_status_change,
                                rc->value,
                                NAN,
                                rc->status,
                                RRDCALC_STATUS_REMOVED,
                                0,
                                rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0);

                        if (ae) {
                            health_log_alert(host, ae);
                            health_alarm_log_add_entry(host, ae);
                            rc->old_status = rc->status;
                            rc->status = RRDCALC_STATUS_REMOVED;
                            rc->last_status_change = now_tmp;
                            rc->last_status_change_value = rc->value;
                            rc->last_updated = now_tmp;
                            rc->value = NAN;
                        }
                    }
                }

                if (unlikely(!rrdcalc_isrunnable(rc, now, &next_run))) {
                    if (unlikely(rc->run_flags & RRDCALC_FLAG_RUNNABLE))
                        rc->run_flags &= ~RRDCALC_FLAG_RUNNABLE;
                    continue;
                }

                runnable++;
                rc->old_value = rc->value;
                rc->run_flags |= RRDCALC_FLAG_RUNNABLE;

                // ------------------------------------------------------------
                // if there is database lookup, do it

                if (unlikely(RRDCALC_HAS_DB_LOOKUP(rc))) {
                    worker_is_busy(WORKER_HEALTH_JOB_DB_QUERY);

                    /* time_t old_db_timestamp = rc->db_before; */
                    int value_is_null = 0;

                    char group_options_buf[100];
                    const char *group_options = group_options_buf;
                    switch(rc->config.time_group) {
                        default:
                            group_options = NULL;
                            break;

                        case RRDR_GROUPING_PERCENTILE:
                        case RRDR_GROUPING_TRIMMED_MEAN:
                        case RRDR_GROUPING_TRIMMED_MEDIAN:
                            snprintfz(group_options_buf, sizeof(group_options_buf),
                                      NETDATA_DOUBLE_FORMAT_AUTO,
                                      rc->config.time_group_value);
                            break;

                        case RRDR_GROUPING_COUNTIF:
                            snprintfz(group_options_buf, sizeof(group_options_buf),
                                      "%s" NETDATA_DOUBLE_FORMAT_AUTO,
                                      alerts_group_conditions_id2txt(rc->config.time_group_condition),
                                      rc->config.time_group_value);
                            break;
                    }

                    int ret = rrdset2value_api_v1(rc->rrdset, NULL, &rc->value, rrdcalc_dimensions(rc), 1,
                                                  rc->config.after, rc->config.before, rc->config.time_group, group_options,
                                                  0, rc->config.options | RRDR_OPTION_SELECTED_TIER,
                                                  &rc->db_after,&rc->db_before,
                                                  NULL, NULL, NULL,
                                                  &value_is_null, NULL, 0, 0,
                                                  QUERY_SOURCE_HEALTH, STORAGE_PRIORITY_SYNCHRONOUS);

                    if (unlikely(ret != 200)) {
                        // database lookup failed
                        rc->value = NAN;
                        rc->run_flags |= RRDCALC_FLAG_DB_ERROR;

                        netdata_log_debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup returned error %d",
                                          rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), ret
                        );
                    } else
                        rc->run_flags &= ~RRDCALC_FLAG_DB_ERROR;

                    if (unlikely(value_is_null)) {
                        // collected value is null
                        rc->value = NAN;
                        rc->run_flags |= RRDCALC_FLAG_DB_NAN;

                        netdata_log_debug(D_HEALTH,
                                          "Health on host '%s', alarm '%s.%s': database lookup returned empty value (possibly value is not collected yet)",
                                          rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc)
                        );
                    } else
                        rc->run_flags &= ~RRDCALC_FLAG_DB_NAN;

                    netdata_log_debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup gave value " NETDATA_DOUBLE_FORMAT,
                                      rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), rc->value
                    );
                }

                // ------------------------------------------------------------
                // if there is calculation expression, run it

                do_eval_expression(rc, rc->config.calculation, "calculation", WORKER_HEALTH_JOB_CALC_EVAL, RRDCALC_FLAG_CALC_ERROR, NULL, &rc->value);
            }
            foreach_rrdcalc_in_rrdhost_done(rc);

            struct health_raised_summary *hrm = alerts_raised_summary_create(host);

            if (unlikely(runnable && service_running(SERVICE_HEALTH))) {
                foreach_rrdcalc_in_rrdhost_read(host, rc) {
                    if(unlikely(!service_running(SERVICE_HEALTH)))
                        break;

                    if (unlikely(!(rc->run_flags & RRDCALC_FLAG_RUNNABLE)))
                        continue;

                    if (rc->run_flags & RRDCALC_FLAG_DISABLED) {
                        continue;
                    }
                    RRDCALC_STATUS warning_status = RRDCALC_STATUS_UNDEFINED;
                    RRDCALC_STATUS critical_status = RRDCALC_STATUS_UNDEFINED;

                    do_eval_expression(rc, rc->config.warning, "warning", WORKER_HEALTH_JOB_WARNING_EVAL, RRDCALC_FLAG_WARN_ERROR, &warning_status, NULL);
                    do_eval_expression(rc, rc->config.critical, "critical", WORKER_HEALTH_JOB_CRITICAL_EVAL, RRDCALC_FLAG_CRIT_ERROR, &critical_status, NULL);

                    // --------------------------------------------------------
                    // decide the final alarm status

                    RRDCALC_STATUS status = RRDCALC_STATUS_UNDEFINED;

                    switch (warning_status) {
                        case RRDCALC_STATUS_CLEAR:
                            status = RRDCALC_STATUS_CLEAR;
                            break;

                        case RRDCALC_STATUS_RAISED:
                            status = RRDCALC_STATUS_WARNING;
                            break;

                        default:
                            break;
                    }

                    switch (critical_status) {
                        case RRDCALC_STATUS_CLEAR:
                            if (status == RRDCALC_STATUS_UNDEFINED)
                                status = RRDCALC_STATUS_CLEAR;
                            break;

                        case RRDCALC_STATUS_RAISED:
                            status = RRDCALC_STATUS_CRITICAL;
                            break;

                        default:
                            break;
                    }

                    // --------------------------------------------------------
                    // check if the new status and the old differ

                    if (status != rc->status) {

                        worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
                        int delay;

                        // apply trigger hysteresis

                        if (now > rc->delay_up_to_timestamp) {
                            rc->delay_up_current = rc->config.delay_up_duration;
                            rc->delay_down_current = rc->config.delay_down_duration;
                            rc->delay_last = 0;
                            rc->delay_up_to_timestamp = 0;
                        } else {
                            rc->delay_up_current = (int)((float)rc->delay_up_current * rc->config.delay_multiplier);
                            if (rc->delay_up_current > rc->config.delay_max_duration)
                                rc->delay_up_current = rc->config.delay_max_duration;

                            rc->delay_down_current = (int)((float)rc->delay_down_current * rc->config.delay_multiplier);
                            if (rc->delay_down_current > rc->config.delay_max_duration)
                                rc->delay_down_current = rc->config.delay_max_duration;
                        }

                        if (status > rc->status)
                            delay = rc->delay_up_current;
                        else
                            delay = rc->delay_down_current;

                        // COMMENTED: because we do need to send raising alarms
                        // if (now + delay < rc->delay_up_to_timestamp)
                        //      delay = (int)(rc->delay_up_to_timestamp - now);

                        rc->delay_last = delay;
                        rc->delay_up_to_timestamp = now + delay;

                        ALARM_ENTRY *ae =
                            health_create_alarm_entry(
                                host,
                                rc,
                                now,
                                now - rc->last_status_change,
                                rc->old_value,
                                rc->value,
                                rc->status,
                                status,
                                rc->delay_last,
                                (
                                    ((rc->config.alert_action_options & ALERT_ACTION_OPTION_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) |
                                    ((rc->run_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) |
                                    (rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0)
                                        )
                            );

                        health_log_alert(host, ae);
                        health_alarm_log_add_entry(host, ae);

                        nd_log(NDLS_DAEMON, NDLP_DEBUG,
                               "[%s]: Alert event for [%s.%s], value [%s], status [%s].",
                               rrdhost_hostname(host), ae_chart_id(ae), ae_name(ae), ae_new_value_string(ae),
                               rrdcalc_status2string(ae->new_status));

                        rc->last_status_change_value = rc->value;
                        rc->last_status_change = now;
                        rc->old_status = rc->status;
                        rc->status = status;

                        if(unlikely(rrdcalc_isrepeating(rc))) {
                            rc->last_repeat = now;
                            if (rc->status == RRDCALC_STATUS_CLEAR)
                                rc->run_flags |= RRDCALC_FLAG_RUN_ONCE;
                        }
                    }

                    rc->last_updated = now;
                    rc->next_update = now + rc->config.update_every;

                    if (next_run > rc->next_update)
                        next_run = rc->next_update;
                }
                foreach_rrdcalc_in_rrdhost_done(rc);

                alerts_raised_summary_populate(hrm);

                // process repeating alarms
                foreach_rrdcalc_in_rrdhost_read(host, rc) {
                    if(unlikely(!service_running(SERVICE_HEALTH)))
                        break;

                    int repeat_every = 0;
                    if(unlikely(rrdcalc_isrepeating(rc) && rc->delay_up_to_timestamp <= now)) {
                        if(unlikely(rc->status == RRDCALC_STATUS_WARNING)) {
                            rc->run_flags &= ~RRDCALC_FLAG_RUN_ONCE;
                            repeat_every = (int)rc->config.warn_repeat_every;
                        }
                        else if(unlikely(rc->status == RRDCALC_STATUS_CRITICAL)) {
                            rc->run_flags &= ~RRDCALC_FLAG_RUN_ONCE;
                            repeat_every = (int)rc->config.crit_repeat_every;
                        }
                        else if(unlikely(rc->status == RRDCALC_STATUS_CLEAR)) {
                            if(!(rc->run_flags & RRDCALC_FLAG_RUN_ONCE) &&
                                (rc->old_status == RRDCALC_STATUS_CRITICAL || rc->old_status == RRDCALC_STATUS_WARNING))
                                repeat_every = 1;
                        }
                    }
                    else
                        continue;

                    if(unlikely(repeat_every > 0 && (rc->last_repeat + repeat_every) <= now)) {
                        worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
                        rc->last_repeat = now;
                        if (likely(rc->times_repeat < UINT32_MAX)) rc->times_repeat++;
                        ALARM_ENTRY *ae =
                            health_create_alarm_entry(
                                host,
                                rc,
                                now,
                                now - rc->last_status_change,
                                rc->old_value,
                                rc->value,
                                rc->old_status,
                                rc->status,
                                rc->delay_last,
                                (
                                    ((rc->config.alert_action_options & ALERT_ACTION_OPTION_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) |
                                    ((rc->run_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) |
                                    (rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0)
                                        )
                            );

                        health_log_alert(host, ae);
                        ae->last_repeat = rc->last_repeat;
                        if (!(rc->run_flags & RRDCALC_FLAG_RUN_ONCE) && rc->status == RRDCALC_STATUS_CLEAR) {
                            ae->flags |= HEALTH_ENTRY_RUN_ONCE;
                        }
                        rc->run_flags |= RRDCALC_FLAG_RUN_ONCE;
                        health_send_notification(host, ae, hrm);
                        netdata_log_debug(D_HEALTH, "Notification sent for the repeating alarm %u.", ae->alarm_id);
                        health_alarm_wait_for_execution(ae);
                        health_alarm_log_free_one_nochecks_nounlink(ae);
                    }
                }
                foreach_rrdcalc_in_rrdhost_done(rc);
            }

            if (unlikely(!service_running(SERVICE_HEALTH)))
                break;

            // execute notifications
            // and cleanup

            worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_PROCESS);
            health_alarm_log_process_to_send_notifications(host, hrm);
            alerts_raised_summary_free(hrm);

            if (unlikely(!service_running(SERVICE_HEALTH))) {
                // wait for all notifications to finish before allowing health to be cleaned up
                wait_for_all_notifications_to_finish_before_allowing_health_to_be_cleaned_up();
                break;
            }
        }
        struct aclk_sync_cfg_t *wc = host->aclk_config;
        if (wc && wc->send_snapshot == 1) {
            wc->send_snapshot = 2;
            rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);
        }
        else
            if (process_alert_pending_queue(host))
                rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);

        dfe_done(host);

        // wait for all notifications to finish before allowing health to be cleaned up
        wait_for_all_notifications_to_finish_before_allowing_health_to_be_cleaned_up();

        if(unlikely(!service_running(SERVICE_HEALTH)))
            break;

        health_sleep(next_run, loop);

    } // forever
}


static void health_main_cleanup(void *pptr) {
    struct netdata_static_thread *static_thread = CLEANUP_FUNCTION_GET_PTR(pptr);
    if(!static_thread) return;

    worker_unregister();
    static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
    netdata_log_info("cleaning up...");
    static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;

    nd_log(NDLS_DAEMON, NDLP_DEBUG, "Health thread ended.");
}

void *health_main(void *ptr) {
    worker_register("HEALTH");
    worker_register_job_name(WORKER_HEALTH_JOB_RRD_LOCK, "rrd lock");
    worker_register_job_name(WORKER_HEALTH_JOB_HOST_LOCK, "host lock");
    worker_register_job_name(WORKER_HEALTH_JOB_DB_QUERY, "db lookup");
    worker_register_job_name(WORKER_HEALTH_JOB_CALC_EVAL, "calc eval");
    worker_register_job_name(WORKER_HEALTH_JOB_WARNING_EVAL, "warning eval");
    worker_register_job_name(WORKER_HEALTH_JOB_CRITICAL_EVAL, "critical eval");
    worker_register_job_name(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY, "alarm log entry");
    worker_register_job_name(WORKER_HEALTH_JOB_ALARM_LOG_PROCESS, "alarm log process");
    worker_register_job_name(WORKER_HEALTH_JOB_DELAYED_INIT_RRDSET, "rrdset init");
    worker_register_job_name(WORKER_HEALTH_JOB_DELAYED_INIT_RRDDIM, "rrddim init");

    CLEANUP_FUNCTION_REGISTER(health_main_cleanup) cleanup_ptr = ptr;
    health_event_loop();
    return NULL;
}