summaryrefslogtreecommitdiffstats
path: root/health/health.c
diff options
context:
space:
mode:
Diffstat (limited to 'health/health.c')
-rw-r--r--health/health.c960
1 files changed, 476 insertions, 484 deletions
diff --git a/health/health.c b/health/health.c
index 3784e0f31..b34f54ab5 100644
--- a/health/health.c
+++ b/health/health.c
@@ -159,9 +159,10 @@ static bool prepare_command(BUFFER *wb,
unsigned int default_health_enabled = 1;
char *silencers_filename;
+SIMPLE_PATTERN *conf_enabled_alarms = NULL;
// the queue of executed alarm notifications that haven't been waited for yet
-static __thread struct {
+static struct {
ALARM_ENTRY *head; // oldest
ALARM_ENTRY *tail; // latest
} alarm_notifications_in_progress = {NULL, NULL};
@@ -301,7 +302,7 @@ void health_init(void) {
* @param host the structure of the host that the function will reload the configuration.
*/
static void health_reload_host(RRDHOST *host) {
- if(unlikely(!host->health_enabled) && !rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH))
+ if(unlikely(!host->health.health_enabled) && !rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH))
return;
log_health("[%s]: Reloading health.", rrdhost_hostname(host));
@@ -345,7 +346,6 @@ static void health_reload_host(RRDHOST *host) {
rrdcalctemplate_link_matching_templates_to_rrdset(st);
}
rrdset_foreach_done(st);
- host->aclk_alert_reloaded = 1;
}
/**
@@ -363,6 +363,12 @@ void health_reload(void) {
health_reload_host(host);
rrd_unlock();
+
+#ifdef ENABLE_ACLK
+ if (netdata_cloud_setting) {
+ aclk_alert_reloaded = 1;
+ }
+#endif
}
// ----------------------------------------------------------------------------
@@ -444,8 +450,8 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
log_health("[%s]: Sending notification for alarm '%s.%s' status %s.", rrdhost_hostname(host), ae_chart_name(ae), ae_name(ae), rrdcalc_status2string(ae->new_status));
- const char *exec = (ae->exec) ? ae_exec(ae) : string2str(host->health_default_exec);
- const char *recipient = (ae->recipient) ? ae_recipient(ae) : string2str(host->health_default_recipient);
+ const char *exec = (ae->exec) ? ae_exec(ae) : string2str(host->health.health_default_exec);
+ const char *recipient = (ae->recipient) ? ae_recipient(ae) : string2str(host->health.health_default_recipient);
int n_warn=0, n_crit=0;
RRDCALC *rc;
@@ -453,8 +459,8 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
BUFFER *warn_alarms, *crit_alarms;
active_alerts_t *active_alerts = callocz(ACTIVE_ALARMS_LIST_EXAMINE, sizeof(active_alerts_t));
- warn_alarms = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE);
- crit_alarms = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE);
+ warn_alarms = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE, &netdata_buffers_statistics.buffers_health);
+ crit_alarms = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE, &netdata_buffers_statistics.buffers_health);
foreach_rrdcalc_in_rrdhost_read(host, rc) {
if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec))
@@ -511,7 +517,7 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
char *edit_command = ae->source ? health_edit_command_from_source(ae_source(ae)) : strdupz("UNKNOWN=0=UNKNOWN");
- BUFFER *wb = buffer_create(8192);
+ BUFFER *wb = buffer_create(8192, &netdata_buffers_statistics.buffers_health);
bool ok = prepare_command(wb,
exec,
recipient,
@@ -692,8 +698,8 @@ static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run)
}
int update_every = rc->rrdset->update_every;
- time_t first = rrdset_first_entry_t(rc->rrdset);
- time_t last = rrdset_last_entry_t(rc->rrdset);
+ time_t first = rrdset_first_entry_s(rc->rrdset);
+ time_t last = rrdset_last_entry_s(rc->rrdset);
if(unlikely(now + update_every < first /* || now - update_every > last */)) {
debug(D_HEALTH
@@ -719,7 +725,7 @@ static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run)
}
static inline int check_if_resumed_from_suspension(void) {
- static __thread usec_t last_realtime = 0, last_monotonic = 0;
+ static usec_t last_realtime = 0, last_monotonic = 0;
usec_t realtime = now_realtime_usec(), monotonic = now_monotonic_usec();
int ret = 0;
@@ -735,25 +741,29 @@ static inline int check_if_resumed_from_suspension(void) {
return ret;
}
-static void health_thread_cleanup(void *ptr) {
+static void health_main_cleanup(void *ptr) {
worker_unregister();
- struct health_state *h = ptr;
- h->host->health_spawn = 0;
+ struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
+ info("cleaning up...");
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
- netdata_thread_cancel(netdata_thread_self());
- log_health("[%s]: Health thread ended.", rrdhost_hostname(h->host));
- debug(D_HEALTH, "HEALTH %s: Health thread ended.", rrdhost_hostname(h->host));
+ log_health("Health thread ended.");
}
static void initialize_health(RRDHOST *host, int is_localhost) {
- if(!host->health_enabled || rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH)) return;
+ if(!host->health.health_enabled ||
+ rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH) ||
+ !service_running(SERVICE_HEALTH))
+ return;
+
rrdhost_flag_set(host, RRDHOST_FLAG_INITIALIZED_HEALTH);
log_health("[%s]: Initializing health.", rrdhost_hostname(host));
- host->health_default_warn_repeat_every = config_get_duration(CONFIG_SECTION_HEALTH, "default repeat warning", "never");
- host->health_default_crit_repeat_every = config_get_duration(CONFIG_SECTION_HEALTH, "default repeat critical", "never");
+ host->health.health_default_warn_repeat_every = config_get_duration(CONFIG_SECTION_HEALTH, "default repeat warning", "never");
+ host->health.health_default_crit_repeat_every = config_get_duration(CONFIG_SECTION_HEALTH, "default repeat critical", "never");
host->health_log.next_log_id = 1;
host->health_log.next_alarm_id = 1;
@@ -769,6 +779,8 @@ static void initialize_health(RRDHOST *host, int is_localhost) {
else
host->health_log.max = (unsigned int)n;
+ conf_enabled_alarms = simple_pattern_create(config_get(CONFIG_SECTION_HEALTH, "enabled alarms", "*"), NULL, SIMPLE_PATTERN_EXACT);
+
netdata_rwlock_init(&host->health_log.alarm_log_rwlock);
char filename[FILENAME_MAX + 1];
@@ -785,30 +797,15 @@ static void initialize_health(RRDHOST *host, int is_localhost) {
if(r != 0 && errno != EEXIST)
error("Host '%s': cannot create directory '%s'", rrdhost_hostname(host), filename);
}
- snprintfz(filename, FILENAME_MAX, "%s/health/health-log.db", host->varlib_dir);
- host->health_log_filename = strdupz(filename);
snprintfz(filename, FILENAME_MAX, "%s/alarm-notify.sh", netdata_configured_primary_plugins_dir);
- host->health_default_exec = string_strdupz(config_get(CONFIG_SECTION_HEALTH, "script to execute on alarm", filename));
- host->health_default_recipient = string_strdupz("root");
-
- if (!file_is_migrated(host->health_log_filename)) {
- int rc = sql_create_health_log_table(host);
- if (unlikely(rc)) {
- log_health("[%s]: Failed to create health log table in the database", rrdhost_hostname(host));
- health_alarm_log_load(host);
- health_alarm_log_open(host);
- }
- else {
- health_alarm_log_load(host);
- add_migrated_file(host->health_log_filename, 0);
- }
- } else {
- // TODO: This needs to go to the metadata thread
- // Health should wait before accessing the table (needs to be created by the metadata thread)
- sql_create_health_log_table(host);
- sql_health_alarm_log_load(host);
- }
+ host->health.health_default_exec = string_strdupz(config_get(CONFIG_SECTION_HEALTH, "script to execute on alarm", filename));
+ host->health.health_default_recipient = string_strdupz("root");
+
+ // TODO: This needs to go to the metadata thread
+ // Health should wait before accessing the table (needs to be created by the metadata thread)
+ sql_create_health_log_table(host);
+ sql_health_alarm_log_load(host);
// ------------------------------------------------------------------------
// load health configuration
@@ -828,16 +825,14 @@ static void initialize_health(RRDHOST *host, int is_localhost) {
//Discard alarms with labels that do not apply to host
rrdcalc_delete_alerts_not_matching_host_labels_from_this_host(host);
-
- health_silencers_init();
}
-static void health_sleep(time_t next_run, unsigned int loop __maybe_unused, RRDHOST *host) {
+static void health_sleep(time_t next_run, unsigned int loop __maybe_unused) {
time_t now = now_realtime_sec();
if(now < next_run) {
worker_is_idle();
debug(D_HEALTH, "Health monitoring iteration no %u done. Next iteration in %d secs", loop, (int) (next_run - now));
- while (now < next_run && host->health_enabled && !netdata_exit) {
+ while (now < next_run && service_running(SERVICE_HEALTH)) {
sleep_usec(USEC_PER_SEC);
now = now_realtime_sec();
}
@@ -995,555 +990,567 @@ void *health_main(void *ptr) {
worker_register_job_name(WORKER_HEALTH_JOB_DELAYED_INIT_RRDSET, "rrdset init");
worker_register_job_name(WORKER_HEALTH_JOB_DELAYED_INIT_RRDDIM, "rrddim init");
- struct health_state *h = ptr;
- netdata_thread_cleanup_push(health_thread_cleanup, ptr);
-
- RRDHOST *host = h->host;
- initialize_health(host, host == localhost);
+ netdata_thread_cleanup_push(health_main_cleanup, ptr);
int min_run_every = (int)config_get_number(CONFIG_SECTION_HEALTH, "run at least every seconds", 10);
if(min_run_every < 1) min_run_every = 1;
- int cleanup_sql_every_loop = 7200 / min_run_every;
-
- time_t now = now_realtime_sec();
time_t hibernation_delay = config_get_number(CONFIG_SECTION_HEALTH, "postpone alarms during hibernation for seconds", 60);
bool health_running_logged = false;
- rrdhost_rdlock(host); //CHECK
- rrdcalc_delete_alerts_not_matching_host_labels_from_this_host(host);
- rrdhost_unlock(host);
+ rrdcalc_delete_alerts_not_matching_host_labels_from_all_hosts();
unsigned int loop = 0;
#ifdef ENABLE_ACLK
unsigned int marked_aclk_reload_loop = 0;
#endif
- while(!netdata_exit && host->health_enabled) {
+ while(service_running(SERVICE_HEALTH)) {
loop++;
debug(D_HEALTH, "Health monitoring iteration no %u started", loop);
- now = now_realtime_sec();
+ time_t now = now_realtime_sec();
int runnable = 0, apply_hibernation_delay = 0;
time_t next_run = now + min_run_every;
RRDCALC *rc;
+ RRDHOST *host;
if (unlikely(check_if_resumed_from_suspension())) {
apply_hibernation_delay = 1;
log_health(
- "[%s]: Postponing alarm checks for %"PRId64" seconds, "
+ "Postponing alarm checks for %"PRId64" seconds, "
"because it seems that the system was just resumed from suspension.",
- rrdhost_hostname(host),
(int64_t)hibernation_delay);
}
if (unlikely(silencers->all_alarms && silencers->stype == STYPE_DISABLE_ALARMS)) {
- static __thread int logged=0;
+ static int logged=0;
if (!logged) {
- log_health("[%s]: Skipping health checks, because all alarms are disabled via a %s command.",
- rrdhost_hostname(host),
+ log_health("Skipping health checks, because all alarms are disabled via a %s command.",
HEALTH_CMDAPI_CMD_DISABLEALL);
logged = 1;
}
}
#ifdef ENABLE_ACLK
- if (host->aclk_alert_reloaded && !marked_aclk_reload_loop)
+ if (aclk_alert_reloaded && !marked_aclk_reload_loop)
marked_aclk_reload_loop = loop;
#endif
- if (unlikely(apply_hibernation_delay)) {
- log_health(
- "[%s]: Postponing health checks for %"PRId64" seconds.",
- rrdhost_hostname(host),
- (int64_t)hibernation_delay);
-
- host->health_delay_up_to = now + hibernation_delay;
- next_run = now + hibernation_delay;
- health_sleep(next_run, loop, host);
- }
+ worker_is_busy(WORKER_HEALTH_JOB_RRD_LOCK);
+ rrd_rdlock();
- if (unlikely(host->health_delay_up_to)) {
- if (unlikely(now < host->health_delay_up_to)) {
- next_run = host->health_delay_up_to;
- health_sleep(next_run, loop, host);
- continue;
- }
+ rrdhost_foreach_read(host) {
- log_health("[%s]: Resuming health checks after delay.", rrdhost_hostname(host));
- host->health_delay_up_to = 0;
- }
+ if(unlikely(!service_running(SERVICE_HEALTH)))
+ break;
- // wait until cleanup of obsolete charts on children is complete
- if (host != localhost) {
- if (unlikely(host->trigger_chart_obsoletion_check == 1)) {
- log_health("[%s]: Waiting for chart obsoletion check.", rrdhost_hostname(host));
- health_sleep(next_run, loop, host);
+ if (unlikely(!host->health.health_enabled))
continue;
- }
- }
- if (!health_running_logged) {
- log_health("[%s]: Health is running.", rrdhost_hostname(host));
- health_running_logged = true;
- }
-
- if(likely(!host->health_log_fp) && (loop == 1 || loop % cleanup_sql_every_loop == 0))
- sql_health_alarm_log_cleanup(host);
+ if (unlikely(!rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH))) {
+ rrd_unlock();
+ initialize_health(host, host == localhost);
+ rrd_rdlock();
+ }
- health_execute_delayed_initializations(host);
+ health_execute_delayed_initializations(host);
- worker_is_busy(WORKER_HEALTH_JOB_HOST_LOCK);
+ rrdcalc_delete_alerts_not_matching_host_labels_from_this_host(host);
- // the first loop is to lookup values from the db
- foreach_rrdcalc_in_rrdhost_read(host, rc) {
+ if (unlikely(apply_hibernation_delay)) {
+ log_health(
+ "[%s]: Postponing health checks for %"PRId64" seconds.",
+ rrdhost_hostname(host),
+ (int64_t)hibernation_delay);
- rrdcalc_update_info_using_rrdset_labels(rc);
+ host->health.health_delay_up_to = now + hibernation_delay;
+ }
- if (update_disabled_silenced(host, rc))
- continue;
+ if (unlikely(host->health.health_delay_up_to)) {
+ if (unlikely(now < host->health.health_delay_up_to)) {
+ continue;
+ }
- // create an alert removed event if the chart is obsolete and
- // has stopped being collected for 60 seconds
- if (unlikely(rc->rrdset && rc->status != RRDCALC_STATUS_REMOVED &&
- rrdset_flag_check(rc->rrdset, RRDSET_FLAG_OBSOLETE) &&
- now > (rc->rrdset->last_collected_time.tv_sec + 60))) {
- if (!rrdcalc_isrepeating(rc)) {
- worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
- time_t now = now_realtime_sec();
-
- ALARM_ENTRY *ae = health_create_alarm_entry(
- host,
- rc->id,
- rc->next_event_id++,
- rc->config_hash_id,
- now,
- rc->name,
- rc->rrdset->id,
- rc->rrdset->context,
- rc->rrdset->family,
- rc->classification,
- rc->component,
- rc->type,
- rc->exec,
- rc->recipient,
- now - rc->last_status_change,
- rc->value,
- NAN,
- rc->status,
- RRDCALC_STATUS_REMOVED,
- rc->source,
- rc->units,
- rc->info,
- 0,
- rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0);
-
- if (ae) {
- health_alarm_log_add_entry(host, ae);
- rc->old_status = rc->status;
- rc->status = RRDCALC_STATUS_REMOVED;
- rc->last_status_change = now;
- rc->last_updated = now;
- rc->value = NAN;
+ log_health("[%s]: Resuming health checks after delay.", rrdhost_hostname(host));
+ host->health.health_delay_up_to = 0;
+ }
-#ifdef ENABLE_ACLK
- if (netdata_cloud_setting && likely(!host->aclk_alert_reloaded))
- sql_queue_alarm_to_aclk(host, ae, 1);
-#endif
- }
+ // wait until cleanup of obsolete charts on children is complete
+ if (host != localhost) {
+ if (unlikely(host->trigger_chart_obsoletion_check == 1)) {
+ log_health("[%s]: Waiting for chart obsoletion check.", rrdhost_hostname(host));
+ continue;
}
}
- if (unlikely(!rrdcalc_isrunnable(rc, now, &next_run))) {
- if (unlikely(rc->run_flags & RRDCALC_FLAG_RUNNABLE))
- rc->run_flags &= ~RRDCALC_FLAG_RUNNABLE;
- continue;
+ if (!health_running_logged) {
+ log_health("[%s]: Health is running.", rrdhost_hostname(host));
+ health_running_logged = true;
}
- runnable++;
- rc->old_value = rc->value;
- rc->run_flags |= RRDCALC_FLAG_RUNNABLE;
+ worker_is_busy(WORKER_HEALTH_JOB_HOST_LOCK);
- // ------------------------------------------------------------
- // if there is database lookup, do it
+ // the first loop is to lookup values from the db
+ foreach_rrdcalc_in_rrdhost_read(host, rc) {
- if (unlikely(RRDCALC_HAS_DB_LOOKUP(rc))) {
- worker_is_busy(WORKER_HEALTH_JOB_DB_QUERY);
+ if(unlikely(!service_running(SERVICE_HEALTH)))
+ break;
- /* time_t old_db_timestamp = rc->db_before; */
- int value_is_null = 0;
+ rrdcalc_update_info_using_rrdset_labels(rc);
- int ret = rrdset2value_api_v1(rc->rrdset, NULL, &rc->value, rrdcalc_dimensions(rc), 1,
- rc->after, rc->before, rc->group, NULL,
- 0, rc->options,
- &rc->db_after,&rc->db_before,
- NULL, NULL, NULL,
- &value_is_null, NULL, 0, 0,
- QUERY_SOURCE_HEALTH);
+ if (update_disabled_silenced(host, rc))
+ continue;
- if (unlikely(ret != 200)) {
- // database lookup failed
- rc->value = NAN;
- rc->run_flags |= RRDCALC_FLAG_DB_ERROR;
+ // create an alert removed event if the chart is obsolete and
+ // has stopped being collected for 60 seconds
+ if (unlikely(rc->rrdset && rc->status != RRDCALC_STATUS_REMOVED &&
+ rrdset_flag_check(rc->rrdset, RRDSET_FLAG_OBSOLETE) &&
+ now > (rc->rrdset->last_collected_time.tv_sec + 60))) {
+ if (!rrdcalc_isrepeating(rc)) {
+ worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
+ time_t now = now_realtime_sec();
+
+ ALARM_ENTRY *ae = health_create_alarm_entry(
+ host,
+ rc->id,
+ rc->next_event_id++,
+ rc->config_hash_id,
+ now,
+ rc->name,
+ rc->rrdset->id,
+ rc->rrdset->context,
+ rc->rrdset->family,
+ rc->classification,
+ rc->component,
+ rc->type,
+ rc->exec,
+ rc->recipient,
+ now - rc->last_status_change,
+ rc->value,
+ NAN,
+ rc->status,
+ RRDCALC_STATUS_REMOVED,
+ rc->source,
+ rc->units,
+ rc->info,
+ 0,
+ rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0);
+
+ if (ae) {
+ health_alarm_log_add_entry(host, ae);
+ rc->old_status = rc->status;
+ rc->status = RRDCALC_STATUS_REMOVED;
+ rc->last_status_change = now;
+ rc->last_updated = now;
+ rc->value = NAN;
- debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup returned error %d",
- rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), ret
- );
- } else
- rc->run_flags &= ~RRDCALC_FLAG_DB_ERROR;
-
- /* - RRDCALC_FLAG_DB_STALE not currently used
- if (unlikely(old_db_timestamp == rc->db_before)) {
- // database is stale
-
- debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database is stale", host->hostname, rc->chart?rc->chart:"NOCHART", rc->name);
-
- if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_DB_STALE))) {
- rc->rrdcalc_flags |= RRDCALC_FLAG_DB_STALE;
- error("Health on host '%s', alarm '%s.%s': database is stale", host->hostname, rc->chart?rc->chart:"NOCHART", rc->name);
- }
- }
- else if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_DB_STALE))
- rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_STALE;
- */
-
- if (unlikely(value_is_null)) {
- // collected value is null
- rc->value = NAN;
- rc->run_flags |= RRDCALC_FLAG_DB_NAN;
-
- debug(D_HEALTH,
- "Health on host '%s', alarm '%s.%s': database lookup returned empty value (possibly value is not collected yet)",
- rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc)
- );
- } else
- rc->run_flags &= ~RRDCALC_FLAG_DB_NAN;
+#ifdef ENABLE_ACLK
+ if (netdata_cloud_setting && likely(!aclk_alert_reloaded))
+ sql_queue_alarm_to_aclk(host, ae, 1);
+#endif
+ }
+ }
+ }
- debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup gave value " NETDATA_DOUBLE_FORMAT,
- rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), rc->value
- );
- }
+ if (unlikely(!rrdcalc_isrunnable(rc, now, &next_run))) {
+ if (unlikely(rc->run_flags & RRDCALC_FLAG_RUNNABLE))
+ rc->run_flags &= ~RRDCALC_FLAG_RUNNABLE;
+ continue;
+ }
- // ------------------------------------------------------------
- // if there is calculation expression, run it
+ runnable++;
+ rc->old_value = rc->value;
+ rc->run_flags |= RRDCALC_FLAG_RUNNABLE;
- if (unlikely(rc->calculation)) {
- worker_is_busy(WORKER_HEALTH_JOB_CALC_EVAL);
+ // ------------------------------------------------------------
+ // if there is database lookup, do it
- if (unlikely(!expression_evaluate(rc->calculation))) {
- // calculation failed
- rc->value = NAN;
- rc->run_flags |= RRDCALC_FLAG_CALC_ERROR;
+ if (unlikely(RRDCALC_HAS_DB_LOOKUP(rc))) {
+ worker_is_busy(WORKER_HEALTH_JOB_DB_QUERY);
- debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': expression '%s' failed: %s",
- rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),
- rc->calculation->parsed_as, buffer_tostring(rc->calculation->error_msg)
- );
- } else {
- rc->run_flags &= ~RRDCALC_FLAG_CALC_ERROR;
+ /* time_t old_db_timestamp = rc->db_before; */
+ int value_is_null = 0;
- debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': expression '%s' gave value "
- NETDATA_DOUBLE_FORMAT
- ": %s (source: %s)", rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),
- rc->calculation->parsed_as, rc->calculation->result,
- buffer_tostring(rc->calculation->error_msg), rrdcalc_source(rc)
- );
+ int ret = rrdset2value_api_v1(rc->rrdset, NULL, &rc->value, rrdcalc_dimensions(rc), 1,
+ rc->after, rc->before, rc->group, NULL,
+ 0, rc->options,
+ &rc->db_after,&rc->db_before,
+ NULL, NULL, NULL,
+ &value_is_null, NULL, 0, 0,
+ QUERY_SOURCE_HEALTH, STORAGE_PRIORITY_LOW);
- rc->value = rc->calculation->result;
- }
- }
- }
- foreach_rrdcalc_in_rrdhost_done(rc);
+ if (unlikely(ret != 200)) {
+ // database lookup failed
+ rc->value = NAN;
+ rc->run_flags |= RRDCALC_FLAG_DB_ERROR;
- if (unlikely(runnable && !netdata_exit)) {
- foreach_rrdcalc_in_rrdhost_read(host, rc) {
- if (unlikely(!(rc->run_flags & RRDCALC_FLAG_RUNNABLE)))
- continue;
+ debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup returned error %d",
+ rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), ret
+ );
+ } else
+ rc->run_flags &= ~RRDCALC_FLAG_DB_ERROR;
- if (rc->run_flags & RRDCALC_FLAG_DISABLED) {
- continue;
+ if (unlikely(value_is_null)) {
+ // collected value is null
+ rc->value = NAN;
+ rc->run_flags |= RRDCALC_FLAG_DB_NAN;
+
+ debug(D_HEALTH,
+ "Health on host '%s', alarm '%s.%s': database lookup returned empty value (possibly value is not collected yet)",
+ rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc)
+ );
+ } else
+ rc->run_flags &= ~RRDCALC_FLAG_DB_NAN;
+
+ debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup gave value " NETDATA_DOUBLE_FORMAT,
+ rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), rc->value
+ );
}
- RRDCALC_STATUS warning_status = RRDCALC_STATUS_UNDEFINED;
- RRDCALC_STATUS critical_status = RRDCALC_STATUS_UNDEFINED;
- // --------------------------------------------------------
- // check the warning expression
+ // ------------------------------------------------------------
+ // if there is calculation expression, run it
- if (likely(rc->warning)) {
- worker_is_busy(WORKER_HEALTH_JOB_WARNING_EVAL);
+ if (unlikely(rc->calculation)) {
+ worker_is_busy(WORKER_HEALTH_JOB_CALC_EVAL);
- if (unlikely(!expression_evaluate(rc->warning))) {
+ if (unlikely(!expression_evaluate(rc->calculation))) {
// calculation failed
- rc->run_flags |= RRDCALC_FLAG_WARN_ERROR;
+ rc->value = NAN;
+ rc->run_flags |= RRDCALC_FLAG_CALC_ERROR;
- debug(D_HEALTH,
- "Health on host '%s', alarm '%s.%s': warning expression failed with error: %s",
+ debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': expression '%s' failed: %s",
rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),
- buffer_tostring(rc->warning->error_msg)
+ rc->calculation->parsed_as, buffer_tostring(rc->calculation->error_msg)
);
} else {
- rc->run_flags &= ~RRDCALC_FLAG_WARN_ERROR;
- debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': warning expression gave value "
+ rc->run_flags &= ~RRDCALC_FLAG_CALC_ERROR;
+
+ debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': expression '%s' gave value "
NETDATA_DOUBLE_FORMAT
- ": %s (source: %s)", rrdhost_hostname(host), rrdcalc_chart_name(rc),
- rrdcalc_name(rc), rc->warning->result, buffer_tostring(rc->warning->error_msg), rrdcalc_source(rc)
+ ": %s (source: %s)", rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),
+ rc->calculation->parsed_as, rc->calculation->result,
+ buffer_tostring(rc->calculation->error_msg), rrdcalc_source(rc)
);
- warning_status = rrdcalc_value2status(rc->warning->result);
+
+ rc->value = rc->calculation->result;
}
}
+ }
+ foreach_rrdcalc_in_rrdhost_done(rc);
- // --------------------------------------------------------
- // check the critical expression
+ if (unlikely(runnable && service_running(SERVICE_HEALTH))) {
+ foreach_rrdcalc_in_rrdhost_read(host, rc) {
+ if(unlikely(!service_running(SERVICE_HEALTH)))
+ break;
- if (likely(rc->critical)) {
- worker_is_busy(WORKER_HEALTH_JOB_CRITICAL_EVAL);
+ if (unlikely(!(rc->run_flags & RRDCALC_FLAG_RUNNABLE)))
+ continue;
- if (unlikely(!expression_evaluate(rc->critical))) {
- // calculation failed
- rc->run_flags |= RRDCALC_FLAG_CRIT_ERROR;
+ if (rc->run_flags & RRDCALC_FLAG_DISABLED) {
+ continue;
+ }
+ RRDCALC_STATUS warning_status = RRDCALC_STATUS_UNDEFINED;
+ RRDCALC_STATUS critical_status = RRDCALC_STATUS_UNDEFINED;
+
+ // --------------------------------------------------------
+ // check the warning expression
+
+ if (likely(rc->warning)) {
+ worker_is_busy(WORKER_HEALTH_JOB_WARNING_EVAL);
+
+ if (unlikely(!expression_evaluate(rc->warning))) {
+ // calculation failed
+ rc->run_flags |= RRDCALC_FLAG_WARN_ERROR;
+
+ debug(D_HEALTH,
+ "Health on host '%s', alarm '%s.%s': warning expression failed with error: %s",
+ rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),
+ buffer_tostring(rc->warning->error_msg)
+ );
+ } else {
+ rc->run_flags &= ~RRDCALC_FLAG_WARN_ERROR;
+ debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': warning expression gave value "
+ NETDATA_DOUBLE_FORMAT
+ ": %s (source: %s)", rrdhost_hostname(host), rrdcalc_chart_name(rc),
+ rrdcalc_name(rc), rc->warning->result, buffer_tostring(rc->warning->error_msg), rrdcalc_source(rc)
+ );
+ warning_status = rrdcalc_value2status(rc->warning->result);
+ }
+ }
- debug(D_HEALTH,
- "Health on host '%s', alarm '%s.%s': critical expression failed with error: %s",
- rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),
- buffer_tostring(rc->critical->error_msg)
- );
- } else {
- rc->run_flags &= ~RRDCALC_FLAG_CRIT_ERROR;
- debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': critical expression gave value "
- NETDATA_DOUBLE_FORMAT
- ": %s (source: %s)", rrdhost_hostname(host), rrdcalc_chart_name(rc),
- rrdcalc_name(rc), rc->critical->result, buffer_tostring(rc->critical->error_msg),
- rrdcalc_source(rc)
- );
- critical_status = rrdcalc_value2status(rc->critical->result);
+ // --------------------------------------------------------
+ // check the critical expression
+
+ if (likely(rc->critical)) {
+ worker_is_busy(WORKER_HEALTH_JOB_CRITICAL_EVAL);
+
+ if (unlikely(!expression_evaluate(rc->critical))) {
+ // calculation failed
+ rc->run_flags |= RRDCALC_FLAG_CRIT_ERROR;
+
+ debug(D_HEALTH,
+ "Health on host '%s', alarm '%s.%s': critical expression failed with error: %s",
+ rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),
+ buffer_tostring(rc->critical->error_msg)
+ );
+ } else {
+ rc->run_flags &= ~RRDCALC_FLAG_CRIT_ERROR;
+ debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': critical expression gave value "
+ NETDATA_DOUBLE_FORMAT
+ ": %s (source: %s)", rrdhost_hostname(host), rrdcalc_chart_name(rc),
+ rrdcalc_name(rc), rc->critical->result, buffer_tostring(rc->critical->error_msg),
+ rrdcalc_source(rc)
+ );
+ critical_status = rrdcalc_value2status(rc->critical->result);
+ }
}
- }
- // --------------------------------------------------------
- // decide the final alarm status
+ // --------------------------------------------------------
+ // decide the final alarm status
- RRDCALC_STATUS status = RRDCALC_STATUS_UNDEFINED;
+ RRDCALC_STATUS status = RRDCALC_STATUS_UNDEFINED;
- switch (warning_status) {
- case RRDCALC_STATUS_CLEAR:
- status = RRDCALC_STATUS_CLEAR;
- break;
+ switch (warning_status) {
+ case RRDCALC_STATUS_CLEAR:
+ status = RRDCALC_STATUS_CLEAR;
+ break;
- case RRDCALC_STATUS_RAISED:
- status = RRDCALC_STATUS_WARNING;
- break;
+ case RRDCALC_STATUS_RAISED:
+ status = RRDCALC_STATUS_WARNING;
+ break;
- default:
- break;
- }
+ default:
+ break;
+ }
- switch (critical_status) {
- case RRDCALC_STATUS_CLEAR:
- if (status == RRDCALC_STATUS_UNDEFINED)
- status = RRDCALC_STATUS_CLEAR;
- break;
+ switch (critical_status) {
+ case RRDCALC_STATUS_CLEAR:
+ if (status == RRDCALC_STATUS_UNDEFINED)
+ status = RRDCALC_STATUS_CLEAR;
+ break;
- case RRDCALC_STATUS_RAISED:
- status = RRDCALC_STATUS_CRITICAL;
- break;
+ case RRDCALC_STATUS_RAISED:
+ status = RRDCALC_STATUS_CRITICAL;
+ break;
- default:
- break;
- }
+ default:
+ break;
+ }
- // --------------------------------------------------------
- // check if the new status and the old differ
+ // --------------------------------------------------------
+ // check if the new status and the old differ
- if (status != rc->status) {
- worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
- int delay = 0;
+ if (status != rc->status) {
+ worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
+ int delay = 0;
- // apply trigger hysteresis
+ // apply trigger hysteresis
- if (now > rc->delay_up_to_timestamp) {
- rc->delay_up_current = rc->delay_up_duration;
- rc->delay_down_current = rc->delay_down_duration;
- rc->delay_last = 0;
- rc->delay_up_to_timestamp = 0;
- } else {
- rc->delay_up_current = (int) (rc->delay_up_current * rc->delay_multiplier);
- if (rc->delay_up_current > rc->delay_max_duration)
- rc->delay_up_current = rc->delay_max_duration;
+ if (now > rc->delay_up_to_timestamp) {
+ rc->delay_up_current = rc->delay_up_duration;
+ rc->delay_down_current = rc->delay_down_duration;
+ rc->delay_last = 0;
+ rc->delay_up_to_timestamp = 0;
+ } else {
+ rc->delay_up_current = (int) (rc->delay_up_current * rc->delay_multiplier);
+ if (rc->delay_up_current > rc->delay_max_duration)
+ rc->delay_up_current = rc->delay_max_duration;
- rc->delay_down_current = (int) (rc->delay_down_current * rc->delay_multiplier);
- if (rc->delay_down_current > rc->delay_max_duration)
- rc->delay_down_current = rc->delay_max_duration;
- }
+ rc->delay_down_current = (int) (rc->delay_down_current * rc->delay_multiplier);
+ if (rc->delay_down_current > rc->delay_max_duration)
+ rc->delay_down_current = rc->delay_max_duration;
+ }
- if (status > rc->status)
- delay = rc->delay_up_current;
- else
- delay = rc->delay_down_current;
-
- // COMMENTED: because we do need to send raising alarms
- // if(now + delay < rc->delay_up_to_timestamp)
- // delay = (int)(rc->delay_up_to_timestamp - now);
-
- rc->delay_last = delay;
- rc->delay_up_to_timestamp = now + delay;
-
- ALARM_ENTRY *ae = health_create_alarm_entry(
- host,
- rc->id,
- rc->next_event_id++,
- rc->config_hash_id,
- now,
- rc->name,
- rc->rrdset->id,
- rc->rrdset->context,
- rc->rrdset->family,
- rc->classification,
- rc->component,
- rc->type,
- rc->exec,
- rc->recipient,
- now - rc->last_status_change,
- rc->old_value,
- rc->value,
- rc->status,
- status,
- rc->source,
- rc->units,
- rc->info,
- rc->delay_last,
- (
- ((rc->options & RRDCALC_OPTION_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) |
- ((rc->run_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) |
- (rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0)
- )
- );
-
- health_alarm_log_add_entry(host, ae);
-
- log_health("[%s]: Alert event for [%s.%s], value [%s], status [%s].", rrdhost_hostname(host), ae_chart_name(ae), ae_name(ae), ae_new_value_string(ae), rrdcalc_status2string(ae->new_status));
-
- rc->last_status_change = now;
- rc->old_status = rc->status;
- rc->status = status;
- }
+ if (status > rc->status)
+ delay = rc->delay_up_current;
+ else
+ delay = rc->delay_down_current;
+
+ // COMMENTED: because we do need to send raising alarms
+ // if(now + delay < rc->delay_up_to_timestamp)
+ // delay = (int)(rc->delay_up_to_timestamp - now);
+
+ rc->delay_last = delay;
+ rc->delay_up_to_timestamp = now + delay;
+
+ ALARM_ENTRY *ae = health_create_alarm_entry(
+ host,
+ rc->id,
+ rc->next_event_id++,
+ rc->config_hash_id,
+ now,
+ rc->name,
+ rc->rrdset->id,
+ rc->rrdset->context,
+ rc->rrdset->family,
+ rc->classification,
+ rc->component,
+ rc->type,
+ rc->exec,
+ rc->recipient,
+ now - rc->last_status_change,
+ rc->old_value,
+ rc->value,
+ rc->status,
+ status,
+ rc->source,
+ rc->units,
+ rc->info,
+ rc->delay_last,
+ (
+ ((rc->options & RRDCALC_OPTION_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) |
+ ((rc->run_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) |
+ (rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0)
+ )
+ );
- rc->last_updated = now;
- rc->next_update = now + rc->update_every;
+ health_alarm_log_add_entry(host, ae);
- if (next_run > rc->next_update)
- next_run = rc->next_update;
- }
- foreach_rrdcalc_in_rrdhost_done(rc);
+ log_health("[%s]: Alert event for [%s.%s], value [%s], status [%s].", rrdhost_hostname(host), ae_chart_name(ae), ae_name(ae), ae_new_value_string(ae), rrdcalc_status2string(ae->new_status));
- // process repeating alarms
- foreach_rrdcalc_in_rrdhost_read(host, rc) {
- int repeat_every = 0;
- if(unlikely(rrdcalc_isrepeating(rc) && rc->delay_up_to_timestamp <= now)) {
- if(unlikely(rc->status == RRDCALC_STATUS_WARNING)) {
- rc->run_flags &= ~RRDCALC_FLAG_RUN_ONCE;
- repeat_every = rc->warn_repeat_every;
- } else if(unlikely(rc->status == RRDCALC_STATUS_CRITICAL)) {
- rc->run_flags &= ~RRDCALC_FLAG_RUN_ONCE;
- repeat_every = rc->crit_repeat_every;
- } else if(unlikely(rc->status == RRDCALC_STATUS_CLEAR)) {
- if(!(rc->run_flags & RRDCALC_FLAG_RUN_ONCE)) {
- if(rc->old_status == RRDCALC_STATUS_CRITICAL) {
- repeat_every = 1;
- } else if (rc->old_status == RRDCALC_STATUS_WARNING) {
- repeat_every = 1;
+ rc->last_status_change = now;
+ rc->old_status = rc->status;
+ rc->status = status;
+ }
+
+ rc->last_updated = now;
+ rc->next_update = now + rc->update_every;
+
+ if (next_run > rc->next_update)
+ next_run = rc->next_update;
+ }
+ foreach_rrdcalc_in_rrdhost_done(rc);
+
+ // process repeating alarms
+ foreach_rrdcalc_in_rrdhost_read(host, rc) {
+ if(unlikely(!service_running(SERVICE_HEALTH)))
+ break;
+
+ int repeat_every = 0;
+ if(unlikely(rrdcalc_isrepeating(rc) && rc->delay_up_to_timestamp <= now)) {
+ if(unlikely(rc->status == RRDCALC_STATUS_WARNING)) {
+ rc->run_flags &= ~RRDCALC_FLAG_RUN_ONCE;
+ repeat_every = rc->warn_repeat_every;
+ } else if(unlikely(rc->status == RRDCALC_STATUS_CRITICAL)) {
+ rc->run_flags &= ~RRDCALC_FLAG_RUN_ONCE;
+ repeat_every = rc->crit_repeat_every;
+ } else if(unlikely(rc->status == RRDCALC_STATUS_CLEAR)) {
+ if(!(rc->run_flags & RRDCALC_FLAG_RUN_ONCE)) {
+ if(rc->old_status == RRDCALC_STATUS_CRITICAL) {
+ repeat_every = 1;
+ } else if (rc->old_status == RRDCALC_STATUS_WARNING) {
+ repeat_every = 1;
+ }
}
}
+ } else {
+ continue;
}
- } else {
- continue;
- }
- if(unlikely(repeat_every > 0 && (rc->last_repeat + repeat_every) <= now)) {
- worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
- rc->last_repeat = now;
- if (likely(rc->times_repeat < UINT32_MAX)) rc->times_repeat++;
-
- ALARM_ENTRY *ae = health_create_alarm_entry(
- host,
- rc->id,
- rc->next_event_id++,
- rc->config_hash_id,
- now,
- rc->name,
- rc->rrdset->id,
- rc->rrdset->context,
- rc->rrdset->family,
- rc->classification,
- rc->component,
- rc->type,
- rc->exec,
- rc->recipient,
- now - rc->last_status_change,
- rc->old_value,
- rc->value,
- rc->old_status,
- rc->status,
- rc->source,
- rc->units,
- rc->info,
- rc->delay_last,
- (
- ((rc->options & RRDCALC_OPTION_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) |
- ((rc->run_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) |
- (rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0)
- )
- );
-
- ae->last_repeat = rc->last_repeat;
- if (!(rc->run_flags & RRDCALC_FLAG_RUN_ONCE) && rc->status == RRDCALC_STATUS_CLEAR) {
- ae->flags |= HEALTH_ENTRY_RUN_ONCE;
+ if(unlikely(repeat_every > 0 && (rc->last_repeat + repeat_every) <= now)) {
+ worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
+ rc->last_repeat = now;
+ if (likely(rc->times_repeat < UINT32_MAX)) rc->times_repeat++;
+
+ ALARM_ENTRY *ae = health_create_alarm_entry(
+ host,
+ rc->id,
+ rc->next_event_id++,
+ rc->config_hash_id,
+ now,
+ rc->name,
+ rc->rrdset->id,
+ rc->rrdset->context,
+ rc->rrdset->family,
+ rc->classification,
+ rc->component,
+ rc->type,
+ rc->exec,
+ rc->recipient,
+ now - rc->last_status_change,
+ rc->old_value,
+ rc->value,
+ rc->old_status,
+ rc->status,
+ rc->source,
+ rc->units,
+ rc->info,
+ rc->delay_last,
+ (
+ ((rc->options & RRDCALC_OPTION_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) |
+ ((rc->run_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) |
+ (rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0)
+ )
+ );
+
+ ae->last_repeat = rc->last_repeat;
+ if (!(rc->run_flags & RRDCALC_FLAG_RUN_ONCE) && rc->status == RRDCALC_STATUS_CLEAR) {
+ ae->flags |= HEALTH_ENTRY_RUN_ONCE;
+ }
+ rc->run_flags |= RRDCALC_FLAG_RUN_ONCE;
+ health_process_notifications(host, ae);
+ debug(D_HEALTH, "Notification sent for the repeating alarm %u.", ae->alarm_id);
+ health_alarm_wait_for_execution(ae);
+ health_alarm_log_free_one_nochecks_nounlink(ae);
}
- rc->run_flags |= RRDCALC_FLAG_RUN_ONCE;
- health_process_notifications(host, ae);
- debug(D_HEALTH, "Notification sent for the repeating alarm %u.", ae->alarm_id);
- health_alarm_wait_for_execution(ae);
- health_alarm_log_free_one_nochecks_nounlink(ae);
}
+ foreach_rrdcalc_in_rrdhost_done(rc);
}
- foreach_rrdcalc_in_rrdhost_done(rc);
- }
- if (unlikely(netdata_exit))
- break;
+ if (unlikely(!service_running(SERVICE_HEALTH)))
+ break;
- // execute notifications
- // and cleanup
- worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_PROCESS);
- health_alarm_log_process(host);
+ // execute notifications
+ // and cleanup
+ worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_PROCESS);
+ health_alarm_log_process(host);
- if (unlikely(netdata_exit)) {
- // wait for all notifications to finish before allowing health to be cleaned up
- ALARM_ENTRY *ae;
- while (NULL != (ae = alarm_notifications_in_progress.head)) {
- health_alarm_wait_for_execution(ae);
+ if (unlikely(!service_running(SERVICE_HEALTH))) {
+ // wait for all notifications to finish before allowing health to be cleaned up
+ ALARM_ENTRY *ae;
+ while (NULL != (ae = alarm_notifications_in_progress.head)) {
+ if(unlikely(!service_running(SERVICE_HEALTH)))
+ break;
+
+ health_alarm_wait_for_execution(ae);
+ }
+ break;
}
- break;
- }
+ } //for each host
+
+ rrd_unlock();
// wait for all notifications to finish before allowing health to be cleaned up
ALARM_ENTRY *ae;
while (NULL != (ae = alarm_notifications_in_progress.head)) {
+ if(unlikely(!service_running(SERVICE_HEALTH)))
+ break;
+
health_alarm_wait_for_execution(ae);
}
#ifdef ENABLE_ACLK
- if (netdata_cloud_setting && unlikely(host->aclk_alert_reloaded) && loop > (marked_aclk_reload_loop + 2)) {
- sql_queue_removed_alerts_to_aclk(host);
- host->aclk_alert_reloaded = 0;
+ if (netdata_cloud_setting && unlikely(aclk_alert_reloaded) && loop > (marked_aclk_reload_loop + 2)) {
+ rrdhost_foreach_read(host) {
+ if(unlikely(!service_running(SERVICE_HEALTH)))
+ break;
+
+ if (unlikely(!host->health.health_enabled))
+ continue;
+
+ sql_queue_removed_alerts_to_aclk(host);
+ }
+ aclk_alert_reloaded = 0;
marked_aclk_reload_loop = 0;
}
#endif
- if(unlikely(netdata_exit))
+ if(unlikely(!service_running(SERVICE_HEALTH)))
break;
- health_sleep(next_run, loop, host);
+ health_sleep(next_run, loop);
} // forever
@@ -1554,28 +1561,13 @@ void *health_main(void *ptr) {
void health_add_host_labels(void) {
DICTIONARY *labels = localhost->rrdlabels;
+ // The source should be CONF, but when it is set, these labels are exported by default ('send configured labels' in exporting.conf).
+ // Their export seems to break exporting to Graphite, see https://github.com/netdata/netdata/issues/14084.
+
int is_ephemeral = appconfig_get_boolean(&netdata_config, CONFIG_SECTION_HEALTH, "is ephemeral", CONFIG_BOOLEAN_NO);
- rrdlabels_add(labels, "_is_ephemeral", is_ephemeral ? "true" : "false", RRDLABEL_SRC_CONFIG);
+ rrdlabels_add(labels, "_is_ephemeral", is_ephemeral ? "true" : "false", RRDLABEL_SRC_AUTO);
int has_unstable_connection = appconfig_get_boolean(&netdata_config, CONFIG_SECTION_HEALTH, "has unstable connection", CONFIG_BOOLEAN_NO);
- rrdlabels_add(labels, "_has_unstable_connection", has_unstable_connection ? "true" : "false", RRDLABEL_SRC_CONFIG);
+ rrdlabels_add(labels, "_has_unstable_connection", has_unstable_connection ? "true" : "false", RRDLABEL_SRC_AUTO);
}
-void health_thread_spawn(RRDHOST * host) {
- if(!host->health_spawn) {
- char tag[NETDATA_THREAD_TAG_MAX + 1];
- snprintfz(tag, NETDATA_THREAD_TAG_MAX, "HEALTH[%s]", rrdhost_hostname(host));
- struct health_state *health = callocz(1, sizeof(*health));
- health->host = host;
-
- if(netdata_thread_create(&host->health_thread, tag, NETDATA_THREAD_OPTION_JOINABLE, health_main, (void *) health)) {
- log_health("[%s]: Failed to create new thread for client.", rrdhost_hostname(host));
- error("HEALTH [%s]: Failed to create new thread for client.", rrdhost_hostname(host));
- }
- else {
- log_health("[%s]: Created new thread for client.", rrdhost_hostname(host));
- host->health_spawn = 1;
- host->aclk_alert_reloaded = 1;
- }
- }
-}
generated by cgit v1.2.3 (git 2.39.1) at 2024-05-05 20:22:58 +0000