diff options
Diffstat (limited to 'lib/icinga/checkable-check.cpp')
| -rw-r--r-- | lib/icinga/checkable-check.cpp | 709 |
1 files changed, 709 insertions, 0 deletions
diff --git a/lib/icinga/checkable-check.cpp b/lib/icinga/checkable-check.cpp new file mode 100644 index 0000000..efa9477 --- /dev/null +++ b/lib/icinga/checkable-check.cpp @@ -0,0 +1,709 @@ +/* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */ + +#include "icinga/checkable.hpp" +#include "icinga/service.hpp" +#include "icinga/host.hpp" +#include "icinga/checkcommand.hpp" +#include "icinga/icingaapplication.hpp" +#include "icinga/cib.hpp" +#include "icinga/clusterevents.hpp" +#include "remote/messageorigin.hpp" +#include "remote/apilistener.hpp" +#include "base/objectlock.hpp" +#include "base/logger.hpp" +#include "base/convert.hpp" +#include "base/utility.hpp" +#include "base/context.hpp" + +using namespace icinga; + +boost::signals2::signal<void (const Checkable::Ptr&, const CheckResult::Ptr&, const MessageOrigin::Ptr&)> Checkable::OnNewCheckResult; +boost::signals2::signal<void (const Checkable::Ptr&, const CheckResult::Ptr&, StateType, const MessageOrigin::Ptr&)> Checkable::OnStateChange; +boost::signals2::signal<void (const Checkable::Ptr&, const CheckResult::Ptr&, std::set<Checkable::Ptr>, const MessageOrigin::Ptr&)> Checkable::OnReachabilityChanged; +boost::signals2::signal<void (const Checkable::Ptr&, NotificationType, const CheckResult::Ptr&, const String&, const String&, const MessageOrigin::Ptr&)> Checkable::OnNotificationsRequested; +boost::signals2::signal<void (const Checkable::Ptr&)> Checkable::OnNextCheckUpdated; + +Atomic<uint_fast64_t> Checkable::CurrentConcurrentChecks (0); + +std::mutex Checkable::m_StatsMutex; +int Checkable::m_PendingChecks = 0; +std::condition_variable Checkable::m_PendingChecksCV; + +CheckCommand::Ptr Checkable::GetCheckCommand() const +{ + return dynamic_pointer_cast<CheckCommand>(NavigateCheckCommandRaw()); +} + +TimePeriod::Ptr Checkable::GetCheckPeriod() const +{ + return TimePeriod::GetByName(GetCheckPeriodRaw()); +} + +void Checkable::SetSchedulingOffset(long offset) +{ + m_SchedulingOffset = offset; +} + +long Checkable::GetSchedulingOffset() +{ + return m_SchedulingOffset; +} + +void Checkable::UpdateNextCheck(const MessageOrigin::Ptr& origin) +{ + double interval; + + if (GetStateType() == StateTypeSoft && GetLastCheckResult() != nullptr) + interval = GetRetryInterval(); + else + interval = GetCheckInterval(); + + double now = Utility::GetTime(); + double adj = 0; + + if (interval > 1) + adj = fmod(now * 100 + GetSchedulingOffset(), interval * 100) / 100.0; + + if (adj != 0.0) + adj = std::min(0.5 + fmod(GetSchedulingOffset(), interval * 5) / 100.0, adj); + + double nextCheck = now - adj + interval; + double lastCheck = GetLastCheck(); + + Log(LogDebug, "Checkable") + << "Update checkable '" << GetName() << "' with check interval '" << GetCheckInterval() + << "' from last check time at " << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", (lastCheck < 0 ? 0 : lastCheck)) + << " (" << GetLastCheck() << ") to next check time at " << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", nextCheck) << " (" << nextCheck << ")."; + + SetNextCheck(nextCheck, false, origin); +} + +bool Checkable::HasBeenChecked() const +{ + return GetLastCheckResult() != nullptr; +} + +double Checkable::GetLastCheck() const +{ + CheckResult::Ptr cr = GetLastCheckResult(); + double schedule_end = -1; + + if (cr) + schedule_end = cr->GetScheduleEnd(); + + return schedule_end; +} + +Checkable::ProcessingResult Checkable::ProcessCheckResult(const CheckResult::Ptr& cr, const MessageOrigin::Ptr& origin) +{ + using Result = Checkable::ProcessingResult; + + { + ObjectLock olock(this); + m_CheckRunning = false; + } + + if (!cr) + return Result::NoCheckResult; + + double now = Utility::GetTime(); + + if (cr->GetScheduleStart() == 0) + cr->SetScheduleStart(now); + + if (cr->GetScheduleEnd() == 0) + cr->SetScheduleEnd(now); + + if (cr->GetExecutionStart() == 0) + cr->SetExecutionStart(now); + + if (cr->GetExecutionEnd() == 0) + cr->SetExecutionEnd(now); + + if (!origin || origin->IsLocal()) + cr->SetSchedulingSource(IcingaApplication::GetInstance()->GetNodeName()); + + Endpoint::Ptr command_endpoint = GetCommandEndpoint(); + + if (cr->GetCheckSource().IsEmpty()) { + if ((!origin || origin->IsLocal())) + cr->SetCheckSource(IcingaApplication::GetInstance()->GetNodeName()); + + /* override check source if command_endpoint was defined */ + if (command_endpoint && !GetExtension("agent_check")) + cr->SetCheckSource(command_endpoint->GetName()); + } + + /* agent checks go through the api */ + if (command_endpoint && GetExtension("agent_check")) { + ApiListener::Ptr listener = ApiListener::GetInstance(); + + if (listener) { + /* send message back to its origin */ + Dictionary::Ptr message = ClusterEvents::MakeCheckResultMessage(this, cr); + listener->SyncSendMessage(command_endpoint, message); + } + + return Result::Ok; + + } + + if (!IsActive()) + return Result::CheckableInactive; + + bool reachable = IsReachable(); + bool notification_reachable = IsReachable(DependencyNotification); + + ObjectLock olock(this); + + CheckResult::Ptr old_cr = GetLastCheckResult(); + ServiceState old_state = GetStateRaw(); + StateType old_stateType = GetStateType(); + long old_attempt = GetCheckAttempt(); + bool recovery = false; + + /* When we have an check result already (not after fresh start), + * prevent to accept old check results and allow overrides for + * CRs happened in the future. + */ + if (old_cr) { + double currentCRTimestamp = old_cr->GetExecutionStart(); + double newCRTimestamp = cr->GetExecutionStart(); + + /* Our current timestamp may be from the future (wrong server time adjusted again). Allow overrides here. */ + if (currentCRTimestamp > now) { + /* our current CR is from the future, let the new CR override it. */ + Log(LogDebug, "Checkable") + << std::fixed << std::setprecision(6) << "Processing check result for checkable '" << GetName() << "' from " + << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", newCRTimestamp) << " (" << newCRTimestamp + << "). Overriding since ours is from the future at " + << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", currentCRTimestamp) << " (" << currentCRTimestamp << ")."; + } else { + /* Current timestamp is from the past, but the new timestamp is even more in the past. Skip it. */ + if (newCRTimestamp < currentCRTimestamp) { + Log(LogDebug, "Checkable") + << std::fixed << std::setprecision(6) << "Skipping check result for checkable '" << GetName() << "' from " + << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", newCRTimestamp) << " (" << newCRTimestamp + << "). It is in the past compared to ours at " + << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", currentCRTimestamp) << " (" << currentCRTimestamp << ")."; + return Result::NewerCheckResultPresent; + } + } + } + + /* The ExecuteCheck function already sets the old state, but we need to do it again + * in case this was a passive check result. */ + SetLastStateRaw(old_state); + SetLastStateType(old_stateType); + SetLastReachable(reachable); + + Host::Ptr host; + Service::Ptr service; + tie(host, service) = GetHostService(this); + + CheckableType checkableType = CheckableHost; + if (service) + checkableType = CheckableService; + + long attempt = 1; + + std::set<Checkable::Ptr> children = GetChildren(); + + if (IsStateOK(cr->GetState())) { + SetStateType(StateTypeHard); // NOT-OK -> HARD OK + + if (!IsStateOK(old_state)) + recovery = true; + + ResetNotificationNumbers(); + SaveLastState(ServiceOK, cr->GetExecutionEnd()); + } else { + /* OK -> NOT-OK change, first SOFT state. Reset attempt counter. */ + if (IsStateOK(old_state)) { + SetStateType(StateTypeSoft); + attempt = 1; + } + + /* SOFT state change, increase attempt counter. */ + if (old_stateType == StateTypeSoft && !IsStateOK(old_state)) { + SetStateType(StateTypeSoft); + attempt = old_attempt + 1; + } + + /* HARD state change (e.g. previously 2/3 and this next attempt). Reset attempt counter. */ + if (attempt >= GetMaxCheckAttempts()) { + SetStateType(StateTypeHard); + attempt = 1; + } + + if (!IsStateOK(cr->GetState())) { + SaveLastState(cr->GetState(), cr->GetExecutionEnd()); + } + } + + if (!reachable) + SetLastStateUnreachable(cr->GetExecutionEnd()); + + SetCheckAttempt(attempt); + + ServiceState new_state = cr->GetState(); + SetStateRaw(new_state); + + bool stateChange; + + /* Exception on state change calculation for hosts. */ + if (checkableType == CheckableService) + stateChange = (old_state != new_state); + else + stateChange = (Host::CalculateState(old_state) != Host::CalculateState(new_state)); + + /* Store the current last state change for the next iteration. */ + SetPreviousStateChange(GetLastStateChange()); + + if (stateChange) { + SetLastStateChange(cr->GetExecutionEnd()); + + /* remove acknowledgements */ + if (GetAcknowledgement() == AcknowledgementNormal || + (GetAcknowledgement() == AcknowledgementSticky && IsStateOK(new_state))) { + ClearAcknowledgement(""); + } + } + + bool remove_acknowledgement_comments = false; + + if (GetAcknowledgement() == AcknowledgementNone) + remove_acknowledgement_comments = true; + + bool hardChange = (GetStateType() == StateTypeHard && old_stateType == StateTypeSoft); + + if (stateChange && old_stateType == StateTypeHard && GetStateType() == StateTypeHard) + hardChange = true; + + bool is_volatile = GetVolatile(); + + if (hardChange || is_volatile) { + SetLastHardStateRaw(new_state); + SetLastHardStateChange(cr->GetExecutionEnd()); + SetLastHardStatesRaw(GetLastHardStatesRaw() / 100u + new_state * 100u); + } + + if (stateChange) { + SetLastSoftStatesRaw(GetLastSoftStatesRaw() / 100u + new_state * 100u); + } + + cr->SetPreviousHardState(ServiceState(GetLastHardStatesRaw() % 100u)); + + if (!IsStateOK(new_state)) + TriggerDowntimes(cr->GetExecutionEnd()); + + /* statistics for external tools */ + Checkable::UpdateStatistics(cr, checkableType); + + bool in_downtime = IsInDowntime(); + + bool send_notification = false; + bool suppress_notification = !notification_reachable || in_downtime || IsAcknowledged(); + + /* Send notifications whether when a hard state change occurred. */ + if (hardChange && !(old_stateType == StateTypeSoft && IsStateOK(new_state))) + send_notification = true; + /* Or if the checkable is volatile and in a HARD state. */ + else if (is_volatile && GetStateType() == StateTypeHard) + send_notification = true; + + if (IsStateOK(old_state) && old_stateType == StateTypeSoft) + send_notification = false; /* Don't send notifications for SOFT-OK -> HARD-OK. */ + + if (is_volatile && IsStateOK(old_state) && IsStateOK(new_state)) + send_notification = false; /* Don't send notifications for volatile OK -> OK changes. */ + + olock.Unlock(); + + if (remove_acknowledgement_comments) + RemoveAckComments(String(), cr->GetExecutionEnd()); + + Dictionary::Ptr vars_after = new Dictionary({ + { "state", new_state }, + { "state_type", GetStateType() }, + { "attempt", GetCheckAttempt() }, + { "reachable", reachable } + }); + + if (old_cr) + cr->SetVarsBefore(old_cr->GetVarsAfter()); + + cr->SetVarsAfter(vars_after); + + olock.Lock(); + + if (service) { + SetLastCheckResult(cr); + } else { + bool wasProblem = GetProblem(); + + SetLastCheckResult(cr); + + if (GetProblem() != wasProblem) { + auto services = host->GetServices(); + olock.Unlock(); + for (auto& service : services) { + Service::OnHostProblemChanged(service, cr, origin); + } + olock.Lock(); + } + } + + bool was_flapping = IsFlapping(); + + UpdateFlappingStatus(cr->GetState()); + + bool is_flapping = IsFlapping(); + + if (cr->GetActive()) { + UpdateNextCheck(origin); + } else { + /* Reschedule the next check for external passive check results. The side effect of + * this is that for as long as we receive results for a service we + * won't execute any active checks. */ + double offset; + double ttl = cr->GetTtl(); + + if (ttl > 0) + offset = ttl; + else + offset = GetCheckInterval(); + + SetNextCheck(Utility::GetTime() + offset, false, origin); + } + + olock.Unlock(); + +#ifdef I2_DEBUG /* I2_DEBUG */ + Log(LogDebug, "Checkable") + << "Flapping: Checkable " << GetName() + << " was: " << was_flapping + << " is: " << is_flapping + << " threshold low: " << GetFlappingThresholdLow() + << " threshold high: " << GetFlappingThresholdHigh() + << "% current: " << GetFlappingCurrent() << "%."; +#endif /* I2_DEBUG */ + + if (recovery) { + for (auto& child : children) { + if (child->GetProblem() && child->GetEnableActiveChecks()) { + auto nextCheck (now + Utility::Random() % 60); + + ObjectLock oLock (child); + + if (nextCheck < child->GetNextCheck()) { + child->SetNextCheck(nextCheck); + } + } + } + } + + if (stateChange) { + /* reschedule direct parents */ + for (const Checkable::Ptr& parent : GetParents()) { + if (parent.get() == this) + continue; + + if (!parent->GetEnableActiveChecks()) + continue; + + if (parent->GetNextCheck() >= now + parent->GetRetryInterval()) { + ObjectLock olock(parent); + parent->SetNextCheck(now); + } + } + } + + OnNewCheckResult(this, cr, origin); + + /* signal status updates to for example db_ido */ + OnStateChanged(this); + + String old_state_str = (service ? Service::StateToString(old_state) : Host::StateToString(Host::CalculateState(old_state))); + String new_state_str = (service ? Service::StateToString(new_state) : Host::StateToString(Host::CalculateState(new_state))); + + /* Whether a hard state change or a volatile state change except OK -> OK happened. */ + if (hardChange || (is_volatile && !(IsStateOK(old_state) && IsStateOK(new_state)))) { + OnStateChange(this, cr, StateTypeHard, origin); + Log(LogNotice, "Checkable") + << "State Change: Checkable '" << GetName() << "' hard state change from " << old_state_str << " to " << new_state_str << " detected." << (is_volatile ? " Checkable is volatile." : ""); + } + /* Whether a state change happened or the state type is SOFT (must be logged too). */ + else if (stateChange || GetStateType() == StateTypeSoft) { + OnStateChange(this, cr, StateTypeSoft, origin); + Log(LogNotice, "Checkable") + << "State Change: Checkable '" << GetName() << "' soft state change from " << old_state_str << " to " << new_state_str << " detected."; + } + + if (GetStateType() == StateTypeSoft || hardChange || recovery || + (is_volatile && !(IsStateOK(old_state) && IsStateOK(new_state)))) + ExecuteEventHandler(); + + int suppressed_types = 0; + + /* Flapping start/end notifications */ + if (!was_flapping && is_flapping) { + /* FlappingStart notifications happen on state changes, not in downtimes */ + if (!IsPaused()) { + if (in_downtime) { + suppressed_types |= NotificationFlappingStart; + } else { + OnNotificationsRequested(this, NotificationFlappingStart, cr, "", "", nullptr); + } + } + + Log(LogNotice, "Checkable") + << "Flapping Start: Checkable '" << GetName() << "' started flapping (Current flapping value " + << GetFlappingCurrent() << "% > high threshold " << GetFlappingThresholdHigh() << "%)."; + + NotifyFlapping(origin); + } else if (was_flapping && !is_flapping) { + /* FlappingEnd notifications are independent from state changes, must not happen in downtine */ + if (!IsPaused()) { + if (in_downtime) { + suppressed_types |= NotificationFlappingEnd; + } else { + OnNotificationsRequested(this, NotificationFlappingEnd, cr, "", "", nullptr); + } + } + + Log(LogNotice, "Checkable") + << "Flapping Stop: Checkable '" << GetName() << "' stopped flapping (Current flapping value " + << GetFlappingCurrent() << "% < low threshold " << GetFlappingThresholdLow() << "%)."; + + NotifyFlapping(origin); + } + + if (send_notification && !is_flapping) { + if (!IsPaused()) { + /* If there are still some pending suppressed state notification, keep the suppression until these are + * handled by Checkable::FireSuppressedNotifications(). + */ + bool pending = GetSuppressedNotifications() & (NotificationRecovery|NotificationProblem); + + if (suppress_notification || pending) { + suppressed_types |= (recovery ? NotificationRecovery : NotificationProblem); + } else { + OnNotificationsRequested(this, recovery ? NotificationRecovery : NotificationProblem, cr, "", "", nullptr); + } + } + } + + if (suppressed_types) { + /* If some notifications were suppressed, but just because of e.g. a downtime, + * stash them into a notification types bitmask for maybe re-sending later. + */ + + ObjectLock olock (this); + int suppressed_types_before (GetSuppressedNotifications()); + int suppressed_types_after (suppressed_types_before | suppressed_types); + + const int conflict = NotificationFlappingStart | NotificationFlappingEnd; + if ((suppressed_types_after & conflict) == conflict) { + /* Flapping start and end cancel out each other. */ + suppressed_types_after &= ~conflict; + } + + const int stateNotifications = NotificationRecovery | NotificationProblem; + if (!(suppressed_types_before & stateNotifications) && (suppressed_types & stateNotifications)) { + /* A state-related notification is suppressed for the first time, store the previous state. When + * notifications are no longer suppressed, this can be compared with the current state to determine + * if a notification must be sent. This is done differently compared to flapping notifications just above + * as for state notifications, problem and recovery don't always cancel each other. For example, + * WARNING -> OK -> CRITICAL generates both types once, but there should still be a notification. + */ + SetStateBeforeSuppression(old_stateType == StateTypeHard ? old_state : ServiceOK); + } + + if (suppressed_types_after != suppressed_types_before) { + SetSuppressedNotifications(suppressed_types_after); + } + } + + /* update reachability for child objects */ + if ((stateChange || hardChange) && !children.empty()) + OnReachabilityChanged(this, cr, children, origin); + + return Result::Ok; +} + +void Checkable::ExecuteRemoteCheck(const Dictionary::Ptr& resolvedMacros) +{ + CONTEXT("Executing remote check for object '" << GetName() << "'"); + + double scheduled_start = GetNextCheck(); + double before_check = Utility::GetTime(); + + CheckResult::Ptr cr = new CheckResult(); + cr->SetScheduleStart(scheduled_start); + cr->SetExecutionStart(before_check); + + GetCheckCommand()->Execute(this, cr, resolvedMacros, true); +} + +void Checkable::ExecuteCheck() +{ + CONTEXT("Executing check for object '" << GetName() << "'"); + + /* keep track of scheduling info in case the check type doesn't provide its own information */ + double scheduled_start = GetNextCheck(); + double before_check = Utility::GetTime(); + + SetLastCheckStarted(Utility::GetTime()); + + /* This calls SetNextCheck() which updates the CheckerComponent's idle/pending + * queues and ensures that checks are not fired multiple times. ProcessCheckResult() + * is called too late. See #6421. + */ + UpdateNextCheck(); + + bool reachable = IsReachable(); + + { + ObjectLock olock(this); + + /* don't run another check if there is one pending */ + if (m_CheckRunning) + return; + + m_CheckRunning = true; + + SetLastStateRaw(GetStateRaw()); + SetLastStateType(GetLastStateType()); + SetLastReachable(reachable); + } + + CheckResult::Ptr cr = new CheckResult(); + + cr->SetScheduleStart(scheduled_start); + cr->SetExecutionStart(before_check); + + Endpoint::Ptr endpoint = GetCommandEndpoint(); + bool local = !endpoint || endpoint == Endpoint::GetLocalEndpoint(); + + if (local) { + GetCheckCommand()->Execute(this, cr, nullptr, false); + } else { + Dictionary::Ptr macros = new Dictionary(); + GetCheckCommand()->Execute(this, cr, macros, false); + + if (endpoint->GetConnected()) { + /* perform check on remote endpoint */ + Dictionary::Ptr message = new Dictionary(); + message->Set("jsonrpc", "2.0"); + message->Set("method", "event::ExecuteCommand"); + + Host::Ptr host; + Service::Ptr service; + tie(host, service) = GetHostService(this); + + Dictionary::Ptr params = new Dictionary(); + message->Set("params", params); + params->Set("command_type", "check_command"); + params->Set("command", GetCheckCommand()->GetName()); + params->Set("host", host->GetName()); + + if (service) + params->Set("service", service->GetShortName()); + + /* + * If the host/service object specifies the 'check_timeout' attribute, + * forward this to the remote endpoint to limit the command execution time. + */ + if (!GetCheckTimeout().IsEmpty()) + params->Set("check_timeout", GetCheckTimeout()); + + params->Set("macros", macros); + + ApiListener::Ptr listener = ApiListener::GetInstance(); + + if (listener) + listener->SyncSendMessage(endpoint, message); + + /* Re-schedule the check so we don't run it again until after we've received + * a check result from the remote instance. The check will be re-scheduled + * using the proper check interval once we've received a check result. + */ + SetNextCheck(Utility::GetTime() + GetCheckCommand()->GetTimeout() + 30); + + /* + * Let the user know that there was a problem with the check if + * 1) The endpoint is not syncing (replay log, etc.) + * 2) Outside of the cold startup window (5min) + */ + } else if (!endpoint->GetSyncing() && Application::GetInstance()->GetStartTime() < Utility::GetTime() - 300) { + /* fail to perform check on unconnected endpoint */ + cr->SetState(ServiceUnknown); + + String output = "Remote Icinga instance '" + endpoint->GetName() + "' is not connected to "; + + Endpoint::Ptr localEndpoint = Endpoint::GetLocalEndpoint(); + + if (localEndpoint) + output += "'" + localEndpoint->GetName() + "'"; + else + output += "this instance"; + + cr->SetOutput(output); + + ProcessCheckResult(cr); + } + + { + ObjectLock olock(this); + m_CheckRunning = false; + } + } +} + +void Checkable::UpdateStatistics(const CheckResult::Ptr& cr, CheckableType type) +{ + time_t ts = cr->GetScheduleEnd(); + + if (type == CheckableHost) { + if (cr->GetActive()) + CIB::UpdateActiveHostChecksStatistics(ts, 1); + else + CIB::UpdatePassiveHostChecksStatistics(ts, 1); + } else if (type == CheckableService) { + if (cr->GetActive()) + CIB::UpdateActiveServiceChecksStatistics(ts, 1); + else + CIB::UpdatePassiveServiceChecksStatistics(ts, 1); + } else { + Log(LogWarning, "Checkable", "Unknown checkable type for statistic update."); + } +} + +void Checkable::IncreasePendingChecks() +{ + std::unique_lock<std::mutex> lock(m_StatsMutex); + m_PendingChecks++; +} + +void Checkable::DecreasePendingChecks() +{ + std::unique_lock<std::mutex> lock(m_StatsMutex); + m_PendingChecks--; + m_PendingChecksCV.notify_one(); +} + +int Checkable::GetPendingChecks() +{ + std::unique_lock<std::mutex> lock(m_StatsMutex); + return m_PendingChecks; +} + +void Checkable::AquirePendingCheckSlot(int maxPendingChecks) +{ + std::unique_lock<std::mutex> lock(m_StatsMutex); + while (m_PendingChecks >= maxPendingChecks) + m_PendingChecksCV.wait(lock); + + m_PendingChecks++; +} |