summaryrefslogtreecommitdiffstats
path: root/lib/icingadb
diff options
context:
space:
mode:
Diffstat (limited to 'lib/icingadb')
-rw-r--r--lib/icingadb/CMakeLists.txt32
-rw-r--r--lib/icingadb/icingadb-itl.conf24
-rw-r--r--lib/icingadb/icingadb-objects.cpp2932
-rw-r--r--lib/icingadb/icingadb-stats.cpp54
-rw-r--r--lib/icingadb/icingadb-utility.cpp313
-rw-r--r--lib/icingadb/icingadb.cpp309
-rw-r--r--lib/icingadb/icingadb.hpp241
-rw-r--r--lib/icingadb/icingadb.ti63
-rw-r--r--lib/icingadb/icingadbchecktask.cpp514
-rw-r--r--lib/icingadb/icingadbchecktask.hpp29
-rw-r--r--lib/icingadb/redisconnection.cpp773
-rw-r--r--lib/icingadb/redisconnection.hpp678
12 files changed, 5962 insertions, 0 deletions
diff --git a/lib/icingadb/CMakeLists.txt b/lib/icingadb/CMakeLists.txt
new file mode 100644
index 0000000..de8e4ad
--- /dev/null
+++ b/lib/icingadb/CMakeLists.txt
@@ -0,0 +1,32 @@
+# Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+
+
+mkclass_target(icingadb.ti icingadb-ti.cpp icingadb-ti.hpp)
+
+mkembedconfig_target(icingadb-itl.conf icingadb-itl.cpp)
+
+set(icingadb_SOURCES
+ icingadb.cpp icingadb-objects.cpp icingadb-stats.cpp icingadb-utility.cpp redisconnection.cpp icingadb-ti.hpp
+ icingadbchecktask.cpp icingadb-itl.cpp
+)
+
+if(ICINGA2_UNITY_BUILD)
+ mkunity_target(icingadb icingadb icingadb_SOURCES)
+endif()
+
+add_library(icingadb OBJECT ${icingadb_SOURCES})
+
+include_directories(${icinga2_SOURCE_DIR}/third-party)
+
+add_dependencies(icingadb base config icinga remote)
+
+set_target_properties (
+ icingadb PROPERTIES
+ FOLDER Components
+)
+
+install_if_not_exists(
+ ${PROJECT_SOURCE_DIR}/etc/icinga2/features-available/icingadb.conf
+ ${CMAKE_INSTALL_SYSCONFDIR}/icinga2/features-available
+)
+
+set(CPACK_NSIS_EXTRA_INSTALL_COMMANDS "${CPACK_NSIS_EXTRA_INSTALL_COMMANDS}" PARENT_SCOPE)
diff --git a/lib/icingadb/icingadb-itl.conf b/lib/icingadb/icingadb-itl.conf
new file mode 100644
index 0000000..5f3950e
--- /dev/null
+++ b/lib/icingadb/icingadb-itl.conf
@@ -0,0 +1,24 @@
+/* Icinga 2 | (c) 2022 Icinga GmbH | GPLv2+ */
+
+System.assert(Internal.run_with_activation_context(function() {
+ template CheckCommand "icingadb-check-command" use (checkFunc = Internal.IcingadbCheck) {
+ execute = checkFunc
+ }
+
+ object CheckCommand "icingadb" {
+ import "icingadb-check-command"
+
+ vars.icingadb_name = "icingadb"
+
+ vars.icingadb_full_dump_duration_warning = 5m
+ vars.icingadb_full_dump_duration_critical = 10m
+ vars.icingadb_full_sync_duration_warning = 5m
+ vars.icingadb_full_sync_duration_critical = 10m
+ vars.icingadb_redis_backlog_warning = 5m
+ vars.icingadb_redis_backlog_critical = 15m
+ vars.icingadb_database_backlog_warning = 5m
+ vars.icingadb_database_backlog_critical = 15m
+ }
+}))
+
+Internal.remove("IcingadbCheck")
diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp
new file mode 100644
index 0000000..ac1469b
--- /dev/null
+++ b/lib/icingadb/icingadb-objects.cpp
@@ -0,0 +1,2932 @@
+/* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */
+
+#include "icingadb/icingadb.hpp"
+#include "icingadb/redisconnection.hpp"
+#include "base/configtype.hpp"
+#include "base/configobject.hpp"
+#include "base/defer.hpp"
+#include "base/json.hpp"
+#include "base/logger.hpp"
+#include "base/serializer.hpp"
+#include "base/shared.hpp"
+#include "base/tlsutility.hpp"
+#include "base/initialize.hpp"
+#include "base/convert.hpp"
+#include "base/array.hpp"
+#include "base/exception.hpp"
+#include "base/utility.hpp"
+#include "base/object-packer.hpp"
+#include "icinga/command.hpp"
+#include "icinga/compatutility.hpp"
+#include "icinga/customvarobject.hpp"
+#include "icinga/host.hpp"
+#include "icinga/service.hpp"
+#include "icinga/hostgroup.hpp"
+#include "icinga/servicegroup.hpp"
+#include "icinga/usergroup.hpp"
+#include "icinga/checkcommand.hpp"
+#include "icinga/eventcommand.hpp"
+#include "icinga/notificationcommand.hpp"
+#include "icinga/timeperiod.hpp"
+#include "icinga/pluginutility.hpp"
+#include "remote/zone.hpp"
+#include <chrono>
+#include <cstdint>
+#include <iterator>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <set>
+#include <utility>
+#include <type_traits>
+
+using namespace icinga;
+
+using Prio = RedisConnection::QueryPriority;
+
+std::unordered_set<Type*> IcingaDB::m_IndexedTypes;
+
+INITIALIZE_ONCE(&IcingaDB::ConfigStaticInitialize);
+
+std::vector<Type::Ptr> IcingaDB::GetTypes()
+{
+ // The initial config sync will queue the types in the following order.
+ return {
+ // Sync them first to get their states ASAP.
+ Host::TypeInstance,
+ Service::TypeInstance,
+
+ // Then sync them for similar reasons.
+ Downtime::TypeInstance,
+ Comment::TypeInstance,
+
+ HostGroup::TypeInstance,
+ ServiceGroup::TypeInstance,
+ CheckCommand::TypeInstance,
+ Endpoint::TypeInstance,
+ EventCommand::TypeInstance,
+ Notification::TypeInstance,
+ NotificationCommand::TypeInstance,
+ TimePeriod::TypeInstance,
+ User::TypeInstance,
+ UserGroup::TypeInstance,
+ Zone::TypeInstance
+ };
+}
+
+void IcingaDB::ConfigStaticInitialize()
+{
+ for (auto& type : GetTypes()) {
+ m_IndexedTypes.emplace(type.get());
+ }
+
+ /* triggered in ProcessCheckResult(), requires UpdateNextCheck() to be called before */
+ Checkable::OnStateChange.connect([](const Checkable::Ptr& checkable, const CheckResult::Ptr& cr, StateType type, const MessageOrigin::Ptr&) {
+ IcingaDB::StateChangeHandler(checkable, cr, type);
+ });
+
+ Checkable::OnAcknowledgementSet.connect([](const Checkable::Ptr& checkable, const String& author, const String& comment, AcknowledgementType type, bool, bool persistent, double changeTime, double expiry, const MessageOrigin::Ptr&) {
+ AcknowledgementSetHandler(checkable, author, comment, type, persistent, changeTime, expiry);
+ });
+ Checkable::OnAcknowledgementCleared.connect([](const Checkable::Ptr& checkable, const String& removedBy, double changeTime, const MessageOrigin::Ptr&) {
+ AcknowledgementClearedHandler(checkable, removedBy, changeTime);
+ });
+
+ Checkable::OnReachabilityChanged.connect([](const Checkable::Ptr&, const CheckResult::Ptr&, std::set<Checkable::Ptr> children, const MessageOrigin::Ptr&) {
+ IcingaDB::ReachabilityChangeHandler(children);
+ });
+
+ /* triggered on create, update and delete objects */
+ ConfigObject::OnActiveChanged.connect([](const ConfigObject::Ptr& object, const Value&) {
+ IcingaDB::VersionChangedHandler(object);
+ });
+ ConfigObject::OnVersionChanged.connect([](const ConfigObject::Ptr& object, const Value&) {
+ IcingaDB::VersionChangedHandler(object);
+ });
+
+ /* downtime start */
+ Downtime::OnDowntimeTriggered.connect(&IcingaDB::DowntimeStartedHandler);
+ /* fixed/flexible downtime end or remove */
+ Downtime::OnDowntimeRemoved.connect(&IcingaDB::DowntimeRemovedHandler);
+
+ Checkable::OnNotificationSentToAllUsers.connect([](
+ const Notification::Ptr& notification, const Checkable::Ptr& checkable, const std::set<User::Ptr>& users,
+ const NotificationType& type, const CheckResult::Ptr& cr, const String& author, const String& text,
+ const MessageOrigin::Ptr&
+ ) {
+ IcingaDB::NotificationSentToAllUsersHandler(notification, checkable, users, type, cr, author, text);
+ });
+
+ Comment::OnCommentAdded.connect(&IcingaDB::CommentAddedHandler);
+ Comment::OnCommentRemoved.connect(&IcingaDB::CommentRemovedHandler);
+
+ Checkable::OnFlappingChange.connect(&IcingaDB::FlappingChangeHandler);
+
+ Checkable::OnNewCheckResult.connect([](const Checkable::Ptr& checkable, const CheckResult::Ptr&, const MessageOrigin::Ptr&) {
+ IcingaDB::NewCheckResultHandler(checkable);
+ });
+
+ Checkable::OnNextCheckUpdated.connect([](const Checkable::Ptr& checkable) {
+ IcingaDB::NextCheckUpdatedHandler(checkable);
+ });
+
+ Service::OnHostProblemChanged.connect([](const Service::Ptr& service, const CheckResult::Ptr&, const MessageOrigin::Ptr&) {
+ IcingaDB::HostProblemChangedHandler(service);
+ });
+
+ Notification::OnUsersRawChangedWithOldValue.connect([](const Notification::Ptr& notification, const Value& oldValues, const Value& newValues) {
+ IcingaDB::NotificationUsersChangedHandler(notification, oldValues, newValues);
+ });
+ Notification::OnUserGroupsRawChangedWithOldValue.connect([](const Notification::Ptr& notification, const Value& oldValues, const Value& newValues) {
+ IcingaDB::NotificationUserGroupsChangedHandler(notification, oldValues, newValues);
+ });
+ TimePeriod::OnRangesChangedWithOldValue.connect([](const TimePeriod::Ptr& timeperiod, const Value& oldValues, const Value& newValues) {
+ IcingaDB::TimePeriodRangesChangedHandler(timeperiod, oldValues, newValues);
+ });
+ TimePeriod::OnIncludesChangedWithOldValue.connect([](const TimePeriod::Ptr& timeperiod, const Value& oldValues, const Value& newValues) {
+ IcingaDB::TimePeriodIncludesChangedHandler(timeperiod, oldValues, newValues);
+ });
+ TimePeriod::OnExcludesChangedWithOldValue.connect([](const TimePeriod::Ptr& timeperiod, const Value& oldValues, const Value& newValues) {
+ IcingaDB::TimePeriodExcludesChangedHandler(timeperiod, oldValues, newValues);
+ });
+ User::OnGroupsChangedWithOldValue.connect([](const User::Ptr& user, const Value& oldValues, const Value& newValues) {
+ IcingaDB::UserGroupsChangedHandler(user, oldValues, newValues);
+ });
+ Host::OnGroupsChangedWithOldValue.connect([](const Host::Ptr& host, const Value& oldValues, const Value& newValues) {
+ IcingaDB::HostGroupsChangedHandler(host, oldValues, newValues);
+ });
+ Service::OnGroupsChangedWithOldValue.connect([](const Service::Ptr& service, const Value& oldValues, const Value& newValues) {
+ IcingaDB::ServiceGroupsChangedHandler(service, oldValues, newValues);
+ });
+ Command::OnEnvChangedWithOldValue.connect([](const ConfigObject::Ptr& command, const Value& oldValues, const Value& newValues) {
+ IcingaDB::CommandEnvChangedHandler(command, oldValues, newValues);
+ });
+ Command::OnArgumentsChangedWithOldValue.connect([](const ConfigObject::Ptr& command, const Value& oldValues, const Value& newValues) {
+ IcingaDB::CommandArgumentsChangedHandler(command, oldValues, newValues);
+ });
+ CustomVarObject::OnVarsChangedWithOldValue.connect([](const ConfigObject::Ptr& object, const Value& oldValues, const Value& newValues) {
+ IcingaDB::CustomVarsChangedHandler(object, oldValues, newValues);
+ });
+}
+
+void IcingaDB::UpdateAllConfigObjects()
+{
+ m_Rcon->Sync();
+ m_Rcon->FireAndForgetQuery({"XADD", "icinga:schema", "MAXLEN", "1", "*", "version", "5"}, Prio::Heartbeat);
+
+ Log(LogInformation, "IcingaDB") << "Starting initial config/status dump";
+ double startTime = Utility::GetTime();
+
+ SetOngoingDumpStart(startTime);
+
+ Defer resetOngoingDumpStart ([this]() {
+ SetOngoingDumpStart(0);
+ });
+
+ // Use a Workqueue to pack objects in parallel
+ WorkQueue upq(25000, Configuration::Concurrency, LogNotice);
+ upq.SetName("IcingaDB:ConfigDump");
+
+ std::vector<Type::Ptr> types = GetTypes();
+
+ m_Rcon->SuppressQueryKind(Prio::CheckResult);
+ m_Rcon->SuppressQueryKind(Prio::RuntimeStateSync);
+
+ Defer unSuppress ([this]() {
+ m_Rcon->UnsuppressQueryKind(Prio::RuntimeStateSync);
+ m_Rcon->UnsuppressQueryKind(Prio::CheckResult);
+ });
+
+ // Add a new type=* state=wip entry to the stream and remove all previous entries (MAXLEN 1).
+ m_Rcon->FireAndForgetQuery({"XADD", "icinga:dump", "MAXLEN", "1", "*", "key", "*", "state", "wip"}, Prio::Config);
+
+ const std::vector<String> globalKeys = {
+ m_PrefixConfigObject + "customvar",
+ m_PrefixConfigObject + "action:url",
+ m_PrefixConfigObject + "notes:url",
+ m_PrefixConfigObject + "icon:image",
+ };
+ DeleteKeys(m_Rcon, globalKeys, Prio::Config);
+ DeleteKeys(m_Rcon, {"icinga:nextupdate:host", "icinga:nextupdate:service"}, Prio::Config);
+ m_Rcon->Sync();
+
+ Defer resetDumpedGlobals ([this]() {
+ m_DumpedGlobals.CustomVar.Reset();
+ m_DumpedGlobals.ActionUrl.Reset();
+ m_DumpedGlobals.NotesUrl.Reset();
+ m_DumpedGlobals.IconImage.Reset();
+ });
+
+ upq.ParallelFor(types, false, [this](const Type::Ptr& type) {
+ String lcType = type->GetName().ToLower();
+ ConfigType *ctype = dynamic_cast<ConfigType *>(type.get());
+ if (!ctype)
+ return;
+
+ auto& rcon (m_Rcons.at(ctype));
+
+ std::vector<String> keys = GetTypeOverwriteKeys(lcType);
+ DeleteKeys(rcon, keys, Prio::Config);
+
+ WorkQueue upqObjectType(25000, Configuration::Concurrency, LogNotice);
+ upqObjectType.SetName("IcingaDB:ConfigDump:" + lcType);
+
+ std::map<String, String> redisCheckSums;
+ String configCheckSum = m_PrefixConfigCheckSum + lcType;
+
+ upqObjectType.Enqueue([&rcon, &configCheckSum, &redisCheckSums]() {
+ String cursor = "0";
+
+ do {
+ Array::Ptr res = rcon->GetResultOfQuery({
+ "HSCAN", configCheckSum, cursor, "COUNT", "1000"
+ }, Prio::Config);
+
+ AddKvsToMap(res->Get(1), redisCheckSums);
+
+ cursor = res->Get(0);
+ } while (cursor != "0");
+ });
+
+ auto objectChunks (ChunkObjects(ctype->GetObjects(), 500));
+ String configObject = m_PrefixConfigObject + lcType;
+
+ // Skimmed away attributes and checksums HMSETs' keys and values by Redis key.
+ std::map<String, std::vector<std::vector<String>>> ourContentRaw {{configCheckSum, {}}, {configObject, {}}};
+ std::mutex ourContentMutex;
+
+ upqObjectType.ParallelFor(objectChunks, [&](decltype(objectChunks)::const_reference chunk) {
+ std::map<String, std::vector<String>> hMSets;
+ // Two values are appended per object: Object ID (Hash encoded) and Object State (IcingaDB::SerializeState() -> JSON encoded)
+ std::vector<String> states = {"HMSET", m_PrefixConfigObject + lcType + ":state"};
+ // Two values are appended per object: Object ID (Hash encoded) and State Checksum ({ "checksum": checksum } -> JSON encoded)
+ std::vector<String> statesChksms = {"HMSET", m_PrefixConfigCheckSum + lcType + ":state"};
+ std::vector<std::vector<String> > transaction = {{"MULTI"}};
+ std::vector<String> hostZAdds = {"ZADD", "icinga:nextupdate:host"}, serviceZAdds = {"ZADD", "icinga:nextupdate:service"};
+
+ auto skimObjects ([&]() {
+ std::lock_guard<std::mutex> l (ourContentMutex);
+
+ for (auto& kv : ourContentRaw) {
+ auto pos (hMSets.find(kv.first));
+
+ if (pos != hMSets.end()) {
+ kv.second.emplace_back(std::move(pos->second));
+ hMSets.erase(pos);
+ }
+ }
+ });
+
+ bool dumpState = (lcType == "host" || lcType == "service");
+
+ size_t bulkCounter = 0;
+ for (const ConfigObject::Ptr& object : chunk) {
+ if (lcType != GetLowerCaseTypeNameDB(object))
+ continue;
+
+ std::vector<Dictionary::Ptr> runtimeUpdates;
+ CreateConfigUpdate(object, lcType, hMSets, runtimeUpdates, false);
+
+ // Write out inital state for checkables
+ if (dumpState) {
+ String objectKey = GetObjectIdentifier(object);
+ Dictionary::Ptr state = SerializeState(dynamic_pointer_cast<Checkable>(object));
+
+ states.emplace_back(objectKey);
+ states.emplace_back(JsonEncode(state));
+
+ statesChksms.emplace_back(objectKey);
+ statesChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", HashValue(state)}})));
+ }
+
+ bulkCounter++;
+ if (!(bulkCounter % 100)) {
+ skimObjects();
+
+ for (auto& kv : hMSets) {
+ if (!kv.second.empty()) {
+ kv.second.insert(kv.second.begin(), {"HMSET", kv.first});
+ transaction.emplace_back(std::move(kv.second));
+ }
+ }
+
+ if (states.size() > 2) {
+ transaction.emplace_back(std::move(states));
+ transaction.emplace_back(std::move(statesChksms));
+ states = {"HMSET", m_PrefixConfigObject + lcType + ":state"};
+ statesChksms = {"HMSET", m_PrefixConfigCheckSum + lcType + ":state"};
+ }
+
+ hMSets = decltype(hMSets)();
+
+ if (transaction.size() > 1) {
+ transaction.push_back({"EXEC"});
+ rcon->FireAndForgetQueries(std::move(transaction), Prio::Config);
+ transaction = {{"MULTI"}};
+ }
+ }
+
+ auto checkable (dynamic_pointer_cast<Checkable>(object));
+
+ if (checkable && checkable->GetEnableActiveChecks()) {
+ auto zAdds (dynamic_pointer_cast<Service>(checkable) ? &serviceZAdds : &hostZAdds);
+
+ zAdds->emplace_back(Convert::ToString(checkable->GetNextUpdate()));
+ zAdds->emplace_back(GetObjectIdentifier(checkable));
+
+ if (zAdds->size() >= 102u) {
+ std::vector<String> header (zAdds->begin(), zAdds->begin() + 2u);
+
+ rcon->FireAndForgetQuery(std::move(*zAdds), Prio::CheckResult);
+
+ *zAdds = std::move(header);
+ }
+ }
+ }
+
+ skimObjects();
+
+ for (auto& kv : hMSets) {
+ if (!kv.second.empty()) {
+ kv.second.insert(kv.second.begin(), {"HMSET", kv.first});
+ transaction.emplace_back(std::move(kv.second));
+ }
+ }
+
+ if (states.size() > 2) {
+ transaction.emplace_back(std::move(states));
+ transaction.emplace_back(std::move(statesChksms));
+ }
+
+ if (transaction.size() > 1) {
+ transaction.push_back({"EXEC"});
+ rcon->FireAndForgetQueries(std::move(transaction), Prio::Config);
+ }
+
+ for (auto zAdds : {&hostZAdds, &serviceZAdds}) {
+ if (zAdds->size() > 2u) {
+ rcon->FireAndForgetQuery(std::move(*zAdds), Prio::CheckResult);
+ }
+ }
+
+ Log(LogNotice, "IcingaDB")
+ << "Dumped " << bulkCounter << " objects of type " << lcType;
+ });
+
+ upqObjectType.Join();
+
+ if (upqObjectType.HasExceptions()) {
+ for (boost::exception_ptr exc : upqObjectType.GetExceptions()) {
+ if (exc) {
+ boost::rethrow_exception(exc);
+ }
+ }
+ }
+
+ std::map<String, std::map<String, String>> ourContent;
+
+ for (auto& source : ourContentRaw) {
+ auto& dest (ourContent[source.first]);
+
+ upqObjectType.Enqueue([&]() {
+ for (auto& hMSet : source.second) {
+ for (decltype(hMSet.size()) i = 0, stop = hMSet.size() - 1u; i < stop; i += 2u) {
+ dest.emplace(std::move(hMSet[i]), std::move(hMSet[i + 1u]));
+ }
+
+ hMSet.clear();
+ }
+
+ source.second.clear();
+ });
+ }
+
+ upqObjectType.Join();
+ ourContentRaw.clear();
+
+ auto& ourCheckSums (ourContent[configCheckSum]);
+ auto& ourObjects (ourContent[configObject]);
+ std::vector<String> setChecksum, setObject, delChecksum, delObject;
+
+ auto redisCurrent (redisCheckSums.begin());
+ auto redisEnd (redisCheckSums.end());
+ auto ourCurrent (ourCheckSums.begin());
+ auto ourEnd (ourCheckSums.end());
+
+ auto flushSets ([&]() {
+ auto affectedConfig (setObject.size() / 2u);
+
+ setChecksum.insert(setChecksum.begin(), {"HMSET", configCheckSum});
+ setObject.insert(setObject.begin(), {"HMSET", configObject});
+
+ std::vector<std::vector<String>> transaction;
+
+ transaction.emplace_back(std::vector<String>{"MULTI"});
+ transaction.emplace_back(std::move(setChecksum));
+ transaction.emplace_back(std::move(setObject));
+ transaction.emplace_back(std::vector<String>{"EXEC"});
+
+ setChecksum.clear();
+ setObject.clear();
+
+ rcon->FireAndForgetQueries(std::move(transaction), Prio::Config, {affectedConfig});
+ });
+
+ auto flushDels ([&]() {
+ auto affectedConfig (delObject.size());
+
+ delChecksum.insert(delChecksum.begin(), {"HDEL", configCheckSum});
+ delObject.insert(delObject.begin(), {"HDEL", configObject});
+
+ std::vector<std::vector<String>> transaction;
+
+ transaction.emplace_back(std::vector<String>{"MULTI"});
+ transaction.emplace_back(std::move(delChecksum));
+ transaction.emplace_back(std::move(delObject));
+ transaction.emplace_back(std::vector<String>{"EXEC"});
+
+ delChecksum.clear();
+ delObject.clear();
+
+ rcon->FireAndForgetQueries(std::move(transaction), Prio::Config, {affectedConfig});
+ });
+
+ auto setOne ([&]() {
+ setChecksum.emplace_back(ourCurrent->first);
+ setChecksum.emplace_back(ourCurrent->second);
+ setObject.emplace_back(ourCurrent->first);
+ setObject.emplace_back(ourObjects[ourCurrent->first]);
+
+ if (setChecksum.size() == 100u) {
+ flushSets();
+ }
+ });
+
+ auto delOne ([&]() {
+ delChecksum.emplace_back(redisCurrent->first);
+ delObject.emplace_back(redisCurrent->first);
+
+ if (delChecksum.size() == 100u) {
+ flushDels();
+ }
+ });
+
+ for (;;) {
+ if (redisCurrent == redisEnd) {
+ for (; ourCurrent != ourEnd; ++ourCurrent) {
+ setOne();
+ }
+
+ break;
+ } else if (ourCurrent == ourEnd) {
+ for (; redisCurrent != redisEnd; ++redisCurrent) {
+ delOne();
+ }
+
+ break;
+ } else if (redisCurrent->first < ourCurrent->first) {
+ delOne();
+ ++redisCurrent;
+ } else if (redisCurrent->first > ourCurrent->first) {
+ setOne();
+ ++ourCurrent;
+ } else {
+ if (redisCurrent->second != ourCurrent->second) {
+ setOne();
+ }
+
+ ++redisCurrent;
+ ++ourCurrent;
+ }
+ }
+
+ if (delChecksum.size()) {
+ flushDels();
+ }
+
+ if (setChecksum.size()) {
+ flushSets();
+ }
+
+ for (auto& key : GetTypeDumpSignalKeys(type)) {
+ rcon->FireAndForgetQuery({"XADD", "icinga:dump", "*", "key", key, "state", "done"}, Prio::Config);
+ }
+ rcon->Sync();
+ });
+
+ upq.Join();
+
+ if (upq.HasExceptions()) {
+ for (boost::exception_ptr exc : upq.GetExceptions()) {
+ try {
+ if (exc) {
+ boost::rethrow_exception(exc);
+ }
+ } catch(const std::exception& e) {
+ Log(LogCritical, "IcingaDB")
+ << "Exception during ConfigDump: " << e.what();
+ }
+ }
+ }
+
+ for (auto& key : globalKeys) {
+ m_Rcon->FireAndForgetQuery({"XADD", "icinga:dump", "*", "key", key, "state", "done"}, Prio::Config);
+ }
+
+ m_Rcon->FireAndForgetQuery({"XADD", "icinga:dump", "*", "key", "*", "state", "done"}, Prio::Config);
+
+ // enqueue a callback that will notify us once all previous queries were executed and wait for this event
+ std::promise<void> p;
+ m_Rcon->EnqueueCallback([&p](boost::asio::yield_context& yc) { p.set_value(); }, Prio::Config);
+ p.get_future().wait();
+
+ auto endTime (Utility::GetTime());
+ auto took (endTime - startTime);
+
+ SetLastdumpTook(took);
+ SetLastdumpEnd(endTime);
+
+ Log(LogInformation, "IcingaDB")
+ << "Initial config/status dump finished in " << took << " seconds.";
+}
+
+std::vector<std::vector<intrusive_ptr<ConfigObject>>> IcingaDB::ChunkObjects(std::vector<intrusive_ptr<ConfigObject>> objects, size_t chunkSize) {
+ std::vector<std::vector<intrusive_ptr<ConfigObject>>> chunks;
+ auto offset (objects.begin());
+ auto end (objects.end());
+
+ chunks.reserve((std::distance(offset, end) + chunkSize - 1) / chunkSize);
+
+ while (std::distance(offset, end) >= chunkSize) {
+ auto until (offset + chunkSize);
+ chunks.emplace_back(offset, until);
+ offset = until;
+ }
+
+ if (offset != end) {
+ chunks.emplace_back(offset, end);
+ }
+
+ return std::move(chunks);
+}
+
+void IcingaDB::DeleteKeys(const RedisConnection::Ptr& conn, const std::vector<String>& keys, RedisConnection::QueryPriority priority) {
+ std::vector<String> query = {"DEL"};
+ for (auto& key : keys) {
+ query.emplace_back(key);
+ }
+
+ conn->FireAndForgetQuery(std::move(query), priority);
+}
+
+std::vector<String> IcingaDB::GetTypeOverwriteKeys(const String& type)
+{
+ std::vector<String> keys = {
+ m_PrefixConfigObject + type + ":customvar",
+ };
+
+ if (type == "host" || type == "service" || type == "user") {
+ keys.emplace_back(m_PrefixConfigObject + type + "group:member");
+ keys.emplace_back(m_PrefixConfigObject + type + ":state");
+ keys.emplace_back(m_PrefixConfigCheckSum + type + ":state");
+ } else if (type == "timeperiod") {
+ keys.emplace_back(m_PrefixConfigObject + type + ":override:include");
+ keys.emplace_back(m_PrefixConfigObject + type + ":override:exclude");
+ keys.emplace_back(m_PrefixConfigObject + type + ":range");
+ } else if (type == "notification") {
+ keys.emplace_back(m_PrefixConfigObject + type + ":user");
+ keys.emplace_back(m_PrefixConfigObject + type + ":usergroup");
+ keys.emplace_back(m_PrefixConfigObject + type + ":recipient");
+ } else if (type == "checkcommand" || type == "notificationcommand" || type == "eventcommand") {
+ keys.emplace_back(m_PrefixConfigObject + type + ":envvar");
+ keys.emplace_back(m_PrefixConfigCheckSum + type + ":envvar");
+ keys.emplace_back(m_PrefixConfigObject + type + ":argument");
+ keys.emplace_back(m_PrefixConfigCheckSum + type + ":argument");
+ }
+
+ return std::move(keys);
+}
+
+std::vector<String> IcingaDB::GetTypeDumpSignalKeys(const Type::Ptr& type)
+{
+ String lcType = type->GetName().ToLower();
+ std::vector<String> keys = {m_PrefixConfigObject + lcType};
+
+ if (CustomVarObject::TypeInstance->IsAssignableFrom(type)) {
+ keys.emplace_back(m_PrefixConfigObject + lcType + ":customvar");
+ }
+
+ if (type == Host::TypeInstance || type == Service::TypeInstance) {
+ keys.emplace_back(m_PrefixConfigObject + lcType + "group:member");
+ keys.emplace_back(m_PrefixConfigObject + lcType + ":state");
+ } else if (type == User::TypeInstance) {
+ keys.emplace_back(m_PrefixConfigObject + lcType + "group:member");
+ } else if (type == TimePeriod::TypeInstance) {
+ keys.emplace_back(m_PrefixConfigObject + lcType + ":override:include");
+ keys.emplace_back(m_PrefixConfigObject + lcType + ":override:exclude");
+ keys.emplace_back(m_PrefixConfigObject + lcType + ":range");
+ } else if (type == Notification::TypeInstance) {
+ keys.emplace_back(m_PrefixConfigObject + lcType + ":user");
+ keys.emplace_back(m_PrefixConfigObject + lcType + ":usergroup");
+ keys.emplace_back(m_PrefixConfigObject + lcType + ":recipient");
+ } else if (type == CheckCommand::TypeInstance || type == NotificationCommand::TypeInstance || type == EventCommand::TypeInstance) {
+ keys.emplace_back(m_PrefixConfigObject + lcType + ":envvar");
+ keys.emplace_back(m_PrefixConfigObject + lcType + ":argument");
+ }
+
+ return std::move(keys);
+}
+
+template<typename ConfigType>
+static ConfigObject::Ptr GetObjectByName(const String& name)
+{
+ return ConfigObject::GetObject<ConfigType>(name);
+}
+
+void IcingaDB::InsertObjectDependencies(const ConfigObject::Ptr& object, const String typeName, std::map<String, std::vector<String>>& hMSets,
+ std::vector<Dictionary::Ptr>& runtimeUpdates, bool runtimeUpdate)
+{
+ String objectKey = GetObjectIdentifier(object);
+ String objectKeyName = typeName + "_id";
+
+ Type::Ptr type = object->GetReflectionType();
+
+ CustomVarObject::Ptr customVarObject = dynamic_pointer_cast<CustomVarObject>(object);
+
+ if (customVarObject) {
+ auto vars(SerializeVars(customVarObject->GetVars()));
+ if (vars) {
+ auto& typeCvs (hMSets[m_PrefixConfigObject + typeName + ":customvar"]);
+ auto& allCvs (hMSets[m_PrefixConfigObject + "customvar"]);
+
+ ObjectLock varsLock(vars);
+ Array::Ptr varsArray(new Array);
+
+ varsArray->Reserve(vars->GetLength());
+
+ for (auto& kv : vars) {
+ if (runtimeUpdate || m_DumpedGlobals.CustomVar.IsNew(kv.first)) {
+ allCvs.emplace_back(kv.first);
+ allCvs.emplace_back(JsonEncode(kv.second));
+
+ if (runtimeUpdate) {
+ AddObjectDataToRuntimeUpdates(runtimeUpdates, kv.first, m_PrefixConfigObject + "customvar", kv.second);
+ }
+ }
+
+ String id = HashValue(new Array({m_EnvironmentId, kv.first, object->GetName()}));
+ typeCvs.emplace_back(id);
+
+ Dictionary::Ptr data = new Dictionary({{objectKeyName, objectKey}, {"environment_id", m_EnvironmentId}, {"customvar_id", kv.first}});
+ typeCvs.emplace_back(JsonEncode(data));
+
+ if (runtimeUpdate) {
+ AddObjectDataToRuntimeUpdates(runtimeUpdates, id, m_PrefixConfigObject + typeName + ":customvar", data);
+ }
+ }
+ }
+ }
+
+ if (type == Host::TypeInstance || type == Service::TypeInstance) {
+ Checkable::Ptr checkable = static_pointer_cast<Checkable>(object);
+
+ String actionUrl = checkable->GetActionUrl();
+ String notesUrl = checkable->GetNotesUrl();
+ String iconImage = checkable->GetIconImage();
+ if (!actionUrl.IsEmpty()) {
+ auto& actionUrls (hMSets[m_PrefixConfigObject + "action:url"]);
+
+ auto id (HashValue(new Array({m_EnvironmentId, actionUrl})));
+
+ if (runtimeUpdate || m_DumpedGlobals.ActionUrl.IsNew(id)) {
+ actionUrls.emplace_back(std::move(id));
+ Dictionary::Ptr data = new Dictionary({{"environment_id", m_EnvironmentId}, {"action_url", actionUrl}});
+ actionUrls.emplace_back(JsonEncode(data));
+
+ if (runtimeUpdate) {
+ AddObjectDataToRuntimeUpdates(runtimeUpdates, actionUrls.at(actionUrls.size() - 2u), m_PrefixConfigObject + "action:url", data);
+ }
+ }
+ }
+ if (!notesUrl.IsEmpty()) {
+ auto& notesUrls (hMSets[m_PrefixConfigObject + "notes:url"]);
+
+ auto id (HashValue(new Array({m_EnvironmentId, notesUrl})));
+
+ if (runtimeUpdate || m_DumpedGlobals.NotesUrl.IsNew(id)) {
+ notesUrls.emplace_back(std::move(id));
+ Dictionary::Ptr data = new Dictionary({{"environment_id", m_EnvironmentId}, {"notes_url", notesUrl}});
+ notesUrls.emplace_back(JsonEncode(data));
+
+ if (runtimeUpdate) {
+ AddObjectDataToRuntimeUpdates(runtimeUpdates, notesUrls.at(notesUrls.size() - 2u), m_PrefixConfigObject + "notes:url", data);
+ }
+ }
+ }
+ if (!iconImage.IsEmpty()) {
+ auto& iconImages (hMSets[m_PrefixConfigObject + "icon:image"]);
+
+ auto id (HashValue(new Array({m_EnvironmentId, iconImage})));
+
+ if (runtimeUpdate || m_DumpedGlobals.IconImage.IsNew(id)) {
+ iconImages.emplace_back(std::move(id));
+ Dictionary::Ptr data = new Dictionary({{"environment_id", m_EnvironmentId}, {"icon_image", iconImage}});
+ iconImages.emplace_back(JsonEncode(data));
+
+ if (runtimeUpdate) {
+ AddObjectDataToRuntimeUpdates(runtimeUpdates, iconImages.at(iconImages.size() - 2u), m_PrefixConfigObject + "icon:image", data);
+ }
+ }
+ }
+
+ Host::Ptr host;
+ Service::Ptr service;
+ tie(host, service) = GetHostService(checkable);
+
+ ConfigObject::Ptr (*getGroup)(const String& name);
+ Array::Ptr groups;
+ if (service) {
+ groups = service->GetGroups();
+ getGroup = &::GetObjectByName<ServiceGroup>;
+ } else {
+ groups = host->GetGroups();
+ getGroup = &::GetObjectByName<HostGroup>;
+ }
+
+ if (groups) {
+ ObjectLock groupsLock(groups);
+ Array::Ptr groupIds(new Array);
+
+ groupIds->Reserve(groups->GetLength());
+
+ auto& members (hMSets[m_PrefixConfigObject + typeName + "group:member"]);
+
+ for (auto& group : groups) {
+ auto groupObj ((*getGroup)(group));
+ String groupId = GetObjectIdentifier(groupObj);
+ String id = HashValue(new Array({m_EnvironmentId, groupObj->GetName(), object->GetName()}));
+ members.emplace_back(id);
+ Dictionary::Ptr data = new Dictionary({{objectKeyName, objectKey}, {"environment_id", m_EnvironmentId}, {typeName + "group_id", groupId}});
+ members.emplace_back(JsonEncode(data));
+
+ if (runtimeUpdate) {
+ AddObjectDataToRuntimeUpdates(runtimeUpdates, id, m_PrefixConfigObject + typeName + "group:member", data);
+ }
+
+ groupIds->Add(groupId);
+ }
+ }
+
+ return;
+ }
+
+ if (type == TimePeriod::TypeInstance) {
+ TimePeriod::Ptr timeperiod = static_pointer_cast<TimePeriod>(object);
+
+ Dictionary::Ptr ranges = timeperiod->GetRanges();
+ if (ranges) {
+ ObjectLock rangesLock(ranges);
+ Array::Ptr rangeIds(new Array);
+ auto& typeRanges (hMSets[m_PrefixConfigObject + typeName + ":range"]);
+
+ rangeIds->Reserve(ranges->GetLength());
+
+ for (auto& kv : ranges) {
+ String rangeId = HashValue(new Array({m_EnvironmentId, kv.first, kv.second}));
+ rangeIds->Add(rangeId);
+
+ String id = HashValue(new Array({m_EnvironmentId, kv.first, kv.second, object->GetName()}));
+ typeRanges.emplace_back(id);
+ Dictionary::Ptr data = new Dictionary({{"environment_id", m_EnvironmentId}, {"timeperiod_id", objectKey}, {"range_key", kv.first}, {"range_value", kv.second}});
+ typeRanges.emplace_back(JsonEncode(data));
+
+ if (runtimeUpdate) {
+ AddObjectDataToRuntimeUpdates(runtimeUpdates, id, m_PrefixConfigObject + typeName + ":range", data);
+ }
+ }
+ }
+
+ Array::Ptr includes;
+ ConfigObject::Ptr (*getInclude)(const String& name);
+ includes = timeperiod->GetIncludes();
+ getInclude = &::GetObjectByName<TimePeriod>;
+
+ Array::Ptr includeChecksums = new Array();
+
+ ObjectLock includesLock(includes);
+ ObjectLock includeChecksumsLock(includeChecksums);
+
+ includeChecksums->Reserve(includes->GetLength());
+
+
+ auto& includs (hMSets[m_PrefixConfigObject + typeName + ":override:include"]);
+ for (auto include : includes) {
+ auto includeTp ((*getInclude)(include.Get<String>()));
+ String includeId = GetObjectIdentifier(includeTp);
+ includeChecksums->Add(includeId);
+
+ String id = HashValue(new Array({m_EnvironmentId, includeTp->GetName(), object->GetName()}));
+ includs.emplace_back(id);
+ Dictionary::Ptr data = new Dictionary({{"environment_id", m_EnvironmentId}, {"timeperiod_id", objectKey}, {"include_id", includeId}});
+ includs.emplace_back(JsonEncode(data));
+
+ if (runtimeUpdate) {
+ AddObjectDataToRuntimeUpdates(runtimeUpdates, id, m_PrefixConfigObject + typeName + ":override:include", data);
+ }
+ }
+
+ Array::Ptr excludes;
+ ConfigObject::Ptr (*getExclude)(const String& name);
+
+ excludes = timeperiod->GetExcludes();
+ getExclude = &::GetObjectByName<TimePeriod>;
+
+ Array::Ptr excludeChecksums = new Array();
+
+ ObjectLock excludesLock(excludes);
+ ObjectLock excludeChecksumsLock(excludeChecksums);
+
+ excludeChecksums->Reserve(excludes->GetLength());
+
+ auto& excluds (hMSets[m_PrefixConfigObject + typeName + ":override:exclude"]);
+
+ for (auto exclude : excludes) {
+ auto excludeTp ((*getExclude)(exclude.Get<String>()));
+ String excludeId = GetObjectIdentifier(excludeTp);
+ excludeChecksums->Add(excludeId);
+
+ String id = HashValue(new Array({m_EnvironmentId, excludeTp->GetName(), object->GetName()}));
+ excluds.emplace_back(id);
+ Dictionary::Ptr data = new Dictionary({{"environment_id", m_EnvironmentId}, {"timeperiod_id", objectKey}, {"exclude_id", excludeId}});
+ excluds.emplace_back(JsonEncode(data));
+
+ if (runtimeUpdate) {
+ AddObjectDataToRuntimeUpdates(runtimeUpdates, id, m_PrefixConfigObject + typeName + ":override:exclude", data);
+ }
+ }
+
+ return;
+ }
+
+ if (type == User::TypeInstance) {
+ User::Ptr user = static_pointer_cast<User>(object);
+ Array::Ptr groups = user->GetGroups();
+
+ if (groups) {
+ ObjectLock groupsLock(groups);
+ Array::Ptr groupIds(new Array);
+
+ groupIds->Reserve(groups->GetLength());
+
+ auto& members (hMSets[m_PrefixConfigObject + typeName + "group:member"]);
+ auto& notificationRecipients (hMSets[m_PrefixConfigObject + "notification:recipient"]);
+
+ for (auto& group : groups) {
+ UserGroup::Ptr groupObj = UserGroup::GetByName(group);
+ String groupId = GetObjectIdentifier(groupObj);
+ String id = HashValue(new Array({m_EnvironmentId, groupObj->GetName(), object->GetName()}));
+ members.emplace_back(id);
+ Dictionary::Ptr data = new Dictionary({{"user_id", objectKey}, {"environment_id", m_EnvironmentId}, {"usergroup_id", groupId}});
+ members.emplace_back(JsonEncode(data));
+
+ if (runtimeUpdate) {
+ AddObjectDataToRuntimeUpdates(runtimeUpdates, id, m_PrefixConfigObject + typeName + "group:member", data);
+
+ // Recipients are handled by notifications during initial dumps and only need to be handled here during runtime (e.g. User creation).
+ for (auto& notification : groupObj->GetNotifications()) {
+ String recipientId = HashValue(new Array({m_EnvironmentId, "usergroupuser", user->GetName(), groupObj->GetName(), notification->GetName()}));
+ notificationRecipients.emplace_back(recipientId);
+ Dictionary::Ptr recipientData = new Dictionary({{"notification_id", GetObjectIdentifier(notification)}, {"environment_id", m_EnvironmentId}, {"user_id", objectKey}, {"usergroup_id", groupId}});
+ notificationRecipients.emplace_back(JsonEncode(recipientData));
+
+ AddObjectDataToRuntimeUpdates(runtimeUpdates, recipientId, m_PrefixConfigObject + "notification:recipient", recipientData);
+ }
+ }
+
+ groupIds->Add(groupId);
+ }
+ }
+
+ return;
+ }
+
+ if (type == Notification::TypeInstance) {
+ Notification::Ptr notification = static_pointer_cast<Notification>(object);
+
+ std::set<User::Ptr> users = notification->GetUsers();
+ Array::Ptr userIds = new Array();
+
+ auto usergroups(notification->GetUserGroups());
+ Array::Ptr usergroupIds = new Array();
+
+ userIds->Reserve(users.size());
+
+ auto& usrs (hMSets[m_PrefixConfigObject + typeName + ":user"]);
+ auto& notificationRecipients (hMSets[m_PrefixConfigObject + typeName + ":recipient"]);
+
+ for (auto& user : users) {
+ String userId = GetObjectIdentifier(user);
+ String id = HashValue(new Array({m_EnvironmentId, "user", user->GetName(), object->GetName()}));
+ usrs.emplace_back(id);
+ notificationRecipients.emplace_back(id);
+
+ Dictionary::Ptr data = new Dictionary({{"notification_id", objectKey}, {"environment_id", m_EnvironmentId}, {"user_id", userId}});
+ String dataJson = JsonEncode(data);
+ usrs.emplace_back(dataJson);
+ notificationRecipients.emplace_back(dataJson);
+
+ if (runtimeUpdate) {
+ AddObjectDataToRuntimeUpdates(runtimeUpdates, id, m_PrefixConfigObject + typeName + ":user", data);
+ AddObjectDataToRuntimeUpdates(runtimeUpdates, id, m_PrefixConfigObject + typeName + ":recipient", data);
+ }
+
+ userIds->Add(userId);
+ }
+
+ usergroupIds->Reserve(usergroups.size());
+
+ auto& groups (hMSets[m_PrefixConfigObject + typeName + ":usergroup"]);
+
+ for (auto& usergroup : usergroups) {
+ String usergroupId = GetObjectIdentifier(usergroup);
+ String id = HashValue(new Array({m_EnvironmentId, "usergroup", usergroup->GetName(), object->GetName()}));
+ groups.emplace_back(id);
+ notificationRecipients.emplace_back(id);
+
+ Dictionary::Ptr groupData = new Dictionary({{"notification_id", objectKey}, {"environment_id", m_EnvironmentId}, {"usergroup_id", usergroupId}});
+ String groupDataJson = JsonEncode(groupData);
+ groups.emplace_back(groupDataJson);
+ notificationRecipients.emplace_back(groupDataJson);
+
+ if (runtimeUpdate) {
+ AddObjectDataToRuntimeUpdates(runtimeUpdates, id, m_PrefixConfigObject + typeName + ":usergroup", groupData);
+ AddObjectDataToRuntimeUpdates(runtimeUpdates, id, m_PrefixConfigObject + typeName + ":recipient", groupData);
+ }
+
+ for (const User::Ptr& user : usergroup->GetMembers()) {
+ String userId = GetObjectIdentifier(user);
+ String recipientId = HashValue(new Array({m_EnvironmentId, "usergroupuser", user->GetName(), usergroup->GetName(), notification->GetName()}));
+ notificationRecipients.emplace_back(recipientId);
+ Dictionary::Ptr userData = new Dictionary({{"notification_id", objectKey}, {"environment_id", m_EnvironmentId}, {"user_id", userId}, {"usergroup_id", usergroupId}});
+ notificationRecipients.emplace_back(JsonEncode(userData));
+
+ if (runtimeUpdate) {
+ AddObjectDataToRuntimeUpdates(runtimeUpdates, recipientId, m_PrefixConfigObject + typeName + ":recipient", userData);
+ }
+ }
+
+ usergroupIds->Add(usergroupId);
+ }
+
+ return;
+ }
+
+ if (type == CheckCommand::TypeInstance || type == NotificationCommand::TypeInstance || type == EventCommand::TypeInstance) {
+ Command::Ptr command = static_pointer_cast<Command>(object);
+
+ Dictionary::Ptr arguments = command->GetArguments();
+ if (arguments) {
+ ObjectLock argumentsLock(arguments);
+ auto& typeArgs (hMSets[m_PrefixConfigObject + typeName + ":argument"]);
+ auto& argChksms (hMSets[m_PrefixConfigCheckSum + typeName + ":argument"]);
+
+ for (auto& kv : arguments) {
+ Dictionary::Ptr values;
+ if (kv.second.IsObjectType<Dictionary>()) {
+ values = kv.second;
+ values = values->ShallowClone();
+ } else if (kv.second.IsObjectType<Array>()) {
+ values = new Dictionary({{"value", JsonEncode(kv.second)}});
+ } else {
+ values = new Dictionary({{"value", kv.second}});
+ }
+
+ for (const char *attr : {"value", "set_if", "separator"}) {
+ Value value;
+
+ // Stringify if set.
+ if (values->Get(attr, &value)) {
+ switch (value.GetType()) {
+ case ValueEmpty:
+ case ValueString:
+ break;
+ case ValueObject:
+ values->Set(attr, value.Get<Object::Ptr>()->ToString());
+ break;
+ default:
+ values->Set(attr, JsonEncode(value));
+ }
+ }
+ }
+
+ {
+ Value order;
+
+ // Intify if set.
+ if (values->Get("order", &order)) {
+ values->Set("order", (int)order);
+ }
+ }
+
+ values->Set(objectKeyName, objectKey);
+ values->Set("argument_key", kv.first);
+ values->Set("environment_id", m_EnvironmentId);
+
+ String id = HashValue(new Array({m_EnvironmentId, kv.first, object->GetName()}));
+
+ typeArgs.emplace_back(id);
+ typeArgs.emplace_back(JsonEncode(values));
+
+ argChksms.emplace_back(id);
+ String checksum = HashValue(kv.second);
+ argChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", checksum}})));
+
+ if (runtimeUpdate) {
+ values->Set("checksum", checksum);
+ AddObjectDataToRuntimeUpdates(runtimeUpdates, id, m_PrefixConfigObject + typeName + ":argument", values);
+ }
+ }
+ }
+
+ Dictionary::Ptr envvars = command->GetEnv();
+ if (envvars) {
+ ObjectLock envvarsLock(envvars);
+ Array::Ptr envvarIds(new Array);
+ auto& typeVars (hMSets[m_PrefixConfigObject + typeName + ":envvar"]);
+ auto& varChksms (hMSets[m_PrefixConfigCheckSum + typeName + ":envvar"]);
+
+ envvarIds->Reserve(envvars->GetLength());
+
+ for (auto& kv : envvars) {
+ Dictionary::Ptr values;
+ if (kv.second.IsObjectType<Dictionary>()) {
+ values = kv.second;
+ values = values->ShallowClone();
+ } else if (kv.second.IsObjectType<Array>()) {
+ values = new Dictionary({{"value", JsonEncode(kv.second)}});
+ } else {
+ values = new Dictionary({{"value", kv.second}});
+ }
+
+ {
+ Value value;
+
+ // JsonEncode() the value if it's set.
+ if (values->Get("value", &value)) {
+ values->Set("value", JsonEncode(value));
+ }
+ }
+
+ values->Set(objectKeyName, objectKey);
+ values->Set("envvar_key", kv.first);
+ values->Set("environment_id", m_EnvironmentId);
+
+ String id = HashValue(new Array({m_EnvironmentId, kv.first, object->GetName()}));
+
+ typeVars.emplace_back(id);
+ typeVars.emplace_back(JsonEncode(values));
+
+ varChksms.emplace_back(id);
+ String checksum = HashValue(kv.second);
+ varChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", checksum}})));
+
+ if (runtimeUpdate) {
+ values->Set("checksum", checksum);
+ AddObjectDataToRuntimeUpdates(runtimeUpdates, id, m_PrefixConfigObject + typeName + ":envvar", values);
+ }
+ }
+ }
+
+ return;
+ }
+}
+
+/**
+ * Update the state information of a checkable in Redis.
+ *
+ * What is updated exactly depends on the mode parameter:
+ * - Volatile: Update the volatile state information stored in icinga:host:state or icinga:service:state as well as
+ * the corresponding checksum stored in icinga:checksum:host:state or icinga:checksum:service:state.
+ * - RuntimeOnly: Write a runtime update to the icinga:runtime:state stream. It is up to the caller to ensure that
+ * identical volatile state information was already written before to avoid inconsistencies. This mode is only
+ * useful to upgrade a previous Volatile to a Full operation, otherwise Full should be used.
+ * - Full: Perform an update of all state information in Redis, that is updating the volatile information and sending
+ * a corresponding runtime update so that this state update gets written through to the persistent database by a
+ * running icingadb process.
+ *
+ * @param checkable State of this checkable is updated in Redis
+ * @param mode Mode of operation (StateUpdate::Volatile, StateUpdate::RuntimeOnly, or StateUpdate::Full)
+ */
+void IcingaDB::UpdateState(const Checkable::Ptr& checkable, StateUpdate mode)
+{
+ if (!m_Rcon || !m_Rcon->IsConnected())
+ return;
+
+ String objectType = GetLowerCaseTypeNameDB(checkable);
+ String objectKey = GetObjectIdentifier(checkable);
+
+ Dictionary::Ptr stateAttrs = SerializeState(checkable);
+
+ String redisStateKey = m_PrefixConfigObject + objectType + ":state";
+ String redisChecksumKey = m_PrefixConfigCheckSum + objectType + ":state";
+ String checksum = HashValue(stateAttrs);
+
+ if (mode & StateUpdate::Volatile) {
+ m_Rcon->FireAndForgetQueries({
+ {"HSET", redisStateKey, objectKey, JsonEncode(stateAttrs)},
+ {"HSET", redisChecksumKey, objectKey, JsonEncode(new Dictionary({{"checksum", checksum}}))},
+ }, Prio::RuntimeStateSync);
+ }
+
+ if (mode & StateUpdate::RuntimeOnly) {
+ ObjectLock olock(stateAttrs);
+
+ std::vector<String> streamadd({
+ "XADD", "icinga:runtime:state", "MAXLEN", "~", "1000000", "*",
+ "runtime_type", "upsert",
+ "redis_key", redisStateKey,
+ "checksum", checksum,
+ });
+
+ for (const Dictionary::Pair& kv : stateAttrs) {
+ streamadd.emplace_back(kv.first);
+ streamadd.emplace_back(IcingaToStreamValue(kv.second));
+ }
+
+ m_Rcon->FireAndForgetQuery(std::move(streamadd), Prio::RuntimeStateStream, {0, 1});
+ }
+}
+
+// Used to update a single object, used for runtime updates
+void IcingaDB::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpdate)
+{
+ if (!m_Rcon || !m_Rcon->IsConnected())
+ return;
+
+ String typeName = GetLowerCaseTypeNameDB(object);
+
+ std::map<String, std::vector<String>> hMSets;
+ std::vector<Dictionary::Ptr> runtimeUpdates;
+
+ CreateConfigUpdate(object, typeName, hMSets, runtimeUpdates, runtimeUpdate);
+ Checkable::Ptr checkable = dynamic_pointer_cast<Checkable>(object);
+ if (checkable) {
+ UpdateState(checkable, runtimeUpdate ? StateUpdate::Full : StateUpdate::Volatile);
+ }
+
+ std::vector<std::vector<String> > transaction = {{"MULTI"}};
+
+ for (auto& kv : hMSets) {
+ if (!kv.second.empty()) {
+ kv.second.insert(kv.second.begin(), {"HMSET", kv.first});
+ transaction.emplace_back(std::move(kv.second));
+ }
+ }
+
+ for (auto& objectAttributes : runtimeUpdates) {
+ std::vector<String> xAdd({"XADD", "icinga:runtime", "MAXLEN", "~", "1000000", "*"});
+ ObjectLock olock(objectAttributes);
+
+ for (const Dictionary::Pair& kv : objectAttributes) {
+ String value = IcingaToStreamValue(kv.second);
+ if (!value.IsEmpty()) {
+ xAdd.emplace_back(kv.first);
+ xAdd.emplace_back(value);
+ }
+ }
+
+ transaction.emplace_back(std::move(xAdd));
+ }
+
+ if (transaction.size() > 1) {
+ transaction.push_back({"EXEC"});
+ m_Rcon->FireAndForgetQueries(std::move(transaction), Prio::Config, {1});
+ }
+
+ if (checkable) {
+ SendNextUpdate(checkable);
+ }
+}
+
+void IcingaDB::AddObjectDataToRuntimeUpdates(std::vector<Dictionary::Ptr>& runtimeUpdates, const String& objectKey,
+ const String& redisKey, const Dictionary::Ptr& data)
+{
+ Dictionary::Ptr dataClone = data->ShallowClone();
+ dataClone->Set("id", objectKey);
+ dataClone->Set("redis_key", redisKey);
+ dataClone->Set("runtime_type", "upsert");
+ runtimeUpdates.emplace_back(dataClone);
+}
+
+// Takes object and collects IcingaDB relevant attributes and computes checksums. Returns whether the object is relevant
+// for IcingaDB.
+bool IcingaDB::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr& attributes, Dictionary::Ptr& checksums)
+{
+ attributes->Set("name_checksum", SHA1(object->GetName()));
+ attributes->Set("environment_id", m_EnvironmentId);
+ attributes->Set("name", object->GetName());
+
+ Zone::Ptr ObjectsZone;
+ Type::Ptr type = object->GetReflectionType();
+
+ if (type == Endpoint::TypeInstance) {
+ ObjectsZone = static_cast<Endpoint*>(object.get())->GetZone();
+ } else {
+ ObjectsZone = static_pointer_cast<Zone>(object->GetZone());
+ }
+
+ if (ObjectsZone) {
+ attributes->Set("zone_id", GetObjectIdentifier(ObjectsZone));
+ attributes->Set("zone_name", ObjectsZone->GetName());
+ }
+
+ if (type == Endpoint::TypeInstance) {
+ return true;
+ }
+
+ if (type == Zone::TypeInstance) {
+ Zone::Ptr zone = static_pointer_cast<Zone>(object);
+
+ attributes->Set("is_global", zone->GetGlobal());
+
+ Zone::Ptr parent = zone->GetParent();
+ if (parent) {
+ attributes->Set("parent_id", GetObjectIdentifier(parent));
+ }
+
+ auto parentsRaw (zone->GetAllParentsRaw());
+ attributes->Set("depth", parentsRaw.size());
+
+ return true;
+ }
+
+ if (type == Host::TypeInstance || type == Service::TypeInstance) {
+ Checkable::Ptr checkable = static_pointer_cast<Checkable>(object);
+
+ attributes->Set("checkcommand_name", checkable->GetCheckCommand()->GetName());
+ attributes->Set("max_check_attempts", checkable->GetMaxCheckAttempts());
+ attributes->Set("check_timeout", checkable->GetCheckTimeout());
+ attributes->Set("check_interval", checkable->GetCheckInterval());
+ attributes->Set("check_retry_interval", checkable->GetRetryInterval());
+ attributes->Set("active_checks_enabled", checkable->GetEnableActiveChecks());
+ attributes->Set("passive_checks_enabled", checkable->GetEnablePassiveChecks());
+ attributes->Set("event_handler_enabled", checkable->GetEnableEventHandler());
+ attributes->Set("notifications_enabled", checkable->GetEnableNotifications());
+ attributes->Set("flapping_enabled", checkable->GetEnableFlapping());
+ attributes->Set("flapping_threshold_low", checkable->GetFlappingThresholdLow());
+ attributes->Set("flapping_threshold_high", checkable->GetFlappingThresholdHigh());
+ attributes->Set("perfdata_enabled", checkable->GetEnablePerfdata());
+ attributes->Set("is_volatile", checkable->GetVolatile());
+ attributes->Set("notes", checkable->GetNotes());
+ attributes->Set("icon_image_alt", checkable->GetIconImageAlt());
+
+ attributes->Set("checkcommand_id", GetObjectIdentifier(checkable->GetCheckCommand()));
+
+ Endpoint::Ptr commandEndpoint = checkable->GetCommandEndpoint();
+ if (commandEndpoint) {
+ attributes->Set("command_endpoint_id", GetObjectIdentifier(commandEndpoint));
+ attributes->Set("command_endpoint_name", commandEndpoint->GetName());
+ }
+
+ TimePeriod::Ptr timePeriod = checkable->GetCheckPeriod();
+ if (timePeriod) {
+ attributes->Set("check_timeperiod_id", GetObjectIdentifier(timePeriod));
+ attributes->Set("check_timeperiod_name", timePeriod->GetName());
+ }
+
+ EventCommand::Ptr eventCommand = checkable->GetEventCommand();
+ if (eventCommand) {
+ attributes->Set("eventcommand_id", GetObjectIdentifier(eventCommand));
+ attributes->Set("eventcommand_name", eventCommand->GetName());
+ }
+
+ String actionUrl = checkable->GetActionUrl();
+ String notesUrl = checkable->GetNotesUrl();
+ String iconImage = checkable->GetIconImage();
+ if (!actionUrl.IsEmpty())
+ attributes->Set("action_url_id", HashValue(new Array({m_EnvironmentId, actionUrl})));
+ if (!notesUrl.IsEmpty())
+ attributes->Set("notes_url_id", HashValue(new Array({m_EnvironmentId, notesUrl})));
+ if (!iconImage.IsEmpty())
+ attributes->Set("icon_image_id", HashValue(new Array({m_EnvironmentId, iconImage})));
+
+
+ Host::Ptr host;
+ Service::Ptr service;
+ tie(host, service) = GetHostService(checkable);
+
+ if (service) {
+ attributes->Set("host_id", GetObjectIdentifier(service->GetHost()));
+ attributes->Set("display_name", service->GetDisplayName());
+
+ // Overwrite name here, `object->name` is 'HostName!ServiceName' but we only want the name of the Service
+ attributes->Set("name", service->GetShortName());
+ } else {
+ attributes->Set("display_name", host->GetDisplayName());
+ attributes->Set("address", host->GetAddress());
+ attributes->Set("address6", host->GetAddress6());
+ }
+
+ return true;
+ }
+
+ if (type == User::TypeInstance) {
+ User::Ptr user = static_pointer_cast<User>(object);
+
+ attributes->Set("display_name", user->GetDisplayName());
+ attributes->Set("email", user->GetEmail());
+ attributes->Set("pager", user->GetPager());
+ attributes->Set("notifications_enabled", user->GetEnableNotifications());
+ attributes->Set("states", user->GetStates());
+ attributes->Set("types", user->GetTypes());
+
+ if (user->GetPeriod())
+ attributes->Set("timeperiod_id", GetObjectIdentifier(user->GetPeriod()));
+
+ return true;
+ }
+
+ if (type == TimePeriod::TypeInstance) {
+ TimePeriod::Ptr timeperiod = static_pointer_cast<TimePeriod>(object);
+
+ attributes->Set("display_name", timeperiod->GetDisplayName());
+ attributes->Set("prefer_includes", timeperiod->GetPreferIncludes());
+ return true;
+ }
+
+ if (type == Notification::TypeInstance) {
+ Notification::Ptr notification = static_pointer_cast<Notification>(object);
+
+ Host::Ptr host;
+ Service::Ptr service;
+
+ tie(host, service) = GetHostService(notification->GetCheckable());
+
+ attributes->Set("notificationcommand_id", GetObjectIdentifier(notification->GetCommand()));
+
+ attributes->Set("host_id", GetObjectIdentifier(host));
+ if (service)
+ attributes->Set("service_id", GetObjectIdentifier(service));
+
+ TimePeriod::Ptr timeperiod = notification->GetPeriod();
+ if (timeperiod)
+ attributes->Set("timeperiod_id", GetObjectIdentifier(timeperiod));
+
+ if (notification->GetTimes()) {
+ attributes->Set("times_begin", notification->GetTimes()->Get("begin"));
+ attributes->Set("times_end",notification->GetTimes()->Get("end"));
+ }
+
+ attributes->Set("notification_interval", notification->GetInterval());
+ attributes->Set("states", notification->GetStates());
+ attributes->Set("types", notification->GetTypes());
+
+ return true;
+ }
+
+ if (type == Comment::TypeInstance) {
+ Comment::Ptr comment = static_pointer_cast<Comment>(object);
+
+ attributes->Set("author", comment->GetAuthor());
+ attributes->Set("text", comment->GetText());
+ attributes->Set("entry_type", comment->GetEntryType());
+ attributes->Set("entry_time", TimestampToMilliseconds(comment->GetEntryTime()));
+ attributes->Set("is_persistent", comment->GetPersistent());
+ attributes->Set("is_sticky", comment->GetSticky());
+
+ Host::Ptr host;
+ Service::Ptr service;
+ tie(host, service) = GetHostService(comment->GetCheckable());
+
+ attributes->Set("host_id", GetObjectIdentifier(host));
+ if (service) {
+ attributes->Set("object_type", "service");
+ attributes->Set("service_id", GetObjectIdentifier(service));
+ } else
+ attributes->Set("object_type", "host");
+
+ auto expireTime (comment->GetExpireTime());
+
+ if (expireTime > 0) {
+ attributes->Set("expire_time", TimestampToMilliseconds(expireTime));
+ }
+
+ return true;
+ }
+
+ if (type == Downtime::TypeInstance) {
+ Downtime::Ptr downtime = static_pointer_cast<Downtime>(object);
+
+ attributes->Set("author", downtime->GetAuthor());
+ attributes->Set("comment", downtime->GetComment());
+ attributes->Set("entry_time", TimestampToMilliseconds(downtime->GetEntryTime()));
+ attributes->Set("scheduled_start_time", TimestampToMilliseconds(downtime->GetStartTime()));
+ attributes->Set("scheduled_end_time", TimestampToMilliseconds(downtime->GetEndTime()));
+ attributes->Set("scheduled_duration", TimestampToMilliseconds(downtime->GetEndTime() - downtime->GetStartTime()));
+ attributes->Set("flexible_duration", TimestampToMilliseconds(downtime->GetDuration()));
+ attributes->Set("is_flexible", !downtime->GetFixed());
+ attributes->Set("is_in_effect", downtime->IsInEffect());
+ if (downtime->IsInEffect()) {
+ attributes->Set("start_time", TimestampToMilliseconds(downtime->GetTriggerTime()));
+ attributes->Set("end_time", TimestampToMilliseconds(downtime->GetFixed() ? downtime->GetEndTime() : (downtime->GetTriggerTime() + downtime->GetDuration())));
+ }
+
+ auto duration = downtime->GetDuration();
+ if (downtime->GetFixed()) {
+ duration = downtime->GetEndTime() - downtime->GetStartTime();
+ }
+ attributes->Set("duration", TimestampToMilliseconds(duration));
+
+ Host::Ptr host;
+ Service::Ptr service;
+ tie(host, service) = GetHostService(downtime->GetCheckable());
+
+ attributes->Set("host_id", GetObjectIdentifier(host));
+ if (service) {
+ attributes->Set("object_type", "service");
+ attributes->Set("service_id", GetObjectIdentifier(service));
+ } else
+ attributes->Set("object_type", "host");
+
+ auto triggeredBy (Downtime::GetByName(downtime->GetTriggeredBy()));
+ if (triggeredBy) {
+ attributes->Set("triggered_by_id", GetObjectIdentifier(triggeredBy));
+ }
+
+ auto scheduledBy (downtime->GetScheduledBy());
+ if (!scheduledBy.IsEmpty()) {
+ attributes->Set("scheduled_by", scheduledBy);
+ }
+
+ auto parent (Downtime::GetByName(downtime->GetParent()));
+ if (parent) {
+ attributes->Set("parent_id", GetObjectIdentifier(parent));
+ }
+
+ return true;
+ }
+
+ if (type == UserGroup::TypeInstance) {
+ UserGroup::Ptr userGroup = static_pointer_cast<UserGroup>(object);
+
+ attributes->Set("display_name", userGroup->GetDisplayName());
+
+ return true;
+ }
+
+ if (type == HostGroup::TypeInstance) {
+ HostGroup::Ptr hostGroup = static_pointer_cast<HostGroup>(object);
+
+ attributes->Set("display_name", hostGroup->GetDisplayName());
+
+ return true;
+ }
+
+ if (type == ServiceGroup::TypeInstance) {
+ ServiceGroup::Ptr serviceGroup = static_pointer_cast<ServiceGroup>(object);
+
+ attributes->Set("display_name", serviceGroup->GetDisplayName());
+
+ return true;
+ }
+
+ if (type == CheckCommand::TypeInstance || type == NotificationCommand::TypeInstance || type == EventCommand::TypeInstance) {
+ Command::Ptr command = static_pointer_cast<Command>(object);
+
+ attributes->Set("command", JsonEncode(command->GetCommandLine()));
+ attributes->Set("timeout", command->GetTimeout());
+
+ return true;
+ }
+
+ return false;
+}
+
+/* Creates a config update with computed checksums etc.
+ * Writes attributes, customVars and checksums into the respective supplied vectors. Adds two values to each vector
+ * (if applicable), first the key then the value. To use in a Redis command the command (e.g. HSET) and the key (e.g.
+ * icinga:config:object:downtime) need to be prepended. There is nothing to indicate success or failure.
+ */
+void
+IcingaDB::CreateConfigUpdate(const ConfigObject::Ptr& object, const String typeName, std::map<String, std::vector<String>>& hMSets,
+ std::vector<Dictionary::Ptr>& runtimeUpdates, bool runtimeUpdate)
+{
+ /* TODO: This isn't essentially correct as we don't keep track of config objects ourselves. This would avoid duplicated config updates at startup.
+ if (!runtimeUpdate && m_ConfigDumpInProgress)
+ return;
+ */
+
+ if (m_Rcon == nullptr)
+ return;
+
+ Dictionary::Ptr attr = new Dictionary;
+ Dictionary::Ptr chksm = new Dictionary;
+
+ if (!PrepareObject(object, attr, chksm))
+ return;
+
+ InsertObjectDependencies(object, typeName, hMSets, runtimeUpdates, runtimeUpdate);
+
+ String objectKey = GetObjectIdentifier(object);
+ auto& attrs (hMSets[m_PrefixConfigObject + typeName]);
+ auto& chksms (hMSets[m_PrefixConfigCheckSum + typeName]);
+
+ attrs.emplace_back(objectKey);
+ attrs.emplace_back(JsonEncode(attr));
+
+ String checksum = HashValue(attr);
+ chksms.emplace_back(objectKey);
+ chksms.emplace_back(JsonEncode(new Dictionary({{"checksum", checksum}})));
+
+ /* Send an update event to subscribers. */
+ if (runtimeUpdate) {
+ attr->Set("checksum", checksum);
+ AddObjectDataToRuntimeUpdates(runtimeUpdates, objectKey, m_PrefixConfigObject + typeName, attr);
+ }
+}
+
+void IcingaDB::SendConfigDelete(const ConfigObject::Ptr& object)
+{
+ Type::Ptr type = object->GetReflectionType();
+ String typeName = type->GetName().ToLower();
+ String objectKey = GetObjectIdentifier(object);
+
+ m_Rcon->FireAndForgetQueries({
+ {"HDEL", m_PrefixConfigObject + typeName, objectKey},
+ {"HDEL", m_PrefixConfigCheckSum + typeName, objectKey},
+ {
+ "XADD", "icinga:runtime", "MAXLEN", "~", "1000000", "*",
+ "redis_key", m_PrefixConfigObject + typeName, "id", objectKey, "runtime_type", "delete"
+ }
+ }, Prio::Config);
+
+ CustomVarObject::Ptr customVarObject = dynamic_pointer_cast<CustomVarObject>(object);
+
+ if (customVarObject) {
+ Dictionary::Ptr vars = customVarObject->GetVars();
+ SendCustomVarsChanged(object, vars, nullptr);
+ }
+
+ if (type == Host::TypeInstance || type == Service::TypeInstance) {
+ Checkable::Ptr checkable = static_pointer_cast<Checkable>(object);
+
+ Host::Ptr host;
+ Service::Ptr service;
+ tie(host, service) = GetHostService(checkable);
+
+ m_Rcon->FireAndForgetQuery({
+ "ZREM",
+ service ? "icinga:nextupdate:service" : "icinga:nextupdate:host",
+ GetObjectIdentifier(checkable)
+ }, Prio::CheckResult);
+
+ m_Rcon->FireAndForgetQueries({
+ {"HDEL", m_PrefixConfigObject + typeName + ":state", objectKey},
+ {"HDEL", m_PrefixConfigCheckSum + typeName + ":state", objectKey}
+ }, Prio::RuntimeStateSync);
+
+ if (service) {
+ SendGroupsChanged<ServiceGroup>(checkable, service->GetGroups(), nullptr);
+ } else {
+ SendGroupsChanged<HostGroup>(checkable, host->GetGroups(), nullptr);
+ }
+
+ return;
+ }
+
+ if (type == TimePeriod::TypeInstance) {
+ TimePeriod::Ptr timeperiod = static_pointer_cast<TimePeriod>(object);
+ SendTimePeriodRangesChanged(timeperiod, timeperiod->GetRanges(), nullptr);
+ SendTimePeriodIncludesChanged(timeperiod, timeperiod->GetIncludes(), nullptr);
+ SendTimePeriodExcludesChanged(timeperiod, timeperiod->GetExcludes(), nullptr);
+ return;
+ }
+
+ if (type == User::TypeInstance) {
+ User::Ptr user = static_pointer_cast<User>(object);
+ SendGroupsChanged<UserGroup>(user, user->GetGroups(), nullptr);
+ return;
+ }
+
+ if (type == Notification::TypeInstance) {
+ Notification::Ptr notification = static_pointer_cast<Notification>(object);
+ SendNotificationUsersChanged(notification, notification->GetUsersRaw(), nullptr);
+ SendNotificationUserGroupsChanged(notification, notification->GetUserGroupsRaw(), nullptr);
+ return;
+ }
+
+ if (type == CheckCommand::TypeInstance || type == NotificationCommand::TypeInstance || type == EventCommand::TypeInstance) {
+ Command::Ptr command = static_pointer_cast<Command>(object);
+ SendCommandArgumentsChanged(command, command->GetArguments(), nullptr);
+ SendCommandEnvChanged(command, command->GetEnv(), nullptr);
+ return;
+ }
+}
+
+static inline
+unsigned short GetPreviousState(const Checkable::Ptr& checkable, const Service::Ptr& service, StateType type)
+{
+ auto phs ((type == StateTypeHard ? checkable->GetLastHardStatesRaw() : checkable->GetLastSoftStatesRaw()) % 100u);
+
+ if (service) {
+ return phs;
+ } else {
+ return phs == 99 ? phs : Host::CalculateState(ServiceState(phs));
+ }
+}
+
+void IcingaDB::SendStateChange(const ConfigObject::Ptr& object, const CheckResult::Ptr& cr, StateType type)
+{
+ if (!GetActive()) {
+ return;
+ }
+
+ Checkable::Ptr checkable = dynamic_pointer_cast<Checkable>(object);
+ if (!checkable)
+ return;
+
+ if (!cr)
+ return;
+
+ Host::Ptr host;
+ Service::Ptr service;
+
+ tie(host, service) = GetHostService(checkable);
+
+ UpdateState(checkable, StateUpdate::RuntimeOnly);
+
+ int hard_state;
+ if (!cr) {
+ hard_state = 99;
+ } else {
+ hard_state = service ? Convert::ToLong(service->GetLastHardState()) : Convert::ToLong(host->GetLastHardState());
+ }
+
+ auto eventTime (cr->GetExecutionEnd());
+ auto eventTs (TimestampToMilliseconds(eventTime));
+
+ Array::Ptr rawId = new Array({m_EnvironmentId, object->GetName()});
+ rawId->Add(eventTs);
+
+ std::vector<String> xAdd ({
+ "XADD", "icinga:history:stream:state", "*",
+ "id", HashValue(rawId),
+ "environment_id", m_EnvironmentId,
+ "host_id", GetObjectIdentifier(host),
+ "state_type", Convert::ToString(type),
+ "soft_state", Convert::ToString(cr ? service ? Convert::ToLong(cr->GetState()) : Convert::ToLong(Host::CalculateState(cr->GetState())) : 99),
+ "hard_state", Convert::ToString(hard_state),
+ "check_attempt", Convert::ToString(checkable->GetCheckAttempt()),
+ "previous_soft_state", Convert::ToString(GetPreviousState(checkable, service, StateTypeSoft)),
+ "previous_hard_state", Convert::ToString(GetPreviousState(checkable, service, StateTypeHard)),
+ "max_check_attempts", Convert::ToString(checkable->GetMaxCheckAttempts()),
+ "event_time", Convert::ToString(eventTs),
+ "event_id", CalcEventID("state_change", object, eventTime),
+ "event_type", "state_change"
+ });
+
+ if (cr) {
+ auto output (cr->GetOutput());
+ auto pos (output.Find("\n"));
+
+ if (pos != String::NPos) {
+ auto longOutput (output.SubStr(pos + 1u));
+ output.erase(output.Begin() + pos, output.End());
+
+ xAdd.emplace_back("long_output");
+ xAdd.emplace_back(Utility::ValidateUTF8(std::move(longOutput)));
+ }
+
+ xAdd.emplace_back("output");
+ xAdd.emplace_back(Utility::ValidateUTF8(std::move(output)));
+ xAdd.emplace_back("check_source");
+ xAdd.emplace_back(cr->GetCheckSource());
+ xAdd.emplace_back("scheduling_source");
+ xAdd.emplace_back(cr->GetSchedulingSource());
+ }
+
+ if (service) {
+ xAdd.emplace_back("object_type");
+ xAdd.emplace_back("service");
+ xAdd.emplace_back("service_id");
+ xAdd.emplace_back(GetObjectIdentifier(checkable));
+ } else {
+ xAdd.emplace_back("object_type");
+ xAdd.emplace_back("host");
+ }
+
+ auto endpoint (Endpoint::GetLocalEndpoint());
+
+ if (endpoint) {
+ xAdd.emplace_back("endpoint_id");
+ xAdd.emplace_back(GetObjectIdentifier(endpoint));
+ }
+
+ m_HistoryBulker.ProduceOne(std::move(xAdd));
+}
+
+void IcingaDB::SendSentNotification(
+ const Notification::Ptr& notification, const Checkable::Ptr& checkable, const std::set<User::Ptr>& users,
+ NotificationType type, const CheckResult::Ptr& cr, const String& author, const String& text, double sendTime
+)
+{
+ if (!GetActive()) {
+ return;
+ }
+
+ Host::Ptr host;
+ Service::Ptr service;
+ tie(host, service) = GetHostService(checkable);
+
+ auto finalText = text;
+ if (finalText == "" && cr) {
+ finalText = cr->GetOutput();
+ }
+
+ auto usersAmount (users.size());
+ auto sendTs (TimestampToMilliseconds(sendTime));
+
+ Array::Ptr rawId = new Array({m_EnvironmentId, notification->GetName()});
+ rawId->Add(GetNotificationTypeByEnum(type));
+ rawId->Add(sendTs);
+
+ auto notificationHistoryId (HashValue(rawId));
+
+ std::vector<String> xAdd ({
+ "XADD", "icinga:history:stream:notification", "*",
+ "id", notificationHistoryId,
+ "environment_id", m_EnvironmentId,
+ "notification_id", GetObjectIdentifier(notification),
+ "host_id", GetObjectIdentifier(host),
+ "type", Convert::ToString(type),
+ "state", Convert::ToString(cr ? service ? Convert::ToLong(cr->GetState()) : Convert::ToLong(Host::CalculateState(cr->GetState())) : 99),
+ "previous_hard_state", Convert::ToString(cr ? Convert::ToLong(service ? cr->GetPreviousHardState() : Host::CalculateState(cr->GetPreviousHardState())) : 99),
+ "author", Utility::ValidateUTF8(author),
+ "text", Utility::ValidateUTF8(finalText),
+ "users_notified", Convert::ToString(usersAmount),
+ "send_time", Convert::ToString(sendTs),
+ "event_id", CalcEventID("notification", notification, sendTime, type),
+ "event_type", "notification"
+ });
+
+ if (service) {
+ xAdd.emplace_back("object_type");
+ xAdd.emplace_back("service");
+ xAdd.emplace_back("service_id");
+ xAdd.emplace_back(GetObjectIdentifier(checkable));
+ } else {
+ xAdd.emplace_back("object_type");
+ xAdd.emplace_back("host");
+ }
+
+ auto endpoint (Endpoint::GetLocalEndpoint());
+
+ if (endpoint) {
+ xAdd.emplace_back("endpoint_id");
+ xAdd.emplace_back(GetObjectIdentifier(endpoint));
+ }
+
+ if (!users.empty()) {
+ Array::Ptr users_notified = new Array();
+ for (const User::Ptr& user : users) {
+ users_notified->Add(GetObjectIdentifier(user));
+ }
+ xAdd.emplace_back("users_notified_ids");
+ xAdd.emplace_back(JsonEncode(users_notified));
+ }
+
+ m_HistoryBulker.ProduceOne(std::move(xAdd));
+}
+
+void IcingaDB::SendStartedDowntime(const Downtime::Ptr& downtime)
+{
+ if (!GetActive()) {
+ return;
+ }
+
+ SendConfigUpdate(downtime, true);
+
+ auto checkable (downtime->GetCheckable());
+ auto triggeredBy (Downtime::GetByName(downtime->GetTriggeredBy()));
+
+ Host::Ptr host;
+ Service::Ptr service;
+ tie(host, service) = GetHostService(checkable);
+
+ /* Update checkable state as in_downtime may have changed. */
+ UpdateState(checkable, StateUpdate::Full);
+
+ std::vector<String> xAdd ({
+ "XADD", "icinga:history:stream:downtime", "*",
+ "downtime_id", GetObjectIdentifier(downtime),
+ "environment_id", m_EnvironmentId,
+ "host_id", GetObjectIdentifier(host),
+ "entry_time", Convert::ToString(TimestampToMilliseconds(downtime->GetEntryTime())),
+ "author", Utility::ValidateUTF8(downtime->GetAuthor()),
+ "comment", Utility::ValidateUTF8(downtime->GetComment()),
+ "is_flexible", Convert::ToString((unsigned short)!downtime->GetFixed()),
+ "flexible_duration", Convert::ToString(TimestampToMilliseconds(downtime->GetDuration())),
+ "scheduled_start_time", Convert::ToString(TimestampToMilliseconds(downtime->GetStartTime())),
+ "scheduled_end_time", Convert::ToString(TimestampToMilliseconds(downtime->GetEndTime())),
+ "has_been_cancelled", Convert::ToString((unsigned short)downtime->GetWasCancelled()),
+ "trigger_time", Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime())),
+ "event_id", CalcEventID("downtime_start", downtime),
+ "event_type", "downtime_start"
+ });
+
+ if (service) {
+ xAdd.emplace_back("object_type");
+ xAdd.emplace_back("service");
+ xAdd.emplace_back("service_id");
+ xAdd.emplace_back(GetObjectIdentifier(checkable));
+ } else {
+ xAdd.emplace_back("object_type");
+ xAdd.emplace_back("host");
+ }
+
+ if (triggeredBy) {
+ xAdd.emplace_back("triggered_by_id");
+ xAdd.emplace_back(GetObjectIdentifier(triggeredBy));
+ }
+
+ if (downtime->GetFixed()) {
+ xAdd.emplace_back("start_time");
+ xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetStartTime())));
+ xAdd.emplace_back("end_time");
+ xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetEndTime())));
+ } else {
+ xAdd.emplace_back("start_time");
+ xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime())));
+ xAdd.emplace_back("end_time");
+ xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime() + downtime->GetDuration())));
+ }
+
+ auto endpoint (Endpoint::GetLocalEndpoint());
+
+ if (endpoint) {
+ xAdd.emplace_back("endpoint_id");
+ xAdd.emplace_back(GetObjectIdentifier(endpoint));
+ }
+
+ auto parent (Downtime::GetByName(downtime->GetParent()));
+
+ if (parent) {
+ xAdd.emplace_back("parent_id");
+ xAdd.emplace_back(GetObjectIdentifier(parent));
+ }
+
+ auto scheduledBy (downtime->GetScheduledBy());
+
+ if (!scheduledBy.IsEmpty()) {
+ xAdd.emplace_back("scheduled_by");
+ xAdd.emplace_back(scheduledBy);
+ }
+
+ m_HistoryBulker.ProduceOne(std::move(xAdd));
+}
+
+void IcingaDB::SendRemovedDowntime(const Downtime::Ptr& downtime)
+{
+ if (!GetActive()) {
+ return;
+ }
+
+ auto checkable (downtime->GetCheckable());
+ auto triggeredBy (Downtime::GetByName(downtime->GetTriggeredBy()));
+
+ Host::Ptr host;
+ Service::Ptr service;
+ tie(host, service) = GetHostService(checkable);
+
+ // Downtime never got triggered (didn't send "downtime_start") so we don't want to send "downtime_end"
+ if (downtime->GetTriggerTime() == 0)
+ return;
+
+ /* Update checkable state as in_downtime may have changed. */
+ UpdateState(checkable, StateUpdate::Full);
+
+ std::vector<String> xAdd ({
+ "XADD", "icinga:history:stream:downtime", "*",
+ "downtime_id", GetObjectIdentifier(downtime),
+ "environment_id", m_EnvironmentId,
+ "host_id", GetObjectIdentifier(host),
+ "entry_time", Convert::ToString(TimestampToMilliseconds(downtime->GetEntryTime())),
+ "author", Utility::ValidateUTF8(downtime->GetAuthor()),
+ "cancelled_by", Utility::ValidateUTF8(downtime->GetRemovedBy()),
+ "comment", Utility::ValidateUTF8(downtime->GetComment()),
+ "is_flexible", Convert::ToString((unsigned short)!downtime->GetFixed()),
+ "flexible_duration", Convert::ToString(TimestampToMilliseconds(downtime->GetDuration())),
+ "scheduled_start_time", Convert::ToString(TimestampToMilliseconds(downtime->GetStartTime())),
+ "scheduled_end_time", Convert::ToString(TimestampToMilliseconds(downtime->GetEndTime())),
+ "has_been_cancelled", Convert::ToString((unsigned short)downtime->GetWasCancelled()),
+ "trigger_time", Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime())),
+ "cancel_time", Convert::ToString(TimestampToMilliseconds(downtime->GetRemoveTime())),
+ "event_id", CalcEventID("downtime_end", downtime),
+ "event_type", "downtime_end"
+ });
+
+ if (service) {
+ xAdd.emplace_back("object_type");
+ xAdd.emplace_back("service");
+ xAdd.emplace_back("service_id");
+ xAdd.emplace_back(GetObjectIdentifier(checkable));
+ } else {
+ xAdd.emplace_back("object_type");
+ xAdd.emplace_back("host");
+ }
+
+ if (triggeredBy) {
+ xAdd.emplace_back("triggered_by_id");
+ xAdd.emplace_back(GetObjectIdentifier(triggeredBy));
+ }
+
+ if (downtime->GetFixed()) {
+ xAdd.emplace_back("start_time");
+ xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetStartTime())));
+ xAdd.emplace_back("end_time");
+ xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetEndTime())));
+ } else {
+ xAdd.emplace_back("start_time");
+ xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime())));
+ xAdd.emplace_back("end_time");
+ xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime() + downtime->GetDuration())));
+ }
+
+ auto endpoint (Endpoint::GetLocalEndpoint());
+
+ if (endpoint) {
+ xAdd.emplace_back("endpoint_id");
+ xAdd.emplace_back(GetObjectIdentifier(endpoint));
+ }
+
+ auto parent (Downtime::GetByName(downtime->GetParent()));
+
+ if (parent) {
+ xAdd.emplace_back("parent_id");
+ xAdd.emplace_back(GetObjectIdentifier(parent));
+ }
+
+ auto scheduledBy (downtime->GetScheduledBy());
+
+ if (!scheduledBy.IsEmpty()) {
+ xAdd.emplace_back("scheduled_by");
+ xAdd.emplace_back(scheduledBy);
+ }
+
+ m_HistoryBulker.ProduceOne(std::move(xAdd));
+}
+
+void IcingaDB::SendAddedComment(const Comment::Ptr& comment)
+{
+ if (comment->GetEntryType() != CommentUser || !GetActive())
+ return;
+
+ auto checkable (comment->GetCheckable());
+
+ Host::Ptr host;
+ Service::Ptr service;
+ tie(host, service) = GetHostService(checkable);
+
+ std::vector<String> xAdd ({
+ "XADD", "icinga:history:stream:comment", "*",
+ "comment_id", GetObjectIdentifier(comment),
+ "environment_id", m_EnvironmentId,
+ "host_id", GetObjectIdentifier(host),
+ "entry_time", Convert::ToString(TimestampToMilliseconds(comment->GetEntryTime())),
+ "author", Utility::ValidateUTF8(comment->GetAuthor()),
+ "comment", Utility::ValidateUTF8(comment->GetText()),
+ "entry_type", Convert::ToString(comment->GetEntryType()),
+ "is_persistent", Convert::ToString((unsigned short)comment->GetPersistent()),
+ "is_sticky", Convert::ToString((unsigned short)comment->GetSticky()),
+ "event_id", CalcEventID("comment_add", comment),
+ "event_type", "comment_add"
+ });
+
+ if (service) {
+ xAdd.emplace_back("object_type");
+ xAdd.emplace_back("service");
+ xAdd.emplace_back("service_id");
+ xAdd.emplace_back(GetObjectIdentifier(checkable));
+ } else {
+ xAdd.emplace_back("object_type");
+ xAdd.emplace_back("host");
+ }
+
+ auto endpoint (Endpoint::GetLocalEndpoint());
+
+ if (endpoint) {
+ xAdd.emplace_back("endpoint_id");
+ xAdd.emplace_back(GetObjectIdentifier(endpoint));
+ }
+
+ {
+ auto expireTime (comment->GetExpireTime());
+
+ if (expireTime > 0) {
+ xAdd.emplace_back("expire_time");
+ xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(expireTime)));
+ }
+ }
+
+ m_HistoryBulker.ProduceOne(std::move(xAdd));
+ UpdateState(checkable, StateUpdate::Full);
+}
+
+void IcingaDB::SendRemovedComment(const Comment::Ptr& comment)
+{
+ if (comment->GetEntryType() != CommentUser || !GetActive()) {
+ return;
+ }
+
+ double removeTime = comment->GetRemoveTime();
+ bool wasRemoved = removeTime > 0;
+
+ double expireTime = comment->GetExpireTime();
+ bool hasExpireTime = expireTime > 0;
+ bool isExpired = hasExpireTime && expireTime <= Utility::GetTime();
+
+ if (!wasRemoved && !isExpired) {
+ /* The comment object disappeared for no apparent reason, most likely because it simply was deleted instead
+ * of using the proper remove-comment API action. In this case, information that should normally be set is
+ * missing and a proper history event cannot be generated.
+ */
+ return;
+ }
+
+ auto checkable (comment->GetCheckable());
+
+ Host::Ptr host;
+ Service::Ptr service;
+ tie(host, service) = GetHostService(checkable);
+
+ std::vector<String> xAdd ({
+ "XADD", "icinga:history:stream:comment", "*",
+ "comment_id", GetObjectIdentifier(comment),
+ "environment_id", m_EnvironmentId,
+ "host_id", GetObjectIdentifier(host),
+ "entry_time", Convert::ToString(TimestampToMilliseconds(comment->GetEntryTime())),
+ "author", Utility::ValidateUTF8(comment->GetAuthor()),
+ "comment", Utility::ValidateUTF8(comment->GetText()),
+ "entry_type", Convert::ToString(comment->GetEntryType()),
+ "is_persistent", Convert::ToString((unsigned short)comment->GetPersistent()),
+ "is_sticky", Convert::ToString((unsigned short)comment->GetSticky()),
+ "event_id", CalcEventID("comment_remove", comment),
+ "event_type", "comment_remove"
+ });
+
+ if (service) {
+ xAdd.emplace_back("object_type");
+ xAdd.emplace_back("service");
+ xAdd.emplace_back("service_id");
+ xAdd.emplace_back(GetObjectIdentifier(checkable));
+ } else {
+ xAdd.emplace_back("object_type");
+ xAdd.emplace_back("host");
+ }
+
+ auto endpoint (Endpoint::GetLocalEndpoint());
+
+ if (endpoint) {
+ xAdd.emplace_back("endpoint_id");
+ xAdd.emplace_back(GetObjectIdentifier(endpoint));
+ }
+
+ if (wasRemoved) {
+ xAdd.emplace_back("remove_time");
+ xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(removeTime)));
+ xAdd.emplace_back("has_been_removed");
+ xAdd.emplace_back("1");
+ xAdd.emplace_back("removed_by");
+ xAdd.emplace_back(Utility::ValidateUTF8(comment->GetRemovedBy()));
+ } else {
+ xAdd.emplace_back("has_been_removed");
+ xAdd.emplace_back("0");
+ }
+
+ if (hasExpireTime) {
+ xAdd.emplace_back("expire_time");
+ xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(expireTime)));
+ }
+
+ m_HistoryBulker.ProduceOne(std::move(xAdd));
+ UpdateState(checkable, StateUpdate::Full);
+}
+
+void IcingaDB::SendFlappingChange(const Checkable::Ptr& checkable, double changeTime, double flappingLastChange)
+{
+ if (!GetActive()) {
+ return;
+ }
+
+ Host::Ptr host;
+ Service::Ptr service;
+ tie(host, service) = GetHostService(checkable);
+
+ std::vector<String> xAdd ({
+ "XADD", "icinga:history:stream:flapping", "*",
+ "environment_id", m_EnvironmentId,
+ "host_id", GetObjectIdentifier(host),
+ "flapping_threshold_low", Convert::ToString(checkable->GetFlappingThresholdLow()),
+ "flapping_threshold_high", Convert::ToString(checkable->GetFlappingThresholdHigh())
+ });
+
+ if (service) {
+ xAdd.emplace_back("object_type");
+ xAdd.emplace_back("service");
+ xAdd.emplace_back("service_id");
+ xAdd.emplace_back(GetObjectIdentifier(checkable));
+ } else {
+ xAdd.emplace_back("object_type");
+ xAdd.emplace_back("host");
+ }
+
+ auto endpoint (Endpoint::GetLocalEndpoint());
+
+ if (endpoint) {
+ xAdd.emplace_back("endpoint_id");
+ xAdd.emplace_back(GetObjectIdentifier(endpoint));
+ }
+
+ long long startTime;
+
+ if (checkable->IsFlapping()) {
+ startTime = TimestampToMilliseconds(changeTime);
+
+ xAdd.emplace_back("event_type");
+ xAdd.emplace_back("flapping_start");
+ xAdd.emplace_back("percent_state_change_start");
+ xAdd.emplace_back(Convert::ToString(checkable->GetFlappingCurrent()));
+ } else {
+ startTime = TimestampToMilliseconds(flappingLastChange);
+
+ xAdd.emplace_back("event_type");
+ xAdd.emplace_back("flapping_end");
+ xAdd.emplace_back("end_time");
+ xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(changeTime)));
+ xAdd.emplace_back("percent_state_change_end");
+ xAdd.emplace_back(Convert::ToString(checkable->GetFlappingCurrent()));
+ }
+
+ xAdd.emplace_back("start_time");
+ xAdd.emplace_back(Convert::ToString(startTime));
+ xAdd.emplace_back("event_id");
+ xAdd.emplace_back(CalcEventID(checkable->IsFlapping() ? "flapping_start" : "flapping_end", checkable, startTime));
+ xAdd.emplace_back("id");
+ xAdd.emplace_back(HashValue(new Array({m_EnvironmentId, checkable->GetName(), startTime})));
+
+ m_HistoryBulker.ProduceOne(std::move(xAdd));
+}
+
+void IcingaDB::SendNextUpdate(const Checkable::Ptr& checkable)
+{
+ if (!m_Rcon || !m_Rcon->IsConnected())
+ return;
+
+ if (checkable->GetEnableActiveChecks()) {
+ m_Rcon->FireAndForgetQuery(
+ {
+ "ZADD",
+ dynamic_pointer_cast<Service>(checkable) ? "icinga:nextupdate:service" : "icinga:nextupdate:host",
+ Convert::ToString(checkable->GetNextUpdate()),
+ GetObjectIdentifier(checkable)
+ },
+ Prio::CheckResult
+ );
+ } else {
+ m_Rcon->FireAndForgetQuery(
+ {
+ "ZREM",
+ dynamic_pointer_cast<Service>(checkable) ? "icinga:nextupdate:service" : "icinga:nextupdate:host",
+ GetObjectIdentifier(checkable)
+ },
+ Prio::CheckResult
+ );
+ }
+}
+
+void IcingaDB::SendAcknowledgementSet(const Checkable::Ptr& checkable, const String& author, const String& comment, AcknowledgementType type, bool persistent, double changeTime, double expiry)
+{
+ if (!GetActive()) {
+ return;
+ }
+
+ Host::Ptr host;
+ Service::Ptr service;
+ tie(host, service) = GetHostService(checkable);
+
+ /* Update checkable state as is_acknowledged may have changed. */
+ UpdateState(checkable, StateUpdate::Full);
+
+ std::vector<String> xAdd ({
+ "XADD", "icinga:history:stream:acknowledgement", "*",
+ "environment_id", m_EnvironmentId,
+ "host_id", GetObjectIdentifier(host),
+ "event_type", "ack_set",
+ "author", author,
+ "comment", comment,
+ "is_sticky", Convert::ToString((unsigned short)(type == AcknowledgementSticky)),
+ "is_persistent", Convert::ToString((unsigned short)persistent)
+ });
+
+ if (service) {
+ xAdd.emplace_back("object_type");
+ xAdd.emplace_back("service");
+ xAdd.emplace_back("service_id");
+ xAdd.emplace_back(GetObjectIdentifier(checkable));
+ } else {
+ xAdd.emplace_back("object_type");
+ xAdd.emplace_back("host");
+ }
+
+ auto endpoint (Endpoint::GetLocalEndpoint());
+
+ if (endpoint) {
+ xAdd.emplace_back("endpoint_id");
+ xAdd.emplace_back(GetObjectIdentifier(endpoint));
+ }
+
+ if (expiry > 0) {
+ xAdd.emplace_back("expire_time");
+ xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(expiry)));
+ }
+
+ long long setTime = TimestampToMilliseconds(changeTime);
+
+ xAdd.emplace_back("set_time");
+ xAdd.emplace_back(Convert::ToString(setTime));
+ xAdd.emplace_back("event_id");
+ xAdd.emplace_back(CalcEventID("ack_set", checkable, setTime));
+ xAdd.emplace_back("id");
+ xAdd.emplace_back(HashValue(new Array({m_EnvironmentId, checkable->GetName(), setTime})));
+
+ m_HistoryBulker.ProduceOne(std::move(xAdd));
+}
+
+void IcingaDB::SendAcknowledgementCleared(const Checkable::Ptr& checkable, const String& removedBy, double changeTime, double ackLastChange)
+{
+ if (!GetActive()) {
+ return;
+ }
+
+ Host::Ptr host;
+ Service::Ptr service;
+ tie(host, service) = GetHostService(checkable);
+
+ /* Update checkable state as is_acknowledged may have changed. */
+ UpdateState(checkable, StateUpdate::Full);
+
+ std::vector<String> xAdd ({
+ "XADD", "icinga:history:stream:acknowledgement", "*",
+ "environment_id", m_EnvironmentId,
+ "host_id", GetObjectIdentifier(host),
+ "clear_time", Convert::ToString(TimestampToMilliseconds(changeTime)),
+ "event_type", "ack_clear"
+ });
+
+ if (service) {
+ xAdd.emplace_back("object_type");
+ xAdd.emplace_back("service");
+ xAdd.emplace_back("service_id");
+ xAdd.emplace_back(GetObjectIdentifier(checkable));
+ } else {
+ xAdd.emplace_back("object_type");
+ xAdd.emplace_back("host");
+ }
+
+ auto endpoint (Endpoint::GetLocalEndpoint());
+
+ if (endpoint) {
+ xAdd.emplace_back("endpoint_id");
+ xAdd.emplace_back(GetObjectIdentifier(endpoint));
+ }
+
+ long long setTime = TimestampToMilliseconds(ackLastChange);
+
+ xAdd.emplace_back("set_time");
+ xAdd.emplace_back(Convert::ToString(setTime));
+ xAdd.emplace_back("event_id");
+ xAdd.emplace_back(CalcEventID("ack_clear", checkable, setTime));
+ xAdd.emplace_back("id");
+ xAdd.emplace_back(HashValue(new Array({m_EnvironmentId, checkable->GetName(), setTime})));
+
+ if (!removedBy.IsEmpty()) {
+ xAdd.emplace_back("cleared_by");
+ xAdd.emplace_back(removedBy);
+ }
+
+ m_HistoryBulker.ProduceOne(std::move(xAdd));
+}
+
+void IcingaDB::ForwardHistoryEntries()
+{
+ using clock = std::chrono::steady_clock;
+
+ const std::chrono::seconds logInterval (10);
+ auto nextLog (clock::now() + logInterval);
+
+ auto logPeriodically ([this, logInterval, &nextLog]() {
+ if (clock::now() > nextLog) {
+ nextLog += logInterval;
+
+ auto size (m_HistoryBulker.Size());
+
+ Log(size > m_HistoryBulker.GetBulkSize() ? LogInformation : LogNotice, "IcingaDB")
+ << "Pending history queries: " << size;
+ }
+ });
+
+ for (;;) {
+ logPeriodically();
+
+ auto haystack (m_HistoryBulker.ConsumeMany());
+
+ if (haystack.empty()) {
+ if (!GetActive()) {
+ break;
+ }
+
+ continue;
+ }
+
+ uintmax_t attempts = 0;
+
+ auto logFailure ([&haystack, &attempts](const char* err = nullptr) {
+ Log msg (LogNotice, "IcingaDB");
+
+ msg << "history: " << haystack.size() << " queries failed temporarily (attempt #" << ++attempts << ")";
+
+ if (err) {
+ msg << ": " << err;
+ }
+ });
+
+ for (;;) {
+ logPeriodically();
+
+ if (m_Rcon && m_Rcon->IsConnected()) {
+ try {
+ m_Rcon->GetResultsOfQueries(haystack, Prio::History, {0, 0, haystack.size()});
+ break;
+ } catch (const std::exception& ex) {
+ logFailure(ex.what());
+ } catch (...) {
+ logFailure();
+ }
+ } else {
+ logFailure("not connected to Redis");
+ }
+
+ if (!GetActive()) {
+ Log(LogCritical, "IcingaDB") << "history: " << haystack.size() << " queries failed (attempt #" << attempts
+ << ") while we're about to shut down. Giving up and discarding additional "
+ << m_HistoryBulker.Size() << " queued history queries.";
+
+ return;
+ }
+
+ Utility::Sleep(2);
+ }
+ }
+}
+
+void IcingaDB::SendNotificationUsersChanged(const Notification::Ptr& notification, const Array::Ptr& oldValues, const Array::Ptr& newValues) {
+ if (!m_Rcon || !m_Rcon->IsConnected() || oldValues == newValues) {
+ return;
+ }
+
+ std::vector<Value> deletedUsers = GetArrayDeletedValues(oldValues, newValues);
+
+ for (const auto& userName : deletedUsers) {
+ String id = HashValue(new Array({m_EnvironmentId, "user", userName, notification->GetName()}));
+ DeleteRelationship(id, "notification:user");
+ DeleteRelationship(id, "notification:recipient");
+ }
+}
+
+void IcingaDB::SendNotificationUserGroupsChanged(const Notification::Ptr& notification, const Array::Ptr& oldValues, const Array::Ptr& newValues) {
+ if (!m_Rcon || !m_Rcon->IsConnected() || oldValues == newValues) {
+ return;
+ }
+
+ std::vector<Value> deletedUserGroups = GetArrayDeletedValues(oldValues, newValues);
+
+ for (const auto& userGroupName : deletedUserGroups) {
+ UserGroup::Ptr userGroup = UserGroup::GetByName(userGroupName);
+ String id = HashValue(new Array({m_EnvironmentId, "usergroup", userGroupName, notification->GetName()}));
+ DeleteRelationship(id, "notification:usergroup");
+ DeleteRelationship(id, "notification:recipient");
+
+ for (const User::Ptr& user : userGroup->GetMembers()) {
+ String userId = HashValue(new Array({m_EnvironmentId, "usergroupuser", user->GetName(), userGroupName, notification->GetName()}));
+ DeleteRelationship(userId, "notification:recipient");
+ }
+ }
+}
+
+void IcingaDB::SendTimePeriodRangesChanged(const TimePeriod::Ptr& timeperiod, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues) {
+ if (!m_Rcon || !m_Rcon->IsConnected() || oldValues == newValues) {
+ return;
+ }
+
+ std::vector<String> deletedKeys = GetDictionaryDeletedKeys(oldValues, newValues);
+ String typeName = GetLowerCaseTypeNameDB(timeperiod);
+
+ for (const auto& rangeKey : deletedKeys) {
+ String id = HashValue(new Array({m_EnvironmentId, rangeKey, oldValues->Get(rangeKey), timeperiod->GetName()}));
+ DeleteRelationship(id, "timeperiod:range");
+ }
+}
+
+void IcingaDB::SendTimePeriodIncludesChanged(const TimePeriod::Ptr& timeperiod, const Array::Ptr& oldValues, const Array::Ptr& newValues) {
+ if (!m_Rcon || !m_Rcon->IsConnected() || oldValues == newValues) {
+ return;
+ }
+
+ std::vector<Value> deletedIncludes = GetArrayDeletedValues(oldValues, newValues);
+
+ for (const auto& includeName : deletedIncludes) {
+ String id = HashValue(new Array({m_EnvironmentId, includeName, timeperiod->GetName()}));
+ DeleteRelationship(id, "timeperiod:override:include");
+ }
+}
+
+void IcingaDB::SendTimePeriodExcludesChanged(const TimePeriod::Ptr& timeperiod, const Array::Ptr& oldValues, const Array::Ptr& newValues) {
+ if (!m_Rcon || !m_Rcon->IsConnected() || oldValues == newValues) {
+ return;
+ }
+
+ std::vector<Value> deletedExcludes = GetArrayDeletedValues(oldValues, newValues);
+
+ for (const auto& excludeName : deletedExcludes) {
+ String id = HashValue(new Array({m_EnvironmentId, excludeName, timeperiod->GetName()}));
+ DeleteRelationship(id, "timeperiod:override:exclude");
+ }
+}
+
+template<typename T>
+void IcingaDB::SendGroupsChanged(const ConfigObject::Ptr& object, const Array::Ptr& oldValues, const Array::Ptr& newValues) {
+ if (!m_Rcon || !m_Rcon->IsConnected() || oldValues == newValues) {
+ return;
+ }
+
+ std::vector<Value> deletedGroups = GetArrayDeletedValues(oldValues, newValues);
+ String typeName = GetLowerCaseTypeNameDB(object);
+
+ for (const auto& groupName : deletedGroups) {
+ typename T::Ptr group = ConfigObject::GetObject<T>(groupName);
+ String id = HashValue(new Array({m_EnvironmentId, group->GetName(), object->GetName()}));
+ DeleteRelationship(id, typeName + "group:member");
+
+ if (std::is_same<T, UserGroup>::value) {
+ UserGroup::Ptr userGroup = dynamic_pointer_cast<UserGroup>(group);
+
+ for (const auto& notification : userGroup->GetNotifications()) {
+ String userId = HashValue(new Array({m_EnvironmentId, "usergroupuser", object->GetName(), groupName, notification->GetName()}));
+ DeleteRelationship(userId, "notification:recipient");
+ }
+ }
+ }
+}
+
+void IcingaDB::SendCommandEnvChanged(const ConfigObject::Ptr& command, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues) {
+ if (!m_Rcon || !m_Rcon->IsConnected() || oldValues == newValues) {
+ return;
+ }
+
+ std::vector<String> deletedKeys = GetDictionaryDeletedKeys(oldValues, newValues);
+ String typeName = GetLowerCaseTypeNameDB(command);
+
+ for (const auto& envvarKey : deletedKeys) {
+ String id = HashValue(new Array({m_EnvironmentId, envvarKey, command->GetName()}));
+ DeleteRelationship(id, typeName + ":envvar", true);
+ }
+}
+
+void IcingaDB::SendCommandArgumentsChanged(const ConfigObject::Ptr& command, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues) {
+ if (!m_Rcon || !m_Rcon->IsConnected() || oldValues == newValues) {
+ return;
+ }
+
+ std::vector<String> deletedKeys = GetDictionaryDeletedKeys(oldValues, newValues);
+ String typeName = GetLowerCaseTypeNameDB(command);
+
+ for (const auto& argumentKey : deletedKeys) {
+ String id = HashValue(new Array({m_EnvironmentId, argumentKey, command->GetName()}));
+ DeleteRelationship(id, typeName + ":argument", true);
+ }
+}
+
+void IcingaDB::SendCustomVarsChanged(const ConfigObject::Ptr& object, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues) {
+ if (m_IndexedTypes.find(object->GetReflectionType().get()) == m_IndexedTypes.end()) {
+ return;
+ }
+
+ if (!m_Rcon || !m_Rcon->IsConnected() || oldValues == newValues) {
+ return;
+ }
+
+ Dictionary::Ptr oldVars = SerializeVars(oldValues);
+ Dictionary::Ptr newVars = SerializeVars(newValues);
+
+ std::vector<String> deletedVars = GetDictionaryDeletedKeys(oldVars, newVars);
+ String typeName = GetLowerCaseTypeNameDB(object);
+
+ for (const auto& varId : deletedVars) {
+ String id = HashValue(new Array({m_EnvironmentId, varId, object->GetName()}));
+ DeleteRelationship(id, typeName + ":customvar");
+ }
+}
+
+Dictionary::Ptr IcingaDB::SerializeState(const Checkable::Ptr& checkable)
+{
+ Dictionary::Ptr attrs = new Dictionary();
+
+ Host::Ptr host;
+ Service::Ptr service;
+
+ tie(host, service) = GetHostService(checkable);
+
+ String id = GetObjectIdentifier(checkable);
+
+ /*
+ * As there is a 1:1 relationship between host and host state, the host ID ('host_id')
+ * is also used as the host state ID ('id'). These are duplicated to 1) avoid having
+ * special handling for this in Icinga DB and 2) to have both a primary key and a foreign key
+ * in the SQL database in the end. In the database 'host_id' ends up as foreign key 'host_state.host_id'
+ * referring to 'host.id' while 'id' ends up as the primary key 'host_state.id'. This also applies for service.
+ */
+ attrs->Set("id", id);
+ attrs->Set("environment_id", m_EnvironmentId);
+ attrs->Set("state_type", checkable->HasBeenChecked() ? checkable->GetStateType() : StateTypeHard);
+
+ // TODO: last_hard/soft_state should be "previous".
+ if (service) {
+ attrs->Set("service_id", id);
+ auto state = service->HasBeenChecked() ? service->GetState() : 99;
+ attrs->Set("soft_state", state);
+ attrs->Set("hard_state", service->HasBeenChecked() ? service->GetLastHardState() : 99);
+ attrs->Set("severity", service->GetSeverity());
+ attrs->Set("host_id", GetObjectIdentifier(host));
+ } else {
+ attrs->Set("host_id", id);
+ auto state = host->HasBeenChecked() ? host->GetState() : 99;
+ attrs->Set("soft_state", state);
+ attrs->Set("hard_state", host->HasBeenChecked() ? host->GetLastHardState() : 99);
+ attrs->Set("severity", host->GetSeverity());
+ }
+
+ attrs->Set("previous_soft_state", GetPreviousState(checkable, service, StateTypeSoft));
+ attrs->Set("previous_hard_state", GetPreviousState(checkable, service, StateTypeHard));
+ attrs->Set("check_attempt", checkable->GetCheckAttempt());
+
+ attrs->Set("is_active", checkable->IsActive());
+
+ CheckResult::Ptr cr = checkable->GetLastCheckResult();
+
+ if (cr) {
+ String rawOutput = cr->GetOutput();
+ if (!rawOutput.IsEmpty()) {
+ size_t lineBreak = rawOutput.Find("\n");
+ String output = rawOutput.SubStr(0, lineBreak);
+ if (!output.IsEmpty())
+ attrs->Set("output", rawOutput.SubStr(0, lineBreak));
+
+ if (lineBreak > 0 && lineBreak != String::NPos) {
+ String longOutput = rawOutput.SubStr(lineBreak+1, rawOutput.GetLength());
+ if (!longOutput.IsEmpty())
+ attrs->Set("long_output", longOutput);
+ }
+ }
+
+ String perfData = PluginUtility::FormatPerfdata(cr->GetPerformanceData());
+ if (!perfData.IsEmpty())
+ attrs->Set("performance_data", perfData);
+
+ String normedPerfData = PluginUtility::FormatPerfdata(cr->GetPerformanceData(), true);
+ if (!normedPerfData.IsEmpty())
+ attrs->Set("normalized_performance_data", normedPerfData);
+
+ if (!cr->GetCommand().IsEmpty())
+ attrs->Set("check_commandline", FormatCommandLine(cr->GetCommand()));
+ attrs->Set("execution_time", TimestampToMilliseconds(fmax(0.0, cr->CalculateExecutionTime())));
+ attrs->Set("latency", TimestampToMilliseconds(cr->CalculateLatency()));
+ attrs->Set("check_source", cr->GetCheckSource());
+ attrs->Set("scheduling_source", cr->GetSchedulingSource());
+ }
+
+ attrs->Set("is_problem", checkable->GetProblem());
+ attrs->Set("is_handled", checkable->GetHandled());
+ attrs->Set("is_reachable", checkable->IsReachable());
+ attrs->Set("is_flapping", checkable->IsFlapping());
+
+ attrs->Set("is_acknowledged", checkable->GetAcknowledgement());
+ if (checkable->IsAcknowledged()) {
+ Timestamp entry = 0;
+ Comment::Ptr AckComment;
+ for (const Comment::Ptr& c : checkable->GetComments()) {
+ if (c->GetEntryType() == CommentAcknowledgement) {
+ if (c->GetEntryTime() > entry) {
+ entry = c->GetEntryTime();
+ AckComment = c;
+ }
+ }
+ }
+ if (AckComment != nullptr) {
+ attrs->Set("acknowledgement_comment_id", GetObjectIdentifier(AckComment));
+ }
+ }
+
+ {
+ auto lastComment (checkable->GetLastComment());
+
+ if (lastComment) {
+ attrs->Set("last_comment_id", GetObjectIdentifier(lastComment));
+ }
+ }
+
+ attrs->Set("in_downtime", checkable->IsInDowntime());
+
+ if (checkable->GetCheckTimeout().IsEmpty())
+ attrs->Set("check_timeout", TimestampToMilliseconds(checkable->GetCheckCommand()->GetTimeout()));
+ else
+ attrs->Set("check_timeout", TimestampToMilliseconds(checkable->GetCheckTimeout()));
+
+ long long lastCheck = TimestampToMilliseconds(checkable->GetLastCheck());
+ if (lastCheck > 0)
+ attrs->Set("last_update", lastCheck);
+
+ attrs->Set("last_state_change", TimestampToMilliseconds(checkable->GetLastStateChange()));
+ attrs->Set("next_check", TimestampToMilliseconds(checkable->GetNextCheck()));
+ attrs->Set("next_update", TimestampToMilliseconds(checkable->GetNextUpdate()));
+
+ return attrs;
+}
+
+std::vector<String>
+IcingaDB::UpdateObjectAttrs(const ConfigObject::Ptr& object, int fieldType,
+ const String& typeNameOverride)
+{
+ Type::Ptr type = object->GetReflectionType();
+ Dictionary::Ptr attrs(new Dictionary);
+
+ for (int fid = 0; fid < type->GetFieldCount(); fid++) {
+ Field field = type->GetFieldInfo(fid);
+
+ if ((field.Attributes & fieldType) == 0)
+ continue;
+
+ Value val = object->GetField(fid);
+
+ /* hide attributes which shouldn't be user-visible */
+ if (field.Attributes & FANoUserView)
+ continue;
+
+ /* hide internal navigation fields */
+ if (field.Attributes & FANavigation && !(field.Attributes & (FAConfig | FAState)))
+ continue;
+
+ attrs->Set(field.Name, Serialize(val));
+ }
+
+ /* Downtimes require in_effect, which is not an attribute */
+ Downtime::Ptr downtime = dynamic_pointer_cast<Downtime>(object);
+ if (downtime) {
+ attrs->Set("in_effect", Serialize(downtime->IsInEffect()));
+ attrs->Set("trigger_time", Serialize(TimestampToMilliseconds(downtime->GetTriggerTime())));
+ }
+
+
+ /* Use the name checksum as unique key. */
+ String typeName = type->GetName().ToLower();
+ if (!typeNameOverride.IsEmpty())
+ typeName = typeNameOverride.ToLower();
+
+ return {GetObjectIdentifier(object), JsonEncode(attrs)};
+ //m_Rcon->FireAndForgetQuery({"HSET", keyPrefix + typeName, GetObjectIdentifier(object), JsonEncode(attrs)});
+}
+
+void IcingaDB::StateChangeHandler(const ConfigObject::Ptr& object, const CheckResult::Ptr& cr, StateType type)
+{
+ for (const IcingaDB::Ptr& rw : ConfigType::GetObjectsByType<IcingaDB>()) {
+ rw->SendStateChange(object, cr, type);
+ }
+}
+
+void IcingaDB::ReachabilityChangeHandler(const std::set<Checkable::Ptr>& children)
+{
+ for (const IcingaDB::Ptr& rw : ConfigType::GetObjectsByType<IcingaDB>()) {
+ for (auto& checkable : children) {
+ rw->UpdateState(checkable, StateUpdate::Full);
+ }
+ }
+}
+
+void IcingaDB::VersionChangedHandler(const ConfigObject::Ptr& object)
+{
+ Type::Ptr type = object->GetReflectionType();
+
+ if (m_IndexedTypes.find(type.get()) == m_IndexedTypes.end()) {
+ return;
+ }
+
+ if (object->IsActive()) {
+ // Create or update the object config
+ for (const IcingaDB::Ptr& rw : ConfigType::GetObjectsByType<IcingaDB>()) {
+ if (rw)
+ rw->SendConfigUpdate(object, true);
+ }
+ } else if (!object->IsActive() &&
+ object->GetExtension("ConfigObjectDeleted")) { // same as in apilistener-configsync.cpp
+ // Delete object config
+ for (const IcingaDB::Ptr& rw : ConfigType::GetObjectsByType<IcingaDB>()) {
+ if (rw)
+ rw->SendConfigDelete(object);
+ }
+ }
+}
+
+void IcingaDB::DowntimeStartedHandler(const Downtime::Ptr& downtime)
+{
+ for (auto& rw : ConfigType::GetObjectsByType<IcingaDB>()) {
+ rw->SendStartedDowntime(downtime);
+ }
+}
+
+void IcingaDB::DowntimeRemovedHandler(const Downtime::Ptr& downtime)
+{
+ for (auto& rw : ConfigType::GetObjectsByType<IcingaDB>()) {
+ rw->SendRemovedDowntime(downtime);
+ }
+}
+
+void IcingaDB::NotificationSentToAllUsersHandler(
+ const Notification::Ptr& notification, const Checkable::Ptr& checkable, const std::set<User::Ptr>& users,
+ NotificationType type, const CheckResult::Ptr& cr, const String& author, const String& text
+)
+{
+ auto rws (ConfigType::GetObjectsByType<IcingaDB>());
+ auto sendTime (notification->GetLastNotification());
+
+ if (!rws.empty()) {
+ for (auto& rw : rws) {
+ rw->SendSentNotification(notification, checkable, users, type, cr, author, text, sendTime);
+ }
+ }
+}
+
+void IcingaDB::CommentAddedHandler(const Comment::Ptr& comment)
+{
+ for (auto& rw : ConfigType::GetObjectsByType<IcingaDB>()) {
+ rw->SendAddedComment(comment);
+ }
+}
+
+void IcingaDB::CommentRemovedHandler(const Comment::Ptr& comment)
+{
+ for (auto& rw : ConfigType::GetObjectsByType<IcingaDB>()) {
+ rw->SendRemovedComment(comment);
+ }
+}
+
+void IcingaDB::FlappingChangeHandler(const Checkable::Ptr& checkable, double changeTime)
+{
+ auto flappingLastChange (checkable->GetFlappingLastChange());
+
+ for (auto& rw : ConfigType::GetObjectsByType<IcingaDB>()) {
+ rw->SendFlappingChange(checkable, changeTime, flappingLastChange);
+ }
+}
+
+void IcingaDB::NewCheckResultHandler(const Checkable::Ptr& checkable)
+{
+ for (auto& rw : ConfigType::GetObjectsByType<IcingaDB>()) {
+ rw->UpdateState(checkable, StateUpdate::Volatile);
+ rw->SendNextUpdate(checkable);
+ }
+}
+
+void IcingaDB::NextCheckUpdatedHandler(const Checkable::Ptr& checkable)
+{
+ for (auto& rw : ConfigType::GetObjectsByType<IcingaDB>()) {
+ rw->UpdateState(checkable, StateUpdate::Volatile);
+ rw->SendNextUpdate(checkable);
+ }
+}
+
+void IcingaDB::HostProblemChangedHandler(const Service::Ptr& service) {
+ for (auto& rw : ConfigType::GetObjectsByType<IcingaDB>()) {
+ /* Host state changes affect is_handled and severity of services. */
+ rw->UpdateState(service, StateUpdate::Full);
+ }
+}
+
+void IcingaDB::AcknowledgementSetHandler(const Checkable::Ptr& checkable, const String& author, const String& comment, AcknowledgementType type, bool persistent, double changeTime, double expiry)
+{
+ auto rws (ConfigType::GetObjectsByType<IcingaDB>());
+
+ if (!rws.empty()) {
+ for (auto& rw : rws) {
+ rw->SendAcknowledgementSet(checkable, author, comment, type, persistent, changeTime, expiry);
+ }
+ }
+}
+
+void IcingaDB::AcknowledgementClearedHandler(const Checkable::Ptr& checkable, const String& removedBy, double changeTime)
+{
+ auto rws (ConfigType::GetObjectsByType<IcingaDB>());
+
+ if (!rws.empty()) {
+ auto rb (Shared<String>::Make(removedBy));
+ auto ackLastChange (checkable->GetAcknowledgementLastChange());
+
+ for (auto& rw : rws) {
+ rw->SendAcknowledgementCleared(checkable, *rb, changeTime, ackLastChange);
+ }
+ }
+}
+
+void IcingaDB::NotificationUsersChangedHandler(const Notification::Ptr& notification, const Array::Ptr& oldValues, const Array::Ptr& newValues) {
+ for (const IcingaDB::Ptr& rw : ConfigType::GetObjectsByType<IcingaDB>()) {
+ rw->SendNotificationUsersChanged(notification, oldValues, newValues);
+ }
+}
+
+void IcingaDB::NotificationUserGroupsChangedHandler(const Notification::Ptr& notification, const Array::Ptr& oldValues, const Array::Ptr& newValues) {
+ for (const IcingaDB::Ptr& rw : ConfigType::GetObjectsByType<IcingaDB>()) {
+ rw->SendNotificationUserGroupsChanged(notification, oldValues, newValues);
+ }
+}
+
+void IcingaDB::TimePeriodRangesChangedHandler(const TimePeriod::Ptr& timeperiod, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues) {
+ for (const IcingaDB::Ptr& rw : ConfigType::GetObjectsByType<IcingaDB>()) {
+ rw->SendTimePeriodRangesChanged(timeperiod, oldValues, newValues);
+ }
+}
+
+void IcingaDB::TimePeriodIncludesChangedHandler(const TimePeriod::Ptr& timeperiod, const Array::Ptr& oldValues, const Array::Ptr& newValues) {
+ for (const IcingaDB::Ptr& rw : ConfigType::GetObjectsByType<IcingaDB>()) {
+ rw->SendTimePeriodIncludesChanged(timeperiod, oldValues, newValues);
+ }
+}
+
+void IcingaDB::TimePeriodExcludesChangedHandler(const TimePeriod::Ptr& timeperiod, const Array::Ptr& oldValues, const Array::Ptr& newValues) {
+ for (const IcingaDB::Ptr& rw : ConfigType::GetObjectsByType<IcingaDB>()) {
+ rw->SendTimePeriodExcludesChanged(timeperiod, oldValues, newValues);
+ }
+}
+
+void IcingaDB::UserGroupsChangedHandler(const User::Ptr& user, const Array::Ptr& oldValues, const Array::Ptr& newValues) {
+ for (const IcingaDB::Ptr& rw : ConfigType::GetObjectsByType<IcingaDB>()) {
+ rw->SendGroupsChanged<UserGroup>(user, oldValues, newValues);
+ }
+}
+
+void IcingaDB::HostGroupsChangedHandler(const Host::Ptr& host, const Array::Ptr& oldValues, const Array::Ptr& newValues) {
+ for (const IcingaDB::Ptr& rw : ConfigType::GetObjectsByType<IcingaDB>()) {
+ rw->SendGroupsChanged<HostGroup>(host, oldValues, newValues);
+ }
+}
+
+void IcingaDB::ServiceGroupsChangedHandler(const Service::Ptr& service, const Array::Ptr& oldValues, const Array::Ptr& newValues) {
+ for (const IcingaDB::Ptr& rw : ConfigType::GetObjectsByType<IcingaDB>()) {
+ rw->SendGroupsChanged<ServiceGroup>(service, oldValues, newValues);
+ }
+}
+
+void IcingaDB::CommandEnvChangedHandler(const ConfigObject::Ptr& command, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues) {
+ for (const IcingaDB::Ptr& rw : ConfigType::GetObjectsByType<IcingaDB>()) {
+ rw->SendCommandEnvChanged(command, oldValues, newValues);
+ }
+}
+
+void IcingaDB::CommandArgumentsChangedHandler(const ConfigObject::Ptr& command, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues) {
+ for (const IcingaDB::Ptr& rw : ConfigType::GetObjectsByType<IcingaDB>()) {
+ rw->SendCommandArgumentsChanged(command, oldValues, newValues);
+ }
+}
+
+void IcingaDB::CustomVarsChangedHandler(const ConfigObject::Ptr& object, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues) {
+ for (const IcingaDB::Ptr& rw : ConfigType::GetObjectsByType<IcingaDB>()) {
+ rw->SendCustomVarsChanged(object, oldValues, newValues);
+ }
+}
+
+void IcingaDB::DeleteRelationship(const String& id, const String& redisKeyWithoutPrefix, bool hasChecksum) {
+ Log(LogNotice, "IcingaDB") << "Deleting relationship '" << redisKeyWithoutPrefix << " -> '" << id << "'";
+
+ String redisKey = m_PrefixConfigObject + redisKeyWithoutPrefix;
+
+ std::vector<std::vector<String>> queries;
+
+ if (hasChecksum) {
+ queries.push_back({"HDEL", m_PrefixConfigCheckSum + redisKeyWithoutPrefix, id});
+ }
+
+ queries.push_back({"HDEL", redisKey, id});
+ queries.push_back({
+ "XADD", "icinga:runtime", "MAXLEN", "~", "1000000", "*",
+ "redis_key", redisKey, "id", id, "runtime_type", "delete"
+ });
+
+ m_Rcon->FireAndForgetQueries(queries, Prio::Config);
+}
diff --git a/lib/icingadb/icingadb-stats.cpp b/lib/icingadb/icingadb-stats.cpp
new file mode 100644
index 0000000..375d0f7
--- /dev/null
+++ b/lib/icingadb/icingadb-stats.cpp
@@ -0,0 +1,54 @@
+/* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */
+
+#include "icingadb/icingadb.hpp"
+#include "base/application.hpp"
+#include "base/json.hpp"
+#include "base/logger.hpp"
+#include "base/serializer.hpp"
+#include "base/statsfunction.hpp"
+#include "base/convert.hpp"
+
+using namespace icinga;
+
+Dictionary::Ptr IcingaDB::GetStats()
+{
+ Dictionary::Ptr stats = new Dictionary();
+
+ //TODO: Figure out if more stats can be useful here.
+ Namespace::Ptr statsFunctions = ScriptGlobal::Get("StatsFunctions", &Empty);
+
+ if (!statsFunctions)
+ Dictionary::Ptr();
+
+ ObjectLock olock(statsFunctions);
+
+ for (auto& kv : statsFunctions)
+ {
+ Function::Ptr func = kv.second->Get();
+
+ if (!func)
+ BOOST_THROW_EXCEPTION(std::invalid_argument("Invalid status function name."));
+
+ Dictionary::Ptr status = new Dictionary();
+ Array::Ptr perfdata = new Array();
+ func->Invoke({ status, perfdata });
+
+ stats->Set(kv.first, new Dictionary({
+ { "status", status },
+ { "perfdata", Serialize(perfdata, FAState) }
+ }));
+ }
+
+ typedef Dictionary::Ptr DP;
+ DP app = DP(DP(DP(stats->Get("IcingaApplication"))->Get("status"))->Get("icingaapplication"))->Get("app");
+
+ app->Set("program_start", TimestampToMilliseconds(Application::GetStartTime()));
+
+ auto localEndpoint (Endpoint::GetLocalEndpoint());
+ if (localEndpoint) {
+ app->Set("endpoint_id", GetObjectIdentifier(localEndpoint));
+ }
+
+ return stats;
+}
+
diff --git a/lib/icingadb/icingadb-utility.cpp b/lib/icingadb/icingadb-utility.cpp
new file mode 100644
index 0000000..bb24e03
--- /dev/null
+++ b/lib/icingadb/icingadb-utility.cpp
@@ -0,0 +1,313 @@
+/* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */
+
+#include "icingadb/icingadb.hpp"
+#include "base/configtype.hpp"
+#include "base/object-packer.hpp"
+#include "base/logger.hpp"
+#include "base/serializer.hpp"
+#include "base/tlsutility.hpp"
+#include "base/initialize.hpp"
+#include "base/objectlock.hpp"
+#include "base/array.hpp"
+#include "base/scriptglobal.hpp"
+#include "base/convert.hpp"
+#include "base/json.hpp"
+#include "icinga/customvarobject.hpp"
+#include "icinga/checkcommand.hpp"
+#include "icinga/notificationcommand.hpp"
+#include "icinga/eventcommand.hpp"
+#include "icinga/host.hpp"
+#include <boost/algorithm/string.hpp>
+#include <map>
+#include <utility>
+#include <vector>
+
+using namespace icinga;
+
+String IcingaDB::FormatCheckSumBinary(const String& str)
+{
+ char output[20*2+1];
+ for (int i = 0; i < 20; i++)
+ sprintf(output + 2 * i, "%02x", str[i]);
+
+ return output;
+}
+
+String IcingaDB::FormatCommandLine(const Value& commandLine)
+{
+ String result;
+ if (commandLine.IsObjectType<Array>()) {
+ Array::Ptr args = commandLine;
+ bool first = true;
+
+ ObjectLock olock(args);
+ for (const Value& arg : args) {
+ String token = "'" + Convert::ToString(arg) + "'";
+
+ if (first)
+ first = false;
+ else
+ result += String(1, ' ');
+
+ result += token;
+ }
+ } else if (!commandLine.IsEmpty()) {
+ result = commandLine;
+ boost::algorithm::replace_all(result, "\'", "\\'");
+ result = "'" + result + "'";
+ }
+
+ return result;
+}
+
+String IcingaDB::GetObjectIdentifier(const ConfigObject::Ptr& object)
+{
+ return HashValue(new Array({m_EnvironmentId, object->GetName()}));
+}
+
+/**
+ * Calculates a deterministic history event ID like SHA1(env, eventType, x...[, nt][, eventTime])
+ *
+ * Where SHA1(env, x...) = GetObjectIdentifier(object)
+ */
+String IcingaDB::CalcEventID(const char* eventType, const ConfigObject::Ptr& object, double eventTime, NotificationType nt)
+{
+ Array::Ptr rawId = new Array({object->GetName()});
+ rawId->Insert(0, m_EnvironmentId);
+ rawId->Insert(1, eventType);
+
+ if (nt) {
+ rawId->Add(GetNotificationTypeByEnum(nt));
+ }
+
+ if (eventTime) {
+ rawId->Add(TimestampToMilliseconds(eventTime));
+ }
+
+ return HashValue(std::move(rawId));
+}
+
+static const std::set<String> metadataWhitelist ({"package", "source_location", "templates"});
+
+/**
+ * Prepare custom vars for being written to Redis
+ *
+ * object.vars = {
+ * "disks": {
+ * "disk": {},
+ * "disk /": {
+ * "disk_partitions": "/"
+ * }
+ * }
+ * }
+ *
+ * return {
+ * SHA1(PackObject([
+ * EnvironmentId,
+ * "disks",
+ * {
+ * "disk": {},
+ * "disk /": {
+ * "disk_partitions": "/"
+ * }
+ * }
+ * ])): {
+ * "environment_id": EnvironmentId,
+ * "name_checksum": SHA1("disks"),
+ * "name": "disks",
+ * "value": {
+ * "disk": {},
+ * "disk /": {
+ * "disk_partitions": "/"
+ * }
+ * }
+ * }
+ * }
+ *
+ * @param Dictionary Config object with custom vars
+ *
+ * @return JSON-like data structure for Redis
+ */
+Dictionary::Ptr IcingaDB::SerializeVars(const Dictionary::Ptr& vars)
+{
+ if (!vars)
+ return nullptr;
+
+ Dictionary::Ptr res = new Dictionary();
+
+ ObjectLock olock(vars);
+
+ for (auto& kv : vars) {
+ res->Set(
+ SHA1(PackObject((Array::Ptr)new Array({m_EnvironmentId, kv.first, kv.second}))),
+ (Dictionary::Ptr)new Dictionary({
+ {"environment_id", m_EnvironmentId},
+ {"name_checksum", SHA1(kv.first)},
+ {"name", kv.first},
+ {"value", JsonEncode(kv.second)},
+ })
+ );
+ }
+
+ return res;
+}
+
+const char* IcingaDB::GetNotificationTypeByEnum(NotificationType type)
+{
+ switch (type) {
+ case NotificationDowntimeStart:
+ return "downtime_start";
+ case NotificationDowntimeEnd:
+ return "downtime_end";
+ case NotificationDowntimeRemoved:
+ return "downtime_removed";
+ case NotificationCustom:
+ return "custom";
+ case NotificationAcknowledgement:
+ return "acknowledgement";
+ case NotificationProblem:
+ return "problem";
+ case NotificationRecovery:
+ return "recovery";
+ case NotificationFlappingStart:
+ return "flapping_start";
+ case NotificationFlappingEnd:
+ return "flapping_end";
+ }
+
+ VERIFY(!"Invalid notification type.");
+}
+
+static const std::set<String> propertiesBlacklistEmpty;
+
+String IcingaDB::HashValue(const Value& value)
+{
+ return HashValue(value, propertiesBlacklistEmpty);
+}
+
+String IcingaDB::HashValue(const Value& value, const std::set<String>& propertiesBlacklist, bool propertiesWhitelist)
+{
+ Value temp;
+ bool mutabl;
+
+ Type::Ptr type = value.GetReflectionType();
+
+ if (ConfigObject::TypeInstance->IsAssignableFrom(type)) {
+ temp = Serialize(value, FAConfig);
+ mutabl = true;
+ } else {
+ temp = value;
+ mutabl = false;
+ }
+
+ if (propertiesBlacklist.size() && temp.IsObject()) {
+ Dictionary::Ptr dict = dynamic_pointer_cast<Dictionary>((Object::Ptr)temp);
+
+ if (dict) {
+ if (!mutabl)
+ dict = dict->ShallowClone();
+
+ ObjectLock olock(dict);
+
+ if (propertiesWhitelist) {
+ auto current = dict->Begin();
+ auto propertiesBlacklistEnd = propertiesBlacklist.end();
+
+ while (current != dict->End()) {
+ if (propertiesBlacklist.find(current->first) == propertiesBlacklistEnd) {
+ dict->Remove(current++);
+ } else {
+ ++current;
+ }
+ }
+ } else {
+ for (auto& property : propertiesBlacklist)
+ dict->Remove(property);
+ }
+
+ if (!mutabl)
+ temp = dict;
+ }
+ }
+
+ return SHA1(PackObject(temp));
+}
+
+String IcingaDB::GetLowerCaseTypeNameDB(const ConfigObject::Ptr& obj)
+{
+ return obj->GetReflectionType()->GetName().ToLower();
+}
+
+long long IcingaDB::TimestampToMilliseconds(double timestamp) {
+ return static_cast<long long>(timestamp * 1000);
+}
+
+String IcingaDB::IcingaToStreamValue(const Value& value)
+{
+ switch (value.GetType()) {
+ case ValueBoolean:
+ return Convert::ToString(int(value));
+ case ValueString:
+ return Utility::ValidateUTF8(value);
+ case ValueNumber:
+ case ValueEmpty:
+ return Convert::ToString(value);
+ default:
+ return JsonEncode(value);
+ }
+}
+
+// Returns the items that exist in "arrayOld" but not in "arrayNew"
+std::vector<Value> IcingaDB::GetArrayDeletedValues(const Array::Ptr& arrayOld, const Array::Ptr& arrayNew) {
+ std::vector<Value> deletedValues;
+
+ if (!arrayOld) {
+ return deletedValues;
+ }
+
+ if (!arrayNew) {
+ ObjectLock olock (arrayOld);
+ return std::vector<Value>(arrayOld->Begin(), arrayOld->End());
+ }
+
+ std::vector<Value> vectorOld;
+ {
+ ObjectLock olock (arrayOld);
+ vectorOld.assign(arrayOld->Begin(), arrayOld->End());
+ }
+ std::sort(vectorOld.begin(), vectorOld.end());
+ vectorOld.erase(std::unique(vectorOld.begin(), vectorOld.end()), vectorOld.end());
+
+ std::vector<Value> vectorNew;
+ {
+ ObjectLock olock (arrayNew);
+ vectorNew.assign(arrayNew->Begin(), arrayNew->End());
+ }
+ std::sort(vectorNew.begin(), vectorNew.end());
+ vectorNew.erase(std::unique(vectorNew.begin(), vectorNew.end()), vectorNew.end());
+
+ std::set_difference(vectorOld.begin(), vectorOld.end(), vectorNew.begin(), vectorNew.end(), std::back_inserter(deletedValues));
+
+ return deletedValues;
+}
+
+// Returns the keys that exist in "dictOld" but not in "dictNew"
+std::vector<String> IcingaDB::GetDictionaryDeletedKeys(const Dictionary::Ptr& dictOld, const Dictionary::Ptr& dictNew) {
+ std::vector<String> deletedKeys;
+
+ if (!dictOld) {
+ return deletedKeys;
+ }
+
+ std::vector<String> oldKeys = dictOld->GetKeys();
+
+ if (!dictNew) {
+ return oldKeys;
+ }
+
+ std::vector<String> newKeys = dictNew->GetKeys();
+
+ std::set_difference(oldKeys.begin(), oldKeys.end(), newKeys.begin(), newKeys.end(), std::back_inserter(deletedKeys));
+
+ return deletedKeys;
+}
diff --git a/lib/icingadb/icingadb.cpp b/lib/icingadb/icingadb.cpp
new file mode 100644
index 0000000..f96656e
--- /dev/null
+++ b/lib/icingadb/icingadb.cpp
@@ -0,0 +1,309 @@
+/* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */
+
+#include "icingadb/icingadb.hpp"
+#include "icingadb/icingadb-ti.cpp"
+#include "icingadb/redisconnection.hpp"
+#include "remote/apilistener.hpp"
+#include "remote/eventqueue.hpp"
+#include "base/configuration.hpp"
+#include "base/json.hpp"
+#include "base/perfdatavalue.hpp"
+#include "base/statsfunction.hpp"
+#include "base/tlsutility.hpp"
+#include "base/utility.hpp"
+#include "icinga/checkable.hpp"
+#include "icinga/host.hpp"
+#include <boost/algorithm/string.hpp>
+#include <fstream>
+#include <memory>
+#include <utility>
+
+using namespace icinga;
+
+#define MAX_EVENTS_DEFAULT 5000
+
+using Prio = RedisConnection::QueryPriority;
+
+String IcingaDB::m_EnvironmentId;
+std::mutex IcingaDB::m_EnvironmentIdInitMutex;
+
+REGISTER_TYPE(IcingaDB);
+
+IcingaDB::IcingaDB()
+ : m_Rcon(nullptr)
+{
+ m_RconLocked.store(nullptr);
+
+ m_WorkQueue.SetName("IcingaDB");
+
+ m_PrefixConfigObject = "icinga:";
+ m_PrefixConfigCheckSum = "icinga:checksum:";
+}
+
+void IcingaDB::Validate(int types, const ValidationUtils& utils)
+{
+ ObjectImpl<IcingaDB>::Validate(types, utils);
+
+ if (!(types & FAConfig))
+ return;
+
+ if (GetEnableTls() && GetCertPath().IsEmpty() != GetKeyPath().IsEmpty()) {
+ BOOST_THROW_EXCEPTION(ValidationError(this, std::vector<String>(), "Validation failed: Either both a client certificate (cert_path) and its private key (key_path) or none of them must be given."));
+ }
+
+ try {
+ InitEnvironmentId();
+ } catch (const std::exception& e) {
+ BOOST_THROW_EXCEPTION(ValidationError(this, std::vector<String>(),
+ String("Validation failed: ") + e.what()));
+ }
+}
+
+/**
+ * Starts the component.
+ */
+void IcingaDB::Start(bool runtimeCreated)
+{
+ ObjectImpl<IcingaDB>::Start(runtimeCreated);
+
+ VERIFY(!m_EnvironmentId.IsEmpty());
+ PersistEnvironmentId();
+
+ Log(LogInformation, "IcingaDB")
+ << "'" << GetName() << "' started.";
+
+ m_ConfigDumpInProgress = false;
+ m_ConfigDumpDone = false;
+
+ m_WorkQueue.SetExceptionCallback([this](boost::exception_ptr exp) { ExceptionHandler(std::move(exp)); });
+
+ m_Rcon = new RedisConnection(GetHost(), GetPort(), GetPath(), GetPassword(), GetDbIndex(),
+ GetEnableTls(), GetInsecureNoverify(), GetCertPath(), GetKeyPath(), GetCaPath(), GetCrlPath(),
+ GetTlsProtocolmin(), GetCipherList(), GetConnectTimeout(), GetDebugInfo());
+ m_RconLocked.store(m_Rcon);
+
+ for (const Type::Ptr& type : GetTypes()) {
+ auto ctype (dynamic_cast<ConfigType*>(type.get()));
+ if (!ctype)
+ continue;
+
+ RedisConnection::Ptr con = new RedisConnection(GetHost(), GetPort(), GetPath(), GetPassword(), GetDbIndex(),
+ GetEnableTls(), GetInsecureNoverify(), GetCertPath(), GetKeyPath(), GetCaPath(), GetCrlPath(),
+ GetTlsProtocolmin(), GetCipherList(), GetConnectTimeout(), GetDebugInfo(), m_Rcon);
+
+ con->SetConnectedCallback([this, con](boost::asio::yield_context& yc) {
+ con->SetConnectedCallback(nullptr);
+
+ size_t pending = --m_PendingRcons;
+ Log(LogDebug, "IcingaDB") << pending << " pending child connections remaining";
+ if (pending == 0) {
+ m_WorkQueue.Enqueue([this]() { OnConnectedHandler(); });
+ }
+ });
+
+ m_Rcons[ctype] = std::move(con);
+ }
+
+ m_PendingRcons = m_Rcons.size();
+
+ m_Rcon->SetConnectedCallback([this](boost::asio::yield_context& yc) {
+ m_Rcon->SetConnectedCallback(nullptr);
+
+ for (auto& kv : m_Rcons) {
+ kv.second->Start();
+ }
+ });
+ m_Rcon->Start();
+
+ m_StatsTimer = new Timer();
+ m_StatsTimer->SetInterval(1);
+ m_StatsTimer->OnTimerExpired.connect([this](const Timer * const&) { PublishStatsTimerHandler(); });
+ m_StatsTimer->Start();
+
+ m_WorkQueue.SetName("IcingaDB");
+
+ m_Rcon->SuppressQueryKind(Prio::CheckResult);
+ m_Rcon->SuppressQueryKind(Prio::RuntimeStateSync);
+
+ Ptr keepAlive (this);
+
+ m_HistoryThread = std::async(std::launch::async, [this, keepAlive]() { ForwardHistoryEntries(); });
+}
+
+void IcingaDB::ExceptionHandler(boost::exception_ptr exp)
+{
+ Log(LogCritical, "IcingaDB", "Exception during redis query. Verify that Redis is operational.");
+
+ Log(LogDebug, "IcingaDB")
+ << "Exception during redis operation: " << DiagnosticInformation(exp);
+}
+
+void IcingaDB::OnConnectedHandler()
+{
+ AssertOnWorkQueue();
+
+ if (m_ConfigDumpInProgress || m_ConfigDumpDone)
+ return;
+
+ /* Config dump */
+ m_ConfigDumpInProgress = true;
+ PublishStats();
+
+ UpdateAllConfigObjects();
+
+ m_ConfigDumpDone = true;
+
+ m_ConfigDumpInProgress = false;
+}
+
+void IcingaDB::PublishStatsTimerHandler(void)
+{
+ PublishStats();
+}
+
+void IcingaDB::PublishStats()
+{
+ if (!m_Rcon || !m_Rcon->IsConnected())
+ return;
+
+ Dictionary::Ptr status = GetStats();
+ status->Set("config_dump_in_progress", m_ConfigDumpInProgress);
+ status->Set("timestamp", TimestampToMilliseconds(Utility::GetTime()));
+ status->Set("icingadb_environment", m_EnvironmentId);
+
+ std::vector<String> query {"XADD", "icinga:stats", "MAXLEN", "1", "*"};
+
+ {
+ ObjectLock statusLock (status);
+ for (auto& kv : status) {
+ query.emplace_back(kv.first);
+ query.emplace_back(JsonEncode(kv.second));
+ }
+ }
+
+ m_Rcon->FireAndForgetQuery(std::move(query), Prio::Heartbeat);
+}
+
+void IcingaDB::Stop(bool runtimeRemoved)
+{
+ Log(LogInformation, "IcingaDB")
+ << "Flushing history data buffer to Redis.";
+
+ if (m_HistoryThread.wait_for(std::chrono::minutes(1)) == std::future_status::timeout) {
+ Log(LogCritical, "IcingaDB")
+ << "Flushing takes more than one minute (while we're about to shut down). Giving up and discarding "
+ << m_HistoryBulker.Size() << " queued history queries.";
+ }
+
+ Log(LogInformation, "IcingaDB")
+ << "'" << GetName() << "' stopped.";
+
+ ObjectImpl<IcingaDB>::Stop(runtimeRemoved);
+}
+
+void IcingaDB::ValidateTlsProtocolmin(const Lazy<String>& lvalue, const ValidationUtils& utils)
+{
+ ObjectImpl<IcingaDB>::ValidateTlsProtocolmin(lvalue, utils);
+
+ try {
+ ResolveTlsProtocolVersion(lvalue());
+ } catch (const std::exception& ex) {
+ BOOST_THROW_EXCEPTION(ValidationError(this, { "tls_protocolmin" }, ex.what()));
+ }
+}
+
+void IcingaDB::ValidateConnectTimeout(const Lazy<double>& lvalue, const ValidationUtils& utils)
+{
+ ObjectImpl<IcingaDB>::ValidateConnectTimeout(lvalue, utils);
+
+ if (lvalue() <= 0) {
+ BOOST_THROW_EXCEPTION(ValidationError(this, { "connect_timeout" }, "Value must be greater than 0."));
+ }
+}
+
+void IcingaDB::AssertOnWorkQueue()
+{
+ ASSERT(m_WorkQueue.IsWorkerThread());
+}
+
+void IcingaDB::DumpedGlobals::Reset()
+{
+ std::lock_guard<std::mutex> l (m_Mutex);
+ m_Ids.clear();
+}
+
+String IcingaDB::GetEnvironmentId() const {
+ return m_EnvironmentId;
+}
+
+bool IcingaDB::DumpedGlobals::IsNew(const String& id)
+{
+ std::lock_guard<std::mutex> l (m_Mutex);
+ return m_Ids.emplace(id).second;
+}
+
+/**
+ * Initializes the m_EnvironmentId attribute or throws an exception on failure to do so. Can be called concurrently.
+ */
+void IcingaDB::InitEnvironmentId()
+{
+ // Initialize m_EnvironmentId once across all IcingaDB objects. In theory, this could be done using
+ // std::call_once, however, due to a bug in libstdc++ (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=66146),
+ // this can result in a deadlock when an exception is thrown (which is explicitly allowed by the standard).
+ std::unique_lock<std::mutex> lock (m_EnvironmentIdInitMutex);
+
+ if (m_EnvironmentId.IsEmpty()) {
+ String path = Configuration::DataDir + "/icingadb.env";
+ String envId;
+
+ if (Utility::PathExists(path)) {
+ envId = Utility::LoadJsonFile(path);
+
+ if (envId.GetLength() != 2*SHA_DIGEST_LENGTH) {
+ throw std::runtime_error("environment ID stored at " + path + " is corrupt: wrong length.");
+ }
+
+ for (unsigned char c : envId) {
+ if (!std::isxdigit(c)) {
+ throw std::runtime_error("environment ID stored at " + path + " is corrupt: invalid hex string.");
+ }
+ }
+ } else {
+ String caPath = ApiListener::GetDefaultCaPath();
+
+ if (!Utility::PathExists(caPath)) {
+ throw std::runtime_error("Cannot find the CA certificate at '" + caPath + "'. "
+ "Please ensure the ApiListener is enabled first using 'icinga2 api setup'.");
+ }
+
+ std::shared_ptr<X509> cert = GetX509Certificate(caPath);
+
+ unsigned int n;
+ unsigned char digest[EVP_MAX_MD_SIZE];
+ if (X509_pubkey_digest(cert.get(), EVP_sha1(), digest, &n) != 1) {
+ BOOST_THROW_EXCEPTION(openssl_error()
+ << boost::errinfo_api_function("X509_pubkey_digest")
+ << errinfo_openssl_error(ERR_peek_error()));
+ }
+
+ envId = BinaryToHex(digest, n);
+ }
+
+ m_EnvironmentId = envId.ToLower();
+ }
+}
+
+/**
+ * Ensures that the environment ID is persisted on disk or throws an exception on failure to do so.
+ * Can be called concurrently.
+ */
+void IcingaDB::PersistEnvironmentId()
+{
+ String path = Configuration::DataDir + "/icingadb.env";
+
+ std::unique_lock<std::mutex> lock (m_EnvironmentIdInitMutex);
+
+ if (!Utility::PathExists(path)) {
+ Utility::SaveJsonFile(path, 0600, m_EnvironmentId);
+ }
+}
diff --git a/lib/icingadb/icingadb.hpp b/lib/icingadb/icingadb.hpp
new file mode 100644
index 0000000..6652d9c
--- /dev/null
+++ b/lib/icingadb/icingadb.hpp
@@ -0,0 +1,241 @@
+/* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */
+
+#ifndef ICINGADB_H
+#define ICINGADB_H
+
+#include "icingadb/icingadb-ti.hpp"
+#include "icingadb/redisconnection.hpp"
+#include "base/atomic.hpp"
+#include "base/bulker.hpp"
+#include "base/timer.hpp"
+#include "base/workqueue.hpp"
+#include "icinga/customvarobject.hpp"
+#include "icinga/checkable.hpp"
+#include "icinga/service.hpp"
+#include "icinga/downtime.hpp"
+#include "remote/messageorigin.hpp"
+#include <atomic>
+#include <chrono>
+#include <future>
+#include <memory>
+#include <mutex>
+#include <set>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+
+namespace icinga
+{
+
+/**
+ * @ingroup icingadb
+ */
+class IcingaDB : public ObjectImpl<IcingaDB>
+{
+public:
+ DECLARE_OBJECT(IcingaDB);
+ DECLARE_OBJECTNAME(IcingaDB);
+
+ IcingaDB();
+
+ static void ConfigStaticInitialize();
+
+ void Validate(int types, const ValidationUtils& utils) override;
+ virtual void Start(bool runtimeCreated) override;
+ virtual void Stop(bool runtimeRemoved) override;
+
+ String GetEnvironmentId() const override;
+
+ inline RedisConnection::Ptr GetConnection()
+ {
+ return m_RconLocked.load();
+ }
+
+ template<class T>
+ static void AddKvsToMap(const Array::Ptr& kvs, T& map)
+ {
+ Value* key = nullptr;
+ ObjectLock oLock (kvs);
+
+ for (auto& kv : kvs) {
+ if (key) {
+ map.emplace(std::move(*key), std::move(kv));
+ key = nullptr;
+ } else {
+ key = &kv;
+ }
+ }
+ }
+
+protected:
+ void ValidateTlsProtocolmin(const Lazy<String>& lvalue, const ValidationUtils& utils) override;
+ void ValidateConnectTimeout(const Lazy<double>& lvalue, const ValidationUtils& utils) override;
+
+private:
+ class DumpedGlobals
+ {
+ public:
+ void Reset();
+ bool IsNew(const String& id);
+
+ private:
+ std::set<String> m_Ids;
+ std::mutex m_Mutex;
+ };
+
+ enum StateUpdate
+ {
+ Volatile = 1ull << 0,
+ RuntimeOnly = 1ull << 1,
+ Full = Volatile | RuntimeOnly,
+ };
+
+ void OnConnectedHandler();
+
+ void PublishStatsTimerHandler();
+ void PublishStats();
+
+ /* config & status dump */
+ void UpdateAllConfigObjects();
+ std::vector<std::vector<intrusive_ptr<ConfigObject>>> ChunkObjects(std::vector<intrusive_ptr<ConfigObject>> objects, size_t chunkSize);
+ void DeleteKeys(const RedisConnection::Ptr& conn, const std::vector<String>& keys, RedisConnection::QueryPriority priority);
+ std::vector<String> GetTypeOverwriteKeys(const String& type);
+ std::vector<String> GetTypeDumpSignalKeys(const Type::Ptr& type);
+ void InsertObjectDependencies(const ConfigObject::Ptr& object, const String typeName, std::map<String, std::vector<String>>& hMSets,
+ std::vector<Dictionary::Ptr>& runtimeUpdates, bool runtimeUpdate);
+ void UpdateState(const Checkable::Ptr& checkable, StateUpdate mode);
+ void SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpdate);
+ void CreateConfigUpdate(const ConfigObject::Ptr& object, const String type, std::map<String, std::vector<String>>& hMSets,
+ std::vector<Dictionary::Ptr>& runtimeUpdates, bool runtimeUpdate);
+ void SendConfigDelete(const ConfigObject::Ptr& object);
+ void SendStateChange(const ConfigObject::Ptr& object, const CheckResult::Ptr& cr, StateType type);
+ void AddObjectDataToRuntimeUpdates(std::vector<Dictionary::Ptr>& runtimeUpdates, const String& objectKey,
+ const String& redisKey, const Dictionary::Ptr& data);
+ void DeleteRelationship(const String& id, const String& redisKeyWithoutPrefix, bool hasChecksum = false);
+
+ void SendSentNotification(
+ const Notification::Ptr& notification, const Checkable::Ptr& checkable, const std::set<User::Ptr>& users,
+ NotificationType type, const CheckResult::Ptr& cr, const String& author, const String& text, double sendTime
+ );
+
+ void SendStartedDowntime(const Downtime::Ptr& downtime);
+ void SendRemovedDowntime(const Downtime::Ptr& downtime);
+ void SendAddedComment(const Comment::Ptr& comment);
+ void SendRemovedComment(const Comment::Ptr& comment);
+ void SendFlappingChange(const Checkable::Ptr& checkable, double changeTime, double flappingLastChange);
+ void SendNextUpdate(const Checkable::Ptr& checkable);
+ void SendAcknowledgementSet(const Checkable::Ptr& checkable, const String& author, const String& comment, AcknowledgementType type, bool persistent, double changeTime, double expiry);
+ void SendAcknowledgementCleared(const Checkable::Ptr& checkable, const String& removedBy, double changeTime, double ackLastChange);
+ void SendNotificationUsersChanged(const Notification::Ptr& notification, const Array::Ptr& oldValues, const Array::Ptr& newValues);
+ void SendNotificationUserGroupsChanged(const Notification::Ptr& notification, const Array::Ptr& oldValues, const Array::Ptr& newValues);
+ void SendTimePeriodRangesChanged(const TimePeriod::Ptr& timeperiod, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues);
+ void SendTimePeriodIncludesChanged(const TimePeriod::Ptr& timeperiod, const Array::Ptr& oldValues, const Array::Ptr& newValues);
+ void SendTimePeriodExcludesChanged(const TimePeriod::Ptr& timeperiod, const Array::Ptr& oldValues, const Array::Ptr& newValues);
+ template<class T>
+ void SendGroupsChanged(const ConfigObject::Ptr& command, const Array::Ptr& oldValues, const Array::Ptr& newValues);
+ void SendCommandEnvChanged(const ConfigObject::Ptr& command, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues);
+ void SendCommandArgumentsChanged(const ConfigObject::Ptr& command, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues);
+ void SendCustomVarsChanged(const ConfigObject::Ptr& object, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues);
+
+ void ForwardHistoryEntries();
+
+ std::vector<String> UpdateObjectAttrs(const ConfigObject::Ptr& object, int fieldType, const String& typeNameOverride);
+ Dictionary::Ptr SerializeState(const Checkable::Ptr& checkable);
+
+ /* Stats */
+ Dictionary::Ptr GetStats();
+
+ /* utilities */
+ static String FormatCheckSumBinary(const String& str);
+ static String FormatCommandLine(const Value& commandLine);
+ static long long TimestampToMilliseconds(double timestamp);
+ static String IcingaToStreamValue(const Value& value);
+ static std::vector<Value> GetArrayDeletedValues(const Array::Ptr& arrayOld, const Array::Ptr& arrayNew);
+ static std::vector<String> GetDictionaryDeletedKeys(const Dictionary::Ptr& dictOld, const Dictionary::Ptr& dictNew);
+
+ static String GetObjectIdentifier(const ConfigObject::Ptr& object);
+ static String CalcEventID(const char* eventType, const ConfigObject::Ptr& object, double eventTime = 0, NotificationType nt = NotificationType(0));
+ static const char* GetNotificationTypeByEnum(NotificationType type);
+ static Dictionary::Ptr SerializeVars(const Dictionary::Ptr& vars);
+
+ static String HashValue(const Value& value);
+ static String HashValue(const Value& value, const std::set<String>& propertiesBlacklist, bool propertiesWhitelist = false);
+
+ static String GetLowerCaseTypeNameDB(const ConfigObject::Ptr& obj);
+ static bool PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr& attributes, Dictionary::Ptr& checkSums);
+
+ static void ReachabilityChangeHandler(const std::set<Checkable::Ptr>& children);
+ static void StateChangeHandler(const ConfigObject::Ptr& object, const CheckResult::Ptr& cr, StateType type);
+ static void VersionChangedHandler(const ConfigObject::Ptr& object);
+ static void DowntimeStartedHandler(const Downtime::Ptr& downtime);
+ static void DowntimeRemovedHandler(const Downtime::Ptr& downtime);
+
+ static void NotificationSentToAllUsersHandler(
+ const Notification::Ptr& notification, const Checkable::Ptr& checkable, const std::set<User::Ptr>& users,
+ NotificationType type, const CheckResult::Ptr& cr, const String& author, const String& text
+ );
+
+ static void CommentAddedHandler(const Comment::Ptr& comment);
+ static void CommentRemovedHandler(const Comment::Ptr& comment);
+ static void FlappingChangeHandler(const Checkable::Ptr& checkable, double changeTime);
+ static void NewCheckResultHandler(const Checkable::Ptr& checkable);
+ static void NextCheckUpdatedHandler(const Checkable::Ptr& checkable);
+ static void HostProblemChangedHandler(const Service::Ptr& service);
+ static void AcknowledgementSetHandler(const Checkable::Ptr& checkable, const String& author, const String& comment, AcknowledgementType type, bool persistent, double changeTime, double expiry);
+ static void AcknowledgementClearedHandler(const Checkable::Ptr& checkable, const String& removedBy, double changeTime);
+ static void NotificationUsersChangedHandler(const Notification::Ptr& notification, const Array::Ptr& oldValues, const Array::Ptr& newValues);
+ static void NotificationUserGroupsChangedHandler(const Notification::Ptr& notification, const Array::Ptr& oldValues, const Array::Ptr& newValues);
+ static void TimePeriodRangesChangedHandler(const TimePeriod::Ptr& timeperiod, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues);
+ static void TimePeriodIncludesChangedHandler(const TimePeriod::Ptr& timeperiod, const Array::Ptr& oldValues, const Array::Ptr& newValues);
+ static void TimePeriodExcludesChangedHandler(const TimePeriod::Ptr& timeperiod, const Array::Ptr& oldValues, const Array::Ptr& newValues);
+ static void UserGroupsChangedHandler(const User::Ptr& user, const Array::Ptr&, const Array::Ptr& newValues);
+ static void HostGroupsChangedHandler(const Host::Ptr& host, const Array::Ptr& oldValues, const Array::Ptr& newValues);
+ static void ServiceGroupsChangedHandler(const Service::Ptr& service, const Array::Ptr& oldValues, const Array::Ptr& newValues);
+ static void CommandEnvChangedHandler(const ConfigObject::Ptr& command, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues);
+ static void CommandArgumentsChangedHandler(const ConfigObject::Ptr& command, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues);
+ static void CustomVarsChangedHandler(const ConfigObject::Ptr& object, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues);
+
+ void AssertOnWorkQueue();
+
+ void ExceptionHandler(boost::exception_ptr exp);
+
+ static std::vector<Type::Ptr> GetTypes();
+
+ static void InitEnvironmentId();
+ static void PersistEnvironmentId();
+
+ Timer::Ptr m_StatsTimer;
+ WorkQueue m_WorkQueue{0, 1, LogNotice};
+
+ std::future<void> m_HistoryThread;
+ Bulker<RedisConnection::Query> m_HistoryBulker {4096, std::chrono::milliseconds(250)};
+
+ String m_PrefixConfigObject;
+ String m_PrefixConfigCheckSum;
+
+ bool m_ConfigDumpInProgress;
+ bool m_ConfigDumpDone;
+
+ RedisConnection::Ptr m_Rcon;
+ // m_RconLocked containes a copy of the value in m_Rcon where all accesses are guarded by a mutex to allow safe
+ // concurrent access like from the icingadb check command. It's a copy to still allow fast access without additional
+ // syncronization to m_Rcon within the IcingaDB feature itself.
+ Locked<RedisConnection::Ptr> m_RconLocked;
+ std::unordered_map<ConfigType*, RedisConnection::Ptr> m_Rcons;
+ std::atomic_size_t m_PendingRcons;
+
+ struct {
+ DumpedGlobals CustomVar, ActionUrl, NotesUrl, IconImage;
+ } m_DumpedGlobals;
+
+ // m_EnvironmentId is shared across all IcingaDB objects (typically there is at most one, but it is perfectly fine
+ // to have multiple ones). It is initialized once (synchronized using m_EnvironmentIdInitMutex). After successful
+ // initialization, the value is read-only and can be accessed without further synchronization.
+ static String m_EnvironmentId;
+ static std::mutex m_EnvironmentIdInitMutex;
+
+ static std::unordered_set<Type*> m_IndexedTypes;
+};
+}
+
+#endif /* ICINGADB_H */
diff --git a/lib/icingadb/icingadb.ti b/lib/icingadb/icingadb.ti
new file mode 100644
index 0000000..1c649c8
--- /dev/null
+++ b/lib/icingadb/icingadb.ti
@@ -0,0 +1,63 @@
+/* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */
+
+#include "base/configobject.hpp"
+#include "base/tlsutility.hpp"
+
+library icingadb;
+
+namespace icinga
+{
+
+class IcingaDB : ConfigObject
+{
+ activation_priority 100;
+
+ [config] String host {
+ default {{{ return "127.0.0.1"; }}}
+ };
+ [config] int port {
+ default {{{ return 6380; }}}
+ };
+ [config] String path;
+ [config, no_user_view, no_user_modify] String password;
+ [config] int db_index;
+
+ [config] bool enable_tls {
+ default {{{ return false; }}}
+ };
+
+ [config] bool insecure_noverify {
+ default {{{ return false; }}}
+ };
+
+ [config] String cert_path;
+ [config] String key_path;
+ [config] String ca_path;
+ [config] String crl_path;
+ [config] String cipher_list {
+ default {{{ return DEFAULT_TLS_CIPHERS; }}}
+ };
+ [config] String tls_protocolmin {
+ default {{{ return DEFAULT_TLS_PROTOCOLMIN; }}}
+ };
+
+ [config] double connect_timeout {
+ default {{{ return DEFAULT_CONNECT_TIMEOUT; }}}
+ };
+
+ [no_storage] String environment_id {
+ get;
+ };
+
+ [set_protected] double ongoing_dump_start {
+ default {{{ return 0; }}}
+ };
+ [state, set_protected] double lastdump_end {
+ default {{{ return 0; }}}
+ };
+ [state, set_protected] double lastdump_took {
+ default {{{ return 0; }}}
+ };
+};
+
+}
diff --git a/lib/icingadb/icingadbchecktask.cpp b/lib/icingadb/icingadbchecktask.cpp
new file mode 100644
index 0000000..c2f1a36
--- /dev/null
+++ b/lib/icingadb/icingadbchecktask.cpp
@@ -0,0 +1,514 @@
+/* Icinga 2 | (c) 2022 Icinga GmbH | GPLv2+ */
+
+#include "icingadb/icingadbchecktask.hpp"
+#include "icinga/host.hpp"
+#include "icinga/checkcommand.hpp"
+#include "icinga/macroprocessor.hpp"
+#include "icinga/pluginutility.hpp"
+#include "base/function.hpp"
+#include "base/utility.hpp"
+#include "base/perfdatavalue.hpp"
+#include "base/convert.hpp"
+#include <utility>
+
+using namespace icinga;
+
+REGISTER_FUNCTION_NONCONST(Internal, IcingadbCheck, &IcingadbCheckTask::ScriptFunc, "checkable:cr:resolvedMacros:useResolvedMacros");
+
+static void ReportIcingadbCheck(
+ const Checkable::Ptr& checkable, const CheckCommand::Ptr& commandObj,
+ const CheckResult::Ptr& cr, String output, ServiceState state)
+{
+ if (Checkable::ExecuteCommandProcessFinishedHandler) {
+ double now = Utility::GetTime();
+ ProcessResult pr;
+ pr.PID = -1;
+ pr.Output = std::move(output);
+ pr.ExecutionStart = now;
+ pr.ExecutionEnd = now;
+ pr.ExitStatus = state;
+
+ Checkable::ExecuteCommandProcessFinishedHandler(commandObj->GetName(), pr);
+ } else {
+ cr->SetState(state);
+ cr->SetOutput(output);
+ checkable->ProcessCheckResult(cr);
+ }
+}
+
+static inline
+double GetXMessageTs(const Array::Ptr& xMessage)
+{
+ return Convert::ToLong(String(xMessage->Get(0)).Split("-")[0]) / 1000.0;
+}
+
+void IcingadbCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr,
+ const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros)
+{
+ CheckCommand::Ptr commandObj = CheckCommand::ExecuteOverride ? CheckCommand::ExecuteOverride : checkable->GetCheckCommand();
+
+ Host::Ptr host;
+ Service::Ptr service;
+ tie(host, service) = GetHostService(checkable);
+
+ MacroProcessor::ResolverList resolvers;
+ String silenceMissingMacroWarning;
+
+ if (MacroResolver::OverrideMacros)
+ resolvers.emplace_back("override", MacroResolver::OverrideMacros);
+
+ if (service)
+ resolvers.emplace_back("service", service);
+ resolvers.emplace_back("host", host);
+ resolvers.emplace_back("command", commandObj);
+ resolvers.emplace_back("icinga", IcingaApplication::GetInstance());
+
+ auto resolve ([&](const String& macro) {
+ return MacroProcessor::ResolveMacros(macro, resolvers, checkable->GetLastCheckResult(),
+ &silenceMissingMacroWarning, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros);
+ });
+
+ struct Thresholds
+ {
+ Value Warning, Critical;
+ };
+
+ auto resolveThresholds ([&resolve](const String& wmacro, const String& cmacro) {
+ return Thresholds{resolve(wmacro), resolve(cmacro)};
+ });
+
+ String icingadbName = resolve("$icingadb_name$");
+
+ auto dumpTakesThresholds (resolveThresholds("$icingadb_full_dump_duration_warning$", "$icingadb_full_dump_duration_critical$"));
+ auto syncTakesThresholds (resolveThresholds("$icingadb_full_sync_duration_warning$", "$icingadb_full_sync_duration_critical$"));
+ auto icingaBacklogThresholds (resolveThresholds("$icingadb_redis_backlog_warning$", "$icingadb_redis_backlog_critical$"));
+ auto icingadbBacklogThresholds (resolveThresholds("$icingadb_database_backlog_warning$", "$icingadb_database_backlog_critical$"));
+
+ if (resolvedMacros && !useResolvedMacros)
+ return;
+
+ if (icingadbName.IsEmpty()) {
+ ReportIcingadbCheck(checkable, commandObj, cr, "Icinga DB UNKNOWN: Attribute 'icingadb_name' must be set.", ServiceUnknown);
+ return;
+ }
+
+ auto conn (IcingaDB::GetByName(icingadbName));
+
+ if (!conn) {
+ ReportIcingadbCheck(checkable, commandObj, cr, "Icinga DB UNKNOWN: Icinga DB connection '" + icingadbName + "' does not exist.", ServiceUnknown);
+ return;
+ }
+
+ auto redis (conn->GetConnection());
+
+ if (!redis || !redis->GetConnected()) {
+ ReportIcingadbCheck(checkable, commandObj, cr, "Icinga DB CRITICAL: Not connected to Redis.", ServiceCritical);
+ return;
+ }
+
+ auto now (Utility::GetTime());
+ Array::Ptr redisTime, xReadHeartbeat, xReadStats, xReadRuntimeBacklog, xReadHistoryBacklog;
+
+ try {
+ auto replies (redis->GetResultsOfQueries(
+ {
+ {"TIME"},
+ {"XREAD", "STREAMS", "icingadb:telemetry:heartbeat", "0-0"},
+ {"XREAD", "STREAMS", "icingadb:telemetry:stats", "0-0"},
+ {"XREAD", "COUNT", "1", "STREAMS", "icinga:runtime", "icinga:runtime:state", "0-0", "0-0"},
+ {
+ "XREAD", "COUNT", "1", "STREAMS",
+ "icinga:history:stream:acknowledgement",
+ "icinga:history:stream:comment",
+ "icinga:history:stream:downtime",
+ "icinga:history:stream:flapping",
+ "icinga:history:stream:notification",
+ "icinga:history:stream:state",
+ "0-0", "0-0", "0-0", "0-0", "0-0", "0-0",
+ }
+ },
+ RedisConnection::QueryPriority::Heartbeat
+ ));
+
+ redisTime = std::move(replies.at(0));
+ xReadHeartbeat = std::move(replies.at(1));
+ xReadStats = std::move(replies.at(2));
+ xReadRuntimeBacklog = std::move(replies.at(3));
+ xReadHistoryBacklog = std::move(replies.at(4));
+ } catch (const std::exception& ex) {
+ ReportIcingadbCheck(
+ checkable, commandObj, cr,
+ String("Icinga DB CRITICAL: Could not query Redis: ") + ex.what(), ServiceCritical
+ );
+ return;
+ }
+
+ if (!xReadHeartbeat) {
+ ReportIcingadbCheck(
+ checkable, commandObj, cr,
+ "Icinga DB CRITICAL: The Icinga DB daemon seems to have never run. (Missing heartbeat)",
+ ServiceCritical
+ );
+
+ return;
+ }
+
+ auto redisOldestPending (redis->GetOldestPendingQueryTs());
+ auto ongoingDumpStart (conn->GetOngoingDumpStart());
+ auto dumpWhen (conn->GetLastdumpEnd());
+ auto dumpTook (conn->GetLastdumpTook());
+
+ auto redisNow (Convert::ToLong(redisTime->Get(0)) + Convert::ToLong(redisTime->Get(1)) / 1000000.0);
+ Array::Ptr heartbeatMessage = Array::Ptr(Array::Ptr(xReadHeartbeat->Get(0))->Get(1))->Get(0);
+ auto heartbeatTime (GetXMessageTs(heartbeatMessage));
+ std::map<String, String> heartbeatData;
+
+ IcingaDB::AddKvsToMap(heartbeatMessage->Get(1), heartbeatData);
+
+ String version = heartbeatData.at("version");
+ auto icingadbNow (Convert::ToLong(heartbeatData.at("time")) / 1000.0 + (redisNow - heartbeatTime));
+ auto icingadbStartTime (Convert::ToLong(heartbeatData.at("start-time")) / 1000.0);
+ String errMsg (heartbeatData.at("error"));
+ auto errSince (Convert::ToLong(heartbeatData.at("error-since")) / 1000.0);
+ String perfdataFromRedis = heartbeatData.at("performance-data");
+ auto heartbeatLastReceived (Convert::ToLong(heartbeatData.at("last-heartbeat-received")) / 1000.0);
+ bool weResponsible = Convert::ToLong(heartbeatData.at("ha-responsible"));
+ auto weResponsibleTs (Convert::ToLong(heartbeatData.at("ha-responsible-ts")) / 1000.0);
+ bool otherResponsible = Convert::ToLong(heartbeatData.at("ha-other-responsible"));
+ auto syncOngoingSince (Convert::ToLong(heartbeatData.at("sync-ongoing-since")) / 1000.0);
+ auto syncSuccessWhen (Convert::ToLong(heartbeatData.at("sync-success-finish")) / 1000.0);
+ auto syncSuccessTook (Convert::ToLong(heartbeatData.at("sync-success-duration")) / 1000.0);
+
+ std::ostringstream i2okmsgs, idbokmsgs, warnmsgs, critmsgs;
+ Array::Ptr perfdata = new Array();
+
+ i2okmsgs << std::fixed << std::setprecision(3);
+ idbokmsgs << std::fixed << std::setprecision(3);
+ warnmsgs << std::fixed << std::setprecision(3);
+ critmsgs << std::fixed << std::setprecision(3);
+
+ const auto downForCritical (10);
+ auto downFor (redisNow - heartbeatTime);
+ bool down = false;
+
+ if (downFor > downForCritical) {
+ down = true;
+
+ critmsgs << " Last seen " << Utility::FormatDuration(downFor)
+ << " ago, greater than CRITICAL threshold (" << Utility::FormatDuration(downForCritical) << ")!";
+ } else {
+ idbokmsgs << "\n* Last seen: " << Utility::FormatDuration(downFor) << " ago";
+ }
+
+ perfdata->Add(new PerfdataValue("icingadb_heartbeat_age", downFor, false, "seconds", Empty, downForCritical, 0));
+
+ const auto errForCritical (10);
+ auto err (!errMsg.IsEmpty());
+ auto errFor (icingadbNow - errSince);
+
+ if (err) {
+ if (errFor > errForCritical) {
+ critmsgs << " ERROR: " << errMsg << "!";
+ }
+
+ perfdata->Add(new PerfdataValue("error_for", errFor * (err ? 1 : -1), false, "seconds", Empty, errForCritical, 0));
+ }
+
+ if (!down) {
+ const auto heartbeatLagWarning (3/* Icinga DB read freq. */ + 1/* Icinga DB write freq. */ + 2/* threshold */);
+ auto heartbeatLag (fmin(icingadbNow - heartbeatLastReceived, 10 * 60));
+
+ if (!heartbeatLastReceived) {
+ critmsgs << " Lost Icinga 2 heartbeat!";
+ } else if (heartbeatLag > heartbeatLagWarning) {
+ warnmsgs << " Icinga 2 heartbeat lag: " << Utility::FormatDuration(heartbeatLag)
+ << ", greater than WARNING threshold (" << Utility::FormatDuration(heartbeatLagWarning) << ").";
+ }
+
+ perfdata->Add(new PerfdataValue("icinga2_heartbeat_age", heartbeatLag, false, "seconds", heartbeatLagWarning, Empty, 0));
+ }
+
+ if (weResponsible) {
+ idbokmsgs << "\n* Responsible";
+ } else if (otherResponsible) {
+ idbokmsgs << "\n* Not responsible, but another instance is";
+ } else {
+ critmsgs << " No instance is responsible!";
+ }
+
+ perfdata->Add(new PerfdataValue("icingadb_responsible_instances", int(weResponsible || otherResponsible), false, "", Empty, Empty, 0, 1));
+
+ const auto clockDriftWarning (5);
+ const auto clockDriftCritical (30);
+ auto clockDrift (std::max({
+ fabs(now - redisNow),
+ fabs(redisNow - icingadbNow),
+ fabs(icingadbNow - now),
+ }));
+
+ if (clockDrift > clockDriftCritical) {
+ critmsgs << " Icinga 2/Redis/Icinga DB clock drift: " << Utility::FormatDuration(clockDrift)
+ << ", greater than CRITICAL threshold (" << Utility::FormatDuration(clockDriftCritical) << ")!";
+ } else if (clockDrift > clockDriftWarning) {
+ warnmsgs << " Icinga 2/Redis/Icinga DB clock drift: " << Utility::FormatDuration(clockDrift)
+ << ", greater than WARNING threshold (" << Utility::FormatDuration(clockDriftWarning) << ").";
+ }
+
+ perfdata->Add(new PerfdataValue("clock_drift", clockDrift, false, "seconds", clockDriftWarning, clockDriftCritical, 0));
+
+ if (ongoingDumpStart) {
+ auto ongoingDumpTakes (now - ongoingDumpStart);
+
+ if (!dumpTakesThresholds.Critical.IsEmpty() && ongoingDumpTakes > dumpTakesThresholds.Critical) {
+ critmsgs << " Current Icinga 2 full dump already takes " << Utility::FormatDuration(ongoingDumpTakes)
+ << ", greater than CRITICAL threshold (" << Utility::FormatDuration(dumpTakesThresholds.Critical) << ")!";
+ } else if (!dumpTakesThresholds.Warning.IsEmpty() && ongoingDumpTakes > dumpTakesThresholds.Warning) {
+ warnmsgs << " Current Icinga 2 full dump already takes " << Utility::FormatDuration(ongoingDumpTakes)
+ << ", greater than WARNING threshold (" << Utility::FormatDuration(dumpTakesThresholds.Warning) << ").";
+ } else {
+ i2okmsgs << "\n* Current full dump running for " << Utility::FormatDuration(ongoingDumpTakes);
+ }
+
+ perfdata->Add(new PerfdataValue("icinga2_current_full_dump_duration", ongoingDumpTakes, false, "seconds",
+ dumpTakesThresholds.Warning, dumpTakesThresholds.Critical, 0));
+ }
+
+ if (!down && syncOngoingSince) {
+ auto ongoingSyncTakes (icingadbNow - syncOngoingSince);
+
+ if (!syncTakesThresholds.Critical.IsEmpty() && ongoingSyncTakes > syncTakesThresholds.Critical) {
+ critmsgs << " Current full sync already takes " << Utility::FormatDuration(ongoingSyncTakes)
+ << ", greater than CRITICAL threshold (" << Utility::FormatDuration(syncTakesThresholds.Critical) << ")!";
+ } else if (!syncTakesThresholds.Warning.IsEmpty() && ongoingSyncTakes > syncTakesThresholds.Warning) {
+ warnmsgs << " Current full sync already takes " << Utility::FormatDuration(ongoingSyncTakes)
+ << ", greater than WARNING threshold (" << Utility::FormatDuration(syncTakesThresholds.Warning) << ").";
+ } else {
+ idbokmsgs << "\n* Current full sync running for " << Utility::FormatDuration(ongoingSyncTakes);
+ }
+
+ perfdata->Add(new PerfdataValue("icingadb_current_full_sync_duration", ongoingSyncTakes, false, "seconds",
+ syncTakesThresholds.Warning, syncTakesThresholds.Critical, 0));
+ }
+
+ auto redisBacklog (now - redisOldestPending);
+
+ if (!redisOldestPending) {
+ redisBacklog = 0;
+ }
+
+ if (!icingaBacklogThresholds.Critical.IsEmpty() && redisBacklog > icingaBacklogThresholds.Critical) {
+ critmsgs << " Icinga 2 Redis query backlog: " << Utility::FormatDuration(redisBacklog)
+ << ", greater than CRITICAL threshold (" << Utility::FormatDuration(icingaBacklogThresholds.Critical) << ")!";
+ } else if (!icingaBacklogThresholds.Warning.IsEmpty() && redisBacklog > icingaBacklogThresholds.Warning) {
+ warnmsgs << " Icinga 2 Redis query backlog: " << Utility::FormatDuration(redisBacklog)
+ << ", greater than WARNING threshold (" << Utility::FormatDuration(icingaBacklogThresholds.Warning) << ").";
+ }
+
+ perfdata->Add(new PerfdataValue("icinga2_redis_query_backlog", redisBacklog, false, "seconds",
+ icingaBacklogThresholds.Warning, icingaBacklogThresholds.Critical, 0));
+
+ if (!down) {
+ auto getBacklog = [redisNow](const Array::Ptr& streams) -> double {
+ if (!streams) {
+ return 0;
+ }
+
+ double minTs = 0;
+ ObjectLock lock (streams);
+
+ for (Array::Ptr stream : streams) {
+ auto ts (GetXMessageTs(Array::Ptr(stream->Get(1))->Get(0)));
+
+ if (minTs == 0 || ts < minTs) {
+ minTs = ts;
+ }
+ }
+
+ if (minTs > 0) {
+ return redisNow - minTs;
+ } else {
+ return 0;
+ }
+ };
+
+ double historyBacklog = getBacklog(xReadHistoryBacklog);
+
+ if (!icingadbBacklogThresholds.Critical.IsEmpty() && historyBacklog > icingadbBacklogThresholds.Critical) {
+ critmsgs << " History backlog: " << Utility::FormatDuration(historyBacklog)
+ << ", greater than CRITICAL threshold (" << Utility::FormatDuration(icingadbBacklogThresholds.Critical) << ")!";
+ } else if (!icingadbBacklogThresholds.Warning.IsEmpty() && historyBacklog > icingadbBacklogThresholds.Warning) {
+ warnmsgs << " History backlog: " << Utility::FormatDuration(historyBacklog)
+ << ", greater than WARNING threshold (" << Utility::FormatDuration(icingadbBacklogThresholds.Warning) << ").";
+ }
+
+ perfdata->Add(new PerfdataValue("icingadb_history_backlog", historyBacklog, false, "seconds",
+ icingadbBacklogThresholds.Warning, icingadbBacklogThresholds.Critical, 0));
+
+ double runtimeBacklog = 0;
+
+ if (weResponsible && !syncOngoingSince) {
+ // These streams are only processed by the responsible instance after the full sync finished,
+ // it's fine for some backlog to exist otherwise.
+ runtimeBacklog = getBacklog(xReadRuntimeBacklog);
+
+ if (!icingadbBacklogThresholds.Critical.IsEmpty() && runtimeBacklog > icingadbBacklogThresholds.Critical) {
+ critmsgs << " Runtime update backlog: " << Utility::FormatDuration(runtimeBacklog)
+ << ", greater than CRITICAL threshold (" << Utility::FormatDuration(icingadbBacklogThresholds.Critical) << ")!";
+ } else if (!icingadbBacklogThresholds.Warning.IsEmpty() && runtimeBacklog > icingadbBacklogThresholds.Warning) {
+ warnmsgs << " Runtime update backlog: " << Utility::FormatDuration(runtimeBacklog)
+ << ", greater than WARNING threshold (" << Utility::FormatDuration(icingadbBacklogThresholds.Warning) << ").";
+ }
+ }
+
+ // Also report the perfdata value on the standby instance or during a full sync (as 0 in this case).
+ perfdata->Add(new PerfdataValue("icingadb_runtime_update_backlog", runtimeBacklog, false, "seconds",
+ icingadbBacklogThresholds.Warning, icingadbBacklogThresholds.Critical, 0));
+ }
+
+ auto dumpAgo (now - dumpWhen);
+
+ if (dumpWhen) {
+ perfdata->Add(new PerfdataValue("icinga2_last_full_dump_ago", dumpAgo, false, "seconds", Empty, Empty, 0));
+ }
+
+ if (dumpTook) {
+ perfdata->Add(new PerfdataValue("icinga2_last_full_dump_duration", dumpTook, false, "seconds", Empty, Empty, 0));
+ }
+
+ if (dumpWhen && dumpTook) {
+ i2okmsgs << "\n* Last full dump: " << Utility::FormatDuration(dumpAgo)
+ << " ago, took " << Utility::FormatDuration(dumpTook);
+ }
+
+ auto icingadbUptime (icingadbNow - icingadbStartTime);
+
+ if (!down) {
+ perfdata->Add(new PerfdataValue("icingadb_uptime", icingadbUptime, false, "seconds", Empty, Empty, 0));
+ }
+
+ {
+ Array::Ptr values = PluginUtility::SplitPerfdata(perfdataFromRedis);
+ ObjectLock lock (values);
+
+ for (auto& v : values) {
+ perfdata->Add(PerfdataValue::Parse(v));
+ }
+ }
+
+ if (weResponsibleTs) {
+ perfdata->Add(new PerfdataValue("icingadb_responsible_for",
+ (weResponsible ? 1 : -1) * (icingadbNow - weResponsibleTs), false, "seconds"));
+ }
+
+ auto syncAgo (icingadbNow - syncSuccessWhen);
+
+ if (syncSuccessWhen) {
+ perfdata->Add(new PerfdataValue("icingadb_last_full_sync_ago", syncAgo, false, "seconds", Empty, Empty, 0));
+ }
+
+ if (syncSuccessTook) {
+ perfdata->Add(new PerfdataValue("icingadb_last_full_sync_duration", syncSuccessTook, false, "seconds", Empty, Empty, 0));
+ }
+
+ if (syncSuccessWhen && syncSuccessTook) {
+ idbokmsgs << "\n* Last full sync: " << Utility::FormatDuration(syncAgo)
+ << " ago, took " << Utility::FormatDuration(syncSuccessTook);
+ }
+
+ std::map<String, RingBuffer> statsPerOp;
+
+ const char * const icingadbKnownStats[] = {
+ "config_sync", "state_sync", "history_sync", "overdue_sync", "history_cleanup"
+ };
+
+ for (auto metric : icingadbKnownStats) {
+ statsPerOp.emplace(std::piecewise_construct, std::forward_as_tuple(metric), std::forward_as_tuple(15 * 60));
+ }
+
+ if (xReadStats) {
+ Array::Ptr messages = Array::Ptr(xReadStats->Get(0))->Get(1);
+ ObjectLock lock (messages);
+
+ for (Array::Ptr message : messages) {
+ auto ts (GetXMessageTs(message));
+ std::map<String, String> opsPerSec;
+
+ IcingaDB::AddKvsToMap(message->Get(1), opsPerSec);
+
+ for (auto& kv : opsPerSec) {
+ auto buf (statsPerOp.find(kv.first));
+
+ if (buf == statsPerOp.end()) {
+ buf = statsPerOp.emplace(
+ std::piecewise_construct,
+ std::forward_as_tuple(kv.first), std::forward_as_tuple(15 * 60)
+ ).first;
+ }
+
+ buf->second.InsertValue(ts, Convert::ToLong(kv.second));
+ }
+ }
+ }
+
+ for (auto& kv : statsPerOp) {
+ perfdata->Add(new PerfdataValue("icingadb_" + kv.first + "_items_1min", kv.second.UpdateAndGetValues(now, 60), false, "", Empty, Empty, 0));
+ perfdata->Add(new PerfdataValue("icingadb_" + kv.first + "_items_5mins", kv.second.UpdateAndGetValues(now, 5 * 60), false, "", Empty, Empty, 0));
+ perfdata->Add(new PerfdataValue("icingadb_" + kv.first + "_items_15mins", kv.second.UpdateAndGetValues(now, 15 * 60), false, "", Empty, Empty, 0));
+ }
+
+ perfdata->Add(new PerfdataValue("icinga2_redis_queries_1min", redis->GetQueryCount(60), false, "", Empty, Empty, 0));
+ perfdata->Add(new PerfdataValue("icinga2_redis_queries_5mins", redis->GetQueryCount(5 * 60), false, "", Empty, Empty, 0));
+ perfdata->Add(new PerfdataValue("icinga2_redis_queries_15mins", redis->GetQueryCount(15 * 60), false, "", Empty, Empty, 0));
+
+ perfdata->Add(new PerfdataValue("icinga2_redis_pending_queries", redis->GetPendingQueryCount(), false, "", Empty, Empty, 0));
+
+ struct {
+ const char * Name;
+ int (RedisConnection::* Getter)(RingBuffer::SizeType span, RingBuffer::SizeType tv);
+ } const icingaWriteSubjects[] = {
+ {"config_dump", &RedisConnection::GetWrittenConfigFor},
+ {"state_dump", &RedisConnection::GetWrittenStateFor},
+ {"history_dump", &RedisConnection::GetWrittenHistoryFor}
+ };
+
+ for (auto subject : icingaWriteSubjects) {
+ perfdata->Add(new PerfdataValue(String("icinga2_") + subject.Name + "_items_1min", (redis.get()->*subject.Getter)(60, now), false, "", Empty, Empty, 0));
+ perfdata->Add(new PerfdataValue(String("icinga2_") + subject.Name + "_items_5mins", (redis.get()->*subject.Getter)(5 * 60, now), false, "", Empty, Empty, 0));
+ perfdata->Add(new PerfdataValue(String("icinga2_") + subject.Name + "_items_15mins", (redis.get()->*subject.Getter)(15 * 60, now), false, "", Empty, Empty, 0));
+ }
+
+ ServiceState state;
+ std::ostringstream msgbuf;
+ auto i2okmsg (i2okmsgs.str());
+ auto idbokmsg (idbokmsgs.str());
+ auto warnmsg (warnmsgs.str());
+ auto critmsg (critmsgs.str());
+
+ msgbuf << "Icinga DB ";
+
+ if (!critmsg.empty()) {
+ state = ServiceCritical;
+ msgbuf << "CRITICAL:" << critmsg;
+
+ if (!warnmsg.empty()) {
+ msgbuf << "\n\nWARNING:" << warnmsg;
+ }
+ } else if (!warnmsg.empty()) {
+ state = ServiceWarning;
+ msgbuf << "WARNING:" << warnmsg;
+ } else {
+ state = ServiceOK;
+ msgbuf << "OK: Uptime: " << Utility::FormatDuration(icingadbUptime) << ". Version: " << version << ".";
+ }
+
+ if (!i2okmsg.empty()) {
+ msgbuf << "\n\nIcinga 2:\n" << i2okmsg;
+ }
+
+ if (!idbokmsg.empty()) {
+ msgbuf << "\n\nIcinga DB:\n" << idbokmsg;
+ }
+
+ cr->SetPerformanceData(perfdata);
+ ReportIcingadbCheck(checkable, commandObj, cr, msgbuf.str(), state);
+}
diff --git a/lib/icingadb/icingadbchecktask.hpp b/lib/icingadb/icingadbchecktask.hpp
new file mode 100644
index 0000000..ba7d61b
--- /dev/null
+++ b/lib/icingadb/icingadbchecktask.hpp
@@ -0,0 +1,29 @@
+/* Icinga 2 | (c) 2022 Icinga GmbH | GPLv2+ */
+
+#ifndef ICINGADBCHECKTASK_H
+#define ICINGADBCHECKTASK_H
+
+#include "icingadb/icingadb.hpp"
+#include "icinga/checkable.hpp"
+
+namespace icinga
+{
+
+/**
+ * Icinga DB check.
+ *
+ * @ingroup icingadb
+ */
+class IcingadbCheckTask
+{
+public:
+ static void ScriptFunc(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr,
+ const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros);
+
+private:
+ IcingadbCheckTask();
+};
+
+}
+
+#endif /* ICINGADBCHECKTASK_H */
diff --git a/lib/icingadb/redisconnection.cpp b/lib/icingadb/redisconnection.cpp
new file mode 100644
index 0000000..798a827
--- /dev/null
+++ b/lib/icingadb/redisconnection.cpp
@@ -0,0 +1,773 @@
+/* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */
+
+#include "icingadb/redisconnection.hpp"
+#include "base/array.hpp"
+#include "base/convert.hpp"
+#include "base/defer.hpp"
+#include "base/exception.hpp"
+#include "base/io-engine.hpp"
+#include "base/logger.hpp"
+#include "base/objectlock.hpp"
+#include "base/string.hpp"
+#include "base/tcpsocket.hpp"
+#include "base/tlsutility.hpp"
+#include "base/utility.hpp"
+#include <boost/asio.hpp>
+#include <boost/coroutine/exceptions.hpp>
+#include <boost/date_time/posix_time/posix_time_duration.hpp>
+#include <boost/utility/string_view.hpp>
+#include <boost/variant/get.hpp>
+#include <exception>
+#include <future>
+#include <iterator>
+#include <memory>
+#include <openssl/ssl.h>
+#include <openssl/x509_vfy.h>
+#include <utility>
+
+using namespace icinga;
+namespace asio = boost::asio;
+
+boost::regex RedisConnection::m_ErrAuth ("\\AERR AUTH ");
+
+RedisConnection::RedisConnection(const String& host, int port, const String& path, const String& password, int db,
+ bool useTls, bool insecure, const String& certPath, const String& keyPath, const String& caPath, const String& crlPath,
+ const String& tlsProtocolmin, const String& cipherList, double connectTimeout, DebugInfo di, const RedisConnection::Ptr& parent)
+ : RedisConnection(IoEngine::Get().GetIoContext(), host, port, path, password, db,
+ useTls, insecure, certPath, keyPath, caPath, crlPath, tlsProtocolmin, cipherList, connectTimeout, std::move(di), parent)
+{
+}
+
+RedisConnection::RedisConnection(boost::asio::io_context& io, String host, int port, String path, String password,
+ int db, bool useTls, bool insecure, String certPath, String keyPath, String caPath, String crlPath,
+ String tlsProtocolmin, String cipherList, double connectTimeout, DebugInfo di, const RedisConnection::Ptr& parent)
+ : m_Host(std::move(host)), m_Port(port), m_Path(std::move(path)), m_Password(std::move(password)),
+ m_DbIndex(db), m_CertPath(std::move(certPath)), m_KeyPath(std::move(keyPath)), m_Insecure(insecure),
+ m_CaPath(std::move(caPath)), m_CrlPath(std::move(crlPath)), m_TlsProtocolmin(std::move(tlsProtocolmin)),
+ m_CipherList(std::move(cipherList)), m_ConnectTimeout(connectTimeout), m_DebugInfo(std::move(di)), m_Connecting(false), m_Connected(false),
+ m_Started(false), m_Strand(io), m_QueuedWrites(io), m_QueuedReads(io), m_LogStatsTimer(io), m_Parent(parent)
+{
+ if (useTls && m_Path.IsEmpty()) {
+ UpdateTLSContext();
+ }
+}
+
+void RedisConnection::UpdateTLSContext()
+{
+ m_TLSContext = SetupSslContext(m_CertPath, m_KeyPath, m_CaPath,
+ m_CrlPath, m_CipherList, m_TlsProtocolmin, m_DebugInfo);
+}
+
+void RedisConnection::Start()
+{
+ if (!m_Started.exchange(true)) {
+ Ptr keepAlive (this);
+
+ IoEngine::SpawnCoroutine(m_Strand, [this, keepAlive](asio::yield_context yc) { ReadLoop(yc); });
+ IoEngine::SpawnCoroutine(m_Strand, [this, keepAlive](asio::yield_context yc) { WriteLoop(yc); });
+
+ if (!m_Parent) {
+ IoEngine::SpawnCoroutine(m_Strand, [this, keepAlive](asio::yield_context yc) { LogStats(yc); });
+ }
+ }
+
+ if (!m_Connecting.exchange(true)) {
+ Ptr keepAlive (this);
+
+ IoEngine::SpawnCoroutine(m_Strand, [this, keepAlive](asio::yield_context yc) { Connect(yc); });
+ }
+}
+
+bool RedisConnection::IsConnected() {
+ return m_Connected.load();
+}
+
+/**
+ * Append a Redis query to a log message
+ *
+ * @param query Redis query
+ * @param msg Log message
+ */
+static inline
+void LogQuery(RedisConnection::Query& query, Log& msg)
+{
+ int i = 0;
+
+ for (auto& arg : query) {
+ if (++i == 8) {
+ msg << " ...";
+ break;
+ }
+
+ if (arg.GetLength() > 64) {
+ msg << " '" << arg.SubStr(0, 61) << "...'";
+ } else {
+ msg << " '" << arg << '\'';
+ }
+ }
+}
+
+/**
+ * Queue a Redis query for sending
+ *
+ * @param query Redis query
+ * @param priority The query's priority
+ */
+void RedisConnection::FireAndForgetQuery(RedisConnection::Query query, RedisConnection::QueryPriority priority, QueryAffects affects)
+{
+ if (LogDebug >= Logger::GetMinLogSeverity()) {
+ Log msg (LogDebug, "IcingaDB", "Firing and forgetting query:");
+ LogQuery(query, msg);
+ }
+
+ auto item (Shared<Query>::Make(std::move(query)));
+ auto ctime (Utility::GetTime());
+
+ asio::post(m_Strand, [this, item, priority, ctime, affects]() {
+ m_Queues.Writes[priority].emplace(WriteQueueItem{item, nullptr, nullptr, nullptr, nullptr, ctime, affects});
+ m_QueuedWrites.Set();
+ IncreasePendingQueries(1);
+ });
+}
+
+/**
+ * Queue Redis queries for sending
+ *
+ * @param queries Redis queries
+ * @param priority The queries' priority
+ */
+void RedisConnection::FireAndForgetQueries(RedisConnection::Queries queries, RedisConnection::QueryPriority priority, QueryAffects affects)
+{
+ if (LogDebug >= Logger::GetMinLogSeverity()) {
+ for (auto& query : queries) {
+ Log msg(LogDebug, "IcingaDB", "Firing and forgetting query:");
+ LogQuery(query, msg);
+ }
+ }
+
+ auto item (Shared<Queries>::Make(std::move(queries)));
+ auto ctime (Utility::GetTime());
+
+ asio::post(m_Strand, [this, item, priority, ctime, affects]() {
+ m_Queues.Writes[priority].emplace(WriteQueueItem{nullptr, item, nullptr, nullptr, nullptr, ctime, affects});
+ m_QueuedWrites.Set();
+ IncreasePendingQueries(item->size());
+ });
+}
+
+/**
+ * Queue a Redis query for sending, wait for the response and return (or throw) it
+ *
+ * @param query Redis query
+ * @param priority The query's priority
+ *
+ * @return The response
+ */
+RedisConnection::Reply RedisConnection::GetResultOfQuery(RedisConnection::Query query, RedisConnection::QueryPriority priority, QueryAffects affects)
+{
+ if (LogDebug >= Logger::GetMinLogSeverity()) {
+ Log msg (LogDebug, "IcingaDB", "Executing query:");
+ LogQuery(query, msg);
+ }
+
+ std::promise<Reply> promise;
+ auto future (promise.get_future());
+ auto item (Shared<std::pair<Query, std::promise<Reply>>>::Make(std::move(query), std::move(promise)));
+ auto ctime (Utility::GetTime());
+
+ asio::post(m_Strand, [this, item, priority, ctime, affects]() {
+ m_Queues.Writes[priority].emplace(WriteQueueItem{nullptr, nullptr, item, nullptr, nullptr, ctime, affects});
+ m_QueuedWrites.Set();
+ IncreasePendingQueries(1);
+ });
+
+ item = nullptr;
+ future.wait();
+ return future.get();
+}
+
+/**
+ * Queue Redis queries for sending, wait for the responses and return (or throw) them
+ *
+ * @param queries Redis queries
+ * @param priority The queries' priority
+ *
+ * @return The responses
+ */
+RedisConnection::Replies RedisConnection::GetResultsOfQueries(RedisConnection::Queries queries, RedisConnection::QueryPriority priority, QueryAffects affects)
+{
+ if (LogDebug >= Logger::GetMinLogSeverity()) {
+ for (auto& query : queries) {
+ Log msg(LogDebug, "IcingaDB", "Executing query:");
+ LogQuery(query, msg);
+ }
+ }
+
+ std::promise<Replies> promise;
+ auto future (promise.get_future());
+ auto item (Shared<std::pair<Queries, std::promise<Replies>>>::Make(std::move(queries), std::move(promise)));
+ auto ctime (Utility::GetTime());
+
+ asio::post(m_Strand, [this, item, priority, ctime, affects]() {
+ m_Queues.Writes[priority].emplace(WriteQueueItem{nullptr, nullptr, nullptr, item, nullptr, ctime, affects});
+ m_QueuedWrites.Set();
+ IncreasePendingQueries(item->first.size());
+ });
+
+ item = nullptr;
+ future.wait();
+ return future.get();
+}
+
+void RedisConnection::EnqueueCallback(const std::function<void(boost::asio::yield_context&)>& callback, RedisConnection::QueryPriority priority)
+{
+ auto ctime (Utility::GetTime());
+
+ asio::post(m_Strand, [this, callback, priority, ctime]() {
+ m_Queues.Writes[priority].emplace(WriteQueueItem{nullptr, nullptr, nullptr, nullptr, callback, ctime});
+ m_QueuedWrites.Set();
+ });
+}
+
+/**
+ * Puts a no-op command with a result at the end of the queue and wait for the result,
+ * i.e. for everything enqueued to be processed by the server.
+ *
+ * @ingroup icingadb
+ */
+void RedisConnection::Sync()
+{
+ GetResultOfQuery({"PING"}, RedisConnection::QueryPriority::SyncConnection);
+}
+
+/**
+ * Get the enqueue time of the oldest still queued Redis query
+ *
+ * @return *nix timestamp or 0
+ */
+double RedisConnection::GetOldestPendingQueryTs()
+{
+ auto promise (Shared<std::promise<double>>::Make());
+ auto future (promise->get_future());
+
+ asio::post(m_Strand, [this, promise]() {
+ double oldest = 0;
+
+ for (auto& queue : m_Queues.Writes) {
+ if (m_SuppressedQueryKinds.find(queue.first) == m_SuppressedQueryKinds.end() && !queue.second.empty()) {
+ auto ctime (queue.second.front().CTime);
+
+ if (ctime < oldest || oldest == 0) {
+ oldest = ctime;
+ }
+ }
+ }
+
+ promise->set_value(oldest);
+ });
+
+ future.wait();
+ return future.get();
+}
+
+/**
+ * Mark kind as kind of queries not to actually send yet
+ *
+ * @param kind Query kind
+ */
+void RedisConnection::SuppressQueryKind(RedisConnection::QueryPriority kind)
+{
+ asio::post(m_Strand, [this, kind]() { m_SuppressedQueryKinds.emplace(kind); });
+}
+
+/**
+ * Unmark kind as kind of queries not to actually send yet
+ *
+ * @param kind Query kind
+ */
+void RedisConnection::UnsuppressQueryKind(RedisConnection::QueryPriority kind)
+{
+ asio::post(m_Strand, [this, kind]() {
+ m_SuppressedQueryKinds.erase(kind);
+ m_QueuedWrites.Set();
+ });
+}
+
+/**
+ * Try to connect to Redis
+ */
+void RedisConnection::Connect(asio::yield_context& yc)
+{
+ Defer notConnecting ([this]() { m_Connecting.store(m_Connected.load()); });
+
+ boost::asio::deadline_timer timer (m_Strand.context());
+
+ auto waitForReadLoop ([this, &yc]() {
+ while (!m_Queues.FutureResponseActions.empty()) {
+ IoEngine::YieldCurrentCoroutine(yc);
+ }
+ });
+
+ for (;;) {
+ try {
+ if (m_Path.IsEmpty()) {
+ if (m_TLSContext) {
+ Log(m_Parent ? LogNotice : LogInformation, "IcingaDB")
+ << "Trying to connect to Redis server (async, TLS) on host '" << m_Host << ":" << m_Port << "'";
+
+ auto conn (Shared<AsioTlsStream>::Make(m_Strand.context(), *m_TLSContext, m_Host));
+ auto& tlsConn (conn->next_layer());
+ auto connectTimeout (MakeTimeout(conn));
+ Defer cancelTimeout ([&connectTimeout]() { connectTimeout->Cancel(); });
+
+ icinga::Connect(conn->lowest_layer(), m_Host, Convert::ToString(m_Port), yc);
+ tlsConn.async_handshake(tlsConn.client, yc);
+
+ if (!m_Insecure) {
+ std::shared_ptr<X509> cert (tlsConn.GetPeerCertificate());
+
+ if (!cert) {
+ BOOST_THROW_EXCEPTION(std::runtime_error(
+ "Redis didn't present any TLS certificate."
+ ));
+ }
+
+ if (!tlsConn.IsVerifyOK()) {
+ BOOST_THROW_EXCEPTION(std::runtime_error(
+ "TLS certificate validation failed: " + std::string(tlsConn.GetVerifyError())
+ ));
+ }
+ }
+
+ Handshake(conn, yc);
+ waitForReadLoop();
+ m_TlsConn = std::move(conn);
+ } else {
+ Log(m_Parent ? LogNotice : LogInformation, "IcingaDB")
+ << "Trying to connect to Redis server (async) on host '" << m_Host << ":" << m_Port << "'";
+
+ auto conn (Shared<TcpConn>::Make(m_Strand.context()));
+ auto connectTimeout (MakeTimeout(conn));
+ Defer cancelTimeout ([&connectTimeout]() { connectTimeout->Cancel(); });
+
+ icinga::Connect(conn->next_layer(), m_Host, Convert::ToString(m_Port), yc);
+ Handshake(conn, yc);
+ waitForReadLoop();
+ m_TcpConn = std::move(conn);
+ }
+ } else {
+ Log(LogInformation, "IcingaDB")
+ << "Trying to connect to Redis server (async) on unix socket path '" << m_Path << "'";
+
+ auto conn (Shared<UnixConn>::Make(m_Strand.context()));
+ auto connectTimeout (MakeTimeout(conn));
+ Defer cancelTimeout ([&connectTimeout]() { connectTimeout->Cancel(); });
+
+ conn->next_layer().async_connect(Unix::endpoint(m_Path.CStr()), yc);
+ Handshake(conn, yc);
+ waitForReadLoop();
+ m_UnixConn = std::move(conn);
+ }
+
+ m_Connected.store(true);
+
+ Log(m_Parent ? LogNotice : LogInformation, "IcingaDB", "Connected to Redis server");
+
+ // Operate on a copy so that the callback can set a new callback without destroying itself while running.
+ auto callback (m_ConnectedCallback);
+ if (callback) {
+ callback(yc);
+ }
+
+ break;
+ } catch (const boost::coroutines::detail::forced_unwind&) {
+ throw;
+ } catch (const std::exception& ex) {
+ Log(LogCritical, "IcingaDB")
+ << "Cannot connect to " << m_Host << ":" << m_Port << ": " << ex.what();
+ }
+
+ timer.expires_from_now(boost::posix_time::seconds(5));
+ timer.async_wait(yc);
+ }
+
+}
+
+/**
+ * Actually receive the responses to the Redis queries send by WriteItem() and handle them
+ */
+void RedisConnection::ReadLoop(asio::yield_context& yc)
+{
+ for (;;) {
+ m_QueuedReads.Wait(yc);
+
+ while (!m_Queues.FutureResponseActions.empty()) {
+ auto item (std::move(m_Queues.FutureResponseActions.front()));
+ m_Queues.FutureResponseActions.pop();
+
+ switch (item.Action) {
+ case ResponseAction::Ignore:
+ try {
+ for (auto i (item.Amount); i; --i) {
+ ReadOne(yc);
+ }
+ } catch (const boost::coroutines::detail::forced_unwind&) {
+ throw;
+ } catch (const std::exception& ex) {
+ Log(LogCritical, "IcingaDB")
+ << "Error during receiving the response to a query which has been fired and forgotten: " << ex.what();
+
+ continue;
+ } catch (...) {
+ Log(LogCritical, "IcingaDB")
+ << "Error during receiving the response to a query which has been fired and forgotten";
+
+ continue;
+ }
+
+ break;
+ case ResponseAction::Deliver:
+ for (auto i (item.Amount); i; --i) {
+ auto promise (std::move(m_Queues.ReplyPromises.front()));
+ m_Queues.ReplyPromises.pop();
+
+ Reply reply;
+
+ try {
+ reply = ReadOne(yc);
+ } catch (const boost::coroutines::detail::forced_unwind&) {
+ throw;
+ } catch (...) {
+ promise.set_exception(std::current_exception());
+
+ continue;
+ }
+
+ promise.set_value(std::move(reply));
+ }
+
+ break;
+ case ResponseAction::DeliverBulk:
+ {
+ auto promise (std::move(m_Queues.RepliesPromises.front()));
+ m_Queues.RepliesPromises.pop();
+
+ Replies replies;
+ replies.reserve(item.Amount);
+
+ for (auto i (item.Amount); i; --i) {
+ try {
+ replies.emplace_back(ReadOne(yc));
+ } catch (const boost::coroutines::detail::forced_unwind&) {
+ throw;
+ } catch (...) {
+ promise.set_exception(std::current_exception());
+ break;
+ }
+ }
+
+ try {
+ promise.set_value(std::move(replies));
+ } catch (const std::future_error&) {
+ // Complaint about the above op is not allowed
+ // due to promise.set_exception() was already called
+ }
+ }
+ }
+ }
+
+ m_QueuedReads.Clear();
+ }
+}
+
+/**
+ * Actually send the Redis queries queued by {FireAndForget,GetResultsOf}{Query,Queries}()
+ */
+void RedisConnection::WriteLoop(asio::yield_context& yc)
+{
+ for (;;) {
+ m_QueuedWrites.Wait(yc);
+
+ WriteFirstOfHighestPrio:
+ for (auto& queue : m_Queues.Writes) {
+ if (m_SuppressedQueryKinds.find(queue.first) != m_SuppressedQueryKinds.end() || queue.second.empty()) {
+ continue;
+ }
+
+ auto next (std::move(queue.second.front()));
+ queue.second.pop();
+
+ WriteItem(yc, std::move(next));
+
+ goto WriteFirstOfHighestPrio;
+ }
+
+ m_QueuedWrites.Clear();
+ }
+}
+
+/**
+ * Periodically log current query performance
+ */
+void RedisConnection::LogStats(asio::yield_context& yc)
+{
+ double lastMessage = 0;
+
+ m_LogStatsTimer.expires_from_now(boost::posix_time::seconds(10));
+
+ for (;;) {
+ m_LogStatsTimer.async_wait(yc);
+ m_LogStatsTimer.expires_from_now(boost::posix_time::seconds(10));
+
+ if (!IsConnected())
+ continue;
+
+ auto now (Utility::GetTime());
+ bool timeoutReached = now - lastMessage >= 5 * 60;
+
+ if (m_PendingQueries < 1 && !timeoutReached)
+ continue;
+
+ auto output (round(m_OutputQueries.CalculateRate(now, 10)));
+
+ if (m_PendingQueries < output * 5 && !timeoutReached)
+ continue;
+
+ Log(LogInformation, "IcingaDB")
+ << "Pending queries: " << m_PendingQueries << " (Input: "
+ << round(m_InputQueries.CalculateRate(now, 10)) << "/s; Output: " << output << "/s)";
+
+ lastMessage = now;
+ }
+}
+
+/**
+ * Send next and schedule receiving the response
+ *
+ * @param next Redis queries
+ */
+void RedisConnection::WriteItem(boost::asio::yield_context& yc, RedisConnection::WriteQueueItem next)
+{
+ if (next.FireAndForgetQuery) {
+ auto& item (*next.FireAndForgetQuery);
+ DecreasePendingQueries(1);
+
+ try {
+ WriteOne(item, yc);
+ } catch (const boost::coroutines::detail::forced_unwind&) {
+ throw;
+ } catch (const std::exception& ex) {
+ Log msg (LogCritical, "IcingaDB", "Error during sending query");
+ LogQuery(item, msg);
+ msg << " which has been fired and forgotten: " << ex.what();
+
+ return;
+ } catch (...) {
+ Log msg (LogCritical, "IcingaDB", "Error during sending query");
+ LogQuery(item, msg);
+ msg << " which has been fired and forgotten";
+
+ return;
+ }
+
+ if (m_Queues.FutureResponseActions.empty() || m_Queues.FutureResponseActions.back().Action != ResponseAction::Ignore) {
+ m_Queues.FutureResponseActions.emplace(FutureResponseAction{1, ResponseAction::Ignore});
+ } else {
+ ++m_Queues.FutureResponseActions.back().Amount;
+ }
+
+ m_QueuedReads.Set();
+ }
+
+ if (next.FireAndForgetQueries) {
+ auto& item (*next.FireAndForgetQueries);
+ size_t i = 0;
+
+ DecreasePendingQueries(item.size());
+
+ try {
+ for (auto& query : item) {
+ WriteOne(query, yc);
+ ++i;
+ }
+ } catch (const boost::coroutines::detail::forced_unwind&) {
+ throw;
+ } catch (const std::exception& ex) {
+ Log msg (LogCritical, "IcingaDB", "Error during sending query");
+ LogQuery(item[i], msg);
+ msg << " which has been fired and forgotten: " << ex.what();
+
+ return;
+ } catch (...) {
+ Log msg (LogCritical, "IcingaDB", "Error during sending query");
+ LogQuery(item[i], msg);
+ msg << " which has been fired and forgotten";
+
+ return;
+ }
+
+ if (m_Queues.FutureResponseActions.empty() || m_Queues.FutureResponseActions.back().Action != ResponseAction::Ignore) {
+ m_Queues.FutureResponseActions.emplace(FutureResponseAction{item.size(), ResponseAction::Ignore});
+ } else {
+ m_Queues.FutureResponseActions.back().Amount += item.size();
+ }
+
+ m_QueuedReads.Set();
+ }
+
+ if (next.GetResultOfQuery) {
+ auto& item (*next.GetResultOfQuery);
+ DecreasePendingQueries(1);
+
+ try {
+ WriteOne(item.first, yc);
+ } catch (const boost::coroutines::detail::forced_unwind&) {
+ throw;
+ } catch (...) {
+ item.second.set_exception(std::current_exception());
+
+ return;
+ }
+
+ m_Queues.ReplyPromises.emplace(std::move(item.second));
+
+ if (m_Queues.FutureResponseActions.empty() || m_Queues.FutureResponseActions.back().Action != ResponseAction::Deliver) {
+ m_Queues.FutureResponseActions.emplace(FutureResponseAction{1, ResponseAction::Deliver});
+ } else {
+ ++m_Queues.FutureResponseActions.back().Amount;
+ }
+
+ m_QueuedReads.Set();
+ }
+
+ if (next.GetResultsOfQueries) {
+ auto& item (*next.GetResultsOfQueries);
+ DecreasePendingQueries(item.first.size());
+
+ try {
+ for (auto& query : item.first) {
+ WriteOne(query, yc);
+ }
+ } catch (const boost::coroutines::detail::forced_unwind&) {
+ throw;
+ } catch (...) {
+ item.second.set_exception(std::current_exception());
+
+ return;
+ }
+
+ m_Queues.RepliesPromises.emplace(std::move(item.second));
+ m_Queues.FutureResponseActions.emplace(FutureResponseAction{item.first.size(), ResponseAction::DeliverBulk});
+
+ m_QueuedReads.Set();
+ }
+
+ if (next.Callback) {
+ next.Callback(yc);
+ }
+
+ RecordAffected(next.Affects, Utility::GetTime());
+}
+
+/**
+ * Receive the response to a Redis query
+ *
+ * @return The response
+ */
+RedisConnection::Reply RedisConnection::ReadOne(boost::asio::yield_context& yc)
+{
+ if (m_Path.IsEmpty()) {
+ if (m_TLSContext) {
+ return ReadOne(m_TlsConn, yc);
+ } else {
+ return ReadOne(m_TcpConn, yc);
+ }
+ } else {
+ return ReadOne(m_UnixConn, yc);
+ }
+}
+
+/**
+ * Send query
+ *
+ * @param query Redis query
+ */
+void RedisConnection::WriteOne(RedisConnection::Query& query, asio::yield_context& yc)
+{
+ if (m_Path.IsEmpty()) {
+ if (m_TLSContext) {
+ WriteOne(m_TlsConn, query, yc);
+ } else {
+ WriteOne(m_TcpConn, query, yc);
+ }
+ } else {
+ WriteOne(m_UnixConn, query, yc);
+ }
+}
+
+/**
+ * Specify a callback that is run each time a connection is successfully established
+ *
+ * The callback is executed from a Boost.Asio coroutine and should therefore not perform blocking operations.
+ *
+ * @param callback Callback to execute
+ */
+void RedisConnection::SetConnectedCallback(std::function<void(asio::yield_context& yc)> callback) {
+ m_ConnectedCallback = std::move(callback);
+}
+
+int RedisConnection::GetQueryCount(RingBuffer::SizeType span)
+{
+ return m_OutputQueries.UpdateAndGetValues(Utility::GetTime(), span);
+}
+
+void RedisConnection::IncreasePendingQueries(int count)
+{
+ if (m_Parent) {
+ auto parent (m_Parent);
+
+ asio::post(parent->m_Strand, [parent, count]() {
+ parent->IncreasePendingQueries(count);
+ });
+ } else {
+ m_PendingQueries += count;
+ m_InputQueries.InsertValue(Utility::GetTime(), count);
+ }
+}
+
+void RedisConnection::DecreasePendingQueries(int count)
+{
+ if (m_Parent) {
+ auto parent (m_Parent);
+
+ asio::post(parent->m_Strand, [parent, count]() {
+ parent->DecreasePendingQueries(count);
+ });
+ } else {
+ m_PendingQueries -= count;
+ m_OutputQueries.InsertValue(Utility::GetTime(), count);
+ }
+}
+
+void RedisConnection::RecordAffected(RedisConnection::QueryAffects affected, double when)
+{
+ if (m_Parent) {
+ auto parent (m_Parent);
+
+ asio::post(parent->m_Strand, [parent, affected, when]() {
+ parent->RecordAffected(affected, when);
+ });
+ } else {
+ if (affected.Config) {
+ m_WrittenConfig.InsertValue(when, affected.Config);
+ }
+
+ if (affected.State) {
+ m_WrittenState.InsertValue(when, affected.State);
+ }
+
+ if (affected.History) {
+ m_WrittenHistory.InsertValue(when, affected.History);
+ }
+ }
+}
diff --git a/lib/icingadb/redisconnection.hpp b/lib/icingadb/redisconnection.hpp
new file mode 100644
index 0000000..f3e05f9
--- /dev/null
+++ b/lib/icingadb/redisconnection.hpp
@@ -0,0 +1,678 @@
+/* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */
+
+#ifndef REDISCONNECTION_H
+#define REDISCONNECTION_H
+
+#include "base/array.hpp"
+#include "base/atomic.hpp"
+#include "base/convert.hpp"
+#include "base/io-engine.hpp"
+#include "base/object.hpp"
+#include "base/ringbuffer.hpp"
+#include "base/shared.hpp"
+#include "base/string.hpp"
+#include "base/tlsstream.hpp"
+#include "base/value.hpp"
+#include <boost/asio/buffer.hpp>
+#include <boost/asio/buffered_stream.hpp>
+#include <boost/asio/deadline_timer.hpp>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/io_context_strand.hpp>
+#include <boost/asio/ip/tcp.hpp>
+#include <boost/asio/local/stream_protocol.hpp>
+#include <boost/asio/read.hpp>
+#include <boost/asio/read_until.hpp>
+#include <boost/asio/ssl/context.hpp>
+#include <boost/asio/streambuf.hpp>
+#include <boost/asio/write.hpp>
+#include <boost/lexical_cast.hpp>
+#include <boost/regex.hpp>
+#include <boost/utility/string_view.hpp>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <future>
+#include <map>
+#include <memory>
+#include <queue>
+#include <set>
+#include <stdexcept>
+#include <utility>
+#include <vector>
+
+namespace icinga
+{
+/**
+ * An Async Redis connection.
+ *
+ * @ingroup icingadb
+ */
+ class RedisConnection final : public Object
+ {
+ public:
+ DECLARE_PTR_TYPEDEFS(RedisConnection);
+
+ typedef std::vector<String> Query;
+ typedef std::vector<Query> Queries;
+ typedef Value Reply;
+ typedef std::vector<Reply> Replies;
+
+ /**
+ * Redis query priorities, highest first.
+ *
+ * @ingroup icingadb
+ */
+ enum class QueryPriority : unsigned char
+ {
+ Heartbeat,
+ RuntimeStateStream, // runtime state updates, doesn't affect initially synced states
+ Config, // includes initially synced states
+ RuntimeStateSync, // updates initially synced states at runtime, in parallel to config dump, therefore must be < Config
+ History,
+ CheckResult,
+ SyncConnection = 255
+ };
+
+ struct QueryAffects
+ {
+ size_t Config;
+ size_t State;
+ size_t History;
+
+ QueryAffects(size_t config = 0, size_t state = 0, size_t history = 0)
+ : Config(config), State(state), History(history) { }
+ };
+
+ RedisConnection(const String& host, int port, const String& path, const String& password, int db,
+ bool useTls, bool insecure, const String& certPath, const String& keyPath, const String& caPath, const String& crlPath,
+ const String& tlsProtocolmin, const String& cipherList, double connectTimeout, DebugInfo di, const Ptr& parent = nullptr);
+
+ void UpdateTLSContext();
+
+ void Start();
+
+ bool IsConnected();
+
+ void FireAndForgetQuery(Query query, QueryPriority priority, QueryAffects affects = {});
+ void FireAndForgetQueries(Queries queries, QueryPriority priority, QueryAffects affects = {});
+
+ Reply GetResultOfQuery(Query query, QueryPriority priority, QueryAffects affects = {});
+ Replies GetResultsOfQueries(Queries queries, QueryPriority priority, QueryAffects affects = {});
+
+ void EnqueueCallback(const std::function<void(boost::asio::yield_context&)>& callback, QueryPriority priority);
+ void Sync();
+ double GetOldestPendingQueryTs();
+
+ void SuppressQueryKind(QueryPriority kind);
+ void UnsuppressQueryKind(QueryPriority kind);
+
+ void SetConnectedCallback(std::function<void(boost::asio::yield_context& yc)> callback);
+
+ inline bool GetConnected()
+ {
+ return m_Connected.load();
+ }
+
+ int GetQueryCount(RingBuffer::SizeType span);
+
+ inline int GetPendingQueryCount()
+ {
+ return m_PendingQueries;
+ }
+
+ inline int GetWrittenConfigFor(RingBuffer::SizeType span, RingBuffer::SizeType tv = Utility::GetTime())
+ {
+ return m_WrittenConfig.UpdateAndGetValues(tv, span);
+ }
+
+ inline int GetWrittenStateFor(RingBuffer::SizeType span, RingBuffer::SizeType tv = Utility::GetTime())
+ {
+ return m_WrittenState.UpdateAndGetValues(tv, span);
+ }
+
+ inline int GetWrittenHistoryFor(RingBuffer::SizeType span, RingBuffer::SizeType tv = Utility::GetTime())
+ {
+ return m_WrittenHistory.UpdateAndGetValues(tv, span);
+ }
+
+ private:
+ /**
+ * What to do with the responses to Redis queries.
+ *
+ * @ingroup icingadb
+ */
+ enum class ResponseAction : unsigned char
+ {
+ Ignore, // discard
+ Deliver, // submit to the requestor
+ DeliverBulk // submit multiple responses to the requestor at once
+ };
+
+ /**
+ * What to do with how many responses to Redis queries.
+ *
+ * @ingroup icingadb
+ */
+ struct FutureResponseAction
+ {
+ size_t Amount;
+ ResponseAction Action;
+ };
+
+ /**
+ * Something to be send to Redis.
+ *
+ * @ingroup icingadb
+ */
+ struct WriteQueueItem
+ {
+ Shared<Query>::Ptr FireAndForgetQuery;
+ Shared<Queries>::Ptr FireAndForgetQueries;
+ Shared<std::pair<Query, std::promise<Reply>>>::Ptr GetResultOfQuery;
+ Shared<std::pair<Queries, std::promise<Replies>>>::Ptr GetResultsOfQueries;
+ std::function<void(boost::asio::yield_context&)> Callback;
+
+ double CTime;
+ QueryAffects Affects;
+ };
+
+ typedef boost::asio::ip::tcp Tcp;
+ typedef boost::asio::local::stream_protocol Unix;
+
+ typedef boost::asio::buffered_stream<Tcp::socket> TcpConn;
+ typedef boost::asio::buffered_stream<Unix::socket> UnixConn;
+
+ Shared<boost::asio::ssl::context>::Ptr m_TLSContext;
+
+ template<class AsyncReadStream>
+ static Value ReadRESP(AsyncReadStream& stream, boost::asio::yield_context& yc);
+
+ template<class AsyncReadStream>
+ static std::vector<char> ReadLine(AsyncReadStream& stream, boost::asio::yield_context& yc, size_t hint = 0);
+
+ template<class AsyncWriteStream>
+ static void WriteRESP(AsyncWriteStream& stream, const Query& query, boost::asio::yield_context& yc);
+
+ static boost::regex m_ErrAuth;
+
+ RedisConnection(boost::asio::io_context& io, String host, int port, String path, String password,
+ int db, bool useTls, bool insecure, String certPath, String keyPath, String caPath, String crlPath,
+ String tlsProtocolmin, String cipherList, double connectTimeout, DebugInfo di, const Ptr& parent);
+
+ void Connect(boost::asio::yield_context& yc);
+ void ReadLoop(boost::asio::yield_context& yc);
+ void WriteLoop(boost::asio::yield_context& yc);
+ void LogStats(boost::asio::yield_context& yc);
+ void WriteItem(boost::asio::yield_context& yc, WriteQueueItem item);
+ Reply ReadOne(boost::asio::yield_context& yc);
+ void WriteOne(Query& query, boost::asio::yield_context& yc);
+
+ template<class StreamPtr>
+ Reply ReadOne(StreamPtr& stream, boost::asio::yield_context& yc);
+
+ template<class StreamPtr>
+ void WriteOne(StreamPtr& stream, Query& query, boost::asio::yield_context& yc);
+
+ void IncreasePendingQueries(int count);
+ void DecreasePendingQueries(int count);
+ void RecordAffected(QueryAffects affected, double when);
+
+ template<class StreamPtr>
+ void Handshake(StreamPtr& stream, boost::asio::yield_context& yc);
+
+ template<class StreamPtr>
+ Timeout::Ptr MakeTimeout(StreamPtr& stream);
+
+ String m_Path;
+ String m_Host;
+ int m_Port;
+ String m_Password;
+ int m_DbIndex;
+
+ String m_CertPath;
+ String m_KeyPath;
+ bool m_Insecure;
+ String m_CaPath;
+ String m_CrlPath;
+ String m_TlsProtocolmin;
+ String m_CipherList;
+ double m_ConnectTimeout;
+ DebugInfo m_DebugInfo;
+
+ boost::asio::io_context::strand m_Strand;
+ Shared<TcpConn>::Ptr m_TcpConn;
+ Shared<UnixConn>::Ptr m_UnixConn;
+ Shared<AsioTlsStream>::Ptr m_TlsConn;
+ Atomic<bool> m_Connecting, m_Connected, m_Started;
+
+ struct {
+ // Items to be send to Redis
+ std::map<QueryPriority, std::queue<WriteQueueItem>> Writes;
+ // Requestors, each waiting for a single response
+ std::queue<std::promise<Reply>> ReplyPromises;
+ // Requestors, each waiting for multiple responses at once
+ std::queue<std::promise<Replies>> RepliesPromises;
+ // Metadata about all of the above
+ std::queue<FutureResponseAction> FutureResponseActions;
+ } m_Queues;
+
+ // Kinds of queries not to actually send yet
+ std::set<QueryPriority> m_SuppressedQueryKinds;
+
+ // Indicate that there's something to send/receive
+ AsioConditionVariable m_QueuedWrites, m_QueuedReads;
+
+ std::function<void(boost::asio::yield_context& yc)> m_ConnectedCallback;
+
+ // Stats
+ RingBuffer m_InputQueries{10};
+ RingBuffer m_OutputQueries{15 * 60};
+ RingBuffer m_WrittenConfig{15 * 60};
+ RingBuffer m_WrittenState{15 * 60};
+ RingBuffer m_WrittenHistory{15 * 60};
+ int m_PendingQueries{0};
+ boost::asio::deadline_timer m_LogStatsTimer;
+ Ptr m_Parent;
+ };
+
+/**
+ * An error response from the Redis server.
+ *
+ * @ingroup icingadb
+ */
+class RedisError final : public Object
+{
+public:
+ DECLARE_PTR_TYPEDEFS(RedisError);
+
+ inline RedisError(String message) : m_Message(std::move(message))
+ {
+ }
+
+ inline const String& GetMessage()
+ {
+ return m_Message;
+ }
+
+private:
+ String m_Message;
+};
+
+/**
+ * Thrown if the connection to the Redis server has already been lost.
+ *
+ * @ingroup icingadb
+ */
+class RedisDisconnected : public std::runtime_error
+{
+public:
+ inline RedisDisconnected() : runtime_error("")
+ {
+ }
+};
+
+/**
+ * Thrown on malformed Redis server responses.
+ *
+ * @ingroup icingadb
+ */
+class RedisProtocolError : public std::runtime_error
+{
+protected:
+ inline RedisProtocolError() : runtime_error("")
+ {
+ }
+};
+
+/**
+ * Thrown on malformed types in Redis server responses.
+ *
+ * @ingroup icingadb
+ */
+class BadRedisType : public RedisProtocolError
+{
+public:
+ inline BadRedisType(char type) : m_What{type, 0}
+ {
+ }
+
+ virtual const char * what() const noexcept override
+ {
+ return m_What;
+ }
+
+private:
+ char m_What[2];
+};
+
+/**
+ * Thrown on malformed ints in Redis server responses.
+ *
+ * @ingroup icingadb
+ */
+class BadRedisInt : public RedisProtocolError
+{
+public:
+ inline BadRedisInt(std::vector<char> intStr) : m_What(std::move(intStr))
+ {
+ m_What.emplace_back(0);
+ }
+
+ virtual const char * what() const noexcept override
+ {
+ return m_What.data();
+ }
+
+private:
+ std::vector<char> m_What;
+};
+
+/**
+ * Read a Redis server response from stream
+ *
+ * @param stream Redis server connection
+ *
+ * @return The response
+ */
+template<class StreamPtr>
+RedisConnection::Reply RedisConnection::ReadOne(StreamPtr& stream, boost::asio::yield_context& yc)
+{
+ namespace asio = boost::asio;
+
+ if (!stream) {
+ throw RedisDisconnected();
+ }
+
+ auto strm (stream);
+
+ try {
+ return ReadRESP(*strm, yc);
+ } catch (const boost::coroutines::detail::forced_unwind&) {
+ throw;
+ } catch (...) {
+ if (m_Connecting.exchange(false)) {
+ m_Connected.store(false);
+ stream = nullptr;
+
+ if (!m_Connecting.exchange(true)) {
+ Ptr keepAlive (this);
+
+ IoEngine::SpawnCoroutine(m_Strand, [this, keepAlive](asio::yield_context yc) { Connect(yc); });
+ }
+ }
+
+ throw;
+ }
+}
+
+/**
+ * Write a Redis query to stream
+ *
+ * @param stream Redis server connection
+ * @param query Redis query
+ */
+template<class StreamPtr>
+void RedisConnection::WriteOne(StreamPtr& stream, RedisConnection::Query& query, boost::asio::yield_context& yc)
+{
+ namespace asio = boost::asio;
+
+ if (!stream) {
+ throw RedisDisconnected();
+ }
+
+ auto strm (stream);
+
+ try {
+ WriteRESP(*strm, query, yc);
+ strm->async_flush(yc);
+ } catch (const boost::coroutines::detail::forced_unwind&) {
+ throw;
+ } catch (...) {
+ if (m_Connecting.exchange(false)) {
+ m_Connected.store(false);
+ stream = nullptr;
+
+ if (!m_Connecting.exchange(true)) {
+ Ptr keepAlive (this);
+
+ IoEngine::SpawnCoroutine(m_Strand, [this, keepAlive](asio::yield_context yc) { Connect(yc); });
+ }
+ }
+
+ throw;
+ }
+}
+
+/**
+ * Initialize a Redis stream
+ *
+ * @param stream Redis server connection
+ * @param query Redis query
+ */
+template<class StreamPtr>
+void RedisConnection::Handshake(StreamPtr& strm, boost::asio::yield_context& yc)
+{
+ if (m_Password.IsEmpty() && !m_DbIndex) {
+ // Trigger NOAUTH
+ WriteRESP(*strm, {"PING"}, yc);
+ } else {
+ if (!m_Password.IsEmpty()) {
+ WriteRESP(*strm, {"AUTH", m_Password}, yc);
+ }
+
+ if (m_DbIndex) {
+ WriteRESP(*strm, {"SELECT", Convert::ToString(m_DbIndex)}, yc);
+ }
+ }
+
+ strm->async_flush(yc);
+
+ if (m_Password.IsEmpty() && !m_DbIndex) {
+ Reply pong (ReadRESP(*strm, yc));
+
+ if (pong.IsObjectType<RedisError>()) {
+ // Likely NOAUTH
+ BOOST_THROW_EXCEPTION(std::runtime_error(RedisError::Ptr(pong)->GetMessage()));
+ }
+ } else {
+ if (!m_Password.IsEmpty()) {
+ Reply auth (ReadRESP(*strm, yc));
+
+ if (auth.IsObjectType<RedisError>()) {
+ auto& authErr (RedisError::Ptr(auth)->GetMessage().GetData());
+ boost::smatch what;
+
+ if (boost::regex_search(authErr, what, m_ErrAuth)) {
+ Log(LogWarning, "IcingaDB") << authErr;
+ } else {
+ // Likely WRONGPASS
+ BOOST_THROW_EXCEPTION(std::runtime_error(authErr));
+ }
+ }
+ }
+
+ if (m_DbIndex) {
+ Reply select (ReadRESP(*strm, yc));
+
+ if (select.IsObjectType<RedisError>()) {
+ // Likely NOAUTH or ERR DB
+ BOOST_THROW_EXCEPTION(std::runtime_error(RedisError::Ptr(select)->GetMessage()));
+ }
+ }
+ }
+}
+
+/**
+ * Creates a Timeout which cancels stream's I/O after m_ConnectTimeout
+ *
+ * @param stream Redis server connection
+ */
+template<class StreamPtr>
+Timeout::Ptr RedisConnection::MakeTimeout(StreamPtr& stream)
+{
+ Ptr keepAlive (this);
+
+ return new Timeout(
+ m_Strand.context(),
+ m_Strand,
+ boost::posix_time::microseconds(intmax_t(m_ConnectTimeout * 1000000)),
+ [keepAlive, stream](boost::asio::yield_context yc) {
+ boost::system::error_code ec;
+ stream->lowest_layer().cancel(ec);
+ }
+ );
+}
+
+/**
+ * Read a Redis protocol value from stream
+ *
+ * @param stream Redis server connection
+ *
+ * @return The value
+ */
+template<class AsyncReadStream>
+Value RedisConnection::ReadRESP(AsyncReadStream& stream, boost::asio::yield_context& yc)
+{
+ namespace asio = boost::asio;
+
+ char type = 0;
+ asio::async_read(stream, asio::mutable_buffer(&type, 1), yc);
+
+ switch (type) {
+ case '+':
+ {
+ auto buf (ReadLine(stream, yc));
+ return String(buf.begin(), buf.end());
+ }
+ case '-':
+ {
+ auto buf (ReadLine(stream, yc));
+ return new RedisError(String(buf.begin(), buf.end()));
+ }
+ case ':':
+ {
+ auto buf (ReadLine(stream, yc, 21));
+ intmax_t i = 0;
+
+ try {
+ i = boost::lexical_cast<intmax_t>(boost::string_view(buf.data(), buf.size()));
+ } catch (...) {
+ throw BadRedisInt(std::move(buf));
+ }
+
+ return (double)i;
+ }
+ case '$':
+ {
+ auto buf (ReadLine(stream, yc, 21));
+ intmax_t i = 0;
+
+ try {
+ i = boost::lexical_cast<intmax_t>(boost::string_view(buf.data(), buf.size()));
+ } catch (...) {
+ throw BadRedisInt(std::move(buf));
+ }
+
+ if (i < 0) {
+ return Value();
+ }
+
+ buf.clear();
+ buf.insert(buf.end(), i, 0);
+ asio::async_read(stream, asio::mutable_buffer(buf.data(), buf.size()), yc);
+
+ {
+ char crlf[2];
+ asio::async_read(stream, asio::mutable_buffer(crlf, 2), yc);
+ }
+
+ return String(buf.begin(), buf.end());
+ }
+ case '*':
+ {
+ auto buf (ReadLine(stream, yc, 21));
+ intmax_t i = 0;
+
+ try {
+ i = boost::lexical_cast<intmax_t>(boost::string_view(buf.data(), buf.size()));
+ } catch (...) {
+ throw BadRedisInt(std::move(buf));
+ }
+
+ if (i < 0) {
+ return Empty;
+ }
+
+ Array::Ptr arr = new Array();
+
+ arr->Reserve(i);
+
+ for (; i; --i) {
+ arr->Add(ReadRESP(stream, yc));
+ }
+
+ return arr;
+ }
+ default:
+ throw BadRedisType(type);
+ }
+}
+
+/**
+ * Read from stream until \r\n
+ *
+ * @param stream Redis server connection
+ * @param hint Expected amount of data
+ *
+ * @return Read data ex. \r\n
+ */
+template<class AsyncReadStream>
+std::vector<char> RedisConnection::ReadLine(AsyncReadStream& stream, boost::asio::yield_context& yc, size_t hint)
+{
+ namespace asio = boost::asio;
+
+ std::vector<char> line;
+ line.reserve(hint);
+
+ char next = 0;
+ asio::mutable_buffer buf (&next, 1);
+
+ for (;;) {
+ asio::async_read(stream, buf, yc);
+
+ if (next == '\r') {
+ asio::async_read(stream, buf, yc);
+ return std::move(line);
+ }
+
+ line.emplace_back(next);
+ }
+}
+
+/**
+ * Write a Redis protocol value to stream
+ *
+ * @param stream Redis server connection
+ * @param query Redis protocol value
+ */
+template<class AsyncWriteStream>
+void RedisConnection::WriteRESP(AsyncWriteStream& stream, const Query& query, boost::asio::yield_context& yc)
+{
+ namespace asio = boost::asio;
+
+ asio::streambuf writeBuffer;
+ std::ostream msg(&writeBuffer);
+
+ msg << "*" << query.size() << "\r\n";
+
+ for (auto& arg : query) {
+ msg << "$" << arg.GetLength() << "\r\n" << arg << "\r\n";
+ }
+
+ asio::async_write(stream, writeBuffer, yc);
+}
+
+}
+
+#endif //REDISCONNECTION_H