From f66ab8dae2f3d0418759f81a3a64dc9517a62449 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 14 Apr 2024 15:17:31 +0200 Subject: Adding upstream version 1.10.2. Signed-off-by: Daniel Baumann --- library/Director/Db/AppliedServiceSetLoader.php | 58 +++ library/Director/Db/Branch/Branch.php | 216 +++++++++++ library/Director/Db/Branch/BranchActivity.php | 390 ++++++++++++++++++++ library/Director/Db/Branch/BranchMerger.php | 157 ++++++++ .../Db/Branch/BranchModificationInspection.php | 93 +++++ library/Director/Db/Branch/BranchSettings.php | 121 ++++++ library/Director/Db/Branch/BranchStore.php | 240 ++++++++++++ library/Director/Db/Branch/BranchSupport.php | 91 +++++ library/Director/Db/Branch/BranchedObject.php | 404 +++++++++++++++++++++ library/Director/Db/Branch/MergeError.php | 37 ++ .../Db/Branch/MergeErrorDeleteMissingObject.php | 15 + .../MergeErrorModificationForMissingObject.php | 15 + .../Db/Branch/MergeErrorRecreateOnMerge.php | 15 + .../Director/Db/Branch/PlainObjectPropertyDiff.php | 50 +++ library/Director/Db/Branch/UuidLookup.php | 141 +++++++ library/Director/Db/Cache/CustomVariableCache.php | 84 +++++ library/Director/Db/Cache/GroupMembershipCache.php | 104 ++++++ library/Director/Db/Cache/PrefetchCache.php | 166 +++++++++ library/Director/Db/DbSelectParenthesis.php | 24 ++ library/Director/Db/DbUtil.php | 96 +++++ library/Director/Db/HostMembershipHousekeeping.php | 8 + library/Director/Db/Housekeeping.php | 249 +++++++++++++ library/Director/Db/IcingaObjectFilterHelper.php | 133 +++++++ library/Director/Db/MembershipHousekeeping.php | 135 +++++++ library/Director/Db/Migration.php | 70 ++++ library/Director/Db/Migrations.php | 239 ++++++++++++ 26 files changed, 3351 insertions(+) create mode 100644 library/Director/Db/AppliedServiceSetLoader.php create mode 100644 library/Director/Db/Branch/Branch.php create mode 100644 library/Director/Db/Branch/BranchActivity.php create mode 100644 library/Director/Db/Branch/BranchMerger.php create mode 100644 library/Director/Db/Branch/BranchModificationInspection.php create mode 100644 library/Director/Db/Branch/BranchSettings.php create mode 100644 library/Director/Db/Branch/BranchStore.php create mode 100644 library/Director/Db/Branch/BranchSupport.php create mode 100644 library/Director/Db/Branch/BranchedObject.php create mode 100644 library/Director/Db/Branch/MergeError.php create mode 100644 library/Director/Db/Branch/MergeErrorDeleteMissingObject.php create mode 100644 library/Director/Db/Branch/MergeErrorModificationForMissingObject.php create mode 100644 library/Director/Db/Branch/MergeErrorRecreateOnMerge.php create mode 100644 library/Director/Db/Branch/PlainObjectPropertyDiff.php create mode 100644 library/Director/Db/Branch/UuidLookup.php create mode 100644 library/Director/Db/Cache/CustomVariableCache.php create mode 100644 library/Director/Db/Cache/GroupMembershipCache.php create mode 100644 library/Director/Db/Cache/PrefetchCache.php create mode 100644 library/Director/Db/DbSelectParenthesis.php create mode 100644 library/Director/Db/DbUtil.php create mode 100644 library/Director/Db/HostMembershipHousekeeping.php create mode 100644 library/Director/Db/Housekeeping.php create mode 100644 library/Director/Db/IcingaObjectFilterHelper.php create mode 100644 library/Director/Db/MembershipHousekeeping.php create mode 100644 library/Director/Db/Migration.php create mode 100644 library/Director/Db/Migrations.php (limited to 'library/Director/Db') diff --git a/library/Director/Db/AppliedServiceSetLoader.php b/library/Director/Db/AppliedServiceSetLoader.php new file mode 100644 index 0000000..b1e9408 --- /dev/null +++ b/library/Director/Db/AppliedServiceSetLoader.php @@ -0,0 +1,58 @@ +host = $host; + } + + /** + * @return IcingaServiceSet[] + */ + public static function fetchForHost(IcingaHost $host) + { + $loader = new static($host); + return $loader->fetchAppliedServiceSets(); + } + + /** + * @return IcingaServiceSet[] + */ + protected function fetchAppliedServiceSets() + { + $sets = array(); + $matcher = HostApplyMatches::prepare($this->host); + foreach ($this->fetchAllServiceSets() as $set) { + $filter = Filter::fromQueryString($set->get('assign_filter')); + if ($matcher->matchesFilter($filter)) { + $sets[] = $set; + } + } + + return $sets; + } + + /** + * @return IcingaServiceSet[] + */ + protected function fetchAllServiceSets() + { + $db = $this->host->getDb(); + $query = $db + ->select() + ->from('icinga_service_set') + ->where('assign_filter IS NOT NULL'); + + return IcingaServiceSet::loadAll($this->host->getConnection(), $query); + } +} diff --git a/library/Director/Db/Branch/Branch.php b/library/Director/Db/Branch/Branch.php new file mode 100644 index 0000000..cd68ff0 --- /dev/null +++ b/library/Director/Db/Branch/Branch.php @@ -0,0 +1,216 @@ +uuid)) { + $row->uuid = stream_get_contents($row->uuid); + } + if (strlen($row->uuid) !== 16) { + throw new RuntimeException('Valid UUID expected, got ' . var_export($row->uuid, 1)); + } + $self->branchUuid = Uuid::fromBytes(Db\DbUtil::binaryResult($row->uuid)); + $self->name = $row->branch_name; + $self->owner = $row->owner; + $self->description = $row->description; + $self->tsMergeRequest = $row->ts_merge_request; + if (isset($row->cnt_activities)) { + $self->cntActivities = $row->cnt_activities; + } else { + $self->cntActivities = 0; + } + + return $self; + } + + /** + * @return Branch + */ + public static function detect(BranchStore $store) + { + try { + return static::forRequest(Icinga::app()->getRequest(), $store, Auth::getInstance()); + } catch (\Exception $e) { + return new static(); + } + } + + /** + * @param Request $request + * @param Db $db + * @param Auth $auth + * @return Branch + */ + public static function forRequest(Request $request, BranchStore $store, Auth $auth) + { + if ($hook = static::optionalHook()) { + return $hook->getBranchForRequest($request, $store, $auth); + } + + return new Branch; + } + + /** + * @return BranchSupportHook + */ + public static function requireHook() + { + if ($hook = static::optionalHook()) { + return $hook; + } + + throw new RuntimeException('BranchSupport Hook requested where not available'); + } + + /** + * @return BranchSupportHook|null + */ + public static function optionalHook() + { + return Hook::first('director/BranchSupport'); + } + + /** + * @param UuidInterface $uuid + * @return Branch + */ + public static function withUuid(UuidInterface $uuid) + { + $self = new static(); + $self->branchUuid = $uuid; + return $self; + } + + /** + * @return bool + */ + public function isBranch() + { + return $this->branchUuid !== null; + } + + public function assertBranch() + { + if ($this->isMain()) { + throw new RuntimeException('Branch expected, but working in main branch'); + } + } + + /** + * @return bool + */ + public function isMain() + { + return $this->branchUuid === null; + } + + /** + * @return bool + */ + public function shouldBeMerged() + { + return $this->tsMergeRequest !== null; + } + + /** + * @return bool + */ + public function isEmpty() + { + return $this->cntActivities === 0; + } + + /** + * @return int + */ + public function getActivityCount() + { + return $this->cntActivities; + } + + /** + * @return UuidInterface|null + */ + public function getUuid() + { + return $this->branchUuid; + } + + /** + * @return string + */ + public function getName() + { + return $this->name; + } + + /** + * @since v1.10.0 + * @return string + */ + public function getDescription() + { + return $this->description; + } + + /** + * @since v1.10.0 + * @param ?string $description + * @return void + */ + public function setDescription($description) + { + $this->description = $description; + } + + /** + * @return string + */ + public function getOwner() + { + return $this->owner; + } + + public function isSyncPreview() + { + return (bool) preg_match('/^' . preg_quote(self::PREFIX_SYNC_PREVIEW, '/') . '\//', $this->getName()); + } +} diff --git a/library/Director/Db/Branch/BranchActivity.php b/library/Director/Db/Branch/BranchActivity.php new file mode 100644 index 0000000..3812e75 --- /dev/null +++ b/library/Director/Db/Branch/BranchActivity.php @@ -0,0 +1,390 @@ +objectUuid = $objectUuid; + $this->branchUuid = $branchUuid; + $this->action = $action; + $this->objectTable = $objectType; + $this->author = $author; + $this->modifiedProperties = $modifiedProperties; + $this->formerProperties = $formerProperties; + } + + public static function deleteObject(DbObject $object, Branch $branch) + { + return new static( + $object->getUniqueId(), + $branch->getUuid(), + self::ACTION_DELETE, + $object->getTableName(), + Auth::getInstance()->getUser()->getUsername(), + SerializableValue::fromSerialization(null), + SerializableValue::fromSerialization(self::getFormerObjectProperties($object)) + ); + } + + public static function forDbObject(DbObject $object, Branch $branch) + { + if (! $object->hasBeenModified()) { + throw new InvalidArgumentException('Cannot get modifications for unmodified object'); + } + if (! $branch->isBranch()) { + throw new InvalidArgumentException('Branch activity requires an active branch'); + } + + $author = Auth::getInstance()->getUser()->getUsername(); + if ($object instanceof IcingaObject && $object->shouldBeRemoved()) { + $action = self::ACTION_DELETE; + $old = self::getFormerObjectProperties($object); + $new = null; + } elseif ($object->hasBeenLoadedFromDb()) { + $action = self::ACTION_MODIFY; + $old = self::getFormerObjectProperties($object); + $new = self::getObjectProperties($object); + } else { + $action = self::ACTION_CREATE; + $old = null; + $new = self::getObjectProperties($object); + } + + if ($new !== null) { + $new = PlainObjectPropertyDiff::calculate( + $old, + $new + ); + } + + return new static( + $object->getUniqueId(), + $branch->getUuid(), + $action, + $object->getTableName(), + $author, + SerializableValue::fromSerialization($new), + SerializableValue::fromSerialization($old) + ); + } + + public static function fixFakeTimestamp($timestampNs) + { + if ($timestampNs < 1600000000 * 1000000) { + // fake TS for cloned branch in sync preview + return (int) $timestampNs * 1000000; + } + + return $timestampNs; + } + + public function applyToDbObject(DbObject $object) + { + if (!$this->isActionModify()) { + throw new RuntimeException('Only BranchActivity instances with action=modify can be applied'); + } + + foreach ($this->getModifiedProperties()->jsonSerialize() as $key => $value) { + $object->set($key, $value); + } + + return $object; + } + + /** + * Hint: $connection is required, because setting groups triggered loading them. + * Should be investigated, as in theory $hostWithoutConnection->groups = 'group' + * is expected to work + * @param Db $connection + * @return DbObject|string + */ + public function createDbObject(Db $connection) + { + if (!$this->isActionCreate()) { + throw new RuntimeException('Only BranchActivity instances with action=create can create objects'); + } + + $class = DbObjectTypeRegistry::classByType($this->getObjectTable()); + $object = $class::create([], $connection); + $object->setUniqueId($this->getObjectUuid()); + foreach ($this->getModifiedProperties()->jsonSerialize() as $key => $value) { + $object->set($key, $value); + } + + return $object; + } + + public function deleteDbObject(DbObject $object) + { + if (!$this->isActionDelete()) { + throw new RuntimeException('Only BranchActivity instances with action=delete can delete objects'); + } + + return $object->delete(); + } + + public static function load($ts, Db $connection) + { + $db = $connection->getDbAdapter(); + $row = $db->fetchRow( + $db->select()->from('director_branch_activity')->where('timestamp_ns = ?', $ts) + ); + + if ($row) { + return static::fromDbRow($row); + } + + throw new NotFoundError('Not found'); + } + + protected static function fixPgResource(&$value) + { + if (is_resource($value)) { + $value = stream_get_contents($value); + } + } + + public static function fromDbRow($row) + { + static::fixPgResource($row->object_uuid); + static::fixPgResource($row->branch_uuid); + $activity = new static( + Uuid::fromBytes($row->object_uuid), + Uuid::fromBytes($row->branch_uuid), + $row->action, + $row->object_table, + $row->author, + SerializableValue::fromSerialization(Json::decodeOptional($row->modified_properties)), + SerializableValue::fromSerialization(Json::decodeOptional($row->former_properties)) + ); + $activity->timestampNs = $row->timestamp_ns; + + return $activity; + } + + /** + * Must be run in a transaction! Repeatable read? + * @param Db $connection + * @throws \Icinga\Module\Director\Exception\JsonEncodeException + * @throws \Zend_Db_Adapter_Exception + */ + public function store(Db $connection) + { + if ($this->timestampNs !== null) { + throw new InvalidArgumentException(sprintf( + 'Cannot store activity with a given timestamp: %s', + $this->timestampNs + )); + } + $db = $connection->getDbAdapter(); + $last = $db->fetchRow( + $db->select()->from('director_branch_activity', ['timestamp_ns' => 'MAX(timestamp_ns)']) + ); + if (PHP_INT_SIZE !== 8) { + throw new RuntimeException('PHP with 64bit integer support is required'); + } + $timestampNs = (int) floor(microtime(true) * 1000000); + if ($last) { + if ($last->timestamp_ns >= $timestampNs) { + $timestampNs = $last + 1; + } + } + $old = Json::encode($this->formerProperties); + $new = Json::encode($this->modifiedProperties); + + $db->insert(self::DB_TABLE, [ + 'timestamp_ns' => $timestampNs, + 'object_uuid' => $connection->quoteBinary($this->objectUuid->getBytes()), + 'branch_uuid' => $connection->quoteBinary($this->branchUuid->getBytes()), + 'action' => $this->action, + 'object_table' => $this->objectTable, + 'author' => $this->author, + 'former_properties' => $old, + 'modified_properties' => $new, + ]); + } + + /** + * @return int + */ + public function getTimestampNs() + { + return $this->timestampNs; + } + + /** + * @return int + */ + public function getTimestamp() + { + return (int) floor(BranchActivity::fixFakeTimestamp($this->timestampNs) / 1000000); + } + + /** + * @return UuidInterface + */ + public function getObjectUuid() + { + return $this->objectUuid; + } + + /** + * @return UuidInterface + */ + public function getBranchUuid() + { + return $this->branchUuid; + } + + /** + * @return string + */ + public function getObjectName() + { + return $this->getProperty('object_name', 'unknown object name'); + } + + /** + * @return string + */ + public function getAction() + { + return $this->action; + } + + public function isActionDelete() + { + return $this->action === self::ACTION_DELETE; + } + + public function isActionCreate() + { + return $this->action === self::ACTION_CREATE; + } + + public function isActionModify() + { + return $this->action === self::ACTION_MODIFY; + } + + /** + * @return string + */ + public function getObjectTable() + { + return $this->objectTable; + } + + /** + * @return string + */ + public function getAuthor() + { + return $this->author; + } + + /** + * @return ?SerializableValue + */ + public function getModifiedProperties() + { + return $this->modifiedProperties; + } + + /** + * @return ?SerializableValue + */ + public function getFormerProperties() + { + return $this->formerProperties; + } + + public function getProperty($key, $default = null) + { + if ($this->modifiedProperties) { + $properties = $this->modifiedProperties->jsonSerialize(); + if (isset($properties->$key)) { + return $properties->$key; + } + } + if ($this->formerProperties) { + $properties = $this->formerProperties->jsonSerialize(); + if (isset($properties->$key)) { + return $properties->$key; + } + } + + return $default; + } + + protected static function getFormerObjectProperties(DbObject $object) + { + if (! $object instanceof IcingaObject) { + throw new RuntimeException('Plain object helpers for DbObject must be implemented'); + } + + return (array) $object->getPlainUnmodifiedObject(); + } + + protected static function getObjectProperties(DbObject $object) + { + if (! $object instanceof IcingaObject) { + throw new RuntimeException('Plain object helpers for DbObject must be implemented'); + } + + return (array) $object->toPlainObject(false, true); + } +} diff --git a/library/Director/Db/Branch/BranchMerger.php b/library/Director/Db/Branch/BranchMerger.php new file mode 100644 index 0000000..2e84863 --- /dev/null +++ b/library/Director/Db/Branch/BranchMerger.php @@ -0,0 +1,157 @@ +branchUuid = $branchUuid; + $this->db = $connection->getDbAdapter(); + $this->connection = $connection; + } + + /** + * Skip a delete operation, when the object to be deleted does not exist + * + * @param bool $ignore + */ + public function ignoreDeleteWhenMissing($ignore = true) + { + $this->ignoreDeleteWhenMissing = $ignore; + } + + /** + * Skip a modification, when the related object does not exist + * @param bool $ignore + */ + public function ignoreModificationWhenMissing($ignore = true) + { + $this->ignoreModificationWhenMissing = $ignore; + } + + /** + * @param int $key + */ + public function ignoreActivity($key) + { + $this->ignoreActivities[$key] = true; + } + + /** + * @param BranchActivity $activity + * @return bool + */ + public function ignoresActivity(BranchActivity $activity) + { + return isset($this->ignoreActivities[$activity->getTimestampNs()]); + } + + /** + * @throws MergeError + */ + public function merge($comment = null) + { + $username = DirectorActivityLog::username(); + $this->connection->runFailSafeTransaction(function () use ($comment, $username) { + $formerActivityId = (int) DirectorActivityLog::loadLatest($this->connection)->get('id'); + $query = $this->db->select() + ->from(BranchActivity::DB_TABLE) + ->where('branch_uuid = ?', $this->connection->quoteBinary($this->branchUuid->getBytes())) + ->order('timestamp_ns ASC'); + $rows = $this->db->fetchAll($query); + foreach ($rows as $row) { + $activity = BranchActivity::fromDbRow($row); + $author = $activity->getAuthor(); + if ($username !== $author) { + DirectorActivityLog::overrideUsername("$author/$username"); + } + $this->applyModification($activity); + } + (new BranchStore($this->connection))->deleteByUuid($this->branchUuid); + $currentActivityId = (int) DirectorActivityLog::loadLatest($this->connection)->get('id'); + $firstActivityId = (int) $this->db->fetchOne( + $this->db->select()->from('director_activity_log', 'MIN(id)')->where('id > ?', $formerActivityId) + ); + if ($comment && strlen($comment)) { + $this->db->insert('director_activity_log_remark', [ + 'first_related_activity' => $firstActivityId, + 'last_related_activity' => $currentActivityId, + 'remark' => $comment, + ]); + } + }); + DirectorActivityLog::restoreUsername(); + } + + /** + * @param BranchActivity $activity + * @throws MergeError + * @throws \Icinga\Exception\NotFoundError + * @throws \Icinga\Module\Director\Exception\DuplicateKeyException + */ + protected function applyModification(BranchActivity $activity) + { + /** @var string|DbObject $class */ + $class = DbObjectTypeRegistry::classByType($activity->getObjectTable()); + $uuid = $activity->getObjectUuid(); + + $exists = $class::uniqueIdExists($uuid, $this->connection); + if ($activity->isActionCreate()) { + if ($exists) { + if (! $this->ignoresActivity($activity)) { + throw new MergeErrorRecreateOnMerge($activity); + } + } else { + $activity->createDbObject($this->connection)->store($this->connection); + } + } elseif ($activity->isActionDelete()) { + if ($exists) { + $activity->deleteDbObject($class::requireWithUniqueId($uuid, $this->connection)); + } elseif (! $this->ignoreDeleteWhenMissing && ! $this->ignoresActivity($activity)) { + throw new MergeErrorDeleteMissingObject($activity); + } + } else { + if ($exists) { + $activity->applyToDbObject($class::requireWithUniqueId($uuid, $this->connection))->store(); + // TODO: you modified an object, and related properties have been changed in the meantime. + // We're able to detect this with the given data, and might want to offer a rebase. + } elseif (! $this->ignoreModificationWhenMissing && ! $this->ignoresActivity($activity)) { + throw new MergeErrorModificationForMissingObject($activity); + } + } + } +} diff --git a/library/Director/Db/Branch/BranchModificationInspection.php b/library/Director/Db/Branch/BranchModificationInspection.php new file mode 100644 index 0000000..978ca5d --- /dev/null +++ b/library/Director/Db/Branch/BranchModificationInspection.php @@ -0,0 +1,93 @@ +connection = $connection; + $this->db = $connection->getDbAdapter(); + } + + public function describe($table, UuidInterface $uuid) + { + return static::describeModificationStatistics($this->loadSingleTableStats($table, $uuid)); + } + + public function describeBranch(UuidInterface $uuid) + { + $tables = [ + $this->translate('API Users') => BranchSupport::BRANCHED_TABLE_ICINGA_APIUSER, + $this->translate('Endpoints') => BranchSupport::BRANCHED_TABLE_ICINGA_COMMAND, + $this->translate('Zones') => BranchSupport::BRANCHED_TABLE_ICINGA_DEPENDENCY, + $this->translate('Commands') => BranchSupport::BRANCHED_TABLE_ICINGA_ENDPOINT, + $this->translate('Hosts') => BranchSupport::BRANCHED_TABLE_ICINGA_HOST, + $this->translate('Hostgroups') => BranchSupport::BRANCHED_TABLE_ICINGA_HOSTGROUP, + $this->translate('Services') => BranchSupport::BRANCHED_TABLE_ICINGA_NOTIFICATION, + $this->translate('Servicegroups') => BranchSupport::BRANCHED_TABLE_ICINGA_SCHEDULED_DOWNTIME, + $this->translate('Servicesets') => BranchSupport::BRANCHED_TABLE_ICINGA_SERVICE_SET, + $this->translate('Users') => BranchSupport::BRANCHED_TABLE_ICINGA_SERVICE, + $this->translate('Usergroups') => BranchSupport::BRANCHED_TABLE_ICINGA_SERVICEGROUP, + $this->translate('Timeperiods') => BranchSupport::BRANCHED_TABLE_ICINGA_TIMEPERIOD, + $this->translate('Notifications') => BranchSupport::BRANCHED_TABLE_ICINGA_USER, + $this->translate('Dependencies') => BranchSupport::BRANCHED_TABLE_ICINGA_USERGROUP, + $this->translate('Scheduled Downtimes') => BranchSupport::BRANCHED_TABLE_ICINGA_ZONE, + ]; + + $parts = new HtmlDocument(); + $parts->setSeparator(Html::tag('br')); + foreach ($tables as $label => $table) { + $info = $this->describe($table, $uuid); + if (! empty($info) && $info !== '-') { + $parts->add("$label: $info"); + } + } + + return $parts; + } + + public static function describeModificationStatistics($stats) + { + $t = StaticTranslator::get(); + $relevantStats = []; + if ($stats->cnt_created > 0) { + $relevantStats[] = sprintf($t->translate('%d created'), $stats->cnt_created); + } + if ($stats->cnt_deleted > 0) { + $relevantStats[] = sprintf($t->translate('%d deleted'), $stats->cnt_deleted); + } + if ($stats->cnt_modified > 0) { + $relevantStats[] = sprintf($t->translate('%d modified'), $stats->cnt_modified); + } + if (empty($relevantStats)) { + return '-'; + } + + return implode(', ', $relevantStats); + } + + public function loadSingleTableStats($table, UuidInterface $uuid) + { + $query = $this->db->select()->from($table, [ + 'cnt_created' => "SUM(CASE WHEN branch_created = 'y' THEN 1 ELSE 0 END)", + 'cnt_deleted' => "SUM(CASE WHEN branch_deleted = 'y' THEN 1 ELSE 0 END)", + 'cnt_modified' => "SUM(CASE WHEN branch_deleted = 'n' AND branch_created = 'n' THEN 1 ELSE 0 END)", + ])->where('branch_uuid = ?', $this->connection->quoteBinary($uuid->getBytes())); + + return $this->db->fetchRow($query); + } +} diff --git a/library/Director/Db/Branch/BranchSettings.php b/library/Director/Db/Branch/BranchSettings.php new file mode 100644 index 0000000..b3fd164 --- /dev/null +++ b/library/Director/Db/Branch/BranchSettings.php @@ -0,0 +1,121 @@ + $value) { + if (! static::propertyIsBranchSpecific($key)) { + if (is_resource($value)) { + $value = stream_get_contents($value); + } + if ($value !== null && static::propertyIsEncodedArray($key)) { + $value = Json::decode($value); + } + if ($value !== null && static::propertyIsRelatedSet($key)) { + // TODO: We might want to combine them (current VS branched) + $value = Json::decode($value); + } + if ($value !== null && static::propertyIsEncodedDictionary($key)) { + $value = Json::decode($value); + } + if ($value !== null) { + $normalized[$key] = $value; + } + } + } + static::flattenEncodedDicationaries($row); + if (isset($row['set_null'])) { + foreach (Json::decode($row['set_null']) as $property) { + $normalized[$property] = null; + } + } + foreach (self::BRANCH_BOOLEANS as $key) { + if ($row[$key] === 'y') { + $row[$key] = true; + } elseif ($row[$key] === 'n') { + $row[$key] = false; + } else { + throw new \RuntimeException(sprintf( + "Boolean DB property expected, got '%s' for '%s'", + $row[$key], + $key + )); + } + } + + return $normalized; + } + + public static function flattenProperty(array &$properties, $property) + { + // TODO: dots in varnames -> throw or escape? + if (isset($properties[$property])) { + foreach ((array) $properties[$property] as $key => $value) { + $properties["$property.$key"] = $value; + } + unset($properties[$property]); + } + } +} diff --git a/library/Director/Db/Branch/BranchStore.php b/library/Director/Db/Branch/BranchStore.php new file mode 100644 index 0000000..196d079 --- /dev/null +++ b/library/Director/Db/Branch/BranchStore.php @@ -0,0 +1,240 @@ +connection = $connection; + $this->db = $connection->getDbAdapter(); + } + + /** + * @param UuidInterface $uuid + * @return ?Branch + */ + public function fetchBranchByUuid(UuidInterface $uuid) + { + return $this->newFromDbResult( + $this->select()->where('b.uuid = ?', $this->connection->quoteBinary($uuid->getBytes())) + ); + } + + /** + * @param string $name + * @return ?Branch + */ + public function fetchBranchByName($name) + { + return $this->newFromDbResult($this->select()->where('b.branch_name = ?', $name)); + } + + public function cloneBranchForSync(Branch $branch, $newName, $owner) + { + $this->runTransaction(function ($db) use ($branch, $newName, $owner) { + $tables = BranchSupport::BRANCHED_TABLES; + $tables[] = self::TABLE_ACTIVITY; + $newBranch = $this->createBranchByName($newName, $owner); + $oldQuotedUuid = DbUtil::quoteBinaryCompat($branch->getUuid()->getBytes(), $db); + $quotedUuid = DbUtil::quoteBinaryCompat($newBranch->getUuid()->getBytes(), $db); + // $timestampNs = (int)floor(microtime(true) * 1000000); + // Hint: would love to do SELECT *, $quotedUuid AS branch_uuid FROM $table INTO $table + foreach ($tables as $table) { + $rows = $db->fetchAll($db->select()->from($table)->where('branch_uuid = ?', $oldQuotedUuid)); + foreach ($rows as $row) { + $modified = (array)$row; + $modified['branch_uuid'] = $quotedUuid; + if ($table === self::TABLE_ACTIVITY) { + $modified['timestamp_ns'] = round($modified['timestamp_ns'] / 1000000); + } + $db->insert($table, $modified); + } + } + }); + + return $this->fetchBranchByName($newName); + } + + protected function runTransaction($callback) + { + $db = $this->db; + $db->beginTransaction(); + try { + $callback($db); + $db->commit(); + } catch (\Exception $e) { + try { + $db->rollBack(); + } catch (\Exception $ignored) { + // + } + throw $e; + } + } + + public function wipeBranch(Branch $branch, $after = null) + { + $this->runTransaction(function ($db) use ($branch, $after) { + $tables = BranchSupport::BRANCHED_TABLES; + $tables[] = self::TABLE_ACTIVITY; + $quotedUuid = DbUtil::quoteBinaryCompat($branch->getUuid()->getBytes(), $db); + $where = $db->quoteInto('branch_uuid = ?', $quotedUuid); + foreach ($tables as $table) { + if ($after && $table === self::TABLE_ACTIVITY) { + $db->delete($table, $where . ' AND timestamp_ns > ' . (int) $after); + } else { + $db->delete($table, $where); + } + } + }); + + } + + protected function newFromDbResult($query) + { + if ($row = $this->db->fetchRow($query)) { + if (is_resource($row->uuid)) { + $row->uuid = stream_get_contents($row->uuid); + } + return Branch::fromDbRow($row); + } + + return null; + } + + public function setReadyForMerge(Branch $branch) + { + $update = [ + 'ts_merge_request' => (int) floor(microtime(true) * 1000000), + 'description' => $branch->getDescription(), + ]; + + $name = $branch->getName(); + if (preg_match('#^/enforced/(.+)$#', $name, $match)) { + $update['branch_name'] = '/merge/' . substr(sha1($branch->getUuid()->getBytes()), 0, 7) . '/' . $match[1]; + } + $this->db->update('director_branch', $update, $this->db->quoteInto( + 'uuid = ?', + $this->connection->quoteBinary($branch->getUuid()->getBytes()) + )); + } + + protected function select() + { + return $this->db->select()->from(['b' => 'director_branch'], [ + 'uuid' => 'b.uuid', + 'owner' => 'b.owner', + 'branch_name' => 'b.branch_name', + 'description' => 'b.description', + 'ts_merge_request' => 'b.ts_merge_request', + 'cnt_activities' => 'COUNT(ba.timestamp_ns)', + ])->joinLeft( + ['ba' => self::TABLE_ACTIVITY], + 'b.uuid = ba.branch_uuid', + [] + )->group('b.uuid'); + } + + /** + * @param string $name + * @return Branch + * @throws \Zend_Db_Adapter_Exception + */ + public function fetchOrCreateByName($name, $owner) + { + if ($branch = $this->fetchBranchByName($name)) { + return $branch; + } + + return $this->createBranchByName($name, $owner); + } + + /** + * @param string $branchName + * @param string $owner + * @return Branch + * @throws \Zend_Db_Adapter_Exception + */ + public function createBranchByName($branchName, $owner) + { + $uuid = Uuid::uuid4(); + $properties = [ + 'uuid' => $this->connection->quoteBinary($uuid->getBytes()), + 'branch_name' => $branchName, + 'owner' => $owner, + 'description' => null, + 'ts_merge_request' => null, + ]; + $this->db->insert(self::TABLE, $properties); + + if ($branch = static::fetchBranchByUuid($uuid)) { + return $branch; + } + + throw new \RuntimeException(sprintf( + 'Branch with UUID=%s has been created, but could not be fetched from DB', + $uuid->toString() + )); + } + + public function deleteByUuid(UuidInterface $uuid) + { + return $this->db->delete(self::TABLE, $this->db->quoteInto( + 'uuid = ?', + $this->connection->quoteBinary($uuid->getBytes()) + )); + } + + /** + * @param string $name + * @return int + */ + public function deleteByName($name) + { + return $this->db->delete(self::TABLE, $this->db->quoteInto( + 'branch_name = ?', + $name + )); + } + + public function delete(Branch $branch) + { + return $this->deleteByUuid($branch->getUuid()); + } + + /** + * @param Branch $branch + * @param ?int $after + * @return float|null + */ + public function getLastActivityTime(Branch $branch, $after = null) + { + $db = $this->db; + $query = $db->select() + ->from(self::TABLE_ACTIVITY, 'MAX(timestamp_ns)') + ->where('branch_uuid = ?', DbUtil::quoteBinaryCompat($branch->getUuid()->getBytes(), $db)); + if ($after) { + $query->where('timestamp_ns > ?', (int) $after); + } + + $last = $db->fetchOne($query); + if ($last) { + return $last / 1000000; + } + + return null; + } +} diff --git a/library/Director/Db/Branch/BranchSupport.php b/library/Director/Db/Branch/BranchSupport.php new file mode 100644 index 0000000..74be021 --- /dev/null +++ b/library/Director/Db/Branch/BranchSupport.php @@ -0,0 +1,91 @@ +get('object_type')) + ); + } +} diff --git a/library/Director/Db/Branch/BranchedObject.php b/library/Director/Db/Branch/BranchedObject.php new file mode 100644 index 0000000..0f276c2 --- /dev/null +++ b/library/Director/Db/Branch/BranchedObject.php @@ -0,0 +1,404 @@ +getObjectTable(), + $activity->getObjectUuid(), + $activity->getBranchUuid() + )->applyActivity($activity, $connection); + } + + public function store(Db $connection) + { + if ($this->object && ! $this->object->hasBeenModified() && empty($this->changes)) { + return false; + } + $db = $connection->getDbAdapter(); + + $properties = [ + 'branch_deleted' => $this->branchDeleted ? 'y' : 'n', + 'branch_created' => $this->branchCreated ? 'y' : 'n', + ] + $this->prepareChangedProperties(); + + $table = 'branched_' . $this->objectTable; + if ($this->loadedAsBranchedObject) { + return $db->update( + $table, + $properties, + $this->prepareWhereString($connection) + ) === 1; + } else { + try { + return $db->insert($table, $this->prepareKeyProperties($connection) + $properties) === 1; + } catch (\Exception $e) { + var_dump($e->getMessage()); + var_dump($this->prepareKeyProperties($connection) + $properties); + exit; + } + } + } + + public function delete(Db $connection) + { + $db = $connection->getDbAdapter(); + $table = 'branched_' . $this->objectTable; + $branchCreated = $db->fetchOne($this->filterQuery($db->select()->from($table, 'branch_created'), $connection)); + // We do not want to nullify all properties, therefore: delete & insert + $db->delete($table, $this->prepareWhereString($connection)); + + if (! $branchCreated) { + // No need to insert a deleted object in case this object lived in this branch only + return $db->insert($table, $this->prepareKeyProperties($connection) + [ + 'branch_deleted' => 'y', + 'branch_created' => 'n', + ]) === 1; + } + + return true; + } + + protected function prepareKeyProperties(Db $connection) + { + return [ + 'uuid' => $connection->quoteBinary($this->objectUuid->getBytes()), + 'branch_uuid' => $connection->quoteBinary($this->branchUuid->getBytes()), + ]; + } + + protected function prepareWhereString(Db $connection) + { + $db = $connection->getDbAdapter(); + $objectUuid = $connection->quoteBinary($this->objectUuid->getBytes()); + $branchUuid = $connection->quoteBinary($this->branchUuid->getBytes()); + + return $db->quoteInto('uuid = ?', $objectUuid) . $db->quoteInto(' AND branch_uuid = ?', $branchUuid); + } + + /** + * @param \Zend_Db_Select $query + * @param Db $connection + * @return \Zend_Db_Select + */ + protected function filterQuery(\Zend_Db_Select $query, Db $connection) + { + return $query->where('uuid = ?', $connection->quoteBinary($this->objectUuid->getBytes())) + ->where('branch_uuid = ?', $connection->quoteBinary($this->branchUuid->getBytes())); + } + + protected function prepareChangedProperties() + { + $properties = (array) $this->changes; + + foreach (BranchSettings::ENCODED_DICTIONARIES as $property) { + $this->combineFlatDictionaries($properties, $property); + } + foreach (BranchSettings::ENCODED_DICTIONARIES as $property) { + if (isset($properties[$property])) { + $properties[$property] = Json::encode($properties[$property]); + } + } + foreach (BranchSettings::ENCODED_ARRAYS as $property) { + if (isset($properties[$property])) { + $properties[$property] = Json::encode($properties[$property]); + } + } + foreach (BranchSettings::RELATED_SETS as $property) { + if (isset($properties[$property])) { + $properties[$property] = Json::encode($properties[$property]); + } + } + $setNull = []; + if (array_key_exists('disabled', $properties) && $properties['disabled'] === null) { + unset($properties['disabled']); + } + foreach ($properties as $key => $value) { + if ($value === null) { + $setNull[] = $key; + } + } + if (empty($setNull)) { + $properties['set_null'] = null; + } else { + $properties['set_null'] = Json::encode($setNull); + } + + return $properties; + } + + protected function combineFlatDictionaries(&$properties, $prefix) + { + $vars = []; + $length = strlen($prefix) + 1; + foreach ($properties as $key => $value) { + if (substr($key, 0, $length) === "$prefix.") { + $vars[substr($key, $length)] = $value; + } + } + if (! empty($vars)) { + foreach (array_keys($vars) as $key) { + unset($properties["$prefix.$key"]); + } + $properties[$prefix] = (object) $vars; + } + } + + public function applyActivity(BranchActivity $activity, Db $connection) + { + if ($activity->isActionDelete()) { + throw new \RuntimeException('Cannot apply a delete action'); + } + if ($activity->isActionCreate()) { + if ($this->hasBeenTouchedByBranch()) { + throw new \RuntimeException('Cannot apply a CREATE activity to an already branched object'); + } else { + $this->branchCreated = true; + } + } + + foreach ($activity->getModifiedProperties()->jsonSerialize() as $key => $value) { + $this->changes[$key] = $value; + } + + return $this; + } + + /** + * @param Db $connection + * @param string $objectTable + * @param UuidInterface $uuid + * @param Branch $branch + * @return static + * @throws NotFoundError + */ + public static function load(Db $connection, $objectTable, UuidInterface $uuid, Branch $branch) + { + $object = static::loadOptional($connection, $objectTable, $uuid, $branch->getUuid()); + if ($object->getOriginalDbObject() === null && ! $object->hasBeenTouchedByBranch()) { + throw new NotFoundError('Not found'); + } + + return $object; + } + + /** + * @return bool + */ + public function hasBeenTouchedByBranch() + { + return $this->loadedAsBranchedObject; + } + + /** + * @return bool + */ + public function hasBeenDeletedByBranch() + { + return $this->branchDeleted; + } + + /** + * @return bool + */ + public function hasBeenCreatedByBranch() + { + return $this->branchCreated; + } + + /** + * @return ?DbObject + */ + public function getOriginalDbObject() + { + return $this->object; + } + + /** + * @return ?DbObject + */ + public function getBranchedDbObject(Db $connection) + { + if ($this->object) { + $branched = DbObjectTypeRegistry::newObject($this->objectTable, [], $connection); + // object_type first, to avoid: + // I can only assign for applied objects or objects with native support for assignments + if ($this->object->hasProperty('object_type')) { + $branched->set('object_type', $this->object->get('object_type')); + } + $branched->set('id', $this->object->get('id')); + $branched->set('uuid', $this->object->get('uuid')); + foreach ((array) $this->object->toPlainObject(false, true) as $key => $value) { + if ($key === 'object_type') { + continue; + } + $branched->set($key, $value); + } + } else { + $branched = DbObjectTypeRegistry::newObject($this->objectTable, [], $connection); + $branched->setUniqueId($this->objectUuid); + } + if ($this->changes === null) { + return $branched; + } + foreach ($this->changes as $key => $value) { + if ($key === 'set_null') { + if ($value !== null) { + foreach ($value as $k) { + $branched->set($k, null); + } + } + } else { + $branched->set($key, $value); + } + } + + return $branched; + } + + /** + * @return UuidInterface + */ + public function getBranchUuid() + { + return $this->branchUuid; + } + + /** + * @param Db $connection + * @param string $table + * @param UuidInterface $uuid + * @param ?UuidInterface $branchUuid + * @return static + */ + protected static function loadOptional( + Db $connection, + $table, + UuidInterface $uuid, + UuidInterface $branchUuid = null + ) { + $class = DbObjectTypeRegistry::classByType($table); + if ($row = static::optionalTableRowByUuid($connection, $table, $uuid)) { + $object = $class::fromDbRow((array) $row, $connection); + } else { + $object = null; + } + + $self = new static(); + $self->object = $object; + $self->objectUuid = $uuid; + $self->branchUuid = $branchUuid; + $self->objectTable = $table; + + if ($branchUuid && $row = static::optionalBranchedTableRowByUuid($connection, $table, $uuid, $branchUuid)) { + if ($row->branch_deleted === 'y') { + $self->branchDeleted = true; + } elseif ($row->branch_created === 'y') { + $self->branchCreated = true; + } + $self->changes = BranchSettings::normalizeBranchedObjectFromDb($row); + $self->loadedAsBranchedObject = true; + } + + return $self; + } + + public static function exists( + Db $connection, + $table, + UuidInterface $uuid, + UuidInterface $branchUuid = null + ) { + if (static::optionalTableRowByUuid($connection, $table, $uuid)) { + return true; + } + + if ($branchUuid && static::optionalBranchedTableRowByUuid($connection, $table, $uuid, $branchUuid)) { + return true; + } + + return false; + } + + /** + * @param Db $connection + * @param string $table + * @param UuidInterface $uuid + * @return stdClass|boolean + */ + protected static function optionalTableRowByUuid(Db $connection, $table, UuidInterface $uuid) + { + $db = $connection->getDbAdapter(); + + return $db->fetchRow( + $db->select()->from($table)->where('uuid = ?', $connection->quoteBinary($uuid->getBytes())) + ); + } + + /** + * @param Db $connection + * @param string $table + * @param UuidInterface $uuid + * @return stdClass|boolean + */ + protected static function optionalBranchedTableRowByUuid( + Db $connection, + $table, + UuidInterface $uuid, + UuidInterface $branchUuid + ) { + $db = $connection->getDbAdapter(); + + $query = $db->select() + ->from("branched_$table") + ->where('uuid = ?', $connection->quoteBinary($uuid->getBytes())) + ->where('branch_uuid = ?', $connection->quoteBinary($branchUuid->getBytes())); + + return $db->fetchRow($query); + } + + protected function __construct() + { + } +} diff --git a/library/Director/Db/Branch/MergeError.php b/library/Director/Db/Branch/MergeError.php new file mode 100644 index 0000000..45c7b5e --- /dev/null +++ b/library/Director/Db/Branch/MergeError.php @@ -0,0 +1,37 @@ +activity = $activity; + parent::__construct($this->prepareMessage()); + } + + abstract protected function prepareMessage(); + + public function getObjectTypeName() + { + return preg_replace('/^icinga_/', '', $this->getActivity()->getObjectTable()); + } + + public function getNiceObjectName() + { + return $this->activity->getObjectName(); + } + + public function getActivity() + { + return $this->activity; + } +} diff --git a/library/Director/Db/Branch/MergeErrorDeleteMissingObject.php b/library/Director/Db/Branch/MergeErrorDeleteMissingObject.php new file mode 100644 index 0000000..71f89d1 --- /dev/null +++ b/library/Director/Db/Branch/MergeErrorDeleteMissingObject.php @@ -0,0 +1,15 @@ +translate('Cannot delete %s %s, it does not exist'), + $this->getObjectTypeName(), + $this->getNiceObjectName() + ); + } +} diff --git a/library/Director/Db/Branch/MergeErrorModificationForMissingObject.php b/library/Director/Db/Branch/MergeErrorModificationForMissingObject.php new file mode 100644 index 0000000..fa4e724 --- /dev/null +++ b/library/Director/Db/Branch/MergeErrorModificationForMissingObject.php @@ -0,0 +1,15 @@ +translate('Cannot apply modification for %s %s, object does not exist'), + $this->getObjectTypeName(), + $this->getNiceObjectName() + ); + } +} diff --git a/library/Director/Db/Branch/MergeErrorRecreateOnMerge.php b/library/Director/Db/Branch/MergeErrorRecreateOnMerge.php new file mode 100644 index 0000000..0bb8c40 --- /dev/null +++ b/library/Director/Db/Branch/MergeErrorRecreateOnMerge.php @@ -0,0 +1,15 @@ +translate('Cannot recreate %s %s'), + $this->getObjectTypeName(), + $this->getNiceObjectName() + ); + } +} diff --git a/library/Director/Db/Branch/PlainObjectPropertyDiff.php b/library/Director/Db/Branch/PlainObjectPropertyDiff.php new file mode 100644 index 0000000..0256798 --- /dev/null +++ b/library/Director/Db/Branch/PlainObjectPropertyDiff.php @@ -0,0 +1,50 @@ + $value) { + if (array_key_exists($key, $new)) { + if ($value === $new[$key]) { + $unchangedKeys[] = $key; + } + } else { + $new[$key] = null; + } + } + foreach ($unchangedKeys as $key) { + unset($new[$key]); + } + + return $new; + } + + protected static function flattenProperty(array &$properties, $property) + { + // TODO: dots in varnames -> throw or escape? + if (isset($properties[$property])) { + foreach ((array) $properties[$property] as $key => $value) { + $properties["$property.$key"] = $value; + } + unset($properties[$property]); + } + } +} diff --git a/library/Director/Db/Branch/UuidLookup.php b/library/Director/Db/Branch/UuidLookup.php new file mode 100644 index 0000000..b340e07 --- /dev/null +++ b/library/Director/Db/Branch/UuidLookup.php @@ -0,0 +1,141 @@ +getDbAdapter(); + $query = $db->select()->from('icinga_service', 'uuid'); + if ($objectType) { + $query->where('object_type = ?', $objectType); + } + $query = self::addKeyToQuery($connection, $query, $key); + if ($host) { + $query->where('host_id = ?', $host->get('id')); + } + if ($set) { + $query->where('service_set_id = ?', $set->get('id')); + } + $uuid = self::fetchOptionalUuid($connection, $query); + + if ($uuid === null && $branch->isBranch()) { + // TODO: use different tables? + $query = $db->select() + ->from('branched_icinga_service', 'uuid') + ->where('branch_uuid = ?', $connection->quoteBinary($branch->getUuid()->getBytes())); + if ($objectType) { + $query->where('object_type = ?', $objectType); + } + $query = self::addKeyToQuery($connection, $query, $key); + if ($host) { + // TODO: uuid? + $query->where('host = ?', $host->getObjectName()); + } + if ($set) { + $query->where('service_set = ?', $set->getObjectName()); + } + + $uuid = self::fetchOptionalUuid($connection, $query); + } + + return $uuid; + } + + /** + * @param int|string|array $key + * @param string $table + * @param Db $connection + * @param Branch $branch + * @return UuidInterface + * @throws NotFoundError + */ + public static function requireUuidForKey($key, $table, Db $connection, Branch $branch) + { + $uuid = self::findUuidForKey($key, $table, $connection, $branch); + if ($uuid === null) { + throw new NotFoundError('No such object available'); + } + + return $uuid; + } + + /** + * @param int|string|array $key + * @param string $table + * @param Db $connection + * @param Branch $branch + * @return ?UuidInterface + */ + public static function findUuidForKey($key, $table, Db $connection, Branch $branch) + { + $db = $connection->getDbAdapter(); + $query = self::addKeyToQuery($connection, $db->select()->from($table, 'uuid'), $key); + $uuid = self::fetchOptionalUuid($connection, $query); + if ($uuid === null && $branch->isBranch()) { + if (is_array($key) && isset($key['host_id'])) { + $key['host'] = IcingaHost::load($key['host_id'], $connection)->getObjectName(); + unset($key['host_id']); + } + $query = self::addKeyToQuery($connection, $db->select()->from("branched_$table", 'uuid'), $key); + $query->where('branch_uuid = ?', $connection->quoteBinary($branch->getUuid()->getBytes())); + $uuid = self::fetchOptionalUuid($connection, $query); + } + + return $uuid; + } + + protected static function addKeyToQuery(Db $connection, $query, $key) + { + if (is_int($key)) { + $query->where('id = ?', $key); + } elseif (is_string($key)) { + $query->where('object_name = ?', $key); + } else { + foreach ($key as $k => $v) { + $query->where($connection->getDbAdapter()->quoteIdentifier($k) . ' = ?', $v); + } + } + + return $query; + } + + protected static function fetchOptionalUuid(Db $connection, $query) + { + $result = $connection->getDbAdapter()->fetchOne($query); + if (is_resource($result)) { + $result = stream_get_contents($result); + } + if (is_string($result)) { + return Uuid::fromBytes($result); + } + + return null; + } +} diff --git a/library/Director/Db/Cache/CustomVariableCache.php b/library/Director/Db/Cache/CustomVariableCache.php new file mode 100644 index 0000000..243ecae --- /dev/null +++ b/library/Director/Db/Cache/CustomVariableCache.php @@ -0,0 +1,84 @@ +getConnection(); + $db = $connection->getDbAdapter(); + + $columns = array( + 'id' => sprintf('v.%s', $object->getVarsIdColumn()), + 'varname' => 'v.varname', + 'varvalue' => 'v.varvalue', + 'format' => 'v.format', + 'checksum' => '(NULL)', + ); + + if ($connection->isPgsql()) { + if ($connection->hasPgExtension('pgcrypto')) { + $columns['checksum'] = "DIGEST(v.varvalue || ';' || v.format, 'sha1')"; + } + } else { + $columns['checksum'] = "UNHEX(SHA1(v.varvalue || ';' || v.format))"; + } + + $query = $db->select()->from( + array('v' => $object->getVarsTableName()), + $columns + ); + + foreach ($db->fetchAll($query) as $row) { + $id = $row->id; + unset($row->id); + + if (is_resource($row->checksum)) { + $row->checksum = stream_get_contents($row->checksum); + } + + if (array_key_exists($id, $this->rowsById)) { + $this->rowsById[$id][] = $row; + } else { + $this->rowsById[$id] = array($row); + } + } + + Benchmark::measure('Filled CustomVariableCache'); + } + + public function getVarsForObject(IcingaObject $object) + { + $id = $object->id; + + if (array_key_exists($id, $this->rowsById)) { + if (! array_key_exists($id, $this->varsById)) { + $this->varsById[$id] = CustomVariables::forStoredRows( + $this->rowsById[$id] + ); + } + + return $this->varsById[$id]; + } else { + return new CustomVariables(); + } + } + + public function __destruct() + { + unset($this->db); + } +} diff --git a/library/Director/Db/Cache/GroupMembershipCache.php b/library/Director/Db/Cache/GroupMembershipCache.php new file mode 100644 index 0000000..d6d9e8b --- /dev/null +++ b/library/Director/Db/Cache/GroupMembershipCache.php @@ -0,0 +1,104 @@ +table = $object->getTableName(); + $this->type = $object->getShortTableName(); + + $this->groupClass = 'Icinga\\Module\\Director\\Objects\\Icinga' + . ucfirst($this->type) . 'Group'; + + Benchmark::measure('Initializing GroupMemberShipCache'); + $this->connection = $object->getConnection(); + $this->loadAllMemberships(); + Benchmark::measure('Filled GroupMemberShipCache'); + } + + protected function loadAllMemberships() + { + $db = $this->connection->getDbAdapter(); + $this->memberships = array(); + + $type = $this->type; + $table = $this->table; + + $query = $db->select()->from( + array('o' => $table), + array( + 'object_id' => 'o.id', + 'group_id' => 'g.id', + 'group_name' => 'g.object_name', + ) + )->join( + array('go' => $table . 'group_' . $type), + 'o.id = go.' . $type . '_id', + array() + )->join( + array('g' => $table . 'group'), + 'go.' . $type . 'group_id = g.id', + array() + )->order('g.object_name'); + + foreach ($db->fetchAll($query) as $row) { + if (! array_key_exists($row->object_id, $this->memberships)) { + $this->memberships[$row->object_id] = array(); + } + + $this->memberships[$row->object_id][$row->group_id] = $row->group_name; + } + } + + public function listGroupNamesForObject(IcingaObject $object) + { + if (array_key_exists($object->id, $this->memberships)) { + return array_values($this->memberships[$object->id]); + } + + return array(); + } + + public function listGroupIdsForObject(IcingaObject $object) + { + if (array_key_exists($object->id, $this->memberships)) { + return array_keys($this->memberships[$object->id]); + } + + return array(); + } + + public function getGroupsForObject(IcingaObject $object) + { + $groups = array(); + $class = $this->groupClass; + foreach ($this->listGroupIdsForObject($object) as $id) { + $object = $class::loadWithAutoIncId($id, $this->connection); + $groups[$object->object_name] = $object; + } + + return $groups; + } + + public function __destruct() + { + unset($this->connection); + } +} diff --git a/library/Director/Db/Cache/PrefetchCache.php b/library/Director/Db/Cache/PrefetchCache.php new file mode 100644 index 0000000..aa9f950 --- /dev/null +++ b/library/Director/Db/Cache/PrefetchCache.php @@ -0,0 +1,166 @@ +db = $db; + } + + /** + * @throws LogicException + * + * @return self + */ + public static function instance() + { + if (static::$instance === null) { + throw new LogicException('Prefetch cache has not been loaded'); + } + + return static::$instance; + } + + public static function forget() + { + DbObject::clearAllPrefetchCaches(); + self::$instance = null; + } + + public static function shouldBeUsed() + { + return self::$instance !== null; + } + + public function vars(IcingaObject $object) + { + return $this->varsCache($object)->getVarsForObject($object); + } + + public function groups(IcingaObject $object) + { + return $this->groupsCache($object)->getGroupsForObject($object); + } + + /* Hint: not implemented, this happens in DbObject right now + public function byObjectType($type) + { + if (! array_key_exists($type, $this->caches)) { + $this->caches[$type] = new ObjectCache($type); + } + + return $this->caches[$type]; + } + */ + + public function renderVar(CustomVariable $var, $renderExpressions = false) + { + $checksum = $var->getChecksum(); + if (null === $checksum) { + return $var->toConfigString($renderExpressions); + } else { + $checksum .= (int) $renderExpressions; + if (! array_key_exists($checksum, $this->renderedVars)) { + $this->renderedVars[$checksum] = $var->toConfigString($renderExpressions); + } + + return $this->renderedVars[$checksum]; + } + } + + public function hostServiceBlacklist() + { + if ($this->hostServiceBlacklist === null) { + $this->hostServiceBlacklist = new HostServiceBlacklist($this->db); + $this->hostServiceBlacklist->preloadMappings(); + } + + return $this->hostServiceBlacklist; + } + + /** + * @param IcingaObject $object + * @return CustomVariableCache + */ + protected function varsCache(IcingaObject $object) + { + $key = $object->getShortTableName(); + + if (! array_key_exists($key, $this->varsCaches)) { + $this->varsCaches[$key] = new CustomVariableCache($object); + } + + return $this->varsCaches[$key]; + } + + protected function groupsCache(IcingaObject $object) + { + $key = $object->getShortTableName(); + + if (! array_key_exists($key, $this->groupsCaches)) { + $this->groupsCaches[$key] = new GroupMembershipCache($object); + } + + return $this->groupsCaches[$key]; + } + + protected function templateTree(IcingaObject $object) + { + $key = $object->getShortTableName(); + if (! array_key_exists($key, $this->templateTrees)) { + $this->templateTrees[$key] = new TemplateTree( + $key, + $object->getConnection() + ); + } + + return $this->templateTrees[$key]; + } + + public function __destruct() + { + unset($this->groupsCaches); + unset($this->varsCaches); + unset($this->templateResolvers); + unset($this->renderedVars); + } +} diff --git a/library/Director/Db/DbSelectParenthesis.php b/library/Director/Db/DbSelectParenthesis.php new file mode 100644 index 0000000..191ad85 --- /dev/null +++ b/library/Director/Db/DbSelectParenthesis.php @@ -0,0 +1,24 @@ +select = $select; + } + + public function getSelect() + { + return $this->select; + } + + public function __toString() + { + return '(' . $this->select . ')'; + } +} diff --git a/library/Director/Db/DbUtil.php b/library/Director/Db/DbUtil.php new file mode 100644 index 0000000..f98e213 --- /dev/null +++ b/library/Director/Db/DbUtil.php @@ -0,0 +1,96 @@ +connection = $connection; + $this->db = $connection->getDbAdapter(); + } + + public function getTaskSummary() + { + $summary = array(); + foreach ($this->listTasks() as $name => $title) { + $func = 'count' . ucfirst($name); + $summary[$name] = (object) array( + 'name' => $name, + 'title' => $title, + 'count' => $this->$func() + ); + } + + return $summary; + } + + public function listTasks() + { + return array( + 'oldUndeployedConfigs' => N_('Undeployed configurations'), + 'unusedFiles' => N_('Unused rendered files'), + 'unlinkedImportedRowSets' => N_('Unlinked imported row sets'), + 'unlinkedImportedRows' => N_('Unlinked imported rows'), + 'unlinkedImportedProperties' => N_('Unlinked imported properties'), + 'resolveCache' => N_('(Host) group resolve cache'), + ); + } + + public function getPendingTaskSummary() + { + return array_filter( + $this->getTaskSummary(), + function ($task) { + return $task->count > 0; + } + ); + } + + public function hasPendingTasks() + { + return count($this->getPendingTaskSummary()) > 0; + } + + public function runAllTasks() + { + $result = array(); + + foreach ($this->listTasks() as $name => $task) { + $this->runTask($name); + } + + return $this; + } + + public function runTask($name) + { + $func = 'wipe' . ucfirst($name); + if (!method_exists($this, $func)) { + throw new NotFoundError( + 'There is no such task: %s', + $name + ); + } + + return $this->$func(); + } + + public function countOldUndeployedConfigs() + { + $conn = $this->connection; + $lastActivity = $conn->getLastActivityChecksum(); + + $sql = 'SELECT COUNT(*) FROM director_generated_config c' + . ' LEFT JOIN director_deployment_log d ON c.checksum = d.config_checksum' + . ' WHERE d.config_checksum IS NULL' + . ' AND ? != ' . $conn->dbHexFunc('c.last_activity_checksum'); + + return $this->db->fetchOne($sql, $lastActivity); + } + + public function wipeOldUndeployedConfigs() + { + $conn = $this->connection; + $lastActivity = $conn->getLastActivityChecksum(); + + if ($this->connection->isPgsql()) { + $sql = 'DELETE FROM director_generated_config' + . ' USING director_generated_config AS c' + . ' LEFT JOIN director_deployment_log d ON c.checksum = d.config_checksum' + . ' WHERE director_generated_config.checksum = c.checksum' + . ' AND d.config_checksum IS NULL' + . ' AND ? != ' . $conn->dbHexFunc('c.last_activity_checksum'); + } else { + $sql = 'DELETE c.* FROM director_generated_config c' + . ' LEFT JOIN director_deployment_log d ON c.checksum = d.config_checksum' + . ' WHERE d.config_checksum IS NULL' + . ' AND ? != ' . $conn->dbHexFunc('c.last_activity_checksum'); + } + + return $this->db->query($sql, $lastActivity); + } + + public function countUnusedFiles() + { + $sql = 'SELECT COUNT(*) FROM director_generated_file f' + . ' LEFT JOIN director_generated_config_file cf ON f.checksum = cf.file_checksum' + . ' WHERE cf.file_checksum IS NULL'; + + return $this->db->fetchOne($sql); + } + + public function wipeUnusedFiles() + { + if ($this->connection->isPgsql()) { + $sql = 'DELETE FROM director_generated_file' + . ' USING director_generated_file AS f' + . ' LEFT JOIN director_generated_config_file cf ON f.checksum = cf.file_checksum' + . ' WHERE director_generated_file.checksum = f.checksum' + . ' AND cf.file_checksum IS NULL'; + } else { + $sql = 'DELETE f FROM director_generated_file f' + . ' LEFT JOIN director_generated_config_file cf ON f.checksum = cf.file_checksum' + . ' WHERE cf.file_checksum IS NULL'; + } + + return $this->db->exec($sql); + } + + public function countUnlinkedImportedRowSets() + { + $sql = 'SELECT COUNT(*) FROM imported_rowset rs LEFT JOIN import_run r' + . ' ON r.rowset_checksum = rs.checksum WHERE r.id IS NULL'; + + return $this->db->fetchOne($sql); + } + + public function wipeUnlinkedImportedRowSets() + { + // This one removes imported_rowset and imported_rowset_row + // entries no longer used by any historic import + if ($this->connection->isPgsql()) { + $sql = 'DELETE FROM imported_rowset' + . ' USING imported_rowset AS rs' + . ' LEFT JOIN import_run r ON r.rowset_checksum = rs.checksum' + . ' WHERE imported_rowset.checksum = rs.checksum' + . ' AND r.id IS NULL'; + } else { + $sql = 'DELETE rs.* FROM imported_rowset rs' + . ' LEFT JOIN import_run r ON r.rowset_checksum = rs.checksum' + . ' WHERE r.id IS NULL'; + } + + return $this->db->exec($sql); + } + + public function countUnlinkedImportedRows() + { + $sql = 'SELECT COUNT(*) FROM imported_row r LEFT JOIN imported_rowset_row rsr' + . ' ON rsr.row_checksum = r.checksum WHERE rsr.row_checksum IS NULL'; + + return $this->db->fetchOne($sql); + } + + public function wipeUnlinkedImportedRows() + { + // This query removes imported_row and imported_row_property columns + // without related rowset + if ($this->connection->isPgsql()) { + $sql = 'DELETE FROM imported_row' + . ' USING imported_row AS r' + . ' LEFT JOIN imported_rowset_row rsr ON rsr.row_checksum = r.checksum' + . ' WHERE imported_row.checksum = r.checksum' + . ' AND rsr.row_checksum IS NULL'; + } else { + $sql = 'DELETE r.* FROM imported_row r' + . ' LEFT JOIN imported_rowset_row rsr ON rsr.row_checksum = r.checksum' + . ' WHERE rsr.row_checksum IS NULL'; + } + + return $this->db->exec($sql); + } + + public function countUnlinkedImportedProperties() + { + $sql = 'SELECT COUNT(*) FROM imported_property p LEFT JOIN imported_row_property rp' + . ' ON rp.property_checksum = p.checksum WHERE rp.property_checksum IS NULL'; + + return $this->db->fetchOne($sql); + } + + public function wipeUnlinkedImportedProperties() + { + // This query removes unlinked imported properties + if ($this->connection->isPgsql()) { + $sql = 'DELETE FROM imported_property' + . ' USING imported_property AS p' + . ' LEFT JOIN imported_row_property rp ON rp.property_checksum = p.checksum' + . ' WHERE imported_property.checksum = p.checksum' + . ' AND rp.property_checksum IS NULL'; + } else { + $sql = 'DELETE p.* FROM imported_property p' + . ' LEFT JOIN imported_row_property rp ON rp.property_checksum = p.checksum' + . ' WHERE rp.property_checksum IS NULL'; + } + + return $this->db->exec($sql); + } + + public function countResolveCache() + { + $helper = MembershipHousekeeping::instance('host', $this->connection); + return array_sum($helper->check()); + } + + public function wipeResolveCache() + { + $helper = MembershipHousekeeping::instance('host', $this->connection); + return $helper->update(); + } +} diff --git a/library/Director/Db/IcingaObjectFilterHelper.php b/library/Director/Db/IcingaObjectFilterHelper.php new file mode 100644 index 0000000..2eef406 --- /dev/null +++ b/library/Director/Db/IcingaObjectFilterHelper.php @@ -0,0 +1,133 @@ +get('id'); + } elseif (is_string($id) && ctype_digit($id)) { + return (int) $id; + } else { + throw new InvalidArgumentException(sprintf( + 'Numeric ID or IcingaObject expected, got %s', + // TODO: just type/class info? + var_export($id, 1) + )); + } + } + + /** + * @param ZfSelect $query + * @param IcingaObject|int|string $template + * @param string $tableAlias + * @param string $inheritanceType + * @return ZfSelect + */ + public static function filterByTemplate( + ZfSelect $query, + $template, + $tableAlias = 'o', + $inheritanceType = self::INHERIT_DIRECT + ) { + $i = $tableAlias . 'i'; + $o = $tableAlias; + $type = $template->getShortTableName(); + $db = $template->getDb(); + $id = static::wantId($template); + $sub = $db->select()->from( + array($i => "icinga_${type}_inheritance"), + array('e' => '(1)') + )->where("$i.${type}_id = $o.id"); + + if ($inheritanceType === self::INHERIT_DIRECT) { + $sub->where("$i.parent_${type}_id = ?", $id); + } elseif ($inheritanceType === self::INHERIT_INDIRECT + || $inheritanceType === self::INHERIT_DIRECT_OR_INDIRECT + ) { + $tree = new TemplateTree($type, $template->getConnection()); + $ids = $tree->listDescendantIdsFor($template); + if ($inheritanceType === self::INHERIT_DIRECT_OR_INDIRECT) { + $ids[] = $template->getAutoincId(); + } + + if (empty($ids)) { + $sub->where('(1 = 0)'); + } else { + $sub->where("$i.parent_${type}_id IN (?)", $ids); + } + } else { + throw new RuntimeException(sprintf( + 'Unable to understand "%s" inheritance', + $inheritanceType + )); + } + + return $query->where('EXISTS ?', $sub); + } + + public static function filterByHostgroups( + ZfSelect $query, + $type, + $groups, + $tableAlias = 'o' + ) { + if (empty($groups)) { + // Asked for an empty set of groups? Give no result + $query->where('(1 = 0)'); + } else { + $sub = $query->getAdapter()->select()->from( + array('go' => "icinga_${type}group_${type}"), + array('e' => '(1)') + )->join( + array('g' => "icinga_${type}group"), + "go.${type}group_id = g.id" + )->where("go.${type}_id = ${tableAlias}.id") + ->where('g.object_name IN (?)', $groups); + + $query->where('EXISTS ?', $sub); + } + } + + public static function filterByResolvedHostgroups( + ZfSelect $query, + $type, + $groups, + $tableAlias = 'o' + ) { + if (empty($groups)) { + // Asked for an empty set of groups? Give no result + $query->where('(1 = 0)'); + } else { + $sub = $query->getAdapter()->select()->from( + array('go' => "icinga_${type}group_${type}_resolved"), + array('e' => '(1)') + )->join( + array('g' => "icinga_${type}group"), + "go.${type}group_id = g.id", + [] + )->where("go.${type}_id = ${tableAlias}.id") + ->where('g.object_name IN (?)', $groups); + + $query->where('EXISTS ?', $sub); + } + } +} diff --git a/library/Director/Db/MembershipHousekeeping.php b/library/Director/Db/MembershipHousekeeping.php new file mode 100644 index 0000000..4d1ae88 --- /dev/null +++ b/library/Director/Db/MembershipHousekeeping.php @@ -0,0 +1,135 @@ +connection = $connection; + + if ($this->groupType === null) { + $this->groupType = $this->type . 'Group'; + } + } + + /** + * @param string $type + * @param DbConnection $connection + * + * @return static + */ + public static function instance($type, $connection) + { + if (! array_key_exists($type, self::$instances)) { + /** @var MembershipHousekeeping $class */ + $class = 'Icinga\\Module\\Director\\Db\\' . ucfirst($type) . 'MembershipHousekeeping'; + + /** @var MembershipHousekeeping $helper */ + self::$instances[$type] = new $class($connection); + } + + return self::$instances[$type]; + } + + protected function prepare() + { + if ($this->prepared) { + return $this; + } + + $this->prepareCache(); + $this->resolver()->defer(); + + $this->objects = IcingaObject::loadAllByType($this->type, $this->connection); + $this->resolver()->addObjects($this->objects); + + $this->groups = IcingaObject::loadAllByType($this->groupType, $this->connection); + $this->resolver()->addGroups($this->groups); + + MemoryLimit::raiseTo('1024M'); + + $this->prepared = true; + + return $this; + } + + public function check() + { + $this->prepare(); + + $resolver = $this->resolver()->checkDb(); + + return array($resolver->getNewMappings(), $resolver->getOutdatedMappings()); + } + + public function update() + { + $this->prepare(); + + $this->resolver()->refreshDb(true); + + return true; + } + + protected function prepareCache() + { + PrefetchCache::initialize($this->connection); + + IcingaObject::prefetchAllRelationsByType($this->type, $this->connection); + } + + protected function resolver() + { + if ($this->resolver === null) { + /** @var GroupMembershipResolver $class */ + $class = 'Icinga\\Module\\Director\\Objects\\' . ucfirst($this->type) . 'GroupMembershipResolver'; + $this->resolver = new $class($this->connection); + } + + return $this->resolver; + } + + /** + * @return IcingaObject[] + */ + public function getObjects() + { + return $this->objects; + } + + /** + * @return IcingaObjectGroup[] + */ + public function getGroups() + { + return $this->groups; + } +} diff --git a/library/Director/Db/Migration.php b/library/Director/Db/Migration.php new file mode 100644 index 0000000..5685121 --- /dev/null +++ b/library/Director/Db/Migration.php @@ -0,0 +1,70 @@ +version = $version; + $this->sql = $sql; + } + + /** + * @param DbConnection $connection + * @return $this + */ + public function apply(DbConnection $connection) + { + /** @var \Zend_Db_Adapter_Pdo_Abstract $db */ + $db = $connection->getDbAdapter(); + + // TODO: this is fragile and depends on accordingly written schema files: + $queries = preg_split( + '/[\n\s\t]*\;[\n\s\t]+/s', + $this->sql, + -1, + PREG_SPLIT_NO_EMPTY + ); + + if (empty($queries)) { + throw new RuntimeException(sprintf( + 'Migration %d has no queries', + $this->version + )); + } + + try { + foreach ($queries as $query) { + if (preg_match('/^(?:OPTIMIZE|EXECUTE) /i', $query)) { + $db->query($query); + } else { + $db->exec($query); + } + } + } catch (Exception $e) { + throw new RuntimeException(sprintf( + 'Migration %d failed (%s) while running %s', + $this->version, + $e->getMessage(), + $query + )); + } + + return $this; + } +} diff --git a/library/Director/Db/Migrations.php b/library/Director/Db/Migrations.php new file mode 100644 index 0000000..2310408 --- /dev/null +++ b/library/Director/Db/Migrations.php @@ -0,0 +1,239 @@ += 1.4.0, you're running %s." + . ' Please either upgrade PHP or downgrade Icinga Director', + PHP_VERSION + ); + } + $this->connection = $connection; + $this->db = $connection->getDbAdapter(); + } + + public function getLastMigrationNumber() + { + try { + $query = $this->db->select()->from( + array('m' => $this->getTableName()), + array('schema_version' => 'MAX(schema_version)') + ); + + return (int) $this->db->fetchOne($query); + } catch (Exception $e) { + return 0; + } + } + + protected function getTableName() + { + return $this->getModuleName() . '_schema_migration'; + } + + public function hasSchema() + { + return $this->listPendingMigrations() !== array(0); + } + + public function hasPendingMigrations() + { + return $this->countPendingMigrations() > 0; + } + + public function countPendingMigrations() + { + return count($this->listPendingMigrations()); + } + + /** + * @return Migration[] + */ + public function getPendingMigrations() + { + $migrations = array(); + foreach ($this->listPendingMigrations() as $version) { + $migrations[] = new Migration( + $version, + $this->loadMigrationFile($version) + ); + } + + return $migrations; + } + + /** + * @return $this + */ + public function applyPendingMigrations() + { + // Ensure we have enough time to migrate + ini_set('max_execution_time', 0); + + foreach ($this->getPendingMigrations() as $migration) { + $migration->apply($this->connection); + } + + return $this; + } + + public function listPendingMigrations() + { + $lastMigration = $this->getLastMigrationNumber(); + if ($lastMigration === 0) { + return array(0); + } + + return $this->listMigrationsAfter($this->getLastMigrationNumber()); + } + + public function listAllMigrations() + { + $dir = $this->getMigrationsDir(); + if (! is_readable($dir)) { + return array(); + } + + $versions = array(); + + foreach (new DirectoryIterator($this->getMigrationsDir()) as $file) { + if ($file->isDot()) { + continue; + } + + $filename = $file->getFilename(); + if (preg_match('/^upgrade_(\d+)\.sql$/', $filename, $match)) { + $versions[] = $match[1]; + } + } + + sort($versions); + + return $versions; + } + + public function loadMigrationFile($version) + { + if ($version === 0) { + $filename = $this->getFullSchemaFile(); + } else { + $filename = $this->getMigrationFileName($version); + } + + return file_get_contents($filename); + } + + public function hasBeenDowngraded() + { + return ! $this->hasMigrationFile($this->getLastMigrationNumber()); + } + + public function hasMigrationFile($version) + { + return \file_exists($this->getMigrationFileName($version)); + } + + protected function getMigrationFileName($version) + { + return sprintf( + '%s/upgrade_%d.sql', + $this->getMigrationsDir(), + $version + ); + } + + protected function listMigrationsAfter($version) + { + $filtered = array(); + foreach ($this->listAllMigrations() as $available) { + if ($available > $version) { + $filtered[] = $available; + } + } + + return $filtered; + } + + protected function getMigrationsDir() + { + if ($this->migrationsDir === null) { + $this->migrationsDir = $this->getSchemaDir( + $this->connection->getDbType() . '-migrations' + ); + } + + return $this->migrationsDir; + } + + protected function getFullSchemaFile() + { + return $this->getSchemaDir( + $this->connection->getDbType() . '.sql' + ); + } + + protected function getSchemaDir($sub = null) + { + try { + $dir = $this->getModuleDir('/schema'); + } catch (ProgrammingError $e) { + throw new RuntimeException( + 'Unable to detect the schema directory for this module', + 0, + $e + ); + } + if ($sub === null) { + return $dir; + } else { + return $dir . '/' . ltrim($sub, '/'); + } + } + + /** + * @param string $sub + * @return string + * @throws ProgrammingError + */ + protected function getModuleDir($sub = '') + { + return Icinga::app()->getModuleManager()->getModuleDir( + $this->getModuleName(), + $sub + ); + } + + protected function getModuleName() + { + return $this->getModuleNameForObject($this); + } + + protected function getModuleNameForObject($object) + { + $class = get_class($object); + // Hint: Icinga\Module\ -> 14 chars + return lcfirst(substr($class, 14, strpos($class, '\\', 15) - 14)); + } +} -- cgit v1.2.3