From f66ab8dae2f3d0418759f81a3a64dc9517a62449 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 14 Apr 2024 15:17:31 +0200 Subject: Adding upstream version 1.10.2. Signed-off-by: Daniel Baumann --- library/Director/Import/Import.php | 481 +++++++++++ library/Director/Import/ImportSourceCoreApi.php | 92 ++ .../Director/Import/ImportSourceDirectorObject.php | 120 +++ library/Director/Import/ImportSourceLdap.php | 90 ++ library/Director/Import/ImportSourceRestApi.php | 380 +++++++++ library/Director/Import/ImportSourceSql.php | 70 ++ .../PurgeStrategy/ImportRunBasedPurgeStrategy.php | 90 ++ .../PurgeStrategy/PurgeNothingPurgeStrategy.php | 11 + .../Import/PurgeStrategy/PurgeStrategy.php | 31 + library/Director/Import/Sync.php | 942 +++++++++++++++++++++ library/Director/Import/SyncUtils.php | 153 ++++ 11 files changed, 2460 insertions(+) create mode 100644 library/Director/Import/Import.php create mode 100644 library/Director/Import/ImportSourceCoreApi.php create mode 100644 library/Director/Import/ImportSourceDirectorObject.php create mode 100644 library/Director/Import/ImportSourceLdap.php create mode 100644 library/Director/Import/ImportSourceRestApi.php create mode 100644 library/Director/Import/ImportSourceSql.php create mode 100644 library/Director/Import/PurgeStrategy/ImportRunBasedPurgeStrategy.php create mode 100644 library/Director/Import/PurgeStrategy/PurgeNothingPurgeStrategy.php create mode 100644 library/Director/Import/PurgeStrategy/PurgeStrategy.php create mode 100644 library/Director/Import/Sync.php create mode 100644 library/Director/Import/SyncUtils.php (limited to 'library/Director/Import') diff --git a/library/Director/Import/Import.php b/library/Director/Import/Import.php new file mode 100644 index 0000000..f82454d --- /dev/null +++ b/library/Director/Import/Import.php @@ -0,0 +1,481 @@ + property + * + * @var array + */ + private $rowProperties; + + /** + * Whether this rowset exists, for caching purposes + * + * @var boolean + */ + private $rowsetExists; + + protected $properties = array(); + + /** + * Checksums of all rows + */ + private $rowChecksums; + + public function __construct(ImportSource $source) + { + $this->source = $source; + $this->connection = $source->getConnection(); + $this->db = $this->connection->getDbAdapter(); + } + + /** + * Whether this import provides modified data + * + * @return boolean + */ + public function providesChanges() + { + return ! $this->rowsetExists() + || ! $this->lastRowsetIs($this->rowsetChecksum()); + } + + /** + * Trigger an import run + * + * @return int Last import run ID + */ + public function run() + { + if ($this->providesChanges() && ! $this->rowsetExists()) { + $this->storeRowset(); + } + + $this->db->insert( + 'import_run', + array( + 'source_id' => $this->source->get('id'), + 'rowset_checksum' => $this->quoteBinary($this->rowsetChecksum()), + 'start_time' => date('Y-m-d H:i:s'), + 'succeeded' => 'y' + ) + ); + if ($this->connection->isPgsql()) { + return $this->db->lastInsertId('import_run', 'id'); + } else { + return $this->db->lastInsertId(); + } + } + + /** + * Whether there are no rows to be fetched from import source + * + * @return boolean + */ + public function isEmpty() + { + $rows = $this->checksummedRows(); + return empty($rows); + } + + /** + * Checksum of all available rows + * + * @return string + */ + protected function & rowsetChecksum() + { + if ($this->rowsetChecksum === null) { + $this->prepareChecksummedRows(); + } + + return $this->rowsetChecksum; + } + + /** + * All rows + * + * @return array + */ + protected function & checksummedRows() + { + if ($this->rows === null) { + $this->prepareChecksummedRows(); + } + + return $this->rows; + } + + /** + * Checksum of all available rows + * + * @return array + */ + protected function & rawData() + { + if ($this->data === null) { + $this->data = ImportSourceHook::forImportSource( + $this->source + )->fetchData(); + Benchmark::measure('Fetched all data from Import Source'); + $this->source->applyModifiers($this->data); + Benchmark::measure('Applied Property Modifiers to imported data'); + } + + return $this->data; + } + + /** + * Prepare and remember an ImportedProperty + * + * @param string $key + * @param mixed $rawValue + * + * @return array + */ + protected function prepareImportedProperty($key, $rawValue) + { + if (is_array($rawValue) || is_bool($rawValue) || is_int($rawValue) || is_float($rawValue)) { + $value = json_encode($rawValue); + $format = 'json'; + } elseif ($rawValue instanceof stdClass) { + $value = json_encode($this->sortObject($rawValue)); + $format = 'json'; + } else { + $value = $rawValue; + $format = 'string'; + } + + $checksum = sha1(sprintf('%s=(%s)%s', $key, $format, $value), true); + + if (! array_key_exists($checksum, $this->properties)) { + $this->properties[$checksum] = array( + 'checksum' => $this->quoteBinary($checksum), + 'property_name' => $key, + 'property_value' => $value, + 'format' => $format + ); + } + + return $this->properties[$checksum]; + } + + /** + * Walk through each row, prepare properties and calculate checksums + */ + protected function prepareChecksummedRows() + { + $keyColumn = $this->source->get('key_column'); + $this->rows = array(); + $this->rowProperties = array(); + $objects = array(); + $rowCount = 0; + + foreach ($this->rawData() as $row) { + $rowCount++; + + // Key column must be set + if (! isset($row->$keyColumn)) { + throw new IcingaException( + 'No key column "%s" in row %d', + $keyColumn, + $rowCount + ); + } + + $object_name = $row->$keyColumn; + + // Check for name collision + if (array_key_exists($object_name, $objects)) { + throw new IcingaException( + 'Duplicate entry: %s', + $object_name + ); + } + + $rowChecksums = array(); + $keys = array_keys((array) $row); + sort($keys); + + foreach ($keys as $key) { + // TODO: Specify how to treat NULL values. Ignoring for now. + // One option might be to import null (checksum '(null)') + // and to provide a flag at sync time + if ($row->$key === null) { + continue; + } + + $property = $this->prepareImportedProperty($key, $row->$key); + $rowChecksums[] = $property['checksum']; + } + + $checksum = sha1($object_name . ';' . implode(';', $rowChecksums), true); + if (array_key_exists($checksum, $this->rows)) { + die('WTF, collision?'); + } + + $this->rows[$checksum] = array( + 'checksum' => $this->quoteBinary($checksum), + 'object_name' => $object_name + ); + + $this->rowProperties[$checksum] = $rowChecksums; + + $objects[$object_name] = $checksum; + } + + $this->rowChecksums = array_keys($this->rows); + $this->rowsetChecksum = sha1(implode(';', $this->rowChecksums), true); + return $this; + } + + /** + * Store our new rowset + */ + protected function storeRowset() + { + $db = $this->db; + $rowset = $this->rowsetChecksum(); + $rows = $this->checksummedRows(); + + $db->beginTransaction(); + + try { + if ($this->isEmpty()) { + $newRows = array(); + $newProperties = array(); + } else { + $newRows = $this->newChecksums('imported_row', $this->rowChecksums); + $newProperties = $this->newChecksums('imported_property', array_keys($this->properties)); + } + + $db->insert('imported_rowset', array('checksum' => $this->quoteBinary($rowset))); + + foreach ($newProperties as $checksum) { + $db->insert('imported_property', $this->properties[$checksum]); + } + + foreach ($newRows as $row) { + try { + $db->insert('imported_row', $rows[$row]); + foreach ($this->rowProperties[$row] as $property) { + $db->insert('imported_row_property', array( + 'row_checksum' => $this->quoteBinary($row), + 'property_checksum' => $property + )); + } + } catch (Exception $e) { + throw new IcingaException( + "Error while storing a row for '%s' into database: %s", + $rows[$row]['object_name'], + $e->getMessage() + ); + } + } + + foreach (array_keys($rows) as $row) { + $db->insert( + 'imported_rowset_row', + array( + 'rowset_checksum' => $this->quoteBinary($rowset), + 'row_checksum' => $this->quoteBinary($row) + ) + ); + } + + $db->commit(); + + $this->rowsetExists = true; + } catch (Exception $e) { + try { + $db->rollBack(); + } catch (Exception $e) { + // Well... + } + // Eventually throws details for invalid UTF8 characters + RecursiveUtf8Validator::validateRows($this->data); + throw $e; + } + } + + /** + * Whether the last run of this import matches the given checksum + * + * @param string $checksum Binary checksum + * + * @return bool + */ + protected function lastRowsetIs($checksum) + { + return $this->connection->getLatestImportedChecksum($this->source->get('id')) + === bin2hex($checksum); + } + + /** + * Whether our rowset already exists in the database + * + * @return boolean + */ + protected function rowsetExists() + { + if (null === $this->rowsetExists) { + $this->rowsetExists = 0 === count( + $this->newChecksums( + 'imported_rowset', + array($this->rowsetChecksum()) + ) + ); + } + + return $this->rowsetExists; + } + + /** + * Finde new checksums for a specific table + * + * Accepts an array of checksums and gives you an array with those checksums + * that are missing in the given table + * + * @param string $table Database table name + * @param array $checksums Array with the checksums that should be verified + * + * @return array + */ + protected function newChecksums($table, $checksums) + { + $db = $this->db; + + // TODO: The following is a quickfix for binary data corrpution reported + // in https://github.com/zendframework/zf1/issues/655 caused by + // https://github.com/zendframework/zf1/commit/2ac9c30f + // + // Should be reverted once fixed, eventually with a check continueing + // to use this workaround for specific ZF versions (1.12.16 and 1.12.17 + // so far). Alternatively we could also use a custom quoteInto method. + + // The former query looked as follows: + // + // $query = $db->select()->from($table, 'checksum') + // ->where('checksum IN (?)', $checksums) + // ... + // return array_diff($checksums, $existing); + + $hexed = array_map('bin2hex', $checksums); + + $conn = $this->connection; + $query = $db + ->select() + ->from( + array('c' => $table), + array('checksum' => $conn->dbHexFunc('c.checksum')) + )->where( + $conn->dbHexFunc('c.checksum') . ' IN (?)', + $hexed + ); + + $existing = $db->fetchCol($query); + $new = array_diff($hexed, $existing); + + return array_map('hex2bin', $new); + } + + /** + * Sort a given stdClass object by property name + * + * @param stdClass $object + * + * @return object + */ + protected function sortObject($object) + { + $array = (array) $object; + foreach ($array as $key => $val) { + $this->sortElement($val); + } + ksort($array); + return (object) $array; + } + + /** + * Walk through a given array and sort all children + * + * Please note that the array itself will NOT be sorted, as arrays must + * keep their ordering + * + * @param array $array + */ + protected function sortArrayObject(&$array) + { + foreach ($array as $key => $val) { + $this->sortElement($val); + } + } + + /** + * Recursively sort a given property + * + * @param mixed $el + */ + protected function sortElement(&$el) + { + if (is_array($el)) { + $this->sortArrayObject($el); + } elseif ($el instanceof stdClass) { + $el = $this->sortObject($el); + } + } + + protected function quoteBinary($bin) + { + return $this->connection->quoteBinary($bin); + } +} diff --git a/library/Director/Import/ImportSourceCoreApi.php b/library/Director/Import/ImportSourceCoreApi.php new file mode 100644 index 0000000..6d590ec --- /dev/null +++ b/library/Director/Import/ImportSourceCoreApi.php @@ -0,0 +1,92 @@ +getSetting('object_type') . 'Objects'; + $objects = $this->api()->$func(); + $result = array(); + foreach ($objects as $object) { + $result[] = $object->toPlainObject(); + } + + return $result; + } + + public function listColumns() + { + $res = $this->fetchData(); + if (empty($data)) { + return array('object_name'); + } + + return array_keys((array) $res[0]); + } + + public static function getDefaultKeyColumnName() + { + return 'object_name'; + } + + public static function addSettingsFormFields(QuickForm $form) + { + $form->addElement('select', 'object_type', array( + 'label' => 'Object type', + 'required' => true, + 'multiOptions' => $form->optionalEnum(self::enumObjectTypes($form)) + )); + } + + protected static function enumObjectTypes($form) + { + $types = array( + 'CheckCommand' => $form->translate('Check Commands'), + 'NotificationCommand' => $form->translate('Notification Commands'), + 'Endpoint' => $form->translate('Endpoints'), + 'Host' => $form->translate('Hosts'), + 'HostGroup' => $form->translate('Hostgroups'), + 'User' => $form->translate('Users'), + 'UserGroup' => $form->translate('Usergroups'), + 'Zone' => $form->translate('Zones'), + ); + + asort($types); + return $types; + } + + protected function api() + { + if ($this->api === null) { + $endpoint = $this->db()->getDeploymentEndpoint(); + $this->api = $endpoint->api()->setDb($this->db()); + } + + return $this->api; + } + + protected function db() + { + if ($this->db === null) { + $resourceName = Config::module('director')->get('db', 'resource'); + if ($resourceName) { + $this->db = Db::fromResourceName($resourceName); + } + } + + return $this->db; + } +} diff --git a/library/Director/Import/ImportSourceDirectorObject.php b/library/Director/Import/ImportSourceDirectorObject.php new file mode 100644 index 0000000..e3f56fc --- /dev/null +++ b/library/Director/Import/ImportSourceDirectorObject.php @@ -0,0 +1,120 @@ +db(); + $objectClass = $this->getSetting('object_class'); + $objectType = $this->getSetting('object_type'); + /** @var IcingaObject $class fake type hint, it's a string */ + $class = DbObjectTypeRegistry::classByType($objectClass); + if ($objectType) { + $dummy = $class::create(); + $query = $db->getDbAdapter()->select() + ->from($dummy->getTableName()) + ->where('object_type = ?', $objectType); + } else { + $query = null; + } + $result = []; + $resolved = $this->getSetting('resolved') === 'y'; + foreach ($class::loadAllByType($objectClass, $db, $query) as $object) { + $result[] = $object->toPlainObject($resolved); + } + if ($objectClass === 'zone') { + $this->enrichZonesWithDeploymentZone($result); + } + return $result; + } + + protected function enrichZonesWithDeploymentZone(&$zones) + { + $masterZone = $this->db()->getMasterZoneName(); + foreach ($zones as $zone) { + $zone->is_master_zone = $zone->object_name === $masterZone; + } + } + + public static function addSettingsFormFields(QuickForm $form) + { + /** @var ImportSourceForm $form */ + Util::addDbResourceFormElement($form, 'resource'); + $form->getElement('resource') + ->setValue(Config::module('director')->get('db', 'resource')); + $form->addElement('select', 'object_class', [ + 'label' => $form->translate('Director Object'), + 'multiOptions' => [ + 'host' => $form->translate('Host'), + 'endpoint' => $form->translate('Endpoint'), + 'zone' => $form->translate('Zone'), + ], + 'required' => true, + ]); + $form->addElement('select', 'object_type', [ + 'label' => $form->translate('Object Type'), + 'multiOptions' => [ + null => $form->translate('All Object Types'), + 'object' => $form->translate('Objects'), + 'template' => $form->translate('Templates'), + 'external_object' => $form->translate('External Objects'), + 'apply' => $form->translate('Apply Rules'), + ], + ]); + + /** @var $form \Icinga\Module\Director\Web\Form\DirectorObjectForm */ + $form->addBoolean('resolved', [ + 'label' => $form->translate('Resolved'), + ], 'n'); + + return $form; + } + + protected function db() + { + if ($this->db === null) { + $this->db = Db::fromResourceName($this->settings['resource']); + } + + return $this->db; + } + + public function listColumns() + { + $rows = $this->fetchData(); + $columns = []; + + foreach ($rows as $object) { + foreach (array_keys((array) $object) as $column) { + if (! isset($columns[$column])) { + $columns[] = $column; + } + } + } + + return $columns; + } +} diff --git a/library/Director/Import/ImportSourceLdap.php b/library/Director/Import/ImportSourceLdap.php new file mode 100644 index 0000000..4518565 --- /dev/null +++ b/library/Director/Import/ImportSourceLdap.php @@ -0,0 +1,90 @@ +listColumns(); + $query = $this->connection() + ->select() + ->setUsePagedResults() + ->from($this->settings['objectclass'], $columns); + + if ($base = $this->settings['base']) { + $query->setBase($base); + } + if ($filter = $this->settings['filter']) { + $query->setNativeFilter($filter); + } + + if (in_array('dn', $columns)) { + $result = $query->fetchAll(); + foreach ($result as $dn => $row) { + $row->dn = $dn; + } + + return $result; + } else { + return $query->fetchAll(); + } + } + + public function listColumns() + { + return preg_split('/,\s*/', $this->settings['query'], -1, PREG_SPLIT_NO_EMPTY); + } + + public static function addSettingsFormFields(QuickForm $form) + { + Util::addLDAPResourceFormElement($form, 'resource'); + $form->addElement('text', 'base', array( + 'label' => $form->translate('LDAP Search Base'), + 'description' => $form->translate( + 'Your LDAP search base. Often something like OU=Users,OU=HQ,DC=your,DC=company,DC=tld' + ) + )); + $form->addElement('text', 'objectclass', array( + 'label' => $form->translate('Object class'), + 'description' => $form->translate( + 'An object class to search for. Might be "user", "group", "computer" or similar' + ) + )); + $form->addElement('text', 'filter', array( + 'label' => 'LDAP filter', + 'description' => $form->translate( + 'A custom LDAP filter to use in addition to the object class. This allows' + . ' for a lot of flexibility but requires LDAP filter skills. Simple filters' + . ' might look as follows: operatingsystem=*server*' + ) + )); + $form->addElement('textarea', 'query', array( + 'label' => $form->translate('Properties'), + 'description' => $form->translate( + 'The LDAP properties that should be fetched. This is required to be a' + . ' comma-separated list like: "cn, dnshostname, operatingsystem, sAMAccountName"' + ), + 'spellcheck' => 'false', + 'required' => true, + 'rows' => 5, + )); + return $form; + } + + protected function connection() + { + if ($this->connection === null) { + $this->connection = ResourceFactory::create($this->settings['resource']); + } + + return $this->connection; + } +} diff --git a/library/Director/Import/ImportSourceRestApi.php b/library/Director/Import/ImportSourceRestApi.php new file mode 100644 index 0000000..dc772e1 --- /dev/null +++ b/library/Director/Import/ImportSourceRestApi.php @@ -0,0 +1,380 @@ +getRestApi()->get( + $this->getUrl(), + null, + $this->buildHeaders() + ); + $result = $this->extractProperty($result); + + return (array) $result; + } + + public function listColumns() + { + $rows = $this->fetchData(); + $columns = []; + + foreach ($rows as $object) { + foreach (array_keys((array) $object) as $column) { + if (! isset($columns[$column])) { + $columns[] = $column; + } + } + } + + return $columns; + } + + /** + * Extract result from a property specified + * + * A simple key, like "objects", will take the final result from key objects + * + * If you have a deeper key like "objects" under the key "results", specify this as "results.objects". + * + * When a key of the JSON object contains a literal ".", this can be escaped as + * + * @param $result + * + * @return mixed + */ + protected function extractProperty($result) + { + $property = $this->getSetting('extract_property'); + if (! $property) { + return $result; + } + + $parts = preg_split('~(?$part; + } else { + throw new \RuntimeException(sprintf( + 'Result has no "%s" property. Available keys: %s', + $part, + implode(', ', array_keys((array) $data)) + )); + } + } + + return $data; + } + + protected function buildHeaders() + { + $headers = []; + + $text = $this->getSetting('headers', ''); + foreach (preg_split('~\r?\n~', $text, -1, PREG_SPLIT_NO_EMPTY) as $header) { + $header = trim($header); + $parts = preg_split('~\s*:\s*~', $header, 2); + if (count($parts) < 2) { + throw new InvalidPropertyException('Could not parse header: "%s"', $header); + } + + $headers[$parts[0]] = $parts[1]; + } + + return $headers; + } + + /** + * @param QuickForm $form + * @throws \Zend_Form_Exception + */ + public static function addSettingsFormFields(QuickForm $form) + { + static::addScheme($form); + static::addSslOptions($form); + static::addUrl($form); + static::addResultProperty($form); + static::addAuthentication($form); + static::addHeader($form); + static::addProxy($form); + } + + /** + * @param QuickForm $form + * @throws \Zend_Form_Exception + */ + protected static function addScheme(QuickForm $form) + { + $form->addElement('select', 'scheme', [ + 'label' => $form->translate('Protocol'), + 'description' => $form->translate( + 'Whether to use encryption when talking to the REST API' + ), + 'multiOptions' => [ + 'HTTPS' => $form->translate('HTTPS (strongly recommended)'), + 'HTTP' => $form->translate('HTTP (this is plaintext!)'), + ], + 'class' => 'autosubmit', + 'value' => 'HTTPS', + 'required' => true, + ]); + } + + /** + * @param QuickForm $form + * @throws \Zend_Form_Exception + */ + protected static function addHeader(QuickForm $form) + { + $form->addElement('textarea', 'headers', [ + 'label' => $form->translate('HTTP Header'), + 'description' => implode(' ', [ + $form->translate('Additional headers for the HTTP request.'), + $form->translate('Specify headers in text format "Header: Value", each header on a new line.'), + ]), + 'class' => 'preformatted', + 'rows' => 4, + ]); + } + + /** + * @param QuickForm $form + * @throws \Zend_Form_Exception + */ + protected static function addSslOptions(QuickForm $form) + { + $ssl = ! ($form->getSentOrObjectSetting('scheme', 'HTTPS') === 'HTTP'); + + if ($ssl) { + static::addBoolean($form, 'ssl_verify_peer', [ + 'label' => $form->translate('Verify Peer'), + 'description' => $form->translate( + 'Whether we should check that our peer\'s certificate has' + . ' been signed by a trusted CA. This is strongly recommended.' + ) + ], 'y'); + static::addBoolean($form, 'ssl_verify_host', [ + 'label' => $form->translate('Verify Host'), + 'description' => $form->translate( + 'Whether we should check that the certificate matches the' + . 'configured host' + ) + ], 'y'); + } + } + + /** + * @param QuickForm $form + * @throws \Zend_Form_Exception + */ + protected static function addUrl(QuickForm $form) + { + $form->addElement('text', 'url', [ + 'label' => 'REST API URL', + 'description' => $form->translate( + 'Something like https://api.example.com/rest/v2/objects' + ), + 'required' => true, + ]); + } + + /** + * @param QuickForm $form + * @throws \Zend_Form_Exception + */ + protected static function addResultProperty(QuickForm $form) + { + $form->addElement('text', 'extract_property', [ + 'label' => 'Extract property', + 'description' => implode("\n", [ + $form->translate('Often the expected result is provided in a property like "objects".' + . ' Please specify this if required.'), + $form->translate('Also deeper keys can be specific by a dot-notation:'), + '"result.objects", "key.deeper_key.very_deep"', + $form->translate('Literal dots in a key name can be written in the escape notation:'), + '"key\.with\.dots"', + ]) + ]); + } + + /** + * @param QuickForm $form + * @throws \Zend_Form_Exception + */ + protected static function addAuthentication(QuickForm $form) + { + $form->addElement('text', 'username', [ + 'label' => $form->translate('Username'), + 'description' => $form->translate( + 'Will be used to authenticate against your REST API' + ), + ]); + + $form->addElement('storedPassword', 'password', [ + 'label' => $form->translate('Password'), + ]); + } + + /** + * @param QuickForm $form + * @throws \Zend_Form_Exception + */ + protected static function addProxy(QuickForm $form) + { + $form->addElement('select', 'proxy_type', [ + 'label' => $form->translate('Proxy'), + 'description' => $form->translate( + 'In case your API is only reachable through a proxy, please' + . ' choose it\'s protocol right here' + ), + 'multiOptions' => $form->optionalEnum([ + 'HTTP' => $form->translate('HTTP proxy'), + 'SOCKS5' => $form->translate('SOCKS5 proxy'), + ]), + 'class' => 'autosubmit' + ]); + + $proxyType = $form->getSentOrObjectSetting('proxy_type'); + + if ($proxyType) { + $form->addElement('text', 'proxy', [ + 'label' => $form->translate('Proxy Address'), + 'description' => $form->translate( + 'Hostname, IP or :' + ), + 'required' => true, + ]); + if ($proxyType === 'HTTP') { + $form->addElement('text', 'proxy_user', [ + 'label' => $form->translate('Proxy Username'), + 'description' => $form->translate( + 'In case your proxy requires authentication, please' + . ' configure this here' + ), + ]); + + $passRequired = strlen($form->getSentOrObjectSetting('proxy_user')) > 0; + + $form->addElement('storedPassword', 'proxy_pass', [ + 'label' => $form->translate('Proxy Password'), + 'required' => $passRequired + ]); + } + } + } + + protected function getUrl() + { + $url = $this->getSetting('url'); + $parts = \parse_url($url); + if (isset($parts['path'])) { + $path = $parts['path']; + } else { + $path = '/'; + } + + if (isset($parts['query'])) { + $url = "$path?" . $parts['query']; + } else { + $url = $path; + } + + return $url; + } + + protected function getRestApi() + { + $url = $this->getSetting('url'); + $parts = \parse_url($url); + if (isset($parts['host'])) { + $host = $parts['host']; + } else { + throw new InvalidArgumentException("URL '$url' has no host"); + } + + $api = new RestApiClient( + $host, + $this->getSetting('username'), + $this->getSetting('password') + ); + + $api->setScheme($this->getSetting('scheme')); + if (isset($parts['port'])) { + $api->setPort($parts['port']); + } + + if ($api->getScheme() === 'HTTPS') { + if ($this->getSetting('ssl_verify_peer', 'y') === 'n') { + $api->disableSslPeerVerification(); + } + if ($this->getSetting('ssl_verify_host', 'y') === 'n') { + $api->disableSslHostVerification(); + } + } + + if ($proxy = $this->getSetting('proxy')) { + if ($proxyType = $this->getSetting('proxy_type')) { + $api->setProxy($proxy, $proxyType); + } else { + $api->setProxy($proxy); + } + + if ($user = $this->getSetting('proxy_user')) { + $api->setProxyAuth($user, $this->getSetting('proxy_pass')); + } + } + + return $api; + } + + /** + * @param QuickForm $form + * @param string $key + * @param array $options + * @param string|null $default + * @throws \Zend_Form_Exception + */ + protected static function addBoolean(QuickForm $form, $key, $options, $default = null) + { + if ($default === null) { + $form->addElement('OptionalYesNo', $key, $options); + } else { + $form->addElement('YesNo', $key, $options); + $form->getElement($key)->setValue($default); + } + } + + /** + * @param QuickForm $form + * @param string $key + * @param string $label + * @param string $description + * @throws \Zend_Form_Exception + */ + protected static function optionalBoolean(QuickForm $form, $key, $label, $description) + { + static::addBoolean($form, $key, [ + 'label' => $label, + 'description' => $description + ]); + } +} diff --git a/library/Director/Import/ImportSourceSql.php b/library/Director/Import/ImportSourceSql.php new file mode 100644 index 0000000..b08a3f3 --- /dev/null +++ b/library/Director/Import/ImportSourceSql.php @@ -0,0 +1,70 @@ +db()->fetchAll($this->settings['query']); + } + + public function listColumns() + { + if ($columns = $this->getSetting('column_cache')) { + return explode(', ', $columns); + } else { + return array_keys((array) current($this->fetchData())); + } + } + + public static function addSettingsFormFields(QuickForm $form) + { + /** @var ImportSourceForm $form */ + Util::addDbResourceFormElement($form, 'resource'); + /** @var ImportSource $current */ + $current = $form->getObject(); + + $form->addElement('textarea', 'query', [ + 'label' => $form->translate('DB Query'), + 'required' => true, + 'rows' => 15, + ]); + $form->addElement('hidden', 'column_cache', [ + 'value' => '', + 'filters' => [new QueryColumnsFromSql($form)], + 'required' => true + ]); + if ($current) { + if ($columns = $current->getSetting('column_cache')) { + $form->addHtmlHint('Columns: ' . $columns); + } else { + $form->addHtmlHint(Hint::warning($form->translate( + 'Please click "Store" once again to determine query columns' + ))); + } + } + return $form; + } + + protected function db() + { + if ($this->db === null) { + $this->db = DbConnection::fromResourceName($this->settings['resource'])->getDbAdapter(); + } + + return $this->db; + } +} diff --git a/library/Director/Import/PurgeStrategy/ImportRunBasedPurgeStrategy.php b/library/Director/Import/PurgeStrategy/ImportRunBasedPurgeStrategy.php new file mode 100644 index 0000000..9f0e8ab --- /dev/null +++ b/library/Director/Import/PurgeStrategy/ImportRunBasedPurgeStrategy.php @@ -0,0 +1,90 @@ +getSyncRule()->fetchInvolvedImportSources() as $source) { + $remove += $this->checkImportSource($source); + } + + return $remove; + } + + protected function getLastSync() + { + return strtotime($this->getSyncRule()->getLastSyncTimestamp()); + } + + // TODO: NAMING! + protected function checkImportSource(ImportSource $source) + { + if (null === ($lastSync = $this->getLastSync())) { + // No last sync, nothing to purge + return array(); + } + + $runA = $source->fetchLastRunBefore($lastSync); + if ($runA === null) { + // Nothing to purge for this source + return array(); + } + + $runB = $source->fetchLastRun(); + if ($runA->rowset_checksum === $runB->rowset_checksum) { + // Same source data, nothing to purge + return array(); + } + + return $this->listKeysRemovedBetween($runA, $runB); + } + + public function listKeysRemovedBetween(ImportRun $runA, ImportRun $runB) + { + $rule = $this->getSyncRule(); + $db = $rule->getDb(); + + $selectA = $runA->prepareImportedObjectQuery(); + $selectB = $runB->prepareImportedObjectQuery(); + + $query = $db->select()->from( + array('a' => $selectA), + 'a.object_name' + )->where('a.object_name NOT IN (?)', $selectB); + + $result = $db->fetchCol($query); + + if (empty($result)) { + return array(); + } + + if ($rule->hasCombinedKey()) { + $pattern = $rule->getSourceKeyPattern(); + $columns = SyncUtils::getRootVariables( + SyncUtils::extractVariableNames($pattern) + ); + $resultForCombinedKey = array(); + foreach (array_chunk($result, 1000) as $keys) { + $rows = $runA->fetchRows($columns, null, $keys); + foreach ($rows as $row) { + $resultForCombinedKey[] = SyncUtils::fillVariables($pattern, $row); + } + } + $result = $resultForCombinedKey; + } + + if (empty($result)) { + return array(); + } + + return array_combine($result, $result); + } +} diff --git a/library/Director/Import/PurgeStrategy/PurgeNothingPurgeStrategy.php b/library/Director/Import/PurgeStrategy/PurgeNothingPurgeStrategy.php new file mode 100644 index 0000000..3da8d4f --- /dev/null +++ b/library/Director/Import/PurgeStrategy/PurgeNothingPurgeStrategy.php @@ -0,0 +1,11 @@ +rule = $rule; + } + + protected function getSyncRule() + { + return $this->rule; + } + + abstract public function listObjectsToPurge(); + + /** + * @return PurgeStrategy + */ + public static function load($name, SyncRule $rule) + { + $class = __NAMESPACE__ . '\\' . $name . 'PurgeStrategy'; + return new $class($rule); + } +} diff --git a/library/Director/Import/Sync.php b/library/Director/Import/Sync.php new file mode 100644 index 0000000..8fea46c --- /dev/null +++ b/library/Director/Import/Sync.php @@ -0,0 +1,942 @@ +> key => [property, property]*/ + protected $setNull = []; + + /** @var bool Whether we already prepared your sync */ + protected $isPrepared = false; + + /** @var bool Whether we applied strtolower() to existing object keys */ + protected $usedLowerCasedKeys = false; + + protected $modify = []; + + protected $remove = []; + + protected $create = []; + + protected $errors = []; + + /** @var SyncProperty[] */ + protected $syncProperties; + + protected $replaceVars = false; + + protected $hasPropertyDisabled = false; + + protected $serviceOverrideKeyName; + + /** + * @var SyncRun + */ + protected $run; + + protected $runStartTime; + + /** @var Filter[] */ + protected $columnFilters = []; + + /** @var HostGroupMembershipResolver|bool */ + protected $hostGroupMembershipResolver; + + /** @var ?DbObjectStore */ + protected $store; + + /** + * @param SyncRule $rule + * @param ?DbObjectStore $store + */ + public function __construct(SyncRule $rule, DbObjectStore $store = null) + { + $this->rule = $rule; + $this->db = $rule->getConnection(); + $this->store = $store; + } + + /** + * Whether the given sync rule would apply modifications + * + * @return boolean + * @throws Exception + */ + public function hasModifications() + { + return count($this->getExpectedModifications()) > 0; + } + + /** + * Retrieve modifications a given SyncRule would apply + * + * @return array Array of IcingaObject elements + * @throws \Icinga\Exception\NotFoundError + * @throws \Icinga\Module\Director\Exception\DuplicateKeyException + */ + public function getExpectedModifications() + { + $modified = []; + $objects = $this->prepare(); + $updateOnly = $this->rule->get('update_policy') === 'update-only'; + $allowCreate = ! $updateOnly; + foreach ($objects as $object) { + if ($object->hasBeenModified()) { + if ($allowCreate || $object->hasBeenLoadedFromDb()) { + $modified[] = $object; + } + } elseif (! $updateOnly && $object->shouldBeRemoved()) { + $modified[] = $object; + } + } + + return $modified; + } + + /** + * Transform the given value to an array + * + * @param array|string|null $value + * + * @return array + */ + protected function wantArray($value) + { + if (is_array($value)) { + return $value; + } elseif ($value === null) { + return []; + } else { + return [$value]; + } + } + + /** + * Raise PHP resource limits + * + * @return self; + */ + protected function raiseLimits() + { + MemoryLimit::raiseTo('1024M'); + ini_set('max_execution_time', 0); + + return $this; + } + + /** + * Initialize run summary measurements + * + * @return self; + */ + protected function startMeasurements() + { + $this->run = SyncRun::start($this->rule); + $this->runStartTime = microtime(true); + Benchmark::measure('Starting sync'); + return $this; + } + + /** + * Fetch the configured properties involved in this sync + * + * @return self + */ + protected function fetchSyncProperties() + { + $this->syncProperties = $this->rule->getSyncProperties(); + foreach ($this->syncProperties as $key => $prop) { + $destinationField = $prop->get('destination_field'); + if ($destinationField === 'vars' && $prop->get('merge_policy') === 'override') { + $this->replaceVars = true; + } + + if ($destinationField === 'disabled') { + $this->hasPropertyDisabled = true; + } + + if ($prop->get('filter_expression') === null || strlen($prop->get('filter_expression')) === 0) { + continue; + } + + $this->columnFilters[$key] = Filter::fromQueryString( + $prop->get('filter_expression') + ); + } + + return $this; + } + + protected function rowMatchesPropertyFilter($row, $key) + { + if (!array_key_exists($key, $this->columnFilters)) { + return true; + } + + return $this->columnFilters[$key]->matches($row); + } + + /** + * Instantiates all related ImportSource objects + * + * @return self + * @throws \Icinga\Exception\NotFoundError + */ + protected function prepareRelatedImportSources() + { + $this->sources = []; + foreach ($this->syncProperties as $p) { + $id = $p->get('source_id'); + if (! array_key_exists($id, $this->sources)) { + $this->sources[$id] = ImportSource::loadWithAutoIncId( + (int) $id, + $this->db + ); + } + } + + return $this; + } + + /** + * Prepare the source columns we want to fetch + * + * @return self + */ + protected function prepareSourceColumns() + { + // $fieldMap = []; + $this->sourceColumns = []; + + foreach ($this->syncProperties as $p) { + $sourceId = $p->get('source_id'); + if (! array_key_exists($sourceId, $this->sourceColumns)) { + $this->sourceColumns[$sourceId] = []; + } + + foreach (SyncUtils::extractVariableNames($p->get('source_expression')) as $varname) { + $this->sourceColumns[$sourceId][$varname] = $varname; + // -> ? $fieldMap[ + } + } + + return $this; + } + + /** + * Fetch latest imported data rows from all involved import sources + * @return Sync + * @throws \Icinga\Exception\NotFoundError + */ + protected function fetchImportedData() + { + Benchmark::measure('Begin loading imported data'); + if ($this->rule->get('object_type') === 'host') { + $this->serviceOverrideKeyName = $this->db->settings()->override_services_varname; + } + + $this->imported = []; + + $sourceKeyPattern = $this->rule->getSourceKeyPattern(); + $combinedKey = $this->rule->hasCombinedKey(); + + foreach ($this->sources as $source) { + /** @var ImportSource $source */ + $sourceId = $source->get('id'); + + // Provide an alias column for our key. TODO: double-check this! + $key = $source->key_column; + $this->sourceColumns[$sourceId][$key] = $key; + $run = $source->fetchLastRun(true); + + $usedColumns = SyncUtils::getRootVariables($this->sourceColumns[$sourceId]); + + $filterColumns = []; + foreach ($this->columnFilters as $filter) { + foreach ($filter->listFilteredColumns() as $column) { + $filterColumns[$column] = $column; + } + } + if (($ruleFilter = $this->rule->filter()) !== null) { + foreach ($ruleFilter->listFilteredColumns() as $column) { + $filterColumns[$column] = $column; + } + } + + if (! empty($filterColumns)) { + foreach (SyncUtils::getRootVariables($filterColumns) as $column) { + $usedColumns[$column] = $column; + } + } + Benchmark::measure(sprintf('Done pre-processing columns for source %s', $source->source_name)); + + $rows = $run->fetchRows($usedColumns); + Benchmark::measure(sprintf('Fetched source %s', $source->source_name)); + + $this->imported[$sourceId] = []; + foreach ($rows as $row) { + if ($combinedKey) { + $key = SyncUtils::fillVariables($sourceKeyPattern, $row); + if ($this->usedLowerCasedKeys) { + $key = strtolower($key); + } + + if (array_key_exists($key, $this->imported[$sourceId])) { + throw new InvalidArgumentException(sprintf( + 'Trying to import row "%s" (%s) twice: %s VS %s', + $key, + $sourceKeyPattern, + json_encode($this->imported[$sourceId][$key]), + json_encode($row) + )); + } + } else { + if (! property_exists($row, $key)) { + throw new InvalidArgumentException(sprintf( + 'There is no key column "%s" in this row from "%s": %s', + $key, + $source->source_name, + json_encode($row) + )); + } + } + + if (! $this->rule->matches($row)) { + continue; + } + + if ($combinedKey) { + $this->imported[$sourceId][$key] = $row; + } else { + if ($this->usedLowerCasedKeys) { + $this->imported[$sourceId][strtolower($row->$key)] = $row; + } else { + $this->imported[$sourceId][$row->$key] = $row; + } + } + } + + unset($rows); + } + + Benchmark::measure('Done loading imported data'); + + return $this; + } + + /** + * TODO: This is rubbish, we need to filter at fetch time + */ + protected function removeForeignListEntries() + { + $listId = null; + foreach ($this->syncProperties as $prop) { + if ($prop->get('destination_field') === 'list_id') { + $listId = (int) $prop->get('source_expression'); + } + } + + if ($listId === null) { + throw new InvalidArgumentException( + 'Cannot sync datalist entry without list_id' + ); + } + + $no = []; + foreach ($this->objects as $k => $o) { + if ((int) $o->get('list_id') !== $listId) { + $no[] = $k; + } + } + + foreach ($no as $k) { + unset($this->objects[$k]); + } + } + + /** + * @return $this + */ + protected function loadExistingObjects() + { + Benchmark::measure('Begin loading existing objects'); + + $ruleObjectType = $this->rule->get('object_type'); + $useLowerCaseKeys = $ruleObjectType !== 'datalistEntry'; + // TODO: Make object_type (template, object...) and object_name mandatory? + if ($this->rule->hasCombinedKey()) { + $this->objects = []; + $destinationKeyPattern = $this->rule->getDestinationKeyPattern(); + $table = DbObjectTypeRegistry::tableNameByType($ruleObjectType); + if ($this->store && BranchSupport::existsForTableName($table)) { + $objects = $this->store->loadAll($table); + } else { + $objects = IcingaObject::loadAllByType($ruleObjectType, $this->db); + } + + foreach ($objects as $object) { + if ($object instanceof IcingaService) { + if (strstr($destinationKeyPattern, '${host}') + && $object->get('host_id') === null + ) { + continue; + } elseif (strstr($destinationKeyPattern, '${service_set}') + && $object->get('service_set_id') === null + ) { + continue; + } + } + + $key = SyncUtils::fillVariables( + $destinationKeyPattern, + $object + ); + if ($useLowerCaseKeys) { + $key = strtolower($key); + } + + if (array_key_exists($key, $this->objects)) { + throw new InvalidArgumentException(sprintf( + 'Combined destination key "%s" is not unique, got "%s" twice', + $destinationKeyPattern, + $key + )); + } + + $this->objects[$key] = $object; + } + } else { + if ($this->store) { + $objects = $this->store->loadAll(DbObjectTypeRegistry::tableNameByType($ruleObjectType), 'object_name'); + } else { + $objects = IcingaObject::loadAllByType($ruleObjectType, $this->db); + } + + if ($useLowerCaseKeys) { + $this->objects = []; + foreach ($objects as $key => $object) { + $this->objects[strtolower($key)] = $object; + } + } else { + $this->objects = $objects; + } + } + + $this->usedLowerCasedKeys = $useLowerCaseKeys; + // TODO: should be obsoleted by a better "loadFiltered" method + if ($ruleObjectType === 'datalistEntry') { + $this->removeForeignListEntries(); + } + + Benchmark::measure('Done loading existing objects'); + + return $this; + } + + /** + * @return array + * @throws \Icinga\Exception\NotFoundError + * @throws \Icinga\Module\Director\Exception\DuplicateKeyException + */ + protected function prepareNewObjects() + { + $objects = []; + $ruleObjectType = $this->rule->get('object_type'); + + foreach ($this->sources as $source) { + $sourceId = $source->id; + $keyColumn = $source->get('key_column'); + + foreach ($this->imported[$sourceId] as $key => $row) { + // Workaround: $a["10"] = "val"; -> array_keys($a) = [(int) 10] + $key = (string) $key; + $originalKey = $row->$keyColumn; + if ($this->usedLowerCasedKeys) { + $key = strtolower($key); + } + if (! array_key_exists($key, $objects)) { + // Safe default values for object_type and object_name + if ($ruleObjectType === 'datalistEntry') { + $props = []; + } else { + $props = [ + 'object_type' => 'object', + 'object_name' => $originalKey, + ]; + } + + $objects[$key] = IcingaObject::createByType( + $ruleObjectType, + $props, + $this->db + ); + } + + $object = $objects[$key]; + $this->prepareNewObject($row, $object, $key, $sourceId); + } + } + + return $objects; + } + + /** + * @param $row + * @param DbObject $object + * @param $sourceId + * @throws \Icinga\Exception\NotFoundError + * @throws \Icinga\Module\Director\Exception\DuplicateKeyException + */ + protected function prepareNewObject($row, DbObject $object, $objectKey, $sourceId) + { + foreach ($this->syncProperties as $propertyKey => $p) { + if ($p->get('source_id') !== $sourceId) { + continue; + } + + if (! $this->rowMatchesPropertyFilter($row, $propertyKey)) { + continue; + } + + $prop = $p->get('destination_field'); + $val = SyncUtils::fillVariables($p->get('source_expression'), $row); + + if ($object instanceof IcingaObject) { + if ($prop === 'import') { + if ($val !== null) { + $object->imports()->add($val); + } + } elseif ($prop === 'groups') { + if ($val !== null) { + $object->groups()->add($val); + } + } elseif (substr($prop, 0, 5) === 'vars.') { + $varName = substr($prop, 5); + if (substr($varName, -2) === '[]') { + $varName = substr($varName, 0, -2); + $current = $this->wantArray($object->vars()->$varName); + $object->vars()->$varName = array_merge( + $current, + $this->wantArray($val) + ); + } else { + if ($val === null) { + $this->setNull[$objectKey][$prop] = $prop; + } else { + unset($this->setNull[$objectKey][$prop]); + $object->vars()->$varName = $val; + } + } + } else { + if ($val === null) { + $this->setNull[$objectKey][$prop] = $prop; + } else { + unset($this->setNull[$objectKey][$prop]); + $object->set($prop, $val); + } + } + } else { + if ($val === null) { + $this->setNull[$objectKey][$prop] = $prop; + } else { + unset($this->setNull[$objectKey][$prop]); + $object->set($prop, $val); + } + } + } + } + + /** + * @return $this + */ + protected function deferResolvers() + { + if (in_array($this->rule->get('object_type'), ['host', 'hostgroup'])) { + $resolver = $this->getHostGroupMembershipResolver(); + $resolver->defer()->setUseTransactions(false); + } + + return $this; + } + + /** + * @param DbObject $object + * @return $this + */ + protected function setResolver($object) + { + if (! ($object instanceof IcingaHost || $object instanceof IcingaHostGroup)) { + return $this; + } + if ($resolver = $this->getHostGroupMembershipResolver()) { + $object->setHostGroupMembershipResolver($resolver); + } + + return $this; + } + + /** + * @return $this + * @throws \Zend_Db_Adapter_Exception + */ + protected function notifyResolvers() + { + if ($resolver = $this->getHostGroupMembershipResolver()) { + $resolver->refreshDb(true); + } + + return $this; + } + + /** + * @return bool|HostGroupMembershipResolver + */ + protected function getHostGroupMembershipResolver() + { + if ($this->hostGroupMembershipResolver === null) { + if (in_array( + $this->rule->get('object_type'), + ['host', 'hostgroup'] + )) { + $this->hostGroupMembershipResolver = new HostGroupMembershipResolver( + $this->db + ); + } else { + $this->hostGroupMembershipResolver = false; + } + } + + return $this->hostGroupMembershipResolver; + } + + /** + * Evaluates a SyncRule and returns a list of modified objects + * + * TODO: Split this into smaller methods + * + * @return DbObject|IcingaObject[] List of modified IcingaObjects + * @throws \Icinga\Exception\NotFoundError + * @throws \Icinga\Module\Director\Exception\DuplicateKeyException + */ + protected function prepare() + { + if ($this->isPrepared) { + return $this->objects; + } + + $this->raiseLimits() + ->startMeasurements() + ->prepareCache() + ->fetchSyncProperties() + ->prepareRelatedImportSources() + ->prepareSourceColumns() + ->loadExistingObjects() + ->fetchImportedData() + ->deferResolvers(); + + Benchmark::measure('Begin preparing updated objects'); + $newObjects = $this->prepareNewObjects(); + + Benchmark::measure('Ready to process objects'); + /** @var DbObject|IcingaObject $object */ + foreach ($newObjects as $key => $object) { + $this->processObject($key, $object); + } + + Benchmark::measure('Modified objects are ready, applying purge strategy'); + $noAction = []; + $purgeAction = $this->rule->get('purge_action'); + foreach ($this->rule->purgeStrategy()->listObjectsToPurge() as $key) { + $key = strtolower($key); + if (array_key_exists($key, $newObjects)) { + // Object has been touched, do not delete + continue; + } + + if (array_key_exists($key, $this->objects)) { + $object = $this->objects[$key]; + if (! $object->hasBeenModified()) { + switch ($purgeAction) { + case 'delete': + $object->markForRemoval(); + break; + case 'disable': + $object->set('disabled', 'y'); + break; + default: + throw new RuntimeException( + "Unsupported purge action: '$purgeAction'" + ); + } + } + } + } + + Benchmark::measure('Done marking objects for purge'); + + foreach ($this->objects as $key => $object) { + if (! $object->hasBeenModified() && ! $object->shouldBeRemoved()) { + $noAction[] = $key; + } + } + + foreach ($noAction as $key) { + unset($this->objects[$key]); + } + + $this->isPrepared = true; + + Benchmark::measure('Done preparing objects'); + + return $this->objects; + } + + /** + * @param $key + * @param DbObject|IcingaObject $object + * @throws \Icinga\Exception\NotFoundError + */ + protected function processObject($key, $object) + { + if (array_key_exists($key, $this->objects)) { + $this->refreshObject($key, $object); + } else { + $this->addNewObject($key, $object); + } + } + + /** + * @param $key + * @param DbObject|IcingaObject $object + * @throws \Icinga\Exception\NotFoundError + */ + protected function refreshObject($key, $object) + { + $policy = $this->rule->get('update_policy'); + + switch ($policy) { + case 'override': + if ($object instanceof IcingaHost + && !in_array('api_key', $this->rule->getSyncProperties()) + ) { + $this->objects[$key]->replaceWith($object, ['api_key']); + } else { + $this->objects[$key]->replaceWith($object); + } + break; + + case 'merge': + case 'update-only': + // TODO: re-evaluate merge settings. vars.x instead of + // just "vars" might suffice. + $this->objects[$key]->merge($object, $this->replaceVars); + if (! $this->hasPropertyDisabled && $object->hasProperty('disabled')) { + $this->objects[$key]->resetProperty('disabled'); + } + break; + + default: + // policy 'ignore', no action + } + + if ($policy === 'override' || $policy === 'merge') { + if ($object instanceof IcingaHost) { + $keyName = $this->serviceOverrideKeyName; + if (! $object->hasInitializedVars() || ! isset($object->vars()->$key)) { + $this->objects[$key]->vars()->restoreStoredVar($keyName); + } + } + } + + if (isset($this->setNull[$key])) { + foreach ($this->setNull[$key] as $property) { + $this->objects[$key]->set($property, null); + } + } + } + + /** + * @param $key + * @param DbObject|IcingaObject $object + */ + protected function addNewObject($key, $object) + { + $this->objects[$key] = $object; + } + + /** + * Runs a SyncRule and applies all resulting changes + * @return int + * @throws Exception + * @throws IcingaException + */ + public function apply() + { + Benchmark::measure('Begin applying objects'); + + $objects = $this->prepare(); + $db = $this->db; + $dba = $db->getDbAdapter(); + if (! $this->store) { // store has it's own transaction + $dba->beginTransaction(); + } + + $object = null; + $updateOnly = $this->rule->get('update_policy') === 'update-only'; + $allowCreate = ! $updateOnly; + + try { + $formerActivityChecksum = hex2bin( + $db->getLastActivityChecksum() + ); + $created = 0; + $modified = 0; + $deleted = 0; + // TODO: Count also failed ones, once we allow such + // $failed = 0; + foreach ($objects as $object) { + $this->setResolver($object); + if (! $updateOnly && $object->shouldBeRemoved()) { + if ($this->store) { + $this->store->delete($object); + } else { + $object->delete(); + } + $deleted++; + continue; + } + + if ($object->hasBeenModified()) { + $existing = $object->hasBeenLoadedFromDb(); + if ($existing) { + if ($this->store) { + $this->store->store($object); + } else { + $object->store($db); + } + $modified++; + } elseif ($allowCreate) { + if ($this->store) { + $this->store->store($object); + } else { + $object->store($db); + } + $created++; + } + } + } + + $runProperties = [ + 'objects_created' => $created, + 'objects_deleted' => $deleted, + 'objects_modified' => $modified, + ]; + + if ($created + $deleted + $modified > 0) { + // TODO: What if this has been the very first activity? + $runProperties['last_former_activity'] = $db->quoteBinary($formerActivityChecksum); + $runProperties['last_related_activity'] = $db->quoteBinary(hex2bin( + $db->getLastActivityChecksum() + )); + } + + $this->run->setProperties($runProperties); + if (!$this->store || !$this->store->getBranch()->isBranch()) { + $this->run->store(); + } + $this->notifyResolvers(); + if (! $this->store) { + $dba->commit(); + } + + // Store duration after commit, as the commit might take some time + $this->run->set('duration_ms', (int) round( + (microtime(true) - $this->runStartTime) * 1000 + )); + if (!$this->store || !$this->store->getBranch()->isBranch()) { + $this->run->store(); + } + + Benchmark::measure('Done applying objects'); + } catch (Exception $e) { + if (! $this->store) { + $dba->rollBack(); + } + + if ($object instanceof IcingaObject) { + throw new IcingaException( + 'Exception while syncing %s %s: %s', + get_class($object), + $object->getObjectName(), + $e->getMessage(), + $e + ); + } else { + throw $e; + } + } + + return $this->run->get('id'); + } + + protected function prepareCache() + { + if ($this->store) { + return $this; + } + PrefetchCache::initialize($this->db); + IcingaTemplateRepository::clear(); + + $ruleObjectType = $this->rule->get('object_type'); + + $dummy = IcingaObject::createByType($ruleObjectType); + if ($dummy instanceof IcingaObject) { + IcingaObject::prefetchAllRelationsByType($ruleObjectType, $this->db); + } + + return $this; + } +} diff --git a/library/Director/Import/SyncUtils.php b/library/Director/Import/SyncUtils.php new file mode 100644 index 0000000..5528b2d --- /dev/null +++ b/library/Director/Import/SyncUtils.php @@ -0,0 +1,153 @@ + { 'disk' => { 'sda' => { 'size' => '256G' } } } } + * + * and a key vars.disk.sda given as [ 'vars', 'disk', 'sda' ] this would + * return { size => '255GB' } + * + * @param string $val The value to extract data from + * @param array $keys A list of nested keys pointing to desired data + * + * @return mixed + */ + public static function getDeepValue($val, array $keys) + { + $key = array_shift($keys); + if (! property_exists($val, $key)) { + return null; + } + + if (empty($keys)) { + return $val->$key; + } + + return static::getDeepValue($val->$key, $keys); + } + + /** + * Return a specific value from a given row object + * + * Supports also keys pointing to nested structures like vars.disk.sda + * + * @param object $row stdClass object providing property values + * @param string $var Variable/property name + * + * @return mixed + */ + public static function getSpecificValue($row, $var) + { + if (strpos($var, '.') === false) { + if ($row instanceof DbObject) { + return $row->$var; + } + if (! property_exists($row, $var)) { + return null; + } + + return $row->$var; + } else { + $parts = explode('.', $var); + $main = array_shift($parts); + if (! property_exists($row, $main)) { + return null; + } + if ($row->$main === null) { + return null; + } + + if (! is_object($row->$main)) { + throw new InvalidArgumentException(sprintf( + 'Data is not nested, cannot access %s: %s', + $var, + var_export($row, 1) + )); + } + + return static::getDeepValue($row->$main, $parts); + } + } + + /** + * Fill variables in the given string pattern + * + * This replaces all occurrences of ${var_name} with the corresponding + * property $row->var_name of the given row object. Missing variables are + * replaced by an empty string. This works also fine in case there are + * multiple variables to be found in your string. + * + * @param string $string String with optional variables/placeholders + * @param object $row stdClass object providing property values + * + * @return string + */ + public static function fillVariables($string, $row) + { + if (preg_match('/^\${([^}]+)}$/', $string, $m)) { + return static::getSpecificValue($row, $m[1]); + } + + $func = function ($match) use ($row) { + return SyncUtils::getSpecificValue($row, $match[1]); + }; + + return preg_replace_callback('/\${([^}]+)}/', $func, $string); + } + + public static function getRootVariables($vars) + { + $res = array(); + foreach ($vars as $p) { + if (false === ($pos = strpos($p, '.')) || $pos === strlen($p) - 1) { + $res[] = $p; + } else { + $res[] = substr($p, 0, $pos); + } + } + + if (empty($res)) { + return array(); + } + + return array_combine($res, $res); + } +} -- cgit v1.2.3