null, 'source_name' => null, 'provider_class' => null, 'key_column' => null, 'import_state' => 'unknown', 'last_error_message' => null, 'last_attempt' => null, 'description' => null, ]; protected $stateProperties = [ 'import_state', 'last_error_message', 'last_attempt', ]; protected $settingsTable = 'import_source_setting'; protected $settingsRemoteId = 'source_id'; private $rowModifiers; private $loadedRowModifiers; private $newRowModifiers; public function setModifiers(array $modifiers) { if ($this->loadedRowModifiers === null && $this->hasBeenLoadedFromDb()) { $this->loadedRowModifiers = $this->fetchRowModifiers(); } $current = (array) $this->loadedRowModifiers; if (\count($current) !== \count($modifiers)) { $this->newRowModifiers = $modifiers; } else { $i = 0; $modified = false; foreach ($modifiers as $props) { $this->loadedRowModifiers[$i]->setProperties((array) $props); if ($this->loadedRowModifiers[$i]->hasBeenModified()) { $modified = true; } $i++; } if ($modified) { $this->newRowModifiers = $modifiers; } } } public function hasBeenModified() { return $this->newRowModifiers !== null || parent::hasBeenModified(); } public function getUniqueIdentifier() { return $this->get('source_name'); } /** * @param $name * @param Db $connection * @return ImportSource * @throws NotFoundError */ public static function loadByName($name, Db $connection) { $db = $connection->getDbAdapter(); $properties = $db->fetchRow( $db->select()->from('import_source')->where('source_name = ?', $name) ); if ($properties === false) { throw new NotFoundError(sprintf( 'There is no such Import Source: "%s"', $name )); } return static::create([], $connection)->setDbProperties($properties); } public static function existsWithName($name, Db $connection) { $db = $connection->getDbAdapter(); return (string) $name === (string) $db->fetchOne( $db->select() ->from('import_source', 'source_name') ->where('source_name = ?', $name) ); } /** * @param string $name * @param int $id * @param Db $connection * @api internal * @return bool */ protected static function existsWithNameAndId($name, $id, Db $connection) { $db = $connection->getDbAdapter(); $dummy = new static; $idCol = $dummy->autoincKeyName; $keyCol = $dummy->keyName; return (string) $id === (string) $db->fetchOne( $db->select() ->from($dummy->table, $idCol) ->where("$idCol = ?", $id) ->where("$keyCol = ?", $name) ); } /** * @deprecated please use \Icinga\Module\Director\Data\FieldReferenceLoader * @return array */ protected function exportRowModifiers() { $modifiers = []; foreach ($this->fetchRowModifiers() as $modifier) { $modifiers[] = $modifier->export(); } return $modifiers; } /** * @param bool $required * @return ImportRun|null * @throws NotFoundError */ public function fetchLastRun($required = false) { return $this->fetchLastRunBefore(time() + 1, $required); } /** * @throws DuplicateKeyException */ protected function onStore() { parent::onStore(); if ($this->newRowModifiers !== null) { $connection = $this->getConnection(); $db = $connection->getDbAdapter(); $myId = $this->get('id'); if ($this->hasBeenLoadedFromDb()) { $db->delete( 'import_row_modifier', $db->quoteInto('source_id = ?', $myId) ); } foreach ($this->newRowModifiers as $modifier) { $modifier = ImportRowModifier::create((array) $modifier, $connection); $modifier->set('source_id', $myId); $modifier->store(); } } } /** * @param $timestamp * @param bool $required * @return ImportRun|null * @throws NotFoundError */ public function fetchLastRunBefore($timestamp, $required = false) { if (! $this->hasBeenLoadedFromDb()) { return $this->nullUnlessRequired($required); } if ($timestamp === null) { $timestamp = time(); } $db = $this->getDb(); $query = $db->select()->from( ['ir' => 'import_run'], 'ir.id' )->where('ir.source_id = ?', $this->get('id')) ->where('ir.start_time < ?', date('Y-m-d H:i:s', $timestamp)) ->order('ir.start_time DESC') ->limit(1); $runId = $db->fetchOne($query); if ($runId) { return ImportRun::load($runId, $this->getConnection()); } else { return $this->nullUnlessRequired($required); } } /** * @param $required * @return null * @throws NotFoundError */ protected function nullUnlessRequired($required) { if ($required) { throw new NotFoundError( 'No data has been imported for "%s" yet', $this->get('source_name') ); } return null; } public function applyModifiers(&$data) { $modifiers = $this->fetchFlatRowModifiers(); if (empty($modifiers)) { return $this; } foreach ($modifiers as $modPair) { /** @var PropertyModifierHook $modifier */ /** @var ?Filter $filter */ list($property, $modifier, $filter) = $modPair; $rejected = []; $newRows = []; foreach ($data as $key => $row) { if ($filter && ! $filter->matches($row)) { continue; } $this->applyPropertyModifierToRow($modifier, $property, $row); if ($modifier->rejectsRow()) { $rejected[] = $key; $modifier->rejectRow(false); } if ($modifier->expandsRows()) { $target = $modifier->getTargetProperty($property); $newValue = $row->$target; if (\is_array($newValue)) { foreach ($newValue as $val) { $newRow = clone $row; $newRow->$target = $val; $newRows[] = $newRow; } $rejected[] = $key; } } } foreach ($rejected as $key) { unset($data[$key]); } foreach ($newRows as $row) { $data[] = $row; } } return $this; } public function getObjectName() { return $this->get('source_name'); } public static function getKeyColumnName() { return 'source_name'; } protected function applyPropertyModifierToRow(PropertyModifierHook $modifier, $key, $row) { if (! is_object($row)) { throw new InvalidArgumentException('Every imported row MUST be an object'); } if ($modifier->requiresRow()) { $modifier->setRow($row); } if (property_exists($row, $key)) { $value = $row->$key; } elseif (strpos($key, '.') !== false) { $value = SyncUtils::getSpecificValue($row, $key); } else { $value = null; } $target = $modifier->getTargetProperty($key); if (strpos($target, '.') !== false) { throw new InvalidArgumentException(sprintf( 'Cannot set value for nested key "%s"', $target )); } if (is_array($value) && ! $modifier->hasArraySupport()) { $new = []; foreach ($value as $k => $v) { $new[$k] = $modifier->transform($v); } $row->$target = $new; } else { $row->$target = $modifier->transform($value); } } public function getRowModifiers() { if ($this->rowModifiers === null) { $this->prepareRowModifiers(); } return $this->rowModifiers; } public function hasRowModifiers() { return count($this->getRowModifiers()) > 0; } /** * @return ImportRowModifier[] */ public function fetchRowModifiers() { $db = $this->getDb(); $modifiers = ImportRowModifier::loadAll( $this->getConnection(), $db->select() ->from('import_row_modifier') ->where('source_id = ?', $this->get('id')) ->order('priority ASC') ); if ($modifiers) { return $modifiers; } else { return []; } } protected function fetchFlatRowModifiers() { $mods = []; foreach ($this->fetchRowModifiers() as $mod) { if ($filterExpression = $mod->get('filter_expression')) { $filter = FilterEnrichment::enrichFilter(Filter::fromQueryString($filterExpression)); } else { $filter = null; } $mods[] = [$mod->get('property_name'), $mod->getInstance(), $filter]; } return $mods; } protected function prepareRowModifiers() { $modifiers = []; foreach ($this->fetchRowModifiers() as $mod) { $name = $mod->get('property_name'); if (! array_key_exists($name, $modifiers)) { $modifiers[$name] = []; } $modifiers[$name][] = $mod->getInstance(); } $this->rowModifiers = $modifiers; } public function listModifierTargetProperties() { $list = []; foreach ($this->getRowModifiers() as $rowMods) { /** @var PropertyModifierHook $mod */ foreach ($rowMods as $mod) { if ($mod->hasTargetProperty()) { $list[$mod->getTargetProperty()] = true; } } } return array_keys($list); } /** * @param bool $runImport * @return bool * @throws DuplicateKeyException */ public function checkForChanges($runImport = false) { $hadChanges = false; $name = $this->get('source_name'); Benchmark::measure("Starting with import $name"); $this->raiseLimits(); try { $import = new Import($this); $this->set('last_attempt', date('Y-m-d H:i:s')); if ($import->providesChanges()) { Benchmark::measure("Found changes for $name"); $hadChanges = true; $this->set('import_state', 'pending-changes'); if ($runImport && $import->run()) { Benchmark::measure("Import succeeded for $name"); $this->set('import_state', 'in-sync'); } } else { $this->set('import_state', 'in-sync'); } $this->set('last_error_message', null); } catch (Exception $e) { $this->set('import_state', 'failing'); Benchmark::measure("Import failed for $name"); $this->set('last_error_message', $e->getMessage()); } if ($this->hasBeenModified()) { $this->store(); } return $hadChanges; } /** * @return bool * @throws DuplicateKeyException */ public function runImport() { return $this->checkForChanges(true); } /** * Raise PHP resource limits * * @return $this; */ protected function raiseLimits() { MemoryLimit::raiseTo('1024M'); ini_set('max_execution_time', '0'); return $this; } }