Adding upstream version 1.10.2.upstream/1.10.2upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

11 files changed, 2460 insertions, 0 deletions

diff --git a/library/Director/Import/Import.php b/library/Director/Import/Import.php
new file mode 100644
index 0000000..f82454d
--- /dev/null
+++ b/library/Director/Import/Import.php
@@ -0,0 +1,481 @@
+<?php
+
+namespace Icinga\Module\Director\Import;
+
+use Exception;
+use Icinga\Application\Benchmark;
+use Icinga\Exception\IcingaException;
+use Icinga\Module\Director\Data\RecursiveUtf8Validator;
+use Icinga\Module\Director\Db;
+use Icinga\Module\Director\Hook\ImportSourceHook;
+use Icinga\Module\Director\Objects\ImportSource;
+use Icinga\Module\Director\Util;
+use stdClass;
+
+class Import
+{
+    /**
+     * @var ImportSource
+     */
+    protected $source;
+
+    /**
+     * @var Db
+     */
+    protected $connection;
+
+    /**
+     * @var \Zend_Db_Adapter_Abstract
+     */
+    protected $db;
+
+    /**
+     * Raw data that should be imported, array of stdClass objects
+     *
+     * @var array
+     */
+    protected $data;
+
+    /**
+     * Checksum of the rowset that should be imported
+     *
+     * @var string
+     */
+    private $rowsetChecksum;
+
+    /**
+     * Checksum-indexed rows
+     *
+     * @var array
+     */
+    private $rows;
+
+    /**
+     * Checksum-indexed row -> property
+     *
+     * @var array
+     */
+    private $rowProperties;
+
+    /**
+     * Whether this rowset exists, for caching purposes
+     *
+     * @var boolean
+     */
+    private $rowsetExists;
+
+    protected $properties = array();
+
+    /**
+     * Checksums of all rows
+     */
+    private $rowChecksums;
+
+    public function __construct(ImportSource $source)
+    {
+        $this->source = $source;
+        $this->connection = $source->getConnection();
+        $this->db = $this->connection->getDbAdapter();
+    }
+
+    /**
+     * Whether this import provides modified data
+     *
+     * @return boolean
+     */
+    public function providesChanges()
+    {
+        return ! $this->rowsetExists()
+            || ! $this->lastRowsetIs($this->rowsetChecksum());
+    }
+
+    /**
+     * Trigger an import run
+     *
+     * @return int Last import run ID
+     */
+    public function run()
+    {
+        if ($this->providesChanges() && ! $this->rowsetExists()) {
+            $this->storeRowset();
+        }
+
+        $this->db->insert(
+            'import_run',
+            array(
+                'source_id'       => $this->source->get('id'),
+                'rowset_checksum' => $this->quoteBinary($this->rowsetChecksum()),
+                'start_time'      => date('Y-m-d H:i:s'),
+                'succeeded'       => 'y'
+            )
+        );
+        if ($this->connection->isPgsql()) {
+            return $this->db->lastInsertId('import_run', 'id');
+        } else {
+            return $this->db->lastInsertId();
+        }
+    }
+
+    /**
+     * Whether there are no rows to be fetched from import source
+     *
+     * @return boolean
+     */
+    public function isEmpty()
+    {
+        $rows = $this->checksummedRows();
+        return empty($rows);
+    }
+
+    /**
+     * Checksum of all available rows
+     *
+     * @return string
+     */
+    protected function & rowsetChecksum()
+    {
+        if ($this->rowsetChecksum === null) {
+            $this->prepareChecksummedRows();
+        }
+
+        return $this->rowsetChecksum;
+    }
+
+    /**
+     * All rows
+     *
+     * @return array
+     */
+    protected function & checksummedRows()
+    {
+        if ($this->rows === null) {
+            $this->prepareChecksummedRows();
+        }
+
+        return $this->rows;
+    }
+
+    /**
+     * Checksum of all available rows
+     *
+     * @return array
+     */
+    protected function & rawData()
+    {
+        if ($this->data === null) {
+            $this->data = ImportSourceHook::forImportSource(
+                $this->source
+            )->fetchData();
+            Benchmark::measure('Fetched all data from Import Source');
+            $this->source->applyModifiers($this->data);
+            Benchmark::measure('Applied Property Modifiers to imported data');
+        }
+
+        return $this->data;
+    }
+
+    /**
+     * Prepare and remember an ImportedProperty
+     *
+     * @param string $key
+     * @param mixed  $rawValue
+     *
+     * @return array
+     */
+    protected function prepareImportedProperty($key, $rawValue)
+    {
+        if (is_array($rawValue) || is_bool($rawValue) || is_int($rawValue) || is_float($rawValue)) {
+            $value = json_encode($rawValue);
+            $format = 'json';
+        } elseif ($rawValue instanceof stdClass) {
+            $value = json_encode($this->sortObject($rawValue));
+            $format = 'json';
+        } else {
+            $value = $rawValue;
+            $format = 'string';
+        }
+
+        $checksum = sha1(sprintf('%s=(%s)%s', $key, $format, $value), true);
+
+        if (! array_key_exists($checksum, $this->properties)) {
+            $this->properties[$checksum] = array(
+                'checksum'       => $this->quoteBinary($checksum),
+                'property_name'  => $key,
+                'property_value' => $value,
+                'format'         => $format
+            );
+        }
+
+        return $this->properties[$checksum];
+    }
+
+    /**
+     * Walk through each row, prepare properties and calculate checksums
+     */
+    protected function prepareChecksummedRows()
+    {
+        $keyColumn = $this->source->get('key_column');
+        $this->rows = array();
+        $this->rowProperties = array();
+        $objects = array();
+        $rowCount = 0;
+
+        foreach ($this->rawData() as $row) {
+            $rowCount++;
+
+            // Key column must be set
+            if (! isset($row->$keyColumn)) {
+                throw new IcingaException(
+                    'No key column "%s" in row %d',
+                    $keyColumn,
+                    $rowCount
+                );
+            }
+
+            $object_name = $row->$keyColumn;
+
+            // Check for name collision
+            if (array_key_exists($object_name, $objects)) {
+                throw new IcingaException(
+                    'Duplicate entry: %s',
+                    $object_name
+                );
+            }
+
+            $rowChecksums = array();
+            $keys = array_keys((array) $row);
+            sort($keys);
+
+            foreach ($keys as $key) {
+                // TODO: Specify how to treat NULL values. Ignoring for now.
+                //       One option might be to import null (checksum '(null)')
+                //       and to provide a flag at sync time
+                if ($row->$key === null) {
+                    continue;
+                }
+
+                $property = $this->prepareImportedProperty($key, $row->$key);
+                $rowChecksums[] = $property['checksum'];
+            }
+
+            $checksum = sha1($object_name . ';' . implode(';', $rowChecksums), true);
+            if (array_key_exists($checksum, $this->rows)) {
+                die('WTF, collision?');
+            }
+
+            $this->rows[$checksum] = array(
+                'checksum'    => $this->quoteBinary($checksum),
+                'object_name' => $object_name
+            );
+
+            $this->rowProperties[$checksum] = $rowChecksums;
+
+            $objects[$object_name] = $checksum;
+        }
+
+        $this->rowChecksums = array_keys($this->rows);
+        $this->rowsetChecksum = sha1(implode(';', $this->rowChecksums), true);
+        return $this;
+    }
+
+    /**
+     * Store our new rowset
+     */
+    protected function storeRowset()
+    {
+        $db = $this->db;
+        $rowset = $this->rowsetChecksum();
+        $rows = $this->checksummedRows();
+
+        $db->beginTransaction();
+
+        try {
+            if ($this->isEmpty()) {
+                $newRows = array();
+                $newProperties = array();
+            } else {
+                $newRows = $this->newChecksums('imported_row', $this->rowChecksums);
+                $newProperties = $this->newChecksums('imported_property', array_keys($this->properties));
+            }
+
+            $db->insert('imported_rowset', array('checksum' => $this->quoteBinary($rowset)));
+
+            foreach ($newProperties as $checksum) {
+                $db->insert('imported_property', $this->properties[$checksum]);
+            }
+
+            foreach ($newRows as $row) {
+                try {
+                    $db->insert('imported_row', $rows[$row]);
+                    foreach ($this->rowProperties[$row] as $property) {
+                        $db->insert('imported_row_property', array(
+                            'row_checksum'      => $this->quoteBinary($row),
+                            'property_checksum' => $property
+                        ));
+                    }
+                } catch (Exception $e) {
+                    throw new IcingaException(
+                        "Error while storing a row for '%s' into database: %s",
+                        $rows[$row]['object_name'],
+                        $e->getMessage()
+                    );
+                }
+            }
+
+            foreach (array_keys($rows) as $row) {
+                $db->insert(
+                    'imported_rowset_row',
+                    array(
+                        'rowset_checksum' => $this->quoteBinary($rowset),
+                        'row_checksum'    => $this->quoteBinary($row)
+                    )
+                );
+            }
+
+            $db->commit();
+
+            $this->rowsetExists = true;
+        } catch (Exception $e) {
+            try {
+                $db->rollBack();
+            } catch (Exception $e) {
+                // Well...
+            }
+            // Eventually throws details for invalid UTF8 characters
+            RecursiveUtf8Validator::validateRows($this->data);
+            throw $e;
+        }
+    }
+
+    /**
+     * Whether the last run of this import matches the given checksum
+     *
+     * @param  string $checksum Binary checksum
+     *
+     * @return bool
+     */
+    protected function lastRowsetIs($checksum)
+    {
+        return $this->connection->getLatestImportedChecksum($this->source->get('id'))
+            === bin2hex($checksum);
+    }
+
+    /**
+     * Whether our rowset already exists in the database
+     *
+     * @return boolean
+     */
+    protected function rowsetExists()
+    {
+        if (null === $this->rowsetExists) {
+            $this->rowsetExists = 0 === count(
+                $this->newChecksums(
+                    'imported_rowset',
+                    array($this->rowsetChecksum())
+                )
+            );
+        }
+
+        return $this->rowsetExists;
+    }
+
+    /**
+     * Finde new checksums for a specific table
+     *
+     * Accepts an array of checksums and gives you an array with those checksums
+     * that are missing in the given table
+     *
+     * @param string $table Database table name
+     * @param array  $checksums Array with the checksums that should be verified
+     *
+     * @return array
+     */
+    protected function newChecksums($table, $checksums)
+    {
+        $db = $this->db;
+
+        // TODO: The following is a quickfix for binary data corrpution reported
+        //       in https://github.com/zendframework/zf1/issues/655 caused by
+        //       https://github.com/zendframework/zf1/commit/2ac9c30f
+        //
+        // Should be reverted once fixed, eventually with a check continueing
+        // to use this workaround for specific ZF versions (1.12.16 and 1.12.17
+        // so far). Alternatively we could also use a custom quoteInto method.
+
+        // The former query looked as follows:
+        //
+        // $query = $db->select()->from($table, 'checksum')
+        //     ->where('checksum IN (?)', $checksums)
+        // ...
+        // return array_diff($checksums, $existing);
+
+        $hexed = array_map('bin2hex', $checksums);
+
+        $conn = $this->connection;
+        $query = $db
+            ->select()
+            ->from(
+                array('c' => $table),
+                array('checksum' => $conn->dbHexFunc('c.checksum'))
+            )->where(
+                $conn->dbHexFunc('c.checksum') . ' IN (?)',
+                $hexed
+            );
+
+        $existing = $db->fetchCol($query);
+        $new = array_diff($hexed, $existing);
+
+        return array_map('hex2bin', $new);
+    }
+
+    /**
+     * Sort a given stdClass object by property name
+     *
+     * @param  stdClass $object
+     *
+     * @return object
+     */
+    protected function sortObject($object)
+    {
+        $array = (array) $object;
+        foreach ($array as $key => $val) {
+            $this->sortElement($val);
+        }
+        ksort($array);
+        return (object) $array;
+    }
+
+    /**
+     * Walk through a given array and sort all children
+     *
+     * Please note that the array itself will NOT be sorted, as arrays must
+     * keep their ordering
+     *
+     * @param array $array
+     */
+    protected function sortArrayObject(&$array)
+    {
+        foreach ($array as $key => $val) {
+            $this->sortElement($val);
+        }
+    }
+
+    /**
+     * Recursively sort a given property
+     *
+     * @param mixed $el
+     */
+    protected function sortElement(&$el)
+    {
+        if (is_array($el)) {
+            $this->sortArrayObject($el);
+        } elseif ($el instanceof stdClass) {
+            $el = $this->sortObject($el);
+        }
+    }
+
+    protected function quoteBinary($bin)
+    {
+        return $this->connection->quoteBinary($bin);
+    }
+}
diff --git a/library/Director/Import/ImportSourceCoreApi.php b/library/Director/Import/ImportSourceCoreApi.php
new file mode 100644
index 0000000..6d590ec
--- /dev/null
+++ b/library/Director/Import/ImportSourceCoreApi.php
@@ -0,0 +1,92 @@
+<?php
+
+namespace Icinga\Module\Director\Import;
+
+use Icinga\Application\Config;
+use Icinga\Module\Director\Db;
+use Icinga\Module\Director\Hook\ImportSourceHook;
+use Icinga\Module\Director\Web\Form\QuickForm;
+
+class ImportSourceCoreApi extends ImportSourceHook
+{
+    protected $connection;
+
+    protected $db;
+
+    protected $api;
+
+    public function fetchData()
+    {
+        $func = 'get' . $this->getSetting('object_type') . 'Objects';
+        $objects = $this->api()->$func();
+        $result = array();
+        foreach ($objects as $object) {
+            $result[] = $object->toPlainObject();
+        }
+
+        return $result;
+    }
+
+    public function listColumns()
+    {
+        $res = $this->fetchData();
+        if (empty($data)) {
+            return array('object_name');
+        }
+
+        return array_keys((array) $res[0]);
+    }
+
+    public static function getDefaultKeyColumnName()
+    {
+        return 'object_name';
+    }
+
+    public static function addSettingsFormFields(QuickForm $form)
+    {
+        $form->addElement('select', 'object_type', array(
+            'label'    => 'Object type',
+            'required' => true,
+            'multiOptions' => $form->optionalEnum(self::enumObjectTypes($form))
+        ));
+    }
+
+    protected static function enumObjectTypes($form)
+    {
+        $types = array(
+            'CheckCommand'        => $form->translate('Check Commands'),
+            'NotificationCommand' => $form->translate('Notification Commands'),
+            'Endpoint'            => $form->translate('Endpoints'),
+            'Host'                => $form->translate('Hosts'),
+            'HostGroup'           => $form->translate('Hostgroups'),
+            'User'                => $form->translate('Users'),
+            'UserGroup'           => $form->translate('Usergroups'),
+            'Zone'                => $form->translate('Zones'),
+        );
+
+        asort($types);
+        return $types;
+    }
+
+    protected function api()
+    {
+        if ($this->api === null) {
+            $endpoint = $this->db()->getDeploymentEndpoint();
+            $this->api = $endpoint->api()->setDb($this->db());
+        }
+
+        return $this->api;
+    }
+
+    protected function db()
+    {
+        if ($this->db === null) {
+            $resourceName = Config::module('director')->get('db', 'resource');
+            if ($resourceName) {
+                $this->db = Db::fromResourceName($resourceName);
+            }
+        }
+
+        return $this->db;
+    }
+}
diff --git a/library/Director/Import/ImportSourceDirectorObject.php b/library/Director/Import/ImportSourceDirectorObject.php
new file mode 100644
index 0000000..e3f56fc
--- /dev/null
+++ b/library/Director/Import/ImportSourceDirectorObject.php
@@ -0,0 +1,120 @@
+<?php
+
+namespace Icinga\Module\Director\Import;
+
+use Icinga\Application\Config;
+use Icinga\Module\Director\Data\Db\DbObjectTypeRegistry;
+use Icinga\Module\Director\Db;
+use Icinga\Module\Director\Forms\ImportSourceForm;
+use Icinga\Module\Director\Hook\ImportSourceHook;
+use Icinga\Module\Director\Objects\IcingaObject;
+use Icinga\Module\Director\Util;
+use Icinga\Module\Director\Web\Form\QuickForm;
+
+class ImportSourceDirectorObject extends ImportSourceHook
+{
+    protected $db;
+
+    public function getName()
+    {
+        return 'Director Objects';
+    }
+
+    public static function getDefaultKeyColumnName()
+    {
+        return 'object_name';
+    }
+
+    public function fetchData()
+    {
+        $db = $this->db();
+        $objectClass = $this->getSetting('object_class');
+        $objectType = $this->getSetting('object_type');
+        /** @var IcingaObject $class fake type hint, it's a string */
+        $class = DbObjectTypeRegistry::classByType($objectClass);
+        if ($objectType) {
+            $dummy = $class::create();
+            $query = $db->getDbAdapter()->select()
+                ->from($dummy->getTableName())
+                ->where('object_type = ?', $objectType);
+        } else {
+            $query = null;
+        }
+        $result = [];
+        $resolved = $this->getSetting('resolved') === 'y';
+        foreach ($class::loadAllByType($objectClass, $db, $query) as $object) {
+            $result[] = $object->toPlainObject($resolved);
+        }
+        if ($objectClass === 'zone') {
+            $this->enrichZonesWithDeploymentZone($result);
+        }
+        return $result;
+    }
+
+    protected function enrichZonesWithDeploymentZone(&$zones)
+    {
+        $masterZone = $this->db()->getMasterZoneName();
+        foreach ($zones as $zone) {
+            $zone->is_master_zone = $zone->object_name === $masterZone;
+        }
+    }
+
+    public static function addSettingsFormFields(QuickForm $form)
+    {
+        /** @var ImportSourceForm $form */
+        Util::addDbResourceFormElement($form, 'resource');
+        $form->getElement('resource')
+            ->setValue(Config::module('director')->get('db', 'resource'));
+        $form->addElement('select', 'object_class', [
+            'label'    => $form->translate('Director Object'),
+            'multiOptions' => [
+                'host'     => $form->translate('Host'),
+                'endpoint' => $form->translate('Endpoint'),
+                'zone'     => $form->translate('Zone'),
+            ],
+            'required' => true,
+        ]);
+        $form->addElement('select', 'object_type', [
+            'label'    => $form->translate('Object Type'),
+            'multiOptions' => [
+                null              => $form->translate('All Object Types'),
+                'object'          => $form->translate('Objects'),
+                'template'        => $form->translate('Templates'),
+                'external_object' => $form->translate('External Objects'),
+                'apply'           => $form->translate('Apply Rules'),
+            ],
+        ]);
+
+        /** @var $form \Icinga\Module\Director\Web\Form\DirectorObjectForm */
+        $form->addBoolean('resolved', [
+            'label' => $form->translate('Resolved'),
+        ], 'n');
+
+        return $form;
+    }
+
+    protected function db()
+    {
+        if ($this->db === null) {
+            $this->db = Db::fromResourceName($this->settings['resource']);
+        }
+
+        return $this->db;
+    }
+
+    public function listColumns()
+    {
+        $rows = $this->fetchData();
+        $columns = [];
+
+        foreach ($rows as $object) {
+            foreach (array_keys((array) $object) as $column) {
+                if (! isset($columns[$column])) {
+                    $columns[] = $column;
+                }
+            }
+        }
+
+        return $columns;
+    }
+}
diff --git a/library/Director/Import/ImportSourceLdap.php b/library/Director/Import/ImportSourceLdap.php
new file mode 100644
index 0000000..4518565
--- /dev/null
+++ b/library/Director/Import/ImportSourceLdap.php
@@ -0,0 +1,90 @@
+<?php
+
+namespace Icinga\Module\Director\Import;
+
+use Icinga\Data\ResourceFactory;
+use Icinga\Module\Director\Hook\ImportSourceHook;
+use Icinga\Module\Director\Util;
+use Icinga\Module\Director\Web\Form\QuickForm;
+
+class ImportSourceLdap extends ImportSourceHook
+{
+    protected $connection;
+
+    public function fetchData()
+    {
+        $columns = $this->listColumns();
+        $query = $this->connection()
+            ->select()
+            ->setUsePagedResults()
+            ->from($this->settings['objectclass'], $columns);
+
+        if ($base = $this->settings['base']) {
+            $query->setBase($base);
+        }
+        if ($filter = $this->settings['filter']) {
+            $query->setNativeFilter($filter);
+        }
+
+        if (in_array('dn', $columns)) {
+            $result = $query->fetchAll();
+            foreach ($result as $dn => $row) {
+                $row->dn = $dn;
+            }
+
+            return $result;
+        } else {
+            return $query->fetchAll();
+        }
+    }
+
+    public function listColumns()
+    {
+        return preg_split('/,\s*/', $this->settings['query'], -1, PREG_SPLIT_NO_EMPTY);
+    }
+
+    public static function addSettingsFormFields(QuickForm $form)
+    {
+        Util::addLDAPResourceFormElement($form, 'resource');
+        $form->addElement('text', 'base', array(
+            'label'       => $form->translate('LDAP Search Base'),
+            'description' => $form->translate(
+                'Your LDAP search base. Often something like OU=Users,OU=HQ,DC=your,DC=company,DC=tld'
+            )
+        ));
+        $form->addElement('text', 'objectclass', array(
+            'label'       => $form->translate('Object class'),
+            'description' => $form->translate(
+                'An object class to search for. Might be "user", "group", "computer" or similar'
+            )
+        ));
+        $form->addElement('text', 'filter', array(
+            'label'    => 'LDAP filter',
+            'description' => $form->translate(
+                'A custom LDAP filter to use in addition to the object class. This allows'
+                . ' for a lot of flexibility but requires LDAP filter skills. Simple filters'
+                . ' might look as follows: operatingsystem=*server*'
+            )
+        ));
+        $form->addElement('textarea', 'query', array(
+            'label'       => $form->translate('Properties'),
+            'description' => $form->translate(
+                'The LDAP properties that should be fetched. This is required to be a'
+                . ' comma-separated list like: "cn, dnshostname, operatingsystem, sAMAccountName"'
+            ),
+            'spellcheck'  => 'false',
+            'required'    => true,
+            'rows'        => 5,
+        ));
+        return $form;
+    }
+
+    protected function connection()
+    {
+        if ($this->connection === null) {
+            $this->connection = ResourceFactory::create($this->settings['resource']);
+        }
+
+        return $this->connection;
+    }
+}
diff --git a/library/Director/Import/ImportSourceRestApi.php b/library/Director/Import/ImportSourceRestApi.php
new file mode 100644
index 0000000..dc772e1
--- /dev/null
+++ b/library/Director/Import/ImportSourceRestApi.php
@@ -0,0 +1,380 @@
+<?php
+
+namespace Icinga\Module\Director\Import;
+
+use Icinga\Exception\InvalidPropertyException;
+use Icinga\Module\Director\Hook\ImportSourceHook;
+use Icinga\Module\Director\RestApi\RestApiClient;
+use Icinga\Module\Director\Web\Form\QuickForm;
+use InvalidArgumentException;
+
+class ImportSourceRestApi extends ImportSourceHook
+{
+    public function getName()
+    {
+        return 'REST API';
+    }
+
+    public function fetchData()
+    {
+        $result = $this->getRestApi()->get(
+            $this->getUrl(),
+            null,
+            $this->buildHeaders()
+        );
+        $result = $this->extractProperty($result);
+
+        return (array) $result;
+    }
+
+    public function listColumns()
+    {
+        $rows = $this->fetchData();
+        $columns = [];
+
+        foreach ($rows as $object) {
+            foreach (array_keys((array) $object) as $column) {
+                if (! isset($columns[$column])) {
+                    $columns[] = $column;
+                }
+            }
+        }
+
+        return $columns;
+    }
+
+    /**
+     * Extract result from a property specified
+     *
+     * A simple key, like "objects", will take the final result from key objects
+     *
+     * If you have a deeper key like "objects" under the key "results", specify this as "results.objects".
+     *
+     * When a key of the JSON object contains a literal ".", this can be escaped as
+     *
+     * @param $result
+     *
+     * @return mixed
+     */
+    protected function extractProperty($result)
+    {
+        $property = $this->getSetting('extract_property');
+        if (! $property) {
+            return $result;
+        }
+
+        $parts = preg_split('~(?<!\\\\)\.~', $property);
+
+        // iterate over parts of the attribute path
+        $data = $result;
+        foreach ($parts as $part) {
+            // un-escape any dots
+            $part = preg_replace('~\\\\.~', '.', $part);
+
+            if (property_exists($data, $part)) {
+                $data = $data->$part;
+            } else {
+                throw new \RuntimeException(sprintf(
+                    'Result has no "%s" property. Available keys: %s',
+                    $part,
+                    implode(', ', array_keys((array) $data))
+                ));
+            }
+        }
+
+        return $data;
+    }
+
+    protected function buildHeaders()
+    {
+        $headers = [];
+
+        $text = $this->getSetting('headers', '');
+        foreach (preg_split('~\r?\n~', $text, -1, PREG_SPLIT_NO_EMPTY) as $header) {
+            $header = trim($header);
+            $parts = preg_split('~\s*:\s*~', $header, 2);
+            if (count($parts) < 2) {
+                throw new InvalidPropertyException('Could not parse header: "%s"', $header);
+            }
+
+            $headers[$parts[0]] = $parts[1];
+        }
+
+        return $headers;
+    }
+
+    /**
+     * @param QuickForm $form
+     * @throws \Zend_Form_Exception
+     */
+    public static function addSettingsFormFields(QuickForm $form)
+    {
+        static::addScheme($form);
+        static::addSslOptions($form);
+        static::addUrl($form);
+        static::addResultProperty($form);
+        static::addAuthentication($form);
+        static::addHeader($form);
+        static::addProxy($form);
+    }
+
+    /**
+     * @param QuickForm $form
+     * @throws \Zend_Form_Exception
+     */
+    protected static function addScheme(QuickForm $form)
+    {
+        $form->addElement('select', 'scheme', [
+            'label' => $form->translate('Protocol'),
+            'description' => $form->translate(
+                'Whether to use encryption when talking to the REST API'
+            ),
+            'multiOptions' => [
+                'HTTPS' => $form->translate('HTTPS (strongly recommended)'),
+                'HTTP'  => $form->translate('HTTP (this is plaintext!)'),
+            ],
+            'class'    => 'autosubmit',
+            'value'    => 'HTTPS',
+            'required' => true,
+        ]);
+    }
+
+    /**
+     * @param QuickForm $form
+     * @throws \Zend_Form_Exception
+     */
+    protected static function addHeader(QuickForm $form)
+    {
+        $form->addElement('textarea', 'headers', [
+            'label'       => $form->translate('HTTP Header'),
+            'description' => implode(' ', [
+                $form->translate('Additional headers for the HTTP request.'),
+                $form->translate('Specify headers in text format "Header: Value", each header on a new line.'),
+            ]),
+            'class'       => 'preformatted',
+            'rows'        => 4,
+        ]);
+    }
+
+    /**
+     * @param QuickForm $form
+     * @throws \Zend_Form_Exception
+     */
+    protected static function addSslOptions(QuickForm $form)
+    {
+        $ssl = ! ($form->getSentOrObjectSetting('scheme', 'HTTPS') === 'HTTP');
+
+        if ($ssl) {
+            static::addBoolean($form, 'ssl_verify_peer', [
+                'label'       => $form->translate('Verify Peer'),
+                'description' => $form->translate(
+                    'Whether we should check that our peer\'s certificate has'
+                    . ' been signed by a trusted CA. This is strongly recommended.'
+                )
+            ], 'y');
+            static::addBoolean($form, 'ssl_verify_host', [
+                'label'       => $form->translate('Verify Host'),
+                'description' => $form->translate(
+                    'Whether we should check that the certificate matches the'
+                    . 'configured host'
+                )
+            ], 'y');
+        }
+    }
+
+    /**
+     * @param QuickForm $form
+     * @throws \Zend_Form_Exception
+     */
+    protected static function addUrl(QuickForm $form)
+    {
+        $form->addElement('text', 'url', [
+            'label'    => 'REST API URL',
+            'description' => $form->translate(
+                'Something like https://api.example.com/rest/v2/objects'
+            ),
+            'required' => true,
+        ]);
+    }
+
+    /**
+     * @param QuickForm $form
+     * @throws \Zend_Form_Exception
+     */
+    protected static function addResultProperty(QuickForm $form)
+    {
+        $form->addElement('text', 'extract_property', [
+            'label'       => 'Extract property',
+            'description' => implode("\n", [
+                $form->translate('Often the expected result is provided in a property like "objects".'
+                    . ' Please specify this if required.'),
+                $form->translate('Also deeper keys can be specific by a dot-notation:'),
+                '"result.objects", "key.deeper_key.very_deep"',
+                $form->translate('Literal dots in a key name can be written in the escape notation:'),
+                '"key\.with\.dots"',
+            ])
+        ]);
+    }
+
+    /**
+     * @param QuickForm $form
+     * @throws \Zend_Form_Exception
+     */
+    protected static function addAuthentication(QuickForm $form)
+    {
+        $form->addElement('text', 'username', [
+            'label' => $form->translate('Username'),
+            'description' => $form->translate(
+                'Will be used to authenticate against your REST API'
+            ),
+        ]);
+
+        $form->addElement('storedPassword', 'password', [
+            'label' => $form->translate('Password'),
+        ]);
+    }
+
+    /**
+     * @param QuickForm $form
+     * @throws \Zend_Form_Exception
+     */
+    protected static function addProxy(QuickForm $form)
+    {
+        $form->addElement('select', 'proxy_type', [
+            'label' => $form->translate('Proxy'),
+            'description' => $form->translate(
+                'In case your API is only reachable through a proxy, please'
+                . ' choose it\'s protocol right here'
+            ),
+            'multiOptions' => $form->optionalEnum([
+                'HTTP'   => $form->translate('HTTP proxy'),
+                'SOCKS5' => $form->translate('SOCKS5 proxy'),
+            ]),
+            'class' => 'autosubmit'
+        ]);
+
+        $proxyType = $form->getSentOrObjectSetting('proxy_type');
+
+        if ($proxyType) {
+            $form->addElement('text', 'proxy', [
+                'label' => $form->translate('Proxy Address'),
+                'description' => $form->translate(
+                    'Hostname, IP or <host>:<port>'
+                ),
+                'required' => true,
+            ]);
+            if ($proxyType === 'HTTP') {
+                $form->addElement('text', 'proxy_user', [
+                    'label'       => $form->translate('Proxy Username'),
+                    'description' => $form->translate(
+                        'In case your proxy requires authentication, please'
+                        . ' configure this here'
+                    ),
+                ]);
+
+                $passRequired = strlen($form->getSentOrObjectSetting('proxy_user')) > 0;
+
+                $form->addElement('storedPassword', 'proxy_pass', [
+                    'label'    => $form->translate('Proxy Password'),
+                    'required' => $passRequired
+                ]);
+            }
+        }
+    }
+
+    protected function getUrl()
+    {
+        $url = $this->getSetting('url');
+        $parts = \parse_url($url);
+        if (isset($parts['path'])) {
+            $path = $parts['path'];
+        } else {
+            $path = '/';
+        }
+
+        if (isset($parts['query'])) {
+            $url = "$path?" . $parts['query'];
+        } else {
+            $url = $path;
+        }
+
+        return $url;
+    }
+
+    protected function getRestApi()
+    {
+        $url = $this->getSetting('url');
+        $parts = \parse_url($url);
+        if (isset($parts['host'])) {
+            $host = $parts['host'];
+        } else {
+            throw new InvalidArgumentException("URL '$url' has no host");
+        }
+
+        $api = new RestApiClient(
+            $host,
+            $this->getSetting('username'),
+            $this->getSetting('password')
+        );
+
+        $api->setScheme($this->getSetting('scheme'));
+        if (isset($parts['port'])) {
+            $api->setPort($parts['port']);
+        }
+
+        if ($api->getScheme() === 'HTTPS') {
+            if ($this->getSetting('ssl_verify_peer', 'y') === 'n') {
+                $api->disableSslPeerVerification();
+            }
+            if ($this->getSetting('ssl_verify_host', 'y') === 'n') {
+                $api->disableSslHostVerification();
+            }
+        }
+
+        if ($proxy = $this->getSetting('proxy')) {
+            if ($proxyType = $this->getSetting('proxy_type')) {
+                $api->setProxy($proxy, $proxyType);
+            } else {
+                $api->setProxy($proxy);
+            }
+
+            if ($user = $this->getSetting('proxy_user')) {
+                $api->setProxyAuth($user, $this->getSetting('proxy_pass'));
+            }
+        }
+
+        return $api;
+    }
+
+    /**
+     * @param QuickForm $form
+     * @param string $key
+     * @param array $options
+     * @param string|null $default
+     * @throws \Zend_Form_Exception
+     */
+    protected static function addBoolean(QuickForm $form, $key, $options, $default = null)
+    {
+        if ($default === null) {
+            $form->addElement('OptionalYesNo', $key, $options);
+        } else {
+            $form->addElement('YesNo', $key, $options);
+            $form->getElement($key)->setValue($default);
+        }
+    }
+
+    /**
+     * @param QuickForm $form
+     * @param string $key
+     * @param string $label
+     * @param string $description
+     * @throws \Zend_Form_Exception
+     */
+    protected static function optionalBoolean(QuickForm $form, $key, $label, $description)
+    {
+        static::addBoolean($form, $key, [
+            'label'       => $label,
+            'description' => $description
+        ]);
+    }
+}
diff --git a/library/Director/Import/ImportSourceSql.php b/library/Director/Import/ImportSourceSql.php
new file mode 100644
index 0000000..b08a3f3
--- /dev/null
+++ b/library/Director/Import/ImportSourceSql.php
@@ -0,0 +1,70 @@
+<?php
+
+namespace Icinga\Module\Director\Import;
+
+use gipfl\Web\Widget\Hint;
+use Icinga\Data\Db\DbConnection;
+use Icinga\Module\Director\Forms\ImportSourceForm;
+use Icinga\Module\Director\Hook\ImportSourceHook;
+use Icinga\Module\Director\Objects\ImportSource;
+use Icinga\Module\Director\Util;
+use Icinga\Module\Director\Web\Form\Filter\QueryColumnsFromSql;
+use Icinga\Module\Director\Web\Form\QuickForm;
+use ipl\Html\Html;
+
+class ImportSourceSql extends ImportSourceHook
+{
+    protected $db;
+
+    public function fetchData()
+    {
+        return $this->db()->fetchAll($this->settings['query']);
+    }
+
+    public function listColumns()
+    {
+        if ($columns = $this->getSetting('column_cache')) {
+            return explode(', ', $columns);
+        } else {
+            return array_keys((array) current($this->fetchData()));
+        }
+    }
+
+    public static function addSettingsFormFields(QuickForm $form)
+    {
+        /** @var ImportSourceForm $form */
+        Util::addDbResourceFormElement($form, 'resource');
+        /** @var ImportSource $current */
+        $current = $form->getObject();
+
+        $form->addElement('textarea', 'query', [
+            'label'    => $form->translate('DB Query'),
+            'required' => true,
+            'rows'     => 15,
+        ]);
+        $form->addElement('hidden', 'column_cache', [
+            'value'    => '',
+            'filters'  => [new QueryColumnsFromSql($form)],
+            'required' => true
+        ]);
+        if ($current) {
+            if ($columns = $current->getSetting('column_cache')) {
+                $form->addHtmlHint('Columns: ' . $columns);
+            } else {
+                $form->addHtmlHint(Hint::warning($form->translate(
+                    'Please click "Store" once again to determine query columns'
+                )));
+            }
+        }
+        return $form;
+    }
+
+    protected function db()
+    {
+        if ($this->db === null) {
+            $this->db = DbConnection::fromResourceName($this->settings['resource'])->getDbAdapter();
+        }
+
+        return $this->db;
+    }
+}
diff --git a/library/Director/Import/PurgeStrategy/ImportRunBasedPurgeStrategy.php b/library/Director/Import/PurgeStrategy/ImportRunBasedPurgeStrategy.php
new file mode 100644
index 0000000..9f0e8ab
--- /dev/null
+++ b/library/Director/Import/PurgeStrategy/ImportRunBasedPurgeStrategy.php
@@ -0,0 +1,90 @@
+<?php
+
+namespace Icinga\Module\Director\Import\PurgeStrategy;
+
+use Icinga\Module\Director\Import\SyncUtils;
+use Icinga\Module\Director\Objects\ImportRun;
+use Icinga\Module\Director\Objects\ImportSource;
+
+class ImportRunBasedPurgeStrategy extends PurgeStrategy
+{
+    public function listObjectsToPurge()
+    {
+        $remove = array();
+
+        foreach ($this->getSyncRule()->fetchInvolvedImportSources() as $source) {
+            $remove += $this->checkImportSource($source);
+        }
+
+        return $remove;
+    }
+
+    protected function getLastSync()
+    {
+        return strtotime($this->getSyncRule()->getLastSyncTimestamp());
+    }
+
+    // TODO: NAMING!
+    protected function checkImportSource(ImportSource $source)
+    {
+        if (null === ($lastSync = $this->getLastSync())) {
+            // No last sync, nothing to purge
+            return array();
+        }
+
+        $runA = $source->fetchLastRunBefore($lastSync);
+        if ($runA === null) {
+            // Nothing to purge for this source
+            return array();
+        }
+
+        $runB = $source->fetchLastRun();
+        if ($runA->rowset_checksum === $runB->rowset_checksum) {
+            // Same source data, nothing to purge
+            return array();
+        }
+
+        return $this->listKeysRemovedBetween($runA, $runB);
+    }
+
+    public function listKeysRemovedBetween(ImportRun $runA, ImportRun $runB)
+    {
+        $rule = $this->getSyncRule();
+        $db = $rule->getDb();
+
+        $selectA = $runA->prepareImportedObjectQuery();
+        $selectB = $runB->prepareImportedObjectQuery();
+
+        $query = $db->select()->from(
+            array('a' => $selectA),
+            'a.object_name'
+        )->where('a.object_name NOT IN (?)', $selectB);
+
+        $result = $db->fetchCol($query);
+
+        if (empty($result)) {
+            return array();
+        }
+
+        if ($rule->hasCombinedKey()) {
+            $pattern = $rule->getSourceKeyPattern();
+            $columns = SyncUtils::getRootVariables(
+                SyncUtils::extractVariableNames($pattern)
+            );
+            $resultForCombinedKey = array();
+            foreach (array_chunk($result, 1000) as $keys) {
+                $rows = $runA->fetchRows($columns, null, $keys);
+                foreach ($rows as $row) {
+                    $resultForCombinedKey[] = SyncUtils::fillVariables($pattern, $row);
+                }
+            }
+            $result = $resultForCombinedKey;
+        }
+
+        if (empty($result)) {
+            return array();
+        }
+
+        return array_combine($result, $result);
+    }
+}
diff --git a/library/Director/Import/PurgeStrategy/PurgeNothingPurgeStrategy.php b/library/Director/Import/PurgeStrategy/PurgeNothingPurgeStrategy.php
new file mode 100644
index 0000000..3da8d4f
--- /dev/null
+++ b/library/Director/Import/PurgeStrategy/PurgeNothingPurgeStrategy.php
@@ -0,0 +1,11 @@
+<?php
+
+namespace Icinga\Module\Director\Import\PurgeStrategy;
+
+class PurgeNothingPurgeStrategy extends PurgeStrategy
+{
+    public function listObjectsToPurge()
+    {
+        return array();
+    }
+}
diff --git a/library/Director/Import/PurgeStrategy/PurgeStrategy.php b/library/Director/Import/PurgeStrategy/PurgeStrategy.php
new file mode 100644
index 0000000..ffbe14f
--- /dev/null
+++ b/library/Director/Import/PurgeStrategy/PurgeStrategy.php
@@ -0,0 +1,31 @@
+<?php
+
+namespace Icinga\Module\Director\Import\PurgeStrategy;
+
+use Icinga\Module\Director\Objects\SyncRule;
+
+abstract class PurgeStrategy
+{
+    private $rule;
+
+    public function __construct(SyncRule $rule)
+    {
+        $this->rule = $rule;
+    }
+
+    protected function getSyncRule()
+    {
+        return $this->rule;
+    }
+
+    abstract public function listObjectsToPurge();
+
+    /**
+     * @return PurgeStrategy
+     */
+    public static function load($name, SyncRule $rule)
+    {
+        $class = __NAMESPACE__ . '\\' . $name . 'PurgeStrategy';
+        return new $class($rule);
+    }
+}
diff --git a/library/Director/Import/Sync.php b/library/Director/Import/Sync.php
new file mode 100644
index 0000000..8fea46c
--- /dev/null
+++ b/library/Director/Import/Sync.php
@@ -0,0 +1,942 @@
+<?php
+
+namespace Icinga\Module\Director\Import;
+
+use Exception;
+use Icinga\Application\Benchmark;
+use Icinga\Data\Filter\Filter;
+use Icinga\Module\Director\Application\MemoryLimit;
+use Icinga\Module\Director\Data\Db\DbObject;
+use Icinga\Module\Director\Data\Db\DbObjectStore;
+use Icinga\Module\Director\Data\Db\DbObjectTypeRegistry;
+use Icinga\Module\Director\Db;
+use Icinga\Module\Director\Db\Branch\BranchSupport;
+use Icinga\Module\Director\Db\Cache\PrefetchCache;
+use Icinga\Module\Director\Objects\HostGroupMembershipResolver;
+use Icinga\Module\Director\Objects\IcingaHost;
+use Icinga\Module\Director\Objects\IcingaHostGroup;
+use Icinga\Module\Director\Objects\IcingaObject;
+use Icinga\Module\Director\Objects\ImportSource;
+use Icinga\Module\Director\Objects\IcingaService;
+use Icinga\Module\Director\Objects\SyncProperty;
+use Icinga\Module\Director\Objects\SyncRule;
+use Icinga\Module\Director\Objects\SyncRun;
+use Icinga\Exception\IcingaException;
+use Icinga\Module\Director\Repository\IcingaTemplateRepository;
+use InvalidArgumentException;
+use RuntimeException;
+
+class Sync
+{
+    /** @var SyncRule */
+    protected $rule;
+
+    /** @var Db */
+    protected $db;
+
+    /** @var array Related ImportSource objects */
+    protected $sources;
+
+    /** @var array Source columns we want to fetch from our sources */
+    protected $sourceColumns;
+
+    /** @var array Imported data */
+    protected $imported;
+
+    /** @var IcingaObject[] Objects to work with */
+    protected $objects;
+
+    /** @var array<mixed, array<int, string>> key => [property, property]*/
+    protected $setNull = [];
+
+    /** @var bool Whether we already prepared your sync */
+    protected $isPrepared = false;
+
+    /** @var bool Whether we applied strtolower() to existing object keys */
+    protected $usedLowerCasedKeys = false;
+
+    protected $modify = [];
+
+    protected $remove = [];
+
+    protected $create = [];
+
+    protected $errors = [];
+
+    /** @var SyncProperty[] */
+    protected $syncProperties;
+
+    protected $replaceVars = false;
+
+    protected $hasPropertyDisabled = false;
+
+    protected $serviceOverrideKeyName;
+
+    /**
+     * @var SyncRun
+     */
+    protected $run;
+
+    protected $runStartTime;
+
+    /** @var Filter[] */
+    protected $columnFilters = [];
+
+    /** @var HostGroupMembershipResolver|bool */
+    protected $hostGroupMembershipResolver;
+
+    /** @var ?DbObjectStore */
+    protected $store;
+
+    /**
+     * @param SyncRule $rule
+     * @param ?DbObjectStore $store
+     */
+    public function __construct(SyncRule $rule, DbObjectStore $store = null)
+    {
+        $this->rule = $rule;
+        $this->db = $rule->getConnection();
+        $this->store = $store;
+    }
+
+    /**
+     * Whether the given sync rule would apply modifications
+     *
+     * @return boolean
+     * @throws Exception
+     */
+    public function hasModifications()
+    {
+        return count($this->getExpectedModifications()) > 0;
+    }
+
+    /**
+     * Retrieve modifications a given SyncRule would apply
+     *
+     * @return array  Array of IcingaObject elements
+     * @throws \Icinga\Exception\NotFoundError
+     * @throws \Icinga\Module\Director\Exception\DuplicateKeyException
+     */
+    public function getExpectedModifications()
+    {
+        $modified = [];
+        $objects = $this->prepare();
+        $updateOnly = $this->rule->get('update_policy') === 'update-only';
+        $allowCreate = ! $updateOnly;
+        foreach ($objects as $object) {
+            if ($object->hasBeenModified()) {
+                if ($allowCreate || $object->hasBeenLoadedFromDb()) {
+                    $modified[] = $object;
+                }
+            } elseif (! $updateOnly && $object->shouldBeRemoved()) {
+                $modified[] = $object;
+            }
+        }
+
+        return $modified;
+    }
+
+    /**
+     * Transform the given value to an array
+     *
+     * @param  array|string|null $value
+     *
+     * @return array
+     */
+    protected function wantArray($value)
+    {
+        if (is_array($value)) {
+            return $value;
+        } elseif ($value === null) {
+            return [];
+        } else {
+            return [$value];
+        }
+    }
+
+    /**
+     * Raise PHP resource limits
+     *
+     * @return self;
+     */
+    protected function raiseLimits()
+    {
+        MemoryLimit::raiseTo('1024M');
+        ini_set('max_execution_time', 0);
+
+        return $this;
+    }
+
+    /**
+     * Initialize run summary measurements
+     *
+     * @return self;
+     */
+    protected function startMeasurements()
+    {
+        $this->run = SyncRun::start($this->rule);
+        $this->runStartTime = microtime(true);
+        Benchmark::measure('Starting sync');
+        return $this;
+    }
+
+    /**
+     * Fetch the configured properties involved in this sync
+     *
+     * @return self
+     */
+    protected function fetchSyncProperties()
+    {
+        $this->syncProperties = $this->rule->getSyncProperties();
+        foreach ($this->syncProperties as $key => $prop) {
+            $destinationField = $prop->get('destination_field');
+            if ($destinationField === 'vars' && $prop->get('merge_policy') === 'override') {
+                $this->replaceVars = true;
+            }
+
+            if ($destinationField === 'disabled') {
+                $this->hasPropertyDisabled = true;
+            }
+
+            if ($prop->get('filter_expression') === null || strlen($prop->get('filter_expression')) === 0) {
+                continue;
+            }
+
+            $this->columnFilters[$key] = Filter::fromQueryString(
+                $prop->get('filter_expression')
+            );
+        }
+
+        return $this;
+    }
+
+    protected function rowMatchesPropertyFilter($row, $key)
+    {
+        if (!array_key_exists($key, $this->columnFilters)) {
+            return true;
+        }
+
+        return $this->columnFilters[$key]->matches($row);
+    }
+
+    /**
+     * Instantiates all related ImportSource objects
+     *
+     * @return self
+     * @throws \Icinga\Exception\NotFoundError
+     */
+    protected function prepareRelatedImportSources()
+    {
+        $this->sources = [];
+        foreach ($this->syncProperties as $p) {
+            $id = $p->get('source_id');
+            if (! array_key_exists($id, $this->sources)) {
+                $this->sources[$id] = ImportSource::loadWithAutoIncId(
+                    (int) $id,
+                    $this->db
+                );
+            }
+        }
+
+        return $this;
+    }
+
+    /**
+     * Prepare the source columns we want to fetch
+     *
+     * @return self
+     */
+    protected function prepareSourceColumns()
+    {
+        // $fieldMap = [];
+        $this->sourceColumns = [];
+
+        foreach ($this->syncProperties as $p) {
+            $sourceId = $p->get('source_id');
+            if (! array_key_exists($sourceId, $this->sourceColumns)) {
+                $this->sourceColumns[$sourceId] = [];
+            }
+
+            foreach (SyncUtils::extractVariableNames($p->get('source_expression')) as $varname) {
+                $this->sourceColumns[$sourceId][$varname] = $varname;
+                // -> ? $fieldMap[
+            }
+        }
+
+        return $this;
+    }
+
+    /**
+     * Fetch latest imported data rows from all involved import sources
+     * @return Sync
+     * @throws \Icinga\Exception\NotFoundError
+     */
+    protected function fetchImportedData()
+    {
+        Benchmark::measure('Begin loading imported data');
+        if ($this->rule->get('object_type') === 'host') {
+            $this->serviceOverrideKeyName = $this->db->settings()->override_services_varname;
+        }
+
+        $this->imported = [];
+
+        $sourceKeyPattern = $this->rule->getSourceKeyPattern();
+        $combinedKey = $this->rule->hasCombinedKey();
+
+        foreach ($this->sources as $source) {
+            /** @var ImportSource $source */
+            $sourceId = $source->get('id');
+
+            // Provide an alias column for our key. TODO: double-check this!
+            $key = $source->key_column;
+            $this->sourceColumns[$sourceId][$key] = $key;
+            $run = $source->fetchLastRun(true);
+
+            $usedColumns = SyncUtils::getRootVariables($this->sourceColumns[$sourceId]);
+
+            $filterColumns = [];
+            foreach ($this->columnFilters as $filter) {
+                foreach ($filter->listFilteredColumns() as $column) {
+                    $filterColumns[$column] = $column;
+                }
+            }
+            if (($ruleFilter = $this->rule->filter()) !== null) {
+                foreach ($ruleFilter->listFilteredColumns() as $column) {
+                    $filterColumns[$column] = $column;
+                }
+            }
+
+            if (! empty($filterColumns)) {
+                foreach (SyncUtils::getRootVariables($filterColumns) as $column) {
+                    $usedColumns[$column] = $column;
+                }
+            }
+            Benchmark::measure(sprintf('Done pre-processing columns for source %s', $source->source_name));
+
+            $rows = $run->fetchRows($usedColumns);
+            Benchmark::measure(sprintf('Fetched source %s', $source->source_name));
+
+            $this->imported[$sourceId] = [];
+            foreach ($rows as $row) {
+                if ($combinedKey) {
+                    $key = SyncUtils::fillVariables($sourceKeyPattern, $row);
+                    if ($this->usedLowerCasedKeys) {
+                        $key = strtolower($key);
+                    }
+
+                    if (array_key_exists($key, $this->imported[$sourceId])) {
+                        throw new InvalidArgumentException(sprintf(
+                            'Trying to import row "%s" (%s) twice: %s VS %s',
+                            $key,
+                            $sourceKeyPattern,
+                            json_encode($this->imported[$sourceId][$key]),
+                            json_encode($row)
+                        ));
+                    }
+                } else {
+                    if (! property_exists($row, $key)) {
+                        throw new InvalidArgumentException(sprintf(
+                            'There is no key column "%s" in this row from "%s": %s',
+                            $key,
+                            $source->source_name,
+                            json_encode($row)
+                        ));
+                    }
+                }
+
+                if (! $this->rule->matches($row)) {
+                    continue;
+                }
+
+                if ($combinedKey) {
+                    $this->imported[$sourceId][$key] = $row;
+                } else {
+                    if ($this->usedLowerCasedKeys) {
+                        $this->imported[$sourceId][strtolower($row->$key)] = $row;
+                    } else {
+                        $this->imported[$sourceId][$row->$key] = $row;
+                    }
+                }
+            }
+
+            unset($rows);
+        }
+
+        Benchmark::measure('Done loading imported data');
+
+        return $this;
+    }
+
+    /**
+     * TODO: This is rubbish, we need to filter at fetch time
+     */
+    protected function removeForeignListEntries()
+    {
+        $listId = null;
+        foreach ($this->syncProperties as $prop) {
+            if ($prop->get('destination_field') === 'list_id') {
+                $listId = (int) $prop->get('source_expression');
+            }
+        }
+
+        if ($listId === null) {
+            throw new InvalidArgumentException(
+                'Cannot sync datalist entry without list_id'
+            );
+        }
+
+        $no = [];
+        foreach ($this->objects as $k => $o) {
+            if ((int) $o->get('list_id') !== $listId) {
+                $no[] = $k;
+            }
+        }
+
+        foreach ($no as $k) {
+            unset($this->objects[$k]);
+        }
+    }
+
+    /**
+     * @return $this
+     */
+    protected function loadExistingObjects()
+    {
+        Benchmark::measure('Begin loading existing objects');
+
+        $ruleObjectType = $this->rule->get('object_type');
+        $useLowerCaseKeys = $ruleObjectType !== 'datalistEntry';
+        // TODO: Make object_type (template, object...) and object_name mandatory?
+        if ($this->rule->hasCombinedKey()) {
+            $this->objects = [];
+            $destinationKeyPattern = $this->rule->getDestinationKeyPattern();
+            $table = DbObjectTypeRegistry::tableNameByType($ruleObjectType);
+            if ($this->store && BranchSupport::existsForTableName($table)) {
+                $objects = $this->store->loadAll($table);
+            } else {
+                $objects = IcingaObject::loadAllByType($ruleObjectType, $this->db);
+            }
+
+            foreach ($objects as $object) {
+                if ($object instanceof IcingaService) {
+                    if (strstr($destinationKeyPattern, '${host}')
+                        && $object->get('host_id') === null
+                    ) {
+                        continue;
+                    } elseif (strstr($destinationKeyPattern, '${service_set}')
+                        && $object->get('service_set_id') === null
+                    ) {
+                        continue;
+                    }
+                }
+
+                $key = SyncUtils::fillVariables(
+                    $destinationKeyPattern,
+                    $object
+                );
+                if ($useLowerCaseKeys) {
+                    $key = strtolower($key);
+                }
+
+                if (array_key_exists($key, $this->objects)) {
+                    throw new InvalidArgumentException(sprintf(
+                        'Combined destination key "%s" is not unique, got "%s" twice',
+                        $destinationKeyPattern,
+                        $key
+                    ));
+                }
+
+                $this->objects[$key] = $object;
+            }
+        } else {
+            if ($this->store) {
+                $objects = $this->store->loadAll(DbObjectTypeRegistry::tableNameByType($ruleObjectType), 'object_name');
+            } else {
+                $objects = IcingaObject::loadAllByType($ruleObjectType, $this->db);
+            }
+
+            if ($useLowerCaseKeys) {
+                $this->objects = [];
+                foreach ($objects as $key => $object) {
+                    $this->objects[strtolower($key)] = $object;
+                }
+            } else {
+                $this->objects = $objects;
+            }
+        }
+
+        $this->usedLowerCasedKeys = $useLowerCaseKeys;
+        // TODO: should be obsoleted by a better "loadFiltered" method
+        if ($ruleObjectType === 'datalistEntry') {
+            $this->removeForeignListEntries();
+        }
+
+        Benchmark::measure('Done loading existing objects');
+
+        return $this;
+    }
+
+    /**
+     * @return array
+     * @throws \Icinga\Exception\NotFoundError
+     * @throws \Icinga\Module\Director\Exception\DuplicateKeyException
+     */
+    protected function prepareNewObjects()
+    {
+        $objects = [];
+        $ruleObjectType = $this->rule->get('object_type');
+
+        foreach ($this->sources as $source) {
+            $sourceId = $source->id;
+            $keyColumn = $source->get('key_column');
+
+            foreach ($this->imported[$sourceId] as $key => $row) {
+                // Workaround: $a["10"] = "val"; -> array_keys($a) = [(int) 10]
+                $key = (string) $key;
+                $originalKey = $row->$keyColumn;
+                if ($this->usedLowerCasedKeys) {
+                    $key = strtolower($key);
+                }
+                if (! array_key_exists($key, $objects)) {
+                    // Safe default values for object_type and object_name
+                    if ($ruleObjectType === 'datalistEntry') {
+                        $props = [];
+                    } else {
+                        $props = [
+                            'object_type' => 'object',
+                            'object_name' => $originalKey,
+                        ];
+                    }
+
+                    $objects[$key] = IcingaObject::createByType(
+                        $ruleObjectType,
+                        $props,
+                        $this->db
+                    );
+                }
+
+                $object = $objects[$key];
+                $this->prepareNewObject($row, $object, $key, $sourceId);
+            }
+        }
+
+        return $objects;
+    }
+
+    /**
+     * @param $row
+     * @param DbObject $object
+     * @param $sourceId
+     * @throws \Icinga\Exception\NotFoundError
+     * @throws \Icinga\Module\Director\Exception\DuplicateKeyException
+     */
+    protected function prepareNewObject($row, DbObject $object, $objectKey, $sourceId)
+    {
+        foreach ($this->syncProperties as $propertyKey => $p) {
+            if ($p->get('source_id') !== $sourceId) {
+                continue;
+            }
+
+            if (! $this->rowMatchesPropertyFilter($row, $propertyKey)) {
+                continue;
+            }
+
+            $prop = $p->get('destination_field');
+            $val = SyncUtils::fillVariables($p->get('source_expression'), $row);
+
+            if ($object instanceof IcingaObject) {
+                if ($prop === 'import') {
+                    if ($val !== null) {
+                        $object->imports()->add($val);
+                    }
+                } elseif ($prop === 'groups') {
+                    if ($val !== null) {
+                        $object->groups()->add($val);
+                    }
+                } elseif (substr($prop, 0, 5) === 'vars.') {
+                    $varName = substr($prop, 5);
+                    if (substr($varName, -2) === '[]') {
+                        $varName = substr($varName, 0, -2);
+                        $current = $this->wantArray($object->vars()->$varName);
+                        $object->vars()->$varName = array_merge(
+                            $current,
+                            $this->wantArray($val)
+                        );
+                    } else {
+                        if ($val === null) {
+                            $this->setNull[$objectKey][$prop] = $prop;
+                        } else {
+                            unset($this->setNull[$objectKey][$prop]);
+                            $object->vars()->$varName = $val;
+                        }
+                    }
+                } else {
+                    if ($val === null) {
+                        $this->setNull[$objectKey][$prop] = $prop;
+                    } else {
+                        unset($this->setNull[$objectKey][$prop]);
+                        $object->set($prop, $val);
+                    }
+                }
+            } else {
+                if ($val === null) {
+                    $this->setNull[$objectKey][$prop] = $prop;
+                } else {
+                    unset($this->setNull[$objectKey][$prop]);
+                    $object->set($prop, $val);
+                }
+            }
+        }
+    }
+
+    /**
+     * @return $this
+     */
+    protected function deferResolvers()
+    {
+        if (in_array($this->rule->get('object_type'), ['host', 'hostgroup'])) {
+            $resolver = $this->getHostGroupMembershipResolver();
+            $resolver->defer()->setUseTransactions(false);
+        }
+
+        return $this;
+    }
+
+    /**
+     * @param DbObject $object
+     * @return $this
+     */
+    protected function setResolver($object)
+    {
+        if (! ($object instanceof IcingaHost || $object instanceof IcingaHostGroup)) {
+            return $this;
+        }
+        if ($resolver = $this->getHostGroupMembershipResolver()) {
+            $object->setHostGroupMembershipResolver($resolver);
+        }
+
+        return $this;
+    }
+
+    /**
+     * @return $this
+     * @throws \Zend_Db_Adapter_Exception
+     */
+    protected function notifyResolvers()
+    {
+        if ($resolver = $this->getHostGroupMembershipResolver()) {
+            $resolver->refreshDb(true);
+        }
+
+        return $this;
+    }
+
+    /**
+     * @return bool|HostGroupMembershipResolver
+     */
+    protected function getHostGroupMembershipResolver()
+    {
+        if ($this->hostGroupMembershipResolver === null) {
+            if (in_array(
+                $this->rule->get('object_type'),
+                ['host', 'hostgroup']
+            )) {
+                $this->hostGroupMembershipResolver = new HostGroupMembershipResolver(
+                    $this->db
+                );
+            } else {
+                $this->hostGroupMembershipResolver = false;
+            }
+        }
+
+        return $this->hostGroupMembershipResolver;
+    }
+
+    /**
+     * Evaluates a SyncRule and returns a list of modified objects
+     *
+     * TODO: Split this into smaller methods
+     *
+     * @return DbObject|IcingaObject[] List of modified IcingaObjects
+     * @throws \Icinga\Exception\NotFoundError
+     * @throws \Icinga\Module\Director\Exception\DuplicateKeyException
+     */
+    protected function prepare()
+    {
+        if ($this->isPrepared) {
+            return $this->objects;
+        }
+
+        $this->raiseLimits()
+             ->startMeasurements()
+             ->prepareCache()
+             ->fetchSyncProperties()
+             ->prepareRelatedImportSources()
+             ->prepareSourceColumns()
+             ->loadExistingObjects()
+             ->fetchImportedData()
+             ->deferResolvers();
+
+        Benchmark::measure('Begin preparing updated objects');
+        $newObjects = $this->prepareNewObjects();
+
+        Benchmark::measure('Ready to process objects');
+        /** @var DbObject|IcingaObject $object */
+        foreach ($newObjects as $key => $object) {
+            $this->processObject($key, $object);
+        }
+
+        Benchmark::measure('Modified objects are ready, applying purge strategy');
+        $noAction = [];
+        $purgeAction = $this->rule->get('purge_action');
+        foreach ($this->rule->purgeStrategy()->listObjectsToPurge() as $key) {
+            $key = strtolower($key);
+            if (array_key_exists($key, $newObjects)) {
+                // Object has been touched, do not delete
+                continue;
+            }
+
+            if (array_key_exists($key, $this->objects)) {
+                $object = $this->objects[$key];
+                if (! $object->hasBeenModified()) {
+                    switch ($purgeAction) {
+                        case 'delete':
+                            $object->markForRemoval();
+                            break;
+                        case 'disable':
+                            $object->set('disabled', 'y');
+                            break;
+                        default:
+                            throw new RuntimeException(
+                                "Unsupported purge action: '$purgeAction'"
+                            );
+                    }
+                }
+            }
+        }
+
+        Benchmark::measure('Done marking objects for purge');
+
+        foreach ($this->objects as $key => $object) {
+            if (! $object->hasBeenModified() && ! $object->shouldBeRemoved()) {
+                $noAction[] = $key;
+            }
+        }
+
+        foreach ($noAction as $key) {
+            unset($this->objects[$key]);
+        }
+
+        $this->isPrepared = true;
+
+        Benchmark::measure('Done preparing objects');
+
+        return $this->objects;
+    }
+
+    /**
+     * @param $key
+     * @param DbObject|IcingaObject $object
+     * @throws \Icinga\Exception\NotFoundError
+     */
+    protected function processObject($key, $object)
+    {
+        if (array_key_exists($key, $this->objects)) {
+            $this->refreshObject($key, $object);
+        } else {
+            $this->addNewObject($key, $object);
+        }
+    }
+
+    /**
+     * @param $key
+     * @param DbObject|IcingaObject $object
+     * @throws \Icinga\Exception\NotFoundError
+     */
+    protected function refreshObject($key, $object)
+    {
+        $policy = $this->rule->get('update_policy');
+
+        switch ($policy) {
+            case 'override':
+                if ($object instanceof IcingaHost
+                    && !in_array('api_key', $this->rule->getSyncProperties())
+                ) {
+                    $this->objects[$key]->replaceWith($object, ['api_key']);
+                } else {
+                    $this->objects[$key]->replaceWith($object);
+                }
+                break;
+
+            case 'merge':
+            case 'update-only':
+                // TODO: re-evaluate merge settings. vars.x instead of
+                //       just "vars" might suffice.
+                $this->objects[$key]->merge($object, $this->replaceVars);
+                if (! $this->hasPropertyDisabled && $object->hasProperty('disabled')) {
+                    $this->objects[$key]->resetProperty('disabled');
+                }
+                break;
+
+            default:
+                // policy 'ignore', no action
+        }
+
+        if ($policy === 'override' || $policy === 'merge') {
+            if ($object instanceof IcingaHost) {
+                $keyName = $this->serviceOverrideKeyName;
+                if (! $object->hasInitializedVars() || ! isset($object->vars()->$key)) {
+                    $this->objects[$key]->vars()->restoreStoredVar($keyName);
+                }
+            }
+        }
+
+        if (isset($this->setNull[$key])) {
+            foreach ($this->setNull[$key] as $property) {
+                $this->objects[$key]->set($property, null);
+            }
+        }
+    }
+
+    /**
+     * @param $key
+     * @param DbObject|IcingaObject $object
+     */
+    protected function addNewObject($key, $object)
+    {
+        $this->objects[$key] = $object;
+    }
+
+    /**
+     * Runs a SyncRule and applies all resulting changes
+     * @return int
+     * @throws Exception
+     * @throws IcingaException
+     */
+    public function apply()
+    {
+        Benchmark::measure('Begin applying objects');
+
+        $objects = $this->prepare();
+        $db = $this->db;
+        $dba = $db->getDbAdapter();
+        if (! $this->store) { // store has it's own transaction
+            $dba->beginTransaction();
+        }
+
+        $object = null;
+        $updateOnly = $this->rule->get('update_policy') === 'update-only';
+        $allowCreate = ! $updateOnly;
+
+        try {
+            $formerActivityChecksum = hex2bin(
+                $db->getLastActivityChecksum()
+            );
+            $created = 0;
+            $modified = 0;
+            $deleted = 0;
+            // TODO: Count also failed ones, once we allow such
+            // $failed = 0;
+            foreach ($objects as $object) {
+                $this->setResolver($object);
+                if (! $updateOnly && $object->shouldBeRemoved()) {
+                    if ($this->store) {
+                        $this->store->delete($object);
+                    } else {
+                        $object->delete();
+                    }
+                    $deleted++;
+                    continue;
+                }
+
+                if ($object->hasBeenModified()) {
+                    $existing = $object->hasBeenLoadedFromDb();
+                    if ($existing) {
+                        if ($this->store) {
+                            $this->store->store($object);
+                        } else {
+                            $object->store($db);
+                        }
+                        $modified++;
+                    } elseif ($allowCreate) {
+                        if ($this->store) {
+                            $this->store->store($object);
+                        } else {
+                            $object->store($db);
+                        }
+                        $created++;
+                    }
+                }
+            }
+
+            $runProperties = [
+                'objects_created'  => $created,
+                'objects_deleted'  => $deleted,
+                'objects_modified' => $modified,
+            ];
+
+            if ($created + $deleted + $modified > 0) {
+                // TODO: What if this has been the very first activity?
+                $runProperties['last_former_activity'] = $db->quoteBinary($formerActivityChecksum);
+                $runProperties['last_related_activity'] = $db->quoteBinary(hex2bin(
+                    $db->getLastActivityChecksum()
+                ));
+            }
+
+            $this->run->setProperties($runProperties);
+            if (!$this->store || !$this->store->getBranch()->isBranch()) {
+                $this->run->store();
+            }
+            $this->notifyResolvers();
+            if (! $this->store) {
+                $dba->commit();
+            }
+
+            // Store duration after commit, as the commit might take some time
+            $this->run->set('duration_ms', (int) round(
+                (microtime(true) - $this->runStartTime) * 1000
+            ));
+            if (!$this->store || !$this->store->getBranch()->isBranch()) {
+                $this->run->store();
+            }
+
+            Benchmark::measure('Done applying objects');
+        } catch (Exception $e) {
+            if (! $this->store) {
+                $dba->rollBack();
+            }
+
+            if ($object instanceof IcingaObject) {
+                throw new IcingaException(
+                    'Exception while syncing %s %s: %s',
+                    get_class($object),
+                    $object->getObjectName(),
+                    $e->getMessage(),
+                    $e
+                );
+            } else {
+                throw $e;
+            }
+        }
+
+        return $this->run->get('id');
+    }
+
+    protected function prepareCache()
+    {
+        if ($this->store) {
+            return $this;
+        }
+        PrefetchCache::initialize($this->db);
+        IcingaTemplateRepository::clear();
+
+        $ruleObjectType = $this->rule->get('object_type');
+
+        $dummy = IcingaObject::createByType($ruleObjectType);
+        if ($dummy instanceof IcingaObject) {
+            IcingaObject::prefetchAllRelationsByType($ruleObjectType, $this->db);
+        }
+
+        return $this;
+    }
+}
diff --git a/library/Director/Import/SyncUtils.php b/library/Director/Import/SyncUtils.php
new file mode 100644
index 0000000..5528b2d
--- /dev/null
+++ b/library/Director/Import/SyncUtils.php
@@ -0,0 +1,153 @@
+<?php
+
+namespace Icinga\Module\Director\Import;
+
+use Icinga\Module\Director\Data\Db\DbObject;
+use InvalidArgumentException;
+
+class SyncUtils
+{
+    /**
+     * Extract variable names in the form ${var_name} from a given string
+     *
+     * @param  string $string
+     *
+     * @return array  List of variable names (without ${})
+     */
+    public static function extractVariableNames($string)
+    {
+        if (preg_match_all('/\${([^}]+)}/', $string, $m, PREG_PATTERN_ORDER)) {
+            return $m[1];
+        } else {
+            return array();
+        }
+    }
+
+    /**
+     * Whether the given string contains variable names in the form ${var_name}
+     *
+     * @param  string $string
+     *
+     * @return bool
+     */
+    public static function hasVariables($string)
+    {
+        return preg_match('/\${([^}]+)}/', $string);
+    }
+
+    /**
+     * Recursively extract a value from a nested structure
+     *
+     * For a $val looking like
+     *
+     * { 'vars' => { 'disk' => { 'sda' => { 'size' => '256G' } } } }
+     *
+     * and a key vars.disk.sda given as [ 'vars', 'disk', 'sda' ] this would
+     * return { size => '255GB' }
+     *
+     * @param  string $val  The value to extract data from
+     * @param  array  $keys A list of nested keys pointing to desired data
+     *
+     * @return mixed
+     */
+    public static function getDeepValue($val, array $keys)
+    {
+        $key = array_shift($keys);
+        if (! property_exists($val, $key)) {
+            return null;
+        }
+
+        if (empty($keys)) {
+            return $val->$key;
+        }
+
+        return static::getDeepValue($val->$key, $keys);
+    }
+
+    /**
+     * Return a specific value from a given row object
+     *
+     * Supports also keys pointing to nested structures like vars.disk.sda
+     *
+     * @param  object $row  stdClass object providing property values
+     * @param  string $var  Variable/property name
+     *
+     * @return mixed
+     */
+    public static function getSpecificValue($row, $var)
+    {
+        if (strpos($var, '.') === false) {
+            if ($row instanceof DbObject) {
+                return $row->$var;
+            }
+            if (! property_exists($row, $var)) {
+                return null;
+            }
+
+            return $row->$var;
+        } else {
+            $parts = explode('.', $var);
+            $main = array_shift($parts);
+            if (! property_exists($row, $main)) {
+                return null;
+            }
+            if ($row->$main === null) {
+                return null;
+            }
+
+            if (! is_object($row->$main)) {
+                throw new InvalidArgumentException(sprintf(
+                    'Data is not nested, cannot access %s: %s',
+                    $var,
+                    var_export($row, 1)
+                ));
+            }
+
+            return static::getDeepValue($row->$main, $parts);
+        }
+    }
+
+    /**
+     * Fill variables in the given string pattern
+     *
+     * This replaces all occurrences of ${var_name} with the corresponding
+     * property $row->var_name of the given row object. Missing variables are
+     * replaced by an empty string. This works also fine in case there are
+     * multiple variables to be found in your string.
+     *
+     * @param  string $string String with optional variables/placeholders
+     * @param  object $row    stdClass object providing property values
+     *
+     * @return string
+     */
+    public static function fillVariables($string, $row)
+    {
+        if (preg_match('/^\${([^}]+)}$/', $string, $m)) {
+            return static::getSpecificValue($row, $m[1]);
+        }
+
+        $func = function ($match) use ($row) {
+            return SyncUtils::getSpecificValue($row, $match[1]);
+        };
+
+        return preg_replace_callback('/\${([^}]+)}/', $func, $string);
+    }
+
+    public static function getRootVariables($vars)
+    {
+        $res = array();
+        foreach ($vars as $p) {
+            if (false === ($pos = strpos($p, '.')) || $pos === strlen($p) - 1) {
+                $res[] = $p;
+            } else {
+                $res[] = substr($p, 0, $pos);
+            }
+        }
+
+        if (empty($res)) {
+            return array();
+        }
+
+        return array_combine($res, $res);
+    }
+}