Adding upstream version 1:1.3.2.upstream/1%1.3.2upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 755 insertions, 0 deletions

diff --git a/library/X509/Job.php b/library/X509/Job.php
new file mode 100644
index 0000000..1e0b3f7
--- /dev/null
+++ b/library/X509/Job.php
@@ -0,0 +1,755 @@
+<?php
+
+// Icinga Web 2 X.509 Module | (c) 2018 Icinga GmbH | GPLv2
+
+namespace Icinga\Module\X509;
+
+use DateTime;
+use Exception;
+use Generator;
+use GMP;
+use Icinga\Application\Logger;
+use Icinga\Module\X509\Common\Database;
+use Icinga\Module\X509\Common\JobOptions;
+use Icinga\Module\X509\Model\X509Certificate;
+use Icinga\Module\X509\Model\X509CertificateChain;
+use Icinga\Module\X509\Model\X509JobRun;
+use Icinga\Module\X509\Model\X509Target;
+use Icinga\Module\X509\React\StreamOptsCaptureConnector;
+use Icinga\Util\Json;
+use ipl\Scheduler\Common\TaskProperties;
+use ipl\Scheduler\Contract\Task;
+use ipl\Sql\Connection;
+use ipl\Sql\Expression;
+use ipl\Stdlib\Filter;
+use LogicException;
+use Ramsey\Uuid\Uuid;
+use Ramsey\Uuid\UuidInterface;
+use React\EventLoop\Loop;
+use React\Promise;
+use React\Socket\ConnectionInterface;
+use React\Socket\Connector;
+use React\Socket\ConnectorInterface;
+use React\Socket\SecureConnector;
+use React\Socket\TimeoutConnector;
+use Throwable;
+
+class Job implements Task
+{
+    use JobOptions;
+    use TaskProperties;
+
+    /** @var int Number of targets to be scanned in parallel by default */
+    public const DEFAULT_PARALLEL = 256;
+
+    /** @var string Default since last scan threshold used to filter out scan targets  */
+    public const DEFAULT_SINCE_LAST_SCAN = '-24 hours';
+
+    /** @var int The database id of this job */
+    protected $id;
+
+    /** @var Connection x509 database connection */
+    private $db;
+
+    /** @var DbTool Database utils for marshalling and unmarshalling binary data */
+    private $dbTool;
+
+    /** @var int Number of pending targets to be scanned */
+    private $pendingTargets = 0;
+
+    /** @var int Total number of scan targets */
+    private $totalTargets = 0;
+
+    /** @var int Number of scanned targets */
+    private $finishedTargets = 0;
+
+    /** @var Generator Scan targets generator */
+    private $targets;
+
+    /** @var array<string, array<string>> The configured SNI maps */
+    private $snimap;
+
+    /** @var int The id of the last inserted job run entry */
+    private $jobRunId;
+
+    /** @var Promise\Deferred React promise deferred instance used to resolve the running promise */
+    protected $deferred;
+
+    /** @var DateTime The start time of this job */
+    protected $jobRunStart;
+
+    /** @var array<string> A list of excluded IP addresses and host names */
+    private $excludedTargets = [];
+
+    /** @var array<string, array<int, int|string>> The configured CIDRs of this job */
+    private $cidrs;
+
+    /** @var array<int, array<string>> The configured ports of this job */
+    private $ports;
+
+    /**
+     * Construct a new Job instance
+     *
+     * @param string $name The name of this job
+     * @param array<string, array<int, int|string>> $cidrs The configured CIDRs to be used by this job
+     * @param array<int, array<string>> $ports The configured ports to be used by this job
+     * @param array<string, array<string>> $snimap The configured SNI maps to be used by this job
+     * @param ?Schedule $schedule
+     */
+    public function __construct(string $name, array $cidrs, array $ports, array $snimap, Schedule $schedule = null)
+    {
+        $this->name = $name;
+        $this->db = Database::get();
+        $this->dbTool = new DbTool($this->db);
+        $this->snimap = $snimap;
+        $this->cidrs = $cidrs;
+        $this->ports = $ports;
+
+        if ($schedule) {
+            $this->setSchedule($schedule);
+        }
+
+        $this->setName($name);
+    }
+
+    /**
+     * Transform the given human-readable IP address into a binary format
+     *
+     * @param string $addr
+     *
+     * @return string
+     */
+    public static function binary(string $addr): string
+    {
+        return str_pad(inet_pton($addr), 16, "\0", STR_PAD_LEFT);
+    }
+
+    /**
+     * Transform the given human-readable IP address into GMP number
+     *
+     * @param string $addr
+     *
+     * @return ?GMP
+     */
+    public static function addrToNumber(string $addr): ?GMP
+    {
+        return gmp_import(static::binary($addr));
+    }
+
+    /**
+     * Transform the given number into human-readable IP address
+     *
+     * @param $num
+     * @param bool $ipv6
+     *
+     * @return false|string
+     */
+    public static function numberToAddr($num, bool $ipv6 = true)
+    {
+        if ($ipv6) {
+            return inet_ntop(str_pad(gmp_export($num), 16, "\0", STR_PAD_LEFT));
+        } else {
+            return inet_ntop(gmp_export($num));
+        }
+    }
+
+    /**
+     * Check whether the given IP is inside the specified CIDR
+     *
+     * @param GMP $addr
+     * @param string $subnet
+     * @param int $mask
+     *
+     * @return bool
+     */
+    public static function isAddrInside(GMP $addr, string $subnet, int $mask): bool
+    {
+        // `gmp_pow()` is like PHP's pow() function, but handles also very large numbers
+        // and `gmp_com()` is like the bitwise NOT (~) operator.
+        $mask = gmp_com(gmp_pow(2, (static::isIPV6($subnet) ? 128 : 32) - $mask) - 1);
+        return gmp_strval(gmp_and($addr, $mask)) === gmp_strval(gmp_and(static::addrToNumber($subnet), $mask));
+    }
+
+    /**
+     * Get whether the given IP address is IPV6 address
+     *
+     * @param $addr
+     *
+     * @return bool
+     */
+    public static function isIPV6($addr): bool
+    {
+        return filter_var($addr, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6) !== false;
+    }
+
+    /**
+     * Get the database id of this job
+     *
+     * @return int
+     */
+    public function getId(): int
+    {
+        return $this->id;
+    }
+
+    /**
+     * Set the database id of this job
+     *
+     * @param int $id
+     *
+     * @return $this
+     */
+    public function setId(int $id): self
+    {
+        $this->id = $id;
+
+        return $this;
+    }
+
+    public function getUuid(): UuidInterface
+    {
+        if (! $this->uuid) {
+            $this->setUuid(Uuid::fromBytes($this->getChecksum()));
+        }
+
+        return $this->uuid;
+    }
+
+    /**
+     * Get the configured job CIDRS
+     *
+     * @return array<string, array<int, int|string>>
+     */
+    public function getCIDRs(): array
+    {
+        return $this->cidrs;
+    }
+
+    /**
+     * Set the CIDRs of this job
+     *
+     * @param array<string, array<int, int|string>> $cidrs
+     *
+     * @return $this
+     */
+    public function setCIDRs(array $cidrs): self
+    {
+        $this->cidrs = $cidrs;
+
+        return $this;
+    }
+
+    /**
+     * Get the configured ports of this job
+     *
+     * @return array<int, array<string>>
+     */
+    public function getPorts(): array
+    {
+        return $this->ports;
+    }
+
+    /**
+     * Set the ports of this job to be scanned
+     *
+     * @param array<int, array<string>> $ports
+     *
+     * @return $this
+     */
+    public function setPorts(array $ports): self
+    {
+        $this->ports = $ports;
+
+        return $this;
+    }
+
+    /**
+     * Get excluded IPs and host names
+     *
+     * @return array<string>
+     */
+    public function getExcludes(): array
+    {
+        return $this->excludedTargets;
+    }
+
+    /**
+     * Set a set of IPs and host names to be excluded from scan
+     *
+     * @param array<string> $targets
+     *
+     * @return $this
+     */
+    public function setExcludes(array $targets): self
+    {
+        $this->excludedTargets = $targets;
+
+        return $this;
+    }
+
+    private function getConnector($peerName): array
+    {
+        $simpleConnector = new Connector();
+        $streamCaptureConnector = new StreamOptsCaptureConnector($simpleConnector);
+        $secureConnector = new SecureConnector($streamCaptureConnector, null, [
+            'verify_peer'             => false,
+            'verify_peer_name'        => false,
+            'capture_peer_cert_chain' => true,
+            'SNI_enabled'             => true,
+            'peer_name'               => $peerName
+        ]);
+        return [new TimeoutConnector($secureConnector, 5.0), $streamCaptureConnector];
+    }
+
+    /**
+     * Get whether this job has been completed scanning all targets
+     *
+     * @return bool
+     */
+    public function isFinished(): bool
+    {
+        return ! $this->targets->valid() && $this->pendingTargets === 0;
+    }
+
+    public function updateLastScan($target)
+    {
+        if (! $this->isRescan() && ! isset($target->id)) {
+            return;
+        }
+
+        $this->db->update('x509_target', [
+            'last_scan' => new Expression('UNIX_TIMESTAMP() * 1000')
+        ], ['id = ?' => $target->id]);
+    }
+
+    public function getChecksum(): string
+    {
+        $data = [
+            'name'            => $this->getName(),
+            'cidrs'           => $this->getCIDRs(),
+            'ports'           => $this->getPorts(),
+            'exclude_targets' => $this->getExcludes(),
+        ];
+
+        $schedule = null;
+        if ($this->schedule) {
+            $schedule = $this->getSchedule();
+        }
+
+        return md5(Json::encode($data) . ($schedule ? bin2hex($schedule->getChecksum()) : ''), true);
+    }
+
+    protected function getScanTargets(): Generator
+    {
+        $generate = $this->fullScan || ! $this->isRescan();
+        if (! $generate) {
+            $run = X509JobRun::on($this->db)
+                ->columns([new Expression('1')])
+                ->filter(Filter::equal('schedule.job_id', $this->getId()))
+                ->filter(Filter::unequal('total_targets', 0))
+                ->limit(1)
+                ->execute();
+
+            $generate = ! $run->hasResult();
+        }
+
+        if ($generate) {
+            yield from $this->generateTargets();
+        }
+
+        $sinceLastScan = $this->getSinceLastScan();
+        if ((! $this->fullScan && $sinceLastScan !== null) || $this->isRescan()) {
+            $targets = X509Target::on($this->db)->columns(['id', 'ip', 'hostname', 'port']);
+            if (! $this->fullScan && $sinceLastScan) {
+                $targets->filter(Filter::lessThan('last_scan', $sinceLastScan));
+            }
+
+            foreach ($targets as $target) {
+                $addr = static::addrToNumber($target->ip);
+                $addrFound = false;
+                foreach ($this->getCIDRs() as $cidr) {
+                    list($subnet, $mask) = $cidr;
+                    if (static::isAddrInside($addr, (string) $subnet, (int) $mask)) {
+                        $target->ip = static::numberToAddr($addr, static::isIPV6($subnet));
+                        $addrFound = true;
+
+                        break;
+                    }
+                }
+
+                if ($addrFound) {
+                    yield $target;
+                }
+            }
+        }
+    }
+
+    private function generateTargets(): Generator
+    {
+        $excludes = $this->getExcludes();
+        foreach ($this->getCIDRs() as $cidr) {
+            list($startIp, $prefix) = $cidr;
+            $ipv6 = static::isIPV6($startIp);
+            $subnet = $ipv6 ? 128 : 32;
+            $numIps = pow(2, ($subnet - (int) $prefix));
+
+            Logger::info('Scanning %d IPs in the CIDR %s', $numIps, implode('/', $cidr));
+
+            $start = static::addrToNumber((string) $startIp);
+            for ($i = 0; $i < $numIps; $i++) {
+                $ip = static::numberToAddr(gmp_add($start, $i), $ipv6);
+                if (isset($excludes[$ip])) {
+                    Logger::debug('Excluding IP %s from scan', $ip);
+                    continue;
+                }
+
+                foreach ($this->getPorts() as $portRange) {
+                    list($startPort, $endPort) = $portRange;
+                    foreach (range($startPort, $endPort) as $port) {
+                        foreach ($this->snimap[$ip] ?? [null] as $hostname) {
+                            if (array_key_exists((string) $hostname, $excludes)) {
+                                Logger::debug('Excluding host %s from scan', $hostname);
+                                continue;
+                            }
+
+                            if (! $this->fullScan) {
+                                $targets = X509Target::on($this->db)
+                                    ->columns([new Expression('1')])
+                                    ->filter(
+                                        Filter::all(
+                                            Filter::equal('ip', $ip),
+                                            Filter::equal('port', $port),
+                                            $hostname !== null
+                                                ? Filter::equal('hostname', $hostname)
+                                                : Filter::unlike('hostname', '*')
+                                        )
+                                    );
+
+                                if ($targets->execute()->hasResult()) {
+                                    continue;
+                                }
+                            }
+
+                            yield (object) [
+                                'ip'       => $ip,
+                                'port'     => $port,
+                                'hostname' => $hostname
+                            ];
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    public function updateJobStats(bool $finished = false): void
+    {
+        $fields = ['finished_targets' => $this->finishedTargets];
+        if ($finished) {
+            $fields['end_time'] = new Expression('UNIX_TIMESTAMP() * 1000');
+            $fields['total_targets'] = $this->totalTargets;
+        }
+
+        $this->db->update('x509_job_run', $fields, ['id = ?' => $this->jobRunId]);
+    }
+
+    private static function formatTarget($target): string
+    {
+        $result = "tls://[{$target->ip}]:{$target->port}";
+
+        if ($target->hostname !== null) {
+            $result .= " [SNI hostname: {$target->hostname}]";
+        }
+
+        return $result;
+    }
+
+    private function finishTarget()
+    {
+        $this->pendingTargets--;
+        $this->finishedTargets++;
+        $this->startNextTarget();
+    }
+
+    private function startNextTarget()
+    {
+        if ($this->isFinished()) {
+            // No targets to process anymore, so we can now resolve the promise
+            $this->deferred->resolve($this->finishedTargets);
+
+            return;
+        }
+
+        if (! $this->targets->valid()) {
+            // When nothing is yielded, and it's still not finished yet, just get the next target
+            return;
+        }
+
+        $target = $this->targets->current();
+        $this->targets->next();
+
+        $this->totalTargets++;
+        $this->pendingTargets++;
+
+        $url = "tls://[{$target->ip}]:{$target->port}";
+        Logger::debug("Connecting to %s", self::formatTarget($target));
+
+        /** @var ConnectorInterface $connector */
+        /** @var StreamOptsCaptureConnector $streamCapture */
+        list($connector, $streamCapture) = $this->getConnector($target->hostname);
+        $connector->connect($url)->then(
+            function (ConnectionInterface $conn) use ($target, $streamCapture) {
+                Logger::info("Connected to %s", self::formatTarget($target));
+
+                // Close connection in order to capture stream context options
+                $conn->close();
+
+                $capturedStreamOptions = $streamCapture->getCapturedStreamOptions();
+
+                $this->processChain($target, $capturedStreamOptions['ssl']['peer_certificate_chain']);
+
+                $this->finishTarget();
+            },
+            function (Exception $exception) use ($target, $streamCapture) {
+                Logger::debug("Cannot connect to server: %s", $exception->getMessage());
+
+                $capturedStreamOptions = $streamCapture->getCapturedStreamOptions();
+
+                if (isset($capturedStreamOptions['ssl']['peer_certificate_chain'])) {
+                    // The scanned target presented its certificate chain despite throwing an error
+                    // This is the case for targets which require client certificates for example
+                    $this->processChain($target, $capturedStreamOptions['ssl']['peer_certificate_chain']);
+                } else {
+                    $this->db->update(
+                        'x509_target',
+                        [
+                            'latest_certificate_chain_id' => null,
+                            'mtime'                       => new Expression('UNIX_TIMESTAMP() * 1000')
+                        ],
+                        [
+                            'hostname = ?' => $target->hostname,
+                            'ip = ?'       => $this->dbTool->marshalBinary(static::binary($target->ip)),
+                            'port = ?'     => $target->port
+                        ]
+                    );
+                }
+
+                $step = max($this->totalTargets / 100, 1);
+
+                if ($this->finishedTargets % (int) $step == 0) {
+                    $this->updateJobStats();
+                }
+
+                $this->finishTarget();
+            }
+        )->always(function () use ($target) {
+            $this->updateLastScan($target);
+        })->otherwise(function (Throwable $e) {
+            Logger::error($e->getMessage());
+            Logger::error($e->getTraceAsString());
+        });
+    }
+
+    public function run(): Promise\ExtendedPromiseInterface
+    {
+        $this->jobRunStart = new DateTime();
+        // Update the job statistics regardless of whether the job was successful, failed, or canceled.
+        // Otherwise, some database columns might remain null.
+        $updateJobStats = function () {
+            $this->updateJobStats(true);
+        };
+        $this->deferred = new Promise\Deferred($updateJobStats);
+        $this->deferred->promise()->always($updateJobStats);
+
+        Loop::futureTick(function () {
+            if (! $this->db->ping()) {
+                $this->deferred->reject(new LogicException('Lost connection to database and failed to reconnect'));
+
+                return;
+            }
+
+            // Reset those statistics for the next run! Is only necessary when
+            // running this job using the scheduler
+            $this->totalTargets = 0;
+            $this->finishedTargets = 0;
+            $this->pendingTargets = 0;
+
+            if ($this->schedule) {
+                $scheduleId = $this->getSchedule()->getId();
+            } else {
+                $scheduleId = new Expression('NULL');
+            }
+
+            $this->db->insert('x509_job_run', [
+                'job_id'           => $this->getId(),
+                'schedule_id'      => $scheduleId,
+                'start_time'       => $this->jobRunStart->getTimestamp() * 1000.0,
+                'total_targets'    => 0,
+                'finished_targets' => 0
+            ]);
+            $this->jobRunId = (int) $this->db->lastInsertId();
+
+            $this->targets = $this->getScanTargets();
+
+            if ($this->isFinished()) {
+                // There are no targets to scan, so we can resolve the promise earlier
+                $this->deferred->resolve(0);
+
+                return;
+            }
+
+            // Start scanning the first couple of targets...
+            for ($i = 0; $i < $this->getParallel() && ! $this->isFinished(); $i++) {
+                $this->startNextTarget();
+            }
+        });
+
+        /** @var Promise\ExtendedPromiseInterface $promise */
+        $promise = $this->deferred->promise();
+        return $promise;
+    }
+
+    protected function processChain($target, $chain)
+    {
+        if ($target->hostname === null) {
+            $hostname = gethostbyaddr($target->ip);
+
+            if ($hostname !== false) {
+                $target->hostname = $hostname;
+            }
+        }
+
+        $this->db->transaction(function () use ($target, $chain) {
+            $row = X509Target::on($this->db)
+                ->columns(['id'])
+                ->filter(
+                    Filter::all(
+                        Filter::equal('ip', $target->ip),
+                        Filter::equal('port', $target->port),
+                        Filter::equal('hostname', $target->hostname)
+                    )
+                )->first();
+
+            if (! $row) {
+                // TODO: https://github.com/Icinga/ipl-orm/pull/78
+                $this->db->insert(
+                    'x509_target',
+                    [
+                        'ip'        => $this->dbTool->marshalBinary(static::binary($target->ip)),
+                        'port'      => $target->port,
+                        'hostname'  => $target->hostname,
+                        'last_scan' => new Expression('UNIX_TIMESTAMP() * 1000'),
+                        'ctime'     => new Expression('UNIX_TIMESTAMP() * 1000')
+                    ]
+                );
+                $targetId = $this->db->lastInsertId();
+            } else {
+                $targetId = $row->id;
+            }
+
+            $chainUptodate = false;
+
+            $lastChain = X509CertificateChain::on($this->db)
+                ->columns(['id'])
+                ->filter(Filter::equal('target_id', $targetId))
+                ->orderBy('id', SORT_DESC)
+                ->limit(1)
+                ->first();
+
+            if ($lastChain) {
+                $lastFingerprints = X509Certificate::on($this->db)->utilize('chain');
+                $lastFingerprints
+                    ->columns(['fingerprint'])
+                    ->getSelectBase()
+                    ->where(new Expression(
+                        'certificate_link.certificate_chain_id = %d',
+                        [$lastChain->id]
+                    ))
+                    ->orderBy('certificate_link.order');
+
+                $lastFingerprintsArr = [];
+                foreach ($lastFingerprints as $lastFingerprint) {
+                    $lastFingerprintsArr[] = $lastFingerprint->fingerprint;
+                }
+
+                $currentFingerprints = [];
+
+                foreach ($chain as $cert) {
+                    $currentFingerprints[] = openssl_x509_fingerprint($cert, 'sha256', true);
+                }
+
+                $chainUptodate = $currentFingerprints === $lastFingerprintsArr;
+            }
+
+            if ($lastChain && $chainUptodate) {
+                $chainId = $lastChain->id;
+            } else {
+                // TODO: https://github.com/Icinga/ipl-orm/pull/78
+                $this->db->insert(
+                    'x509_certificate_chain',
+                    [
+                        'target_id' => $targetId,
+                        'length'    => count($chain),
+                        'ctime'     => new Expression('UNIX_TIMESTAMP() * 1000')
+                    ]
+                );
+
+                $chainId = $this->db->lastInsertId();
+
+                $lastCertInfo = [];
+                foreach ($chain as $index => $cert) {
+                    $lastCertInfo = CertificateUtils::findOrInsertCert($this->db, $cert);
+                    list($certId, $_) = $lastCertInfo;
+
+                    $this->db->insert(
+                        'x509_certificate_chain_link',
+                        [
+                            'certificate_chain_id'              => $chainId,
+                            $this->db->quoteIdentifier('order') => $index,
+                            'certificate_id'                    => $certId,
+                            'ctime'                             => new Expression('UNIX_TIMESTAMP() * 1000')
+                        ]
+                    );
+
+                    $lastCertInfo[] = $index;
+                }
+
+                // There might be chains that do not include the self-signed top-level Ca,
+                // so we need to include it manually here, as we need to display the full
+                // chain in the UI.
+                $rootCa = X509Certificate::on($this->db)
+                    ->columns(['id'])
+                    ->filter(Filter::equal('subject_hash', $lastCertInfo[1]))
+                    ->filter(Filter::equal('self_signed', true))
+                    ->first();
+
+                if ($rootCa && $rootCa->id !== $lastCertInfo[0]) {
+                    $this->db->update(
+                        'x509_certificate_chain',
+                        ['length' => count($chain) + 1],
+                        ['id = ?' => $chainId]
+                    );
+
+                    $this->db->insert(
+                        'x509_certificate_chain_link',
+                        [
+                            'certificate_chain_id'              => $chainId,
+                            $this->db->quoteIdentifier('order') => $lastCertInfo[2] + 1,
+                            'certificate_id'                    => $rootCa->id,
+                            'ctime'                             => new Expression('UNIX_TIMESTAMP() * 1000')
+                        ]
+                    );
+                }
+            }
+
+            $this->db->update(
+                'x509_target',
+                [
+                    'latest_certificate_chain_id' => $chainId,
+                    'mtime'                       => new Expression('UNIX_TIMESTAMP() * 1000')
+                ],
+                ['id = ?' => $targetId]
+            );
+        });
+    }
+}