summaryrefslogtreecommitdiffstats
path: root/library/X509/Job.php
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-14 13:31:28 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-14 13:31:28 +0000
commit067008c5f094ba9606daacbe540f6b929dc124ea (patch)
tree3092ce2cd8bf1ac6db6c97f4c98c7f71a51c6ac8 /library/X509/Job.php
parentInitial commit. (diff)
downloadicingaweb2-module-x509-067008c5f094ba9606daacbe540f6b929dc124ea.tar.xz
icingaweb2-module-x509-067008c5f094ba9606daacbe540f6b929dc124ea.zip
Adding upstream version 1:1.3.2.upstream/1%1.3.2upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '')
-rw-r--r--library/X509/Job.php755
1 files changed, 755 insertions, 0 deletions
diff --git a/library/X509/Job.php b/library/X509/Job.php
new file mode 100644
index 0000000..1e0b3f7
--- /dev/null
+++ b/library/X509/Job.php
@@ -0,0 +1,755 @@
+<?php
+
+// Icinga Web 2 X.509 Module | (c) 2018 Icinga GmbH | GPLv2
+
+namespace Icinga\Module\X509;
+
+use DateTime;
+use Exception;
+use Generator;
+use GMP;
+use Icinga\Application\Logger;
+use Icinga\Module\X509\Common\Database;
+use Icinga\Module\X509\Common\JobOptions;
+use Icinga\Module\X509\Model\X509Certificate;
+use Icinga\Module\X509\Model\X509CertificateChain;
+use Icinga\Module\X509\Model\X509JobRun;
+use Icinga\Module\X509\Model\X509Target;
+use Icinga\Module\X509\React\StreamOptsCaptureConnector;
+use Icinga\Util\Json;
+use ipl\Scheduler\Common\TaskProperties;
+use ipl\Scheduler\Contract\Task;
+use ipl\Sql\Connection;
+use ipl\Sql\Expression;
+use ipl\Stdlib\Filter;
+use LogicException;
+use Ramsey\Uuid\Uuid;
+use Ramsey\Uuid\UuidInterface;
+use React\EventLoop\Loop;
+use React\Promise;
+use React\Socket\ConnectionInterface;
+use React\Socket\Connector;
+use React\Socket\ConnectorInterface;
+use React\Socket\SecureConnector;
+use React\Socket\TimeoutConnector;
+use Throwable;
+
+class Job implements Task
+{
+ use JobOptions;
+ use TaskProperties;
+
+ /** @var int Number of targets to be scanned in parallel by default */
+ public const DEFAULT_PARALLEL = 256;
+
+ /** @var string Default since last scan threshold used to filter out scan targets */
+ public const DEFAULT_SINCE_LAST_SCAN = '-24 hours';
+
+ /** @var int The database id of this job */
+ protected $id;
+
+ /** @var Connection x509 database connection */
+ private $db;
+
+ /** @var DbTool Database utils for marshalling and unmarshalling binary data */
+ private $dbTool;
+
+ /** @var int Number of pending targets to be scanned */
+ private $pendingTargets = 0;
+
+ /** @var int Total number of scan targets */
+ private $totalTargets = 0;
+
+ /** @var int Number of scanned targets */
+ private $finishedTargets = 0;
+
+ /** @var Generator Scan targets generator */
+ private $targets;
+
+ /** @var array<string, array<string>> The configured SNI maps */
+ private $snimap;
+
+ /** @var int The id of the last inserted job run entry */
+ private $jobRunId;
+
+ /** @var Promise\Deferred React promise deferred instance used to resolve the running promise */
+ protected $deferred;
+
+ /** @var DateTime The start time of this job */
+ protected $jobRunStart;
+
+ /** @var array<string> A list of excluded IP addresses and host names */
+ private $excludedTargets = [];
+
+ /** @var array<string, array<int, int|string>> The configured CIDRs of this job */
+ private $cidrs;
+
+ /** @var array<int, array<string>> The configured ports of this job */
+ private $ports;
+
+ /**
+ * Construct a new Job instance
+ *
+ * @param string $name The name of this job
+ * @param array<string, array<int, int|string>> $cidrs The configured CIDRs to be used by this job
+ * @param array<int, array<string>> $ports The configured ports to be used by this job
+ * @param array<string, array<string>> $snimap The configured SNI maps to be used by this job
+ * @param ?Schedule $schedule
+ */
+ public function __construct(string $name, array $cidrs, array $ports, array $snimap, Schedule $schedule = null)
+ {
+ $this->name = $name;
+ $this->db = Database::get();
+ $this->dbTool = new DbTool($this->db);
+ $this->snimap = $snimap;
+ $this->cidrs = $cidrs;
+ $this->ports = $ports;
+
+ if ($schedule) {
+ $this->setSchedule($schedule);
+ }
+
+ $this->setName($name);
+ }
+
+ /**
+ * Transform the given human-readable IP address into a binary format
+ *
+ * @param string $addr
+ *
+ * @return string
+ */
+ public static function binary(string $addr): string
+ {
+ return str_pad(inet_pton($addr), 16, "\0", STR_PAD_LEFT);
+ }
+
+ /**
+ * Transform the given human-readable IP address into GMP number
+ *
+ * @param string $addr
+ *
+ * @return ?GMP
+ */
+ public static function addrToNumber(string $addr): ?GMP
+ {
+ return gmp_import(static::binary($addr));
+ }
+
+ /**
+ * Transform the given number into human-readable IP address
+ *
+ * @param $num
+ * @param bool $ipv6
+ *
+ * @return false|string
+ */
+ public static function numberToAddr($num, bool $ipv6 = true)
+ {
+ if ($ipv6) {
+ return inet_ntop(str_pad(gmp_export($num), 16, "\0", STR_PAD_LEFT));
+ } else {
+ return inet_ntop(gmp_export($num));
+ }
+ }
+
+ /**
+ * Check whether the given IP is inside the specified CIDR
+ *
+ * @param GMP $addr
+ * @param string $subnet
+ * @param int $mask
+ *
+ * @return bool
+ */
+ public static function isAddrInside(GMP $addr, string $subnet, int $mask): bool
+ {
+ // `gmp_pow()` is like PHP's pow() function, but handles also very large numbers
+ // and `gmp_com()` is like the bitwise NOT (~) operator.
+ $mask = gmp_com(gmp_pow(2, (static::isIPV6($subnet) ? 128 : 32) - $mask) - 1);
+ return gmp_strval(gmp_and($addr, $mask)) === gmp_strval(gmp_and(static::addrToNumber($subnet), $mask));
+ }
+
+ /**
+ * Get whether the given IP address is IPV6 address
+ *
+ * @param $addr
+ *
+ * @return bool
+ */
+ public static function isIPV6($addr): bool
+ {
+ return filter_var($addr, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6) !== false;
+ }
+
+ /**
+ * Get the database id of this job
+ *
+ * @return int
+ */
+ public function getId(): int
+ {
+ return $this->id;
+ }
+
+ /**
+ * Set the database id of this job
+ *
+ * @param int $id
+ *
+ * @return $this
+ */
+ public function setId(int $id): self
+ {
+ $this->id = $id;
+
+ return $this;
+ }
+
+ public function getUuid(): UuidInterface
+ {
+ if (! $this->uuid) {
+ $this->setUuid(Uuid::fromBytes($this->getChecksum()));
+ }
+
+ return $this->uuid;
+ }
+
+ /**
+ * Get the configured job CIDRS
+ *
+ * @return array<string, array<int, int|string>>
+ */
+ public function getCIDRs(): array
+ {
+ return $this->cidrs;
+ }
+
+ /**
+ * Set the CIDRs of this job
+ *
+ * @param array<string, array<int, int|string>> $cidrs
+ *
+ * @return $this
+ */
+ public function setCIDRs(array $cidrs): self
+ {
+ $this->cidrs = $cidrs;
+
+ return $this;
+ }
+
+ /**
+ * Get the configured ports of this job
+ *
+ * @return array<int, array<string>>
+ */
+ public function getPorts(): array
+ {
+ return $this->ports;
+ }
+
+ /**
+ * Set the ports of this job to be scanned
+ *
+ * @param array<int, array<string>> $ports
+ *
+ * @return $this
+ */
+ public function setPorts(array $ports): self
+ {
+ $this->ports = $ports;
+
+ return $this;
+ }
+
+ /**
+ * Get excluded IPs and host names
+ *
+ * @return array<string>
+ */
+ public function getExcludes(): array
+ {
+ return $this->excludedTargets;
+ }
+
+ /**
+ * Set a set of IPs and host names to be excluded from scan
+ *
+ * @param array<string> $targets
+ *
+ * @return $this
+ */
+ public function setExcludes(array $targets): self
+ {
+ $this->excludedTargets = $targets;
+
+ return $this;
+ }
+
+ private function getConnector($peerName): array
+ {
+ $simpleConnector = new Connector();
+ $streamCaptureConnector = new StreamOptsCaptureConnector($simpleConnector);
+ $secureConnector = new SecureConnector($streamCaptureConnector, null, [
+ 'verify_peer' => false,
+ 'verify_peer_name' => false,
+ 'capture_peer_cert_chain' => true,
+ 'SNI_enabled' => true,
+ 'peer_name' => $peerName
+ ]);
+ return [new TimeoutConnector($secureConnector, 5.0), $streamCaptureConnector];
+ }
+
+ /**
+ * Get whether this job has been completed scanning all targets
+ *
+ * @return bool
+ */
+ public function isFinished(): bool
+ {
+ return ! $this->targets->valid() && $this->pendingTargets === 0;
+ }
+
+ public function updateLastScan($target)
+ {
+ if (! $this->isRescan() && ! isset($target->id)) {
+ return;
+ }
+
+ $this->db->update('x509_target', [
+ 'last_scan' => new Expression('UNIX_TIMESTAMP() * 1000')
+ ], ['id = ?' => $target->id]);
+ }
+
+ public function getChecksum(): string
+ {
+ $data = [
+ 'name' => $this->getName(),
+ 'cidrs' => $this->getCIDRs(),
+ 'ports' => $this->getPorts(),
+ 'exclude_targets' => $this->getExcludes(),
+ ];
+
+ $schedule = null;
+ if ($this->schedule) {
+ $schedule = $this->getSchedule();
+ }
+
+ return md5(Json::encode($data) . ($schedule ? bin2hex($schedule->getChecksum()) : ''), true);
+ }
+
+ protected function getScanTargets(): Generator
+ {
+ $generate = $this->fullScan || ! $this->isRescan();
+ if (! $generate) {
+ $run = X509JobRun::on($this->db)
+ ->columns([new Expression('1')])
+ ->filter(Filter::equal('schedule.job_id', $this->getId()))
+ ->filter(Filter::unequal('total_targets', 0))
+ ->limit(1)
+ ->execute();
+
+ $generate = ! $run->hasResult();
+ }
+
+ if ($generate) {
+ yield from $this->generateTargets();
+ }
+
+ $sinceLastScan = $this->getSinceLastScan();
+ if ((! $this->fullScan && $sinceLastScan !== null) || $this->isRescan()) {
+ $targets = X509Target::on($this->db)->columns(['id', 'ip', 'hostname', 'port']);
+ if (! $this->fullScan && $sinceLastScan) {
+ $targets->filter(Filter::lessThan('last_scan', $sinceLastScan));
+ }
+
+ foreach ($targets as $target) {
+ $addr = static::addrToNumber($target->ip);
+ $addrFound = false;
+ foreach ($this->getCIDRs() as $cidr) {
+ list($subnet, $mask) = $cidr;
+ if (static::isAddrInside($addr, (string) $subnet, (int) $mask)) {
+ $target->ip = static::numberToAddr($addr, static::isIPV6($subnet));
+ $addrFound = true;
+
+ break;
+ }
+ }
+
+ if ($addrFound) {
+ yield $target;
+ }
+ }
+ }
+ }
+
+ private function generateTargets(): Generator
+ {
+ $excludes = $this->getExcludes();
+ foreach ($this->getCIDRs() as $cidr) {
+ list($startIp, $prefix) = $cidr;
+ $ipv6 = static::isIPV6($startIp);
+ $subnet = $ipv6 ? 128 : 32;
+ $numIps = pow(2, ($subnet - (int) $prefix));
+
+ Logger::info('Scanning %d IPs in the CIDR %s', $numIps, implode('/', $cidr));
+
+ $start = static::addrToNumber((string) $startIp);
+ for ($i = 0; $i < $numIps; $i++) {
+ $ip = static::numberToAddr(gmp_add($start, $i), $ipv6);
+ if (isset($excludes[$ip])) {
+ Logger::debug('Excluding IP %s from scan', $ip);
+ continue;
+ }
+
+ foreach ($this->getPorts() as $portRange) {
+ list($startPort, $endPort) = $portRange;
+ foreach (range($startPort, $endPort) as $port) {
+ foreach ($this->snimap[$ip] ?? [null] as $hostname) {
+ if (array_key_exists((string) $hostname, $excludes)) {
+ Logger::debug('Excluding host %s from scan', $hostname);
+ continue;
+ }
+
+ if (! $this->fullScan) {
+ $targets = X509Target::on($this->db)
+ ->columns([new Expression('1')])
+ ->filter(
+ Filter::all(
+ Filter::equal('ip', $ip),
+ Filter::equal('port', $port),
+ $hostname !== null
+ ? Filter::equal('hostname', $hostname)
+ : Filter::unlike('hostname', '*')
+ )
+ );
+
+ if ($targets->execute()->hasResult()) {
+ continue;
+ }
+ }
+
+ yield (object) [
+ 'ip' => $ip,
+ 'port' => $port,
+ 'hostname' => $hostname
+ ];
+ }
+ }
+ }
+ }
+ }
+ }
+
+ public function updateJobStats(bool $finished = false): void
+ {
+ $fields = ['finished_targets' => $this->finishedTargets];
+ if ($finished) {
+ $fields['end_time'] = new Expression('UNIX_TIMESTAMP() * 1000');
+ $fields['total_targets'] = $this->totalTargets;
+ }
+
+ $this->db->update('x509_job_run', $fields, ['id = ?' => $this->jobRunId]);
+ }
+
+ private static function formatTarget($target): string
+ {
+ $result = "tls://[{$target->ip}]:{$target->port}";
+
+ if ($target->hostname !== null) {
+ $result .= " [SNI hostname: {$target->hostname}]";
+ }
+
+ return $result;
+ }
+
+ private function finishTarget()
+ {
+ $this->pendingTargets--;
+ $this->finishedTargets++;
+ $this->startNextTarget();
+ }
+
+ private function startNextTarget()
+ {
+ if ($this->isFinished()) {
+ // No targets to process anymore, so we can now resolve the promise
+ $this->deferred->resolve($this->finishedTargets);
+
+ return;
+ }
+
+ if (! $this->targets->valid()) {
+ // When nothing is yielded, and it's still not finished yet, just get the next target
+ return;
+ }
+
+ $target = $this->targets->current();
+ $this->targets->next();
+
+ $this->totalTargets++;
+ $this->pendingTargets++;
+
+ $url = "tls://[{$target->ip}]:{$target->port}";
+ Logger::debug("Connecting to %s", self::formatTarget($target));
+
+ /** @var ConnectorInterface $connector */
+ /** @var StreamOptsCaptureConnector $streamCapture */
+ list($connector, $streamCapture) = $this->getConnector($target->hostname);
+ $connector->connect($url)->then(
+ function (ConnectionInterface $conn) use ($target, $streamCapture) {
+ Logger::info("Connected to %s", self::formatTarget($target));
+
+ // Close connection in order to capture stream context options
+ $conn->close();
+
+ $capturedStreamOptions = $streamCapture->getCapturedStreamOptions();
+
+ $this->processChain($target, $capturedStreamOptions['ssl']['peer_certificate_chain']);
+
+ $this->finishTarget();
+ },
+ function (Exception $exception) use ($target, $streamCapture) {
+ Logger::debug("Cannot connect to server: %s", $exception->getMessage());
+
+ $capturedStreamOptions = $streamCapture->getCapturedStreamOptions();
+
+ if (isset($capturedStreamOptions['ssl']['peer_certificate_chain'])) {
+ // The scanned target presented its certificate chain despite throwing an error
+ // This is the case for targets which require client certificates for example
+ $this->processChain($target, $capturedStreamOptions['ssl']['peer_certificate_chain']);
+ } else {
+ $this->db->update(
+ 'x509_target',
+ [
+ 'latest_certificate_chain_id' => null,
+ 'mtime' => new Expression('UNIX_TIMESTAMP() * 1000')
+ ],
+ [
+ 'hostname = ?' => $target->hostname,
+ 'ip = ?' => $this->dbTool->marshalBinary(static::binary($target->ip)),
+ 'port = ?' => $target->port
+ ]
+ );
+ }
+
+ $step = max($this->totalTargets / 100, 1);
+
+ if ($this->finishedTargets % (int) $step == 0) {
+ $this->updateJobStats();
+ }
+
+ $this->finishTarget();
+ }
+ )->always(function () use ($target) {
+ $this->updateLastScan($target);
+ })->otherwise(function (Throwable $e) {
+ Logger::error($e->getMessage());
+ Logger::error($e->getTraceAsString());
+ });
+ }
+
+ public function run(): Promise\ExtendedPromiseInterface
+ {
+ $this->jobRunStart = new DateTime();
+ // Update the job statistics regardless of whether the job was successful, failed, or canceled.
+ // Otherwise, some database columns might remain null.
+ $updateJobStats = function () {
+ $this->updateJobStats(true);
+ };
+ $this->deferred = new Promise\Deferred($updateJobStats);
+ $this->deferred->promise()->always($updateJobStats);
+
+ Loop::futureTick(function () {
+ if (! $this->db->ping()) {
+ $this->deferred->reject(new LogicException('Lost connection to database and failed to reconnect'));
+
+ return;
+ }
+
+ // Reset those statistics for the next run! Is only necessary when
+ // running this job using the scheduler
+ $this->totalTargets = 0;
+ $this->finishedTargets = 0;
+ $this->pendingTargets = 0;
+
+ if ($this->schedule) {
+ $scheduleId = $this->getSchedule()->getId();
+ } else {
+ $scheduleId = new Expression('NULL');
+ }
+
+ $this->db->insert('x509_job_run', [
+ 'job_id' => $this->getId(),
+ 'schedule_id' => $scheduleId,
+ 'start_time' => $this->jobRunStart->getTimestamp() * 1000.0,
+ 'total_targets' => 0,
+ 'finished_targets' => 0
+ ]);
+ $this->jobRunId = (int) $this->db->lastInsertId();
+
+ $this->targets = $this->getScanTargets();
+
+ if ($this->isFinished()) {
+ // There are no targets to scan, so we can resolve the promise earlier
+ $this->deferred->resolve(0);
+
+ return;
+ }
+
+ // Start scanning the first couple of targets...
+ for ($i = 0; $i < $this->getParallel() && ! $this->isFinished(); $i++) {
+ $this->startNextTarget();
+ }
+ });
+
+ /** @var Promise\ExtendedPromiseInterface $promise */
+ $promise = $this->deferred->promise();
+ return $promise;
+ }
+
+ protected function processChain($target, $chain)
+ {
+ if ($target->hostname === null) {
+ $hostname = gethostbyaddr($target->ip);
+
+ if ($hostname !== false) {
+ $target->hostname = $hostname;
+ }
+ }
+
+ $this->db->transaction(function () use ($target, $chain) {
+ $row = X509Target::on($this->db)
+ ->columns(['id'])
+ ->filter(
+ Filter::all(
+ Filter::equal('ip', $target->ip),
+ Filter::equal('port', $target->port),
+ Filter::equal('hostname', $target->hostname)
+ )
+ )->first();
+
+ if (! $row) {
+ // TODO: https://github.com/Icinga/ipl-orm/pull/78
+ $this->db->insert(
+ 'x509_target',
+ [
+ 'ip' => $this->dbTool->marshalBinary(static::binary($target->ip)),
+ 'port' => $target->port,
+ 'hostname' => $target->hostname,
+ 'last_scan' => new Expression('UNIX_TIMESTAMP() * 1000'),
+ 'ctime' => new Expression('UNIX_TIMESTAMP() * 1000')
+ ]
+ );
+ $targetId = $this->db->lastInsertId();
+ } else {
+ $targetId = $row->id;
+ }
+
+ $chainUptodate = false;
+
+ $lastChain = X509CertificateChain::on($this->db)
+ ->columns(['id'])
+ ->filter(Filter::equal('target_id', $targetId))
+ ->orderBy('id', SORT_DESC)
+ ->limit(1)
+ ->first();
+
+ if ($lastChain) {
+ $lastFingerprints = X509Certificate::on($this->db)->utilize('chain');
+ $lastFingerprints
+ ->columns(['fingerprint'])
+ ->getSelectBase()
+ ->where(new Expression(
+ 'certificate_link.certificate_chain_id = %d',
+ [$lastChain->id]
+ ))
+ ->orderBy('certificate_link.order');
+
+ $lastFingerprintsArr = [];
+ foreach ($lastFingerprints as $lastFingerprint) {
+ $lastFingerprintsArr[] = $lastFingerprint->fingerprint;
+ }
+
+ $currentFingerprints = [];
+
+ foreach ($chain as $cert) {
+ $currentFingerprints[] = openssl_x509_fingerprint($cert, 'sha256', true);
+ }
+
+ $chainUptodate = $currentFingerprints === $lastFingerprintsArr;
+ }
+
+ if ($lastChain && $chainUptodate) {
+ $chainId = $lastChain->id;
+ } else {
+ // TODO: https://github.com/Icinga/ipl-orm/pull/78
+ $this->db->insert(
+ 'x509_certificate_chain',
+ [
+ 'target_id' => $targetId,
+ 'length' => count($chain),
+ 'ctime' => new Expression('UNIX_TIMESTAMP() * 1000')
+ ]
+ );
+
+ $chainId = $this->db->lastInsertId();
+
+ $lastCertInfo = [];
+ foreach ($chain as $index => $cert) {
+ $lastCertInfo = CertificateUtils::findOrInsertCert($this->db, $cert);
+ list($certId, $_) = $lastCertInfo;
+
+ $this->db->insert(
+ 'x509_certificate_chain_link',
+ [
+ 'certificate_chain_id' => $chainId,
+ $this->db->quoteIdentifier('order') => $index,
+ 'certificate_id' => $certId,
+ 'ctime' => new Expression('UNIX_TIMESTAMP() * 1000')
+ ]
+ );
+
+ $lastCertInfo[] = $index;
+ }
+
+ // There might be chains that do not include the self-signed top-level Ca,
+ // so we need to include it manually here, as we need to display the full
+ // chain in the UI.
+ $rootCa = X509Certificate::on($this->db)
+ ->columns(['id'])
+ ->filter(Filter::equal('subject_hash', $lastCertInfo[1]))
+ ->filter(Filter::equal('self_signed', true))
+ ->first();
+
+ if ($rootCa && $rootCa->id !== $lastCertInfo[0]) {
+ $this->db->update(
+ 'x509_certificate_chain',
+ ['length' => count($chain) + 1],
+ ['id = ?' => $chainId]
+ );
+
+ $this->db->insert(
+ 'x509_certificate_chain_link',
+ [
+ 'certificate_chain_id' => $chainId,
+ $this->db->quoteIdentifier('order') => $lastCertInfo[2] + 1,
+ 'certificate_id' => $rootCa->id,
+ 'ctime' => new Expression('UNIX_TIMESTAMP() * 1000')
+ ]
+ );
+ }
+ }
+
+ $this->db->update(
+ 'x509_target',
+ [
+ 'latest_certificate_chain_id' => $chainId,
+ 'mtime' => new Expression('UNIX_TIMESTAMP() * 1000')
+ ],
+ ['id = ?' => $targetId]
+ );
+ });
+ }
+}