Adding upstream version 1:9.16.44.upstream/1%9.16.44

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 194 insertions, 0 deletions

diff --git a/bin/tests/system/geoip2/data/write-test-data.pl b/bin/tests/system/geoip2/data/write-test-data.pl
new file mode 100755
index 0000000..d12a014
--- /dev/null
+++ b/bin/tests/system/geoip2/data/write-test-data.pl
@@ -0,0 +1,194 @@
+#!/usr/bin/env perl
+
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+use strict;
+use warnings;
+use autodie;
+use utf8;
+
+use Carp qw( croak );
+use Cwd qw( abs_path );
+use File::Basename qw( dirname );
+use File::Slurper qw( read_binary write_binary );
+use Cpanel::JSON::XS qw( decode_json );
+use Math::Int128 qw( MAX_UINT128 string_to_uint128 uint128 );
+use MaxMind::DB::Writer::Serializer 0.100004;
+use MaxMind::DB::Writer::Tree 0.100004;
+use MaxMind::DB::Writer::Util qw( key_for_data );
+use Net::Works::Network;
+use Test::MaxMind::DB::Common::Util qw( standard_test_metadata );
+
+my $Dir = dirname( abs_path($0) );
+
+sub main {
+    write_geoip2_dbs();
+}
+
+sub write_geoip2_dbs {
+    _write_geoip2_db( @{$_}[ 0, 1 ], 'Test' )
+        for (
+        ['GeoIP2-City'],
+        ['GeoIP2-Country'],
+        ['GeoIP2-Domain'],
+        ['GeoIP2-ISP'],
+        ['GeoLite2-ASN'],
+        );
+}
+
+sub _universal_map_key_type_callback {
+    my $map = {
+
+        # languages
+        de      => 'utf8_string',
+        en      => 'utf8_string',
+        es      => 'utf8_string',
+        fr      => 'utf8_string',
+        ja      => 'utf8_string',
+        'pt-BR' => 'utf8_string',
+        ru      => 'utf8_string',
+        'zh-CN' => 'utf8_string',
+
+        # production
+        accuracy_radius                => 'uint16',
+        autonomous_system_number       => 'uint32',
+        autonomous_system_organization => 'utf8_string',
+        average_income                 => 'uint32',
+        city                           => 'map',
+        code                           => 'utf8_string',
+        confidence                     => 'uint16',
+        connection_type                => 'utf8_string',
+        continent                      => 'map',
+        country                        => 'map',
+        domain                         => 'utf8_string',
+        geoname_id                     => 'uint32',
+        ipv4_24                        => 'uint32',
+        ipv4_32                        => 'uint32',
+        ipv6_32                        => 'uint32',
+        ipv6_48                        => 'uint32',
+        ipv6_64                        => 'uint32',
+        is_anonymous                   => 'boolean',
+        is_anonymous_proxy             => 'boolean',
+        is_anonymous_vpn               => 'boolean',
+        is_hosting_provider            => 'boolean',
+        is_in_european_union           => 'boolean',
+        is_legitimate_proxy            => 'boolean',
+        is_public_proxy                => 'boolean',
+        is_satellite_provider          => 'boolean',
+        is_tor_exit_node               => 'boolean',
+        iso_code                       => 'utf8_string',
+        isp                            => 'utf8_string',
+        latitude                       => 'double',
+        location                       => 'map',
+        longitude                      => 'double',
+        metro_code                     => 'uint16',
+        names                          => 'map',
+        organization                   => 'utf8_string',
+        population_density             => 'uint32',
+        postal                         => 'map',
+        registered_country             => 'map',
+        represented_country            => 'map',
+        subdivisions                   => [ 'array', 'map' ],
+        time_zone                      => 'utf8_string',
+        traits                         => 'map',
+        traits                         => 'map',
+        type                           => 'utf8_string',
+        user_type                      => 'utf8_string',
+
+        # for testing only
+        foo       => 'utf8_string',
+        bar       => 'utf8_string',
+        buzz      => 'utf8_string',
+        our_value => 'utf8_string',
+    };
+
+    my $callback = sub {
+        my $key = shift;
+
+        return $map->{$key} || die <<"ERROR";
+Unknown tree key '$key'.
+
+The universal_map_key_type_callback doesn't know what type to use for the passed
+key.  If you are adding a new key that will be used in a frozen tree / mmdb then
+you should update the mapping in both our internal code and here.
+ERROR
+    };
+
+    return $callback;
+}
+
+sub _write_geoip2_db {
+    my $type                            = shift;
+    my $populate_all_networks_with_data = shift;
+    my $description                     = shift;
+
+    my $writer = MaxMind::DB::Writer::Tree->new(
+        ip_version    => 6,
+        record_size   => 28,
+        ip_version    => 6,
+        database_type => $type,
+        languages     => [ 'en', $type eq 'GeoIP2-City' ? ('zh') : () ],
+        description   => {
+            en => ( $type =~ s/-/ /gr )
+                . " $description Database (fake GeoIP2 data, for example purposes only)",
+            $type eq 'GeoIP2-City' ? ( zh => '小型数据库' ) : (),
+        },
+        alias_ipv6_to_ipv4    => 1,
+        map_key_type_callback => _universal_map_key_type_callback(),
+        remove_reserved_networks => 0,
+    );
+
+    _populate_all_networks( $writer, $populate_all_networks_with_data )
+        if $populate_all_networks_with_data;
+
+    my $value = shift;
+    my $nodes
+        = decode_json( read_binary("$Dir/$type.json") );
+
+    for my $node (@$nodes) {
+        for my $network ( keys %$node ) {
+            $writer->insert_network(
+                Net::Works::Network->new_from_string( string => $network ),
+                $node->{$network}
+            );
+        }
+    }
+
+    open my $output_fh, '>', "$Dir/$type.mmdb";
+    $writer->write_tree($output_fh);
+    close $output_fh;
+
+    return;
+}
+
+sub _populate_all_networks {
+    my $writer = shift;
+    my $data   = shift;
+
+    my $max_uint128 = uint128(0) - 1;
+    my @networks    = Net::Works::Network->range_as_subnets(
+        Net::Works::Address->new_from_integer(
+            integer => 0,
+            version => 6,
+        ),
+        Net::Works::Address->new_from_integer(
+            integer => $max_uint128,
+            version => 6,
+        ),
+    );
+
+    for my $network (@networks) {
+        $writer->insert_network( $network => $data );
+    }
+}
+
+main();