summaryrefslogtreecommitdiffstats
path: root/bin/tests/system/geoip2/data/write-test-data.pl
diff options
context:
space:
mode:
Diffstat (limited to '')
-rwxr-xr-xbin/tests/system/geoip2/data/write-test-data.pl194
1 files changed, 194 insertions, 0 deletions
diff --git a/bin/tests/system/geoip2/data/write-test-data.pl b/bin/tests/system/geoip2/data/write-test-data.pl
new file mode 100755
index 0000000..d12a014
--- /dev/null
+++ b/bin/tests/system/geoip2/data/write-test-data.pl
@@ -0,0 +1,194 @@
+#!/usr/bin/env perl
+
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+use strict;
+use warnings;
+use autodie;
+use utf8;
+
+use Carp qw( croak );
+use Cwd qw( abs_path );
+use File::Basename qw( dirname );
+use File::Slurper qw( read_binary write_binary );
+use Cpanel::JSON::XS qw( decode_json );
+use Math::Int128 qw( MAX_UINT128 string_to_uint128 uint128 );
+use MaxMind::DB::Writer::Serializer 0.100004;
+use MaxMind::DB::Writer::Tree 0.100004;
+use MaxMind::DB::Writer::Util qw( key_for_data );
+use Net::Works::Network;
+use Test::MaxMind::DB::Common::Util qw( standard_test_metadata );
+
+my $Dir = dirname( abs_path($0) );
+
+sub main {
+ write_geoip2_dbs();
+}
+
+sub write_geoip2_dbs {
+ _write_geoip2_db( @{$_}[ 0, 1 ], 'Test' )
+ for (
+ ['GeoIP2-City'],
+ ['GeoIP2-Country'],
+ ['GeoIP2-Domain'],
+ ['GeoIP2-ISP'],
+ ['GeoLite2-ASN'],
+ );
+}
+
+sub _universal_map_key_type_callback {
+ my $map = {
+
+ # languages
+ de => 'utf8_string',
+ en => 'utf8_string',
+ es => 'utf8_string',
+ fr => 'utf8_string',
+ ja => 'utf8_string',
+ 'pt-BR' => 'utf8_string',
+ ru => 'utf8_string',
+ 'zh-CN' => 'utf8_string',
+
+ # production
+ accuracy_radius => 'uint16',
+ autonomous_system_number => 'uint32',
+ autonomous_system_organization => 'utf8_string',
+ average_income => 'uint32',
+ city => 'map',
+ code => 'utf8_string',
+ confidence => 'uint16',
+ connection_type => 'utf8_string',
+ continent => 'map',
+ country => 'map',
+ domain => 'utf8_string',
+ geoname_id => 'uint32',
+ ipv4_24 => 'uint32',
+ ipv4_32 => 'uint32',
+ ipv6_32 => 'uint32',
+ ipv6_48 => 'uint32',
+ ipv6_64 => 'uint32',
+ is_anonymous => 'boolean',
+ is_anonymous_proxy => 'boolean',
+ is_anonymous_vpn => 'boolean',
+ is_hosting_provider => 'boolean',
+ is_in_european_union => 'boolean',
+ is_legitimate_proxy => 'boolean',
+ is_public_proxy => 'boolean',
+ is_satellite_provider => 'boolean',
+ is_tor_exit_node => 'boolean',
+ iso_code => 'utf8_string',
+ isp => 'utf8_string',
+ latitude => 'double',
+ location => 'map',
+ longitude => 'double',
+ metro_code => 'uint16',
+ names => 'map',
+ organization => 'utf8_string',
+ population_density => 'uint32',
+ postal => 'map',
+ registered_country => 'map',
+ represented_country => 'map',
+ subdivisions => [ 'array', 'map' ],
+ time_zone => 'utf8_string',
+ traits => 'map',
+ traits => 'map',
+ type => 'utf8_string',
+ user_type => 'utf8_string',
+
+ # for testing only
+ foo => 'utf8_string',
+ bar => 'utf8_string',
+ buzz => 'utf8_string',
+ our_value => 'utf8_string',
+ };
+
+ my $callback = sub {
+ my $key = shift;
+
+ return $map->{$key} || die <<"ERROR";
+Unknown tree key '$key'.
+
+The universal_map_key_type_callback doesn't know what type to use for the passed
+key. If you are adding a new key that will be used in a frozen tree / mmdb then
+you should update the mapping in both our internal code and here.
+ERROR
+ };
+
+ return $callback;
+}
+
+sub _write_geoip2_db {
+ my $type = shift;
+ my $populate_all_networks_with_data = shift;
+ my $description = shift;
+
+ my $writer = MaxMind::DB::Writer::Tree->new(
+ ip_version => 6,
+ record_size => 28,
+ ip_version => 6,
+ database_type => $type,
+ languages => [ 'en', $type eq 'GeoIP2-City' ? ('zh') : () ],
+ description => {
+ en => ( $type =~ s/-/ /gr )
+ . " $description Database (fake GeoIP2 data, for example purposes only)",
+ $type eq 'GeoIP2-City' ? ( zh => '小型数据库' ) : (),
+ },
+ alias_ipv6_to_ipv4 => 1,
+ map_key_type_callback => _universal_map_key_type_callback(),
+ remove_reserved_networks => 0,
+ );
+
+ _populate_all_networks( $writer, $populate_all_networks_with_data )
+ if $populate_all_networks_with_data;
+
+ my $value = shift;
+ my $nodes
+ = decode_json( read_binary("$Dir/$type.json") );
+
+ for my $node (@$nodes) {
+ for my $network ( keys %$node ) {
+ $writer->insert_network(
+ Net::Works::Network->new_from_string( string => $network ),
+ $node->{$network}
+ );
+ }
+ }
+
+ open my $output_fh, '>', "$Dir/$type.mmdb";
+ $writer->write_tree($output_fh);
+ close $output_fh;
+
+ return;
+}
+
+sub _populate_all_networks {
+ my $writer = shift;
+ my $data = shift;
+
+ my $max_uint128 = uint128(0) - 1;
+ my @networks = Net::Works::Network->range_as_subnets(
+ Net::Works::Address->new_from_integer(
+ integer => 0,
+ version => 6,
+ ),
+ Net::Works::Address->new_from_integer(
+ integer => $max_uint128,
+ version => 6,
+ ),
+ );
+
+ for my $network (@networks) {
+ $writer->insert_network( $network => $data );
+ }
+}
+
+main();