diff options
Diffstat (limited to 'bin/tests/system/geoip2/data/write-test-data.pl')
| -rwxr-xr-x | bin/tests/system/geoip2/data/write-test-data.pl | 194 |
1 files changed, 194 insertions, 0 deletions
diff --git a/bin/tests/system/geoip2/data/write-test-data.pl b/bin/tests/system/geoip2/data/write-test-data.pl new file mode 100755 index 0000000..d12a014 --- /dev/null +++ b/bin/tests/system/geoip2/data/write-test-data.pl @@ -0,0 +1,194 @@ +#!/usr/bin/env perl + +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# SPDX-License-Identifier: MPL-2.0 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at https://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. + +use strict; +use warnings; +use autodie; +use utf8; + +use Carp qw( croak ); +use Cwd qw( abs_path ); +use File::Basename qw( dirname ); +use File::Slurper qw( read_binary write_binary ); +use Cpanel::JSON::XS qw( decode_json ); +use Math::Int128 qw( MAX_UINT128 string_to_uint128 uint128 ); +use MaxMind::DB::Writer::Serializer 0.100004; +use MaxMind::DB::Writer::Tree 0.100004; +use MaxMind::DB::Writer::Util qw( key_for_data ); +use Net::Works::Network; +use Test::MaxMind::DB::Common::Util qw( standard_test_metadata ); + +my $Dir = dirname( abs_path($0) ); + +sub main { + write_geoip2_dbs(); +} + +sub write_geoip2_dbs { + _write_geoip2_db( @{$_}[ 0, 1 ], 'Test' ) + for ( + ['GeoIP2-City'], + ['GeoIP2-Country'], + ['GeoIP2-Domain'], + ['GeoIP2-ISP'], + ['GeoLite2-ASN'], + ); +} + +sub _universal_map_key_type_callback { + my $map = { + + # languages + de => 'utf8_string', + en => 'utf8_string', + es => 'utf8_string', + fr => 'utf8_string', + ja => 'utf8_string', + 'pt-BR' => 'utf8_string', + ru => 'utf8_string', + 'zh-CN' => 'utf8_string', + + # production + accuracy_radius => 'uint16', + autonomous_system_number => 'uint32', + autonomous_system_organization => 'utf8_string', + average_income => 'uint32', + city => 'map', + code => 'utf8_string', + confidence => 'uint16', + connection_type => 'utf8_string', + continent => 'map', + country => 'map', + domain => 'utf8_string', + geoname_id => 'uint32', + ipv4_24 => 'uint32', + ipv4_32 => 'uint32', + ipv6_32 => 'uint32', + ipv6_48 => 'uint32', + ipv6_64 => 'uint32', + is_anonymous => 'boolean', + is_anonymous_proxy => 'boolean', + is_anonymous_vpn => 'boolean', + is_hosting_provider => 'boolean', + is_in_european_union => 'boolean', + is_legitimate_proxy => 'boolean', + is_public_proxy => 'boolean', + is_satellite_provider => 'boolean', + is_tor_exit_node => 'boolean', + iso_code => 'utf8_string', + isp => 'utf8_string', + latitude => 'double', + location => 'map', + longitude => 'double', + metro_code => 'uint16', + names => 'map', + organization => 'utf8_string', + population_density => 'uint32', + postal => 'map', + registered_country => 'map', + represented_country => 'map', + subdivisions => [ 'array', 'map' ], + time_zone => 'utf8_string', + traits => 'map', + traits => 'map', + type => 'utf8_string', + user_type => 'utf8_string', + + # for testing only + foo => 'utf8_string', + bar => 'utf8_string', + buzz => 'utf8_string', + our_value => 'utf8_string', + }; + + my $callback = sub { + my $key = shift; + + return $map->{$key} || die <<"ERROR"; +Unknown tree key '$key'. + +The universal_map_key_type_callback doesn't know what type to use for the passed +key. If you are adding a new key that will be used in a frozen tree / mmdb then +you should update the mapping in both our internal code and here. +ERROR + }; + + return $callback; +} + +sub _write_geoip2_db { + my $type = shift; + my $populate_all_networks_with_data = shift; + my $description = shift; + + my $writer = MaxMind::DB::Writer::Tree->new( + ip_version => 6, + record_size => 28, + ip_version => 6, + database_type => $type, + languages => [ 'en', $type eq 'GeoIP2-City' ? ('zh') : () ], + description => { + en => ( $type =~ s/-/ /gr ) + . " $description Database (fake GeoIP2 data, for example purposes only)", + $type eq 'GeoIP2-City' ? ( zh => '小型数据库' ) : (), + }, + alias_ipv6_to_ipv4 => 1, + map_key_type_callback => _universal_map_key_type_callback(), + remove_reserved_networks => 0, + ); + + _populate_all_networks( $writer, $populate_all_networks_with_data ) + if $populate_all_networks_with_data; + + my $value = shift; + my $nodes + = decode_json( read_binary("$Dir/$type.json") ); + + for my $node (@$nodes) { + for my $network ( keys %$node ) { + $writer->insert_network( + Net::Works::Network->new_from_string( string => $network ), + $node->{$network} + ); + } + } + + open my $output_fh, '>', "$Dir/$type.mmdb"; + $writer->write_tree($output_fh); + close $output_fh; + + return; +} + +sub _populate_all_networks { + my $writer = shift; + my $data = shift; + + my $max_uint128 = uint128(0) - 1; + my @networks = Net::Works::Network->range_as_subnets( + Net::Works::Address->new_from_integer( + integer => 0, + version => 6, + ), + Net::Works::Address->new_from_integer( + integer => $max_uint128, + version => 6, + ), + ); + + for my $network (@networks) { + $writer->insert_network( $network => $data ); + } +} + +main(); |