diff options
Diffstat (limited to 'src/fluent-bit/plugins/filter_geoip2/libmaxminddb')
16 files changed, 0 insertions, 3977 deletions
diff --git a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/CMakeLists.txt b/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/CMakeLists.txt deleted file mode 100644 index 2ed068af2..000000000 --- a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/CMakeLists.txt +++ /dev/null @@ -1,101 +0,0 @@ -project(maxminddb - LANGUAGES C -) -set(MAXMINDDB_SOVERSION 0.0.7) - -option(BUILD_SHARED_LIBS "Build shared libraries (.dll/.so) instead of static ones (.lib/.a)" OFF) -option(BUILD_TESTING "Build test programs" ON) - -include(CheckTypeSize) -check_type_size("unsigned __int128" UINT128) -check_type_size("unsigned int __attribute__((mode(TI)))" UINT128_USING_MODE) -if(HAVE_UINT128) - set(MMDB_UINT128_USING_MODE 0) - set(MMDB_UINT128_IS_BYTE_ARRAY 0) -elseif(HAVE_UINT128_USING_MODE) - set(MMDB_UINT128_USING_MODE 1) - set(MMDB_UINT128_IS_BYTE_ARRAY 0) -else() - set(MMDB_UINT128_USING_MODE 0) - set(MMDB_UINT128_IS_BYTE_ARRAY 1) -endif() - -include (TestBigEndian) -TEST_BIG_ENDIAN(IS_BIG_ENDIAN) - -if (${CMAKE_SYSTEM_NAME} MATCHES "Linux") - set(CMAKE_POSITION_INDEPENDENT_CODE ON) -endif() - -configure_file(${PROJECT_SOURCE_DIR}/include/maxminddb_config.h.cmake.in - ${PROJECT_SOURCE_DIR}/include/maxminddb_config.h) - -add_library(maxminddb STATIC - src/maxminddb.c - src/data-pool.c -) -add_library(maxminddb::maxminddb ALIAS maxminddb) - -set_target_properties(maxminddb PROPERTIES VERSION ${MAXMINDDB_SOVERSION}) - -target_compile_definitions(maxminddb PUBLIC PACKAGE_VERSION="${PROJECT_VERSION}") - -if(NOT IS_BIG_ENDIAN) - target_compile_definitions(maxminddb PRIVATE MMDB_LITTLE_ENDIAN=1) -endif() - -if(MSVC) - target_compile_definitions(maxminddb PRIVATE _CRT_SECURE_NO_WARNINGS) -endif() - -if(WIN32) - target_link_libraries(maxminddb ws2_32) -endif() - -set(CMAKE_SHARED_LIBRARY_PREFIX lib) -set(CMAKE_STATIC_LIBRARY_PREFIX lib) - -set(MAXMINDB_INCLUDE_DIR - . - include -) - - - -# -# NOTE: This function call was modified for Fluent Bit. -# The original first argument was the following: -# -# $<BUILD_INTERFACE:${MAXMINDB_INCLUDE_DIR}> -# -target_include_directories(maxminddb PUBLIC - $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include> - $<INSTALL_INTERFACE:include> -) - -set(MAXMINDB_HEADERS - include/maxminddb.h - include/maxminddb_config.h -) -set_target_properties(maxminddb PROPERTIES PUBLIC_HEADER "${MAXMINDB_HEADERS}") - -#install(TARGETS maxminddb -# EXPORT maxminddb -# ARCHIVE DESTINATION lib -# PUBLIC_HEADER DESTINATION include/ -#) -# -## This is required to work with FetchContent -#install(EXPORT maxminddb -# FILE maxminddb-config.cmake -# NAMESPACE maxminddb:: -# DESTINATION lib/cmake/maxminddb) - -# We always want to build mmdblookup -add_subdirectory(bin) - -if (BUILD_TESTING) - enable_testing() - add_subdirectory(t) -endif() - diff --git a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/LICENSE b/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/LICENSE deleted file mode 100644 index d64569567..000000000 --- a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/NOTICE b/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/NOTICE deleted file mode 100644 index 6b8694752..000000000 --- a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/NOTICE +++ /dev/null @@ -1,13 +0,0 @@ -Copyright 2013-2014 MaxMind, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License.
\ No newline at end of file diff --git a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/VERSION b/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/VERSION deleted file mode 100644 index dbdbf984c..000000000 --- a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/VERSION +++ /dev/null @@ -1 +0,0 @@ -ad35e6af diff --git a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/bin/CMakeLists.txt b/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/bin/CMakeLists.txt deleted file mode 100644 index 9026be83b..000000000 --- a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/bin/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -# getopt is required by mmdblookup which is not available by default on Windows -if(NOT WIN32) - add_executable(mmdblookup - mmdblookup.c - ) - - target_link_libraries(mmdblookup maxminddb pthread) - - install( - TARGETS mmdblookup - RUNTIME DESTINATION bin - ) -endif() diff --git a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/bin/Makefile.am b/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/bin/Makefile.am deleted file mode 100644 index c00ba95b5..000000000 --- a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/bin/Makefile.am +++ /dev/null @@ -1,10 +0,0 @@ -include $(top_srcdir)/common.mk - -AM_LDFLAGS = $(top_builddir)/src/libmaxminddb.la - -bin_PROGRAMS = mmdblookup - -if !WINDOWS -AM_CPPFLAGS += -pthread -AM_LDFLAGS += -pthread -endif diff --git a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/bin/mmdblookup.c b/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/bin/mmdblookup.c deleted file mode 100644 index d7ec3fff2..000000000 --- a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/bin/mmdblookup.c +++ /dev/null @@ -1,762 +0,0 @@ -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif -#include "maxminddb.h" -#include <errno.h> -#include <getopt.h> -#ifndef _WIN32 -#include <pthread.h> -#endif -#include <stdbool.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <time.h> - -#ifdef _WIN32 -#ifndef UNICODE -#define UNICODE -#endif -#include <malloc.h> -#else -#include <libgen.h> -#include <unistd.h> -#endif - -#define LOCAL static - -LOCAL void usage(char *program, int exit_code, const char *error); -LOCAL const char **get_options( - int argc, - char **argv, - char **mmdb_file, - char **ip_address, - int *verbose, - int *iterations, - int *lookup_path_length, - int *const thread_count, - char **const ip_file); -LOCAL MMDB_s open_or_die(const char *fname); -LOCAL void dump_meta(MMDB_s *mmdb); -LOCAL bool lookup_from_file(MMDB_s *const mmdb, - char const *const ip_file, - bool const dump); -LOCAL int lookup_and_print(MMDB_s *mmdb, const char *ip_address, - const char **lookup_path, - int lookup_path_length, - bool verbose); -LOCAL int benchmark(MMDB_s *mmdb, int iterations); -LOCAL MMDB_lookup_result_s lookup_or_die(MMDB_s *mmdb, const char *ipstr); -LOCAL void random_ipv4(char *ip); - -#ifndef _WIN32 -// These aren't with the automatically generated prototypes as we'd lose the -// enclosing macros. -static bool start_threaded_benchmark( - MMDB_s *const mmdb, - int const thread_count, - int const iterations); -static long double get_time(void); -static void *thread(void *arg); -#endif - -#ifdef _WIN32 -int wmain(int argc, wchar_t **wargv) -{ - // Convert our argument list from UTF-16 to UTF-8. - char **argv = (char **)calloc(argc, sizeof(char *)); - if (!argv) { - fprintf(stderr, "calloc(): %s\n", strerror(errno)); - exit(1); - } - for (int i = 0; i < argc; i++) { - int utf8_width; - char *utf8_string; - utf8_width = WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, NULL, 0, - NULL, NULL); - if (utf8_width < 1) { - fprintf(stderr, "WideCharToMultiByte() failed: %d\n", - GetLastError()); - exit(1); - } - utf8_string = calloc(utf8_width, sizeof(char)); - if (!utf8_string) { - fprintf(stderr, "calloc(): %s\n", strerror(errno)); - exit(1); - } - if (WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, utf8_string, - utf8_width, NULL, NULL) < 1) { - fprintf(stderr, "WideCharToMultiByte() failed: %d\n", - GetLastError()); - exit(1); - } - argv[i] = utf8_string; - } -#else // _WIN32 -int main(int argc, char **argv) -{ -#endif // _WIN32 - char *mmdb_file = NULL; - char *ip_address = NULL; - int verbose = 0; - int iterations = 0; - int lookup_path_length = 0; - int thread_count = 0; - char *ip_file = NULL; - - const char **lookup_path = - get_options(argc, argv, &mmdb_file, &ip_address, &verbose, &iterations, - &lookup_path_length, &thread_count, &ip_file); - - MMDB_s mmdb = open_or_die(mmdb_file); - - if (verbose) { - dump_meta(&mmdb); - } - - // The benchmarking and lookup from file modes are hidden features mainly - // intended for development right now. This means there are several flags - // that exist but are intentionally not mentioned in the usage or man page. - - // The lookup from file mode may be useful to expose publicly in the usage, - // but we should have it respect the lookup_path functionality if we do so. - if (ip_file) { - free((void *)lookup_path); - if (!lookup_from_file(&mmdb, ip_file, verbose == 1)) { - MMDB_close(&mmdb); - return 1; - } - MMDB_close(&mmdb); - return 0; - } - - if (0 == iterations) { - exit(lookup_and_print(&mmdb, ip_address, lookup_path, - lookup_path_length, verbose)); - } - - free((void *)lookup_path); - - srand( (int)time(NULL) ); - -#ifndef _WIN32 - if (thread_count > 0) { - if (!start_threaded_benchmark(&mmdb, thread_count, iterations)) { - MMDB_close(&mmdb); - exit(1); - } - MMDB_close(&mmdb); - exit(0); - } -#endif - - exit(benchmark(&mmdb, iterations)); -} - -LOCAL void usage(char *program, int exit_code, const char *error) -{ - if (NULL != error) { - fprintf(stderr, "\n *ERROR: %s\n", error); - } - - char *usage = "\n" - " %s --file /path/to/file.mmdb --ip 1.2.3.4 [path to lookup]\n" - "\n" - " This application accepts the following options:\n" - "\n" - " --file (-f) The path to the MMDB file. Required.\n" - "\n" - " --ip (-i) The IP address to look up. Required.\n" - "\n" - " --verbose (-v) Turns on verbose output. Specifically, this causes this\n" - " application to output the database metadata.\n" - "\n" - " --version Print the program's version number and exit.\n" - "\n" - " --help (-h -?) Show usage information.\n" - "\n" - " If an IP's data entry resolves to a map or array, you can provide\n" - " a lookup path to only show part of that data.\n" - "\n" - " For example, given a JSON structure like this:\n" - "\n" - " {\n" - " \"names\": {\n" - " \"en\": \"Germany\",\n" - " \"de\": \"Deutschland\"\n" - " },\n" - " \"cities\": [ \"Berlin\", \"Frankfurt\" ]\n" - " }\n" - "\n" - " You could look up just the English name by calling mmdblookup with a lookup path of:\n" - "\n" - " mmdblookup --file ... --ip ... names en\n" - "\n" - " Or you could look up the second city in the list with:\n" - "\n" - " mmdblookup --file ... --ip ... cities 1\n" - "\n" - " Array numbering begins with zero (0).\n" - "\n" - " If you do not provide a path to lookup, all of the information for a given IP\n" - " will be shown.\n" - "\n"; - - fprintf(stdout, usage, program); - exit(exit_code); -} - -LOCAL const char **get_options( - int argc, - char **argv, - char **mmdb_file, - char **ip_address, - int *verbose, - int *iterations, - int *lookup_path_length, - int *const thread_count, - char **const ip_file) -{ - static int help = 0; - static int version = 0; - - while (1) { - static struct option options[] = { - { "file", required_argument, 0, 'f' }, - { "ip", required_argument, 0, 'i' }, - { "verbose", no_argument, 0, 'v' }, - { "version", no_argument, 0, 'n' }, - { "benchmark", required_argument, 0, 'b' }, -#ifndef _WIN32 - { "threads", required_argument, 0, 't' }, -#endif - { "ip-file", required_argument, 0, 'I' }, - { "help", no_argument, 0, 'h' }, - { "?", no_argument, 0, 1 }, - { 0, 0, 0, 0 } - }; - - int opt_index; -#ifdef _WIN32 - char const * const optstring = "f:i:b:I:vnh?"; -#else - char const * const optstring = "f:i:b:t:I:vnh?"; -#endif - int opt_char = getopt_long(argc, argv, optstring, options, - &opt_index); - - if (-1 == opt_char) { - break; - } - - if ('f' == opt_char) { - *mmdb_file = optarg; - } else if ('i' == opt_char) { - *ip_address = optarg; - } else if ('v' == opt_char) { - *verbose = 1; - } else if ('n' == opt_char) { - version = 1; - } else if ('b' == opt_char) { - *iterations = strtol(optarg, NULL, 10); - } else if ('h' == opt_char || '?' == opt_char) { - help = 1; - } else if (opt_char == 't') { - *thread_count = strtol(optarg, NULL, 10); - } else if (opt_char == 'I') { - *ip_file = optarg; - } - } - -#ifdef _WIN32 - char *program = alloca(strlen(argv[0])); - _splitpath(argv[0], NULL, NULL, program, NULL); - _splitpath(argv[0], NULL, NULL, NULL, program + strlen(program)); -#else - char *program = basename(argv[0]); -#endif - - if (help) { - usage(program, 0, NULL); - } - - if (version) { - fprintf(stdout, "\n %s version %s\n\n", program, PACKAGE_VERSION); - exit(0); - } - - if (NULL == *mmdb_file) { - usage(program, 1, "You must provide a filename with --file"); - } - - if (*ip_address == NULL && *iterations == 0 && !*ip_file) { - usage(program, 1, "You must provide an IP address with --ip"); - } - - const char **lookup_path = - calloc((argc - optind) + 1, sizeof(const char *)); - int i; - for (i = 0; i < argc - optind; i++) { - lookup_path[i] = argv[i + optind]; - (*lookup_path_length)++; - } - lookup_path[i] = NULL; - - return lookup_path; -} - -LOCAL MMDB_s open_or_die(const char *fname) -{ - MMDB_s mmdb; - int status = MMDB_open(fname, MMDB_MODE_MMAP, &mmdb); - - if (MMDB_SUCCESS != status) { - fprintf(stderr, "\n Can't open %s - %s\n", fname, - MMDB_strerror(status)); - - if (MMDB_IO_ERROR == status) { - fprintf(stderr, " IO error: %s\n", strerror(errno)); - } - - fprintf(stderr, "\n"); - - exit(2); - } - - return mmdb; -} - -LOCAL void dump_meta(MMDB_s *mmdb) -{ - const char *meta_dump = "\n" - " Database metadata\n" - " Node count: %i\n" - " Record size: %i bits\n" - " IP version: IPv%i\n" - " Binary format: %i.%i\n" - " Build epoch: %llu (%s)\n" - " Type: %s\n" - " Languages: "; - - char date[40]; - const time_t epoch = (const time_t)mmdb->metadata.build_epoch; - strftime(date, 40, "%F %T UTC", gmtime(&epoch)); - - fprintf(stdout, meta_dump, - mmdb->metadata.node_count, - mmdb->metadata.record_size, - mmdb->metadata.ip_version, - mmdb->metadata.binary_format_major_version, - mmdb->metadata.binary_format_minor_version, - mmdb->metadata.build_epoch, - date, - mmdb->metadata.database_type); - - for (size_t i = 0; i < mmdb->metadata.languages.count; i++) { - fprintf(stdout, "%s", mmdb->metadata.languages.names[i]); - if (i < mmdb->metadata.languages.count - 1) { - fprintf(stdout, " "); - } - } - fprintf(stdout, "\n"); - - fprintf(stdout, " Description:\n"); - for (size_t i = 0; i < mmdb->metadata.description.count; i++) { - fprintf(stdout, " %s: %s\n", - mmdb->metadata.description.descriptions[i]->language, - mmdb->metadata.description.descriptions[i]->description); - } - fprintf(stdout, "\n"); -} - -// The input file should have one IP per line. -// -// We look up each IP. -// -// If dump is true, we dump the data for each IP to stderr. This is useful for -// comparison in that you can dump out the data for the IPs before and after -// making changes. It goes to stderr rather than stdout so that the report does -// not get included in what you will compare (since it will almost always be -// different). -// -// In addition to being useful for comparisons, this function provides a way to -// have a more deterministic set of lookups for benchmarking. -LOCAL bool lookup_from_file(MMDB_s *const mmdb, - char const *const ip_file, - bool const dump) -{ - FILE *const fh = fopen(ip_file, "r"); - if (!fh) { - fprintf(stderr, "fopen(): %s: %s\n", ip_file, strerror(errno)); - return false; - } - - clock_t const clock_start = clock(); - char buf[1024] = { 0 }; - // I'd normally use uint64_t, but support for it is optional in C99. - unsigned long long i = 0; - while (1) { - if (fgets(buf, sizeof(buf), fh) == NULL) { - if (!feof(fh)) { - fprintf(stderr, "fgets(): %s\n", strerror(errno)); - fclose(fh); - return false; - } - if (fclose(fh) != 0) { - fprintf(stderr, "fclose(): %s\n", strerror(errno)); - return false; - } - break; - } - - char *ptr = buf; - while (*ptr != '\0') { - if (*ptr == '\n') { - *ptr = '\0'; - break; - } - ptr++; - } - if (strlen(buf) == 0) { - continue; - } - - i++; - - MMDB_lookup_result_s result = lookup_or_die(mmdb, buf); - if (!result.found_entry) { - continue; - } - - MMDB_entry_data_list_s *entry_data_list = NULL; - int const status = MMDB_get_entry_data_list(&result.entry, - &entry_data_list); - if (status != MMDB_SUCCESS) { - fprintf(stderr, "MMDB_get_entry_data_list(): %s\n", - MMDB_strerror(status)); - fclose(fh); - MMDB_free_entry_data_list(entry_data_list); - return false; - } - - if (!entry_data_list) { - fprintf(stderr, "entry_data_list is NULL\n"); - fclose(fh); - return false; - } - - if (dump) { - fprintf(stdout, "%s:\n", buf); - int const status = MMDB_dump_entry_data_list(stderr, - entry_data_list, 0); - if (status != MMDB_SUCCESS) { - fprintf(stderr, "MMDB_dump_entry_data_list(): %s\n", - MMDB_strerror(status)); - fclose(fh); - MMDB_free_entry_data_list(entry_data_list); - return false; - } - } - - MMDB_free_entry_data_list(entry_data_list); - } - - clock_t const clock_diff = clock() - clock_start; - double const seconds = (double)clock_diff / CLOCKS_PER_SEC; - - fprintf( - stdout, - "Looked up %llu addresses in %.2f seconds. %.2f lookups per second.\n", - i, seconds, i / seconds); - - return true; -} - -LOCAL int lookup_and_print(MMDB_s *mmdb, const char *ip_address, - const char **lookup_path, - int lookup_path_length, - bool verbose) -{ - - MMDB_lookup_result_s result = lookup_or_die(mmdb, ip_address); - MMDB_entry_data_list_s *entry_data_list = NULL; - - int exit_code = 0; - - if (verbose) { - fprintf( - stdout, - "\n Record prefix length: %d\n", - result.netmask - ); - } - - if (result.found_entry) { - int status; - if (lookup_path_length) { - MMDB_entry_data_s entry_data; - status = MMDB_aget_value(&result.entry, &entry_data, - lookup_path); - if (MMDB_SUCCESS == status) { - if (entry_data.offset) { - MMDB_entry_s entry = - { .mmdb = mmdb, .offset = entry_data.offset }; - status = MMDB_get_entry_data_list(&entry, - &entry_data_list); - } else { - fprintf( - stdout, - "\n No data was found at the lookup path you provided\n\n"); - } - } - } else { - status = MMDB_get_entry_data_list(&result.entry, - &entry_data_list); - } - - if (MMDB_SUCCESS != status) { - fprintf(stderr, "Got an error looking up the entry data - %s\n", - MMDB_strerror(status)); - exit_code = 5; - goto end; - } - - if (NULL != entry_data_list) { - fprintf(stdout, "\n"); - MMDB_dump_entry_data_list(stdout, entry_data_list, 2); - fprintf(stdout, "\n"); - } - } else { - fprintf(stderr, - "\n Could not find an entry for this IP address (%s)\n\n", - ip_address); - exit_code = 6; - } - - end: - MMDB_free_entry_data_list(entry_data_list); - MMDB_close(mmdb); - free((void *)lookup_path); - - return exit_code; -} - -LOCAL int benchmark(MMDB_s *mmdb, int iterations) -{ - char ip_address[16]; - int exit_code = 0; - - clock_t time = clock(); - - for (int i = 0; i < iterations; i++) { - random_ipv4(ip_address); - - MMDB_lookup_result_s result = lookup_or_die(mmdb, ip_address); - MMDB_entry_data_list_s *entry_data_list = NULL; - - if (result.found_entry) { - - int status = MMDB_get_entry_data_list(&result.entry, - &entry_data_list); - - if (MMDB_SUCCESS != status) { - fprintf(stderr, "Got an error looking up the entry data - %s\n", - MMDB_strerror(status)); - exit_code = 5; - MMDB_free_entry_data_list(entry_data_list); - goto end; - } - } - - MMDB_free_entry_data_list(entry_data_list); - } - - time = clock() - time; - double seconds = ((double)time / CLOCKS_PER_SEC); - fprintf( - stdout, - "\n Looked up %i addresses in %.2f seconds. %.2f lookups per second.\n\n", - iterations, seconds, iterations / seconds); - - end: - MMDB_close(mmdb); - - return exit_code; -} - -LOCAL MMDB_lookup_result_s lookup_or_die(MMDB_s *mmdb, const char *ipstr) -{ - int gai_error, mmdb_error; - MMDB_lookup_result_s result = - MMDB_lookup_string(mmdb, ipstr, &gai_error, &mmdb_error); - - if (0 != gai_error) { - fprintf(stderr, - "\n Error from call to getaddrinfo for %s - %s\n\n", - ipstr, -#ifdef _WIN32 - gai_strerrorA(gai_error) -#else - gai_strerror(gai_error) -#endif - ); - exit(3); - } - - if (MMDB_SUCCESS != mmdb_error) { - fprintf(stderr, "\n Got an error from the maxminddb library: %s\n\n", - MMDB_strerror(mmdb_error)); - exit(4); - } - - return result; -} - -LOCAL void random_ipv4(char *ip) -{ - // rand() is perfectly fine for this use case - // coverity[dont_call] - int ip_int = rand(); - uint8_t *bytes = (uint8_t *)&ip_int; - - snprintf(ip, 16, "%u.%u.%u.%u", - *bytes, *(bytes + 1), *(bytes + 2), *(bytes + 3)); -} - -#ifndef _WIN32 -struct thread_info { - pthread_t id; - int num; - MMDB_s *mmdb; - int iterations; -}; - -static bool start_threaded_benchmark( - MMDB_s *const mmdb, - int const thread_count, - int const iterations) -{ - struct thread_info *const tinfo = calloc(thread_count, - sizeof(struct thread_info)); - if (!tinfo) { - fprintf(stderr, "calloc(): %s\n", strerror(errno)); - return false; - } - - // Using clock() isn't appropriate for multiple threads. It's CPU time, not - // wall time. - long double const start_time = get_time(); - if (start_time == -1) { - free(tinfo); - return false; - } - - for (int i = 0; i < thread_count; i++) { - tinfo[i].num = i; - tinfo[i].mmdb = mmdb; - tinfo[i].iterations = iterations; - - if (pthread_create(&tinfo[i].id, NULL, &thread, &tinfo[i]) != 0) { - fprintf(stderr, "pthread_create() failed\n"); - free(tinfo); - return false; - } - } - - for (int i = 0; i < thread_count; i++) { - if (pthread_join(tinfo[i].id, NULL) != 0) { - fprintf(stderr, "pthread_join() failed\n"); - free(tinfo); - return false; - } - } - - free(tinfo); - - long double const end_time = get_time(); - if (end_time == -1) { - return false; - } - - long double const elapsed = end_time - start_time; - unsigned long long const total_ips = iterations * thread_count; - long double rate = total_ips; - if (elapsed != 0) { - rate = total_ips / elapsed; - } - - fprintf( - stdout, - "Looked up %llu addresses using %d threads in %.2Lf seconds. %.2Lf lookups per second.\n", - total_ips, thread_count, elapsed, rate); - - return true; -} - -static long double get_time(void) -{ - // clock_gettime() is not present on OSX until 10.12. -#ifdef HAVE_CLOCK_GETTIME - struct timespec tp = { - .tv_sec = 0, - .tv_nsec = 0, - }; - clockid_t clk_id = CLOCK_REALTIME; -#ifdef _POSIX_MONOTONIC_CLOCK - clk_id = CLOCK_MONOTONIC; -#endif - if (clock_gettime(clk_id, &tp) != 0) { - fprintf(stderr, "clock_gettime(): %s\n", strerror(errno)); - return -1; - } - return tp.tv_sec + ((float)tp.tv_nsec / 1e9); -#else - time_t t = time(NULL); - if (t == (time_t)-1) { - fprintf(stderr, "time(): %s\n", strerror(errno)); - return -1; - } - return (long double)t; -#endif -} - -static void *thread(void *arg) -{ - const struct thread_info *const tinfo = arg; - if (!tinfo) { - fprintf(stderr, "thread(): %s\n", strerror(EINVAL)); - return NULL; - } - - char ip_address[16] = { 0 }; - - for (int i = 0; i < tinfo->iterations; i++) { - memset(ip_address, 0, 16); - random_ipv4(ip_address); - - MMDB_lookup_result_s result = lookup_or_die(tinfo->mmdb, ip_address); - if (!result.found_entry) { - continue; - } - - MMDB_entry_data_list_s *entry_data_list = NULL; - int const status = MMDB_get_entry_data_list(&result.entry, - &entry_data_list); - if (status != MMDB_SUCCESS) { - fprintf(stderr, "MMDB_get_entry_data_list(): %s\n", - MMDB_strerror(status)); - MMDB_free_entry_data_list(entry_data_list); - return NULL; - } - - if (!entry_data_list) { - fprintf(stderr, "entry_data_list is NULL\n"); - return NULL; - } - - MMDB_free_entry_data_list(entry_data_list); - } - - return NULL; -} -#endif diff --git a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/include/maxminddb.h b/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/include/maxminddb.h deleted file mode 100644 index 13b276f14..000000000 --- a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/include/maxminddb.h +++ /dev/null @@ -1,255 +0,0 @@ -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef MAXMINDDB_H -#define MAXMINDDB_H - -/* Request POSIX.1-2008. However, we want to remain compatible with - * POSIX.1-2001 (since we have been historically and see no reason to drop - * compatibility). By requesting POSIX.1-2008, we can conditionally use - * features provided by that standard if the implementation provides it. We can - * check for what the implementation provides by checking the _POSIX_VERSION - * macro after including unistd.h. If a feature is in POSIX.1-2008 but not - * POSIX.1-2001, check that macro before using the feature (or check for the - * feature directly if possible). */ -#ifndef _POSIX_C_SOURCE -#define _POSIX_C_SOURCE 200809L -#endif - -#include "maxminddb_config.h" -#include <stdarg.h> -#include <stdbool.h> -#include <stdint.h> -#include <stdio.h> -#include <sys/types.h> - -#ifdef _WIN32 -#include <winsock2.h> -#include <ws2tcpip.h> -/* libmaxminddb package version from configure */ -#define PACKAGE_VERSION "1.4.3" - -typedef ADDRESS_FAMILY sa_family_t; - -#if defined(_MSC_VER) -/* MSVC doesn't define signed size_t, copy it from configure */ -#define ssize_t SSIZE_T - -/* MSVC doesn't support restricted pointers */ -#define restrict -#endif -#else -#include <netdb.h> -#include <netinet/in.h> -#include <sys/socket.h> -#endif - -#define MMDB_DATA_TYPE_EXTENDED (0) -#define MMDB_DATA_TYPE_POINTER (1) -#define MMDB_DATA_TYPE_UTF8_STRING (2) -#define MMDB_DATA_TYPE_DOUBLE (3) -#define MMDB_DATA_TYPE_BYTES (4) -#define MMDB_DATA_TYPE_UINT16 (5) -#define MMDB_DATA_TYPE_UINT32 (6) -#define MMDB_DATA_TYPE_MAP (7) -#define MMDB_DATA_TYPE_INT32 (8) -#define MMDB_DATA_TYPE_UINT64 (9) -#define MMDB_DATA_TYPE_UINT128 (10) -#define MMDB_DATA_TYPE_ARRAY (11) -#define MMDB_DATA_TYPE_CONTAINER (12) -#define MMDB_DATA_TYPE_END_MARKER (13) -#define MMDB_DATA_TYPE_BOOLEAN (14) -#define MMDB_DATA_TYPE_FLOAT (15) - -#define MMDB_RECORD_TYPE_SEARCH_NODE (0) -#define MMDB_RECORD_TYPE_EMPTY (1) -#define MMDB_RECORD_TYPE_DATA (2) -#define MMDB_RECORD_TYPE_INVALID (3) - -/* flags for open */ -#define MMDB_MODE_MMAP (1) -#define MMDB_MODE_MASK (7) - -/* error codes */ -#define MMDB_SUCCESS (0) -#define MMDB_FILE_OPEN_ERROR (1) -#define MMDB_CORRUPT_SEARCH_TREE_ERROR (2) -#define MMDB_INVALID_METADATA_ERROR (3) -#define MMDB_IO_ERROR (4) -#define MMDB_OUT_OF_MEMORY_ERROR (5) -#define MMDB_UNKNOWN_DATABASE_FORMAT_ERROR (6) -#define MMDB_INVALID_DATA_ERROR (7) -#define MMDB_INVALID_LOOKUP_PATH_ERROR (8) -#define MMDB_LOOKUP_PATH_DOES_NOT_MATCH_DATA_ERROR (9) -#define MMDB_INVALID_NODE_NUMBER_ERROR (10) -#define MMDB_IPV6_LOOKUP_IN_IPV4_DATABASE_ERROR (11) - -#if !(MMDB_UINT128_IS_BYTE_ARRAY) -#if MMDB_UINT128_USING_MODE -typedef unsigned int mmdb_uint128_t __attribute__ ((__mode__(TI))); -#else -typedef unsigned __int128 mmdb_uint128_t; -#endif -#endif - -/* This is a pointer into the data section for a given IP address lookup */ -typedef struct MMDB_entry_s { - const struct MMDB_s *mmdb; - uint32_t offset; -} MMDB_entry_s; - -typedef struct MMDB_lookup_result_s { - bool found_entry; - MMDB_entry_s entry; - uint16_t netmask; -} MMDB_lookup_result_s; - -typedef struct MMDB_entry_data_s { - bool has_data; - union { - uint32_t pointer; - const char *utf8_string; - double double_value; - const uint8_t *bytes; - uint16_t uint16; - uint32_t uint32; - int32_t int32; - uint64_t uint64; -#if MMDB_UINT128_IS_BYTE_ARRAY - uint8_t uint128[16]; -#else - mmdb_uint128_t uint128; -#endif - bool boolean; - float float_value; - }; - /* This is a 0 if a given entry cannot be found. This can only happen - * when a call to MMDB_(v)get_value() asks for hash keys or array - * indices that don't exist. */ - uint32_t offset; - /* This is the next entry in the data section, but it's really only - * relevant for entries that part of a larger map or array - * struct. There's no good reason for an end user to look at this - * directly. */ - uint32_t offset_to_next; - /* This is only valid for strings, utf8_strings or binary data */ - uint32_t data_size; - /* This is an MMDB_DATA_TYPE_* constant */ - uint32_t type; -} MMDB_entry_data_s; - -/* This is the return type when someone asks for all the entry data in a map or array */ -typedef struct MMDB_entry_data_list_s { - MMDB_entry_data_s entry_data; - struct MMDB_entry_data_list_s *next; - void *pool; -} MMDB_entry_data_list_s; - -typedef struct MMDB_description_s { - const char *language; - const char *description; -} MMDB_description_s; - -/* WARNING: do not add new fields to this struct without bumping the SONAME. - * The struct is allocated by the users of this library and increasing the - * size will cause existing users to allocate too little space when the shared - * library is upgraded */ -typedef struct MMDB_metadata_s { - uint32_t node_count; - uint16_t record_size; - uint16_t ip_version; - const char *database_type; - struct { - size_t count; - const char **names; - } languages; - uint16_t binary_format_major_version; - uint16_t binary_format_minor_version; - uint64_t build_epoch; - struct { - size_t count; - MMDB_description_s **descriptions; - } description; - /* See above warning before adding fields */ -} MMDB_metadata_s; - -/* WARNING: do not add new fields to this struct without bumping the SONAME. - * The struct is allocated by the users of this library and increasing the - * size will cause existing users to allocate too little space when the shared - * library is upgraded */ -typedef struct MMDB_ipv4_start_node_s { - uint16_t netmask; - uint32_t node_value; - /* See above warning before adding fields */ -} MMDB_ipv4_start_node_s; - -/* WARNING: do not add new fields to this struct without bumping the SONAME. - * The struct is allocated by the users of this library and increasing the - * size will cause existing users to allocate too little space when the shared - * library is upgraded */ -typedef struct MMDB_s { - uint32_t flags; - const char *filename; - ssize_t file_size; - const uint8_t *file_content; - const uint8_t *data_section; - uint32_t data_section_size; - const uint8_t *metadata_section; - uint32_t metadata_section_size; - uint16_t full_record_byte_size; - uint16_t depth; - MMDB_ipv4_start_node_s ipv4_start_node; - MMDB_metadata_s metadata; - /* See above warning before adding fields */ -} MMDB_s; - -typedef struct MMDB_search_node_s { - uint64_t left_record; - uint64_t right_record; - uint8_t left_record_type; - uint8_t right_record_type; - MMDB_entry_s left_record_entry; - MMDB_entry_s right_record_entry; -} MMDB_search_node_s; - -extern int MMDB_open(const char *const filename, uint32_t flags, - MMDB_s *const mmdb); -extern MMDB_lookup_result_s MMDB_lookup_string(const MMDB_s *const mmdb, - const char *const ipstr, - int *const gai_error, - int *const mmdb_error); -extern MMDB_lookup_result_s MMDB_lookup_sockaddr( - const MMDB_s *const mmdb, - const struct sockaddr *const sockaddr, - int *const mmdb_error); -extern int MMDB_read_node(const MMDB_s *const mmdb, - uint32_t node_number, - MMDB_search_node_s *const node); -extern int MMDB_get_value(MMDB_entry_s *const start, - MMDB_entry_data_s *const entry_data, - ...); -extern int MMDB_vget_value(MMDB_entry_s *const start, - MMDB_entry_data_s *const entry_data, - va_list va_path); -extern int MMDB_aget_value(MMDB_entry_s *const start, - MMDB_entry_data_s *const entry_data, - const char *const *const path); -extern int MMDB_get_metadata_as_entry_data_list( - const MMDB_s *const mmdb, MMDB_entry_data_list_s **const entry_data_list); -extern int MMDB_get_entry_data_list( - MMDB_entry_s *start, MMDB_entry_data_list_s **const entry_data_list); -extern void MMDB_free_entry_data_list( - MMDB_entry_data_list_s *const entry_data_list); -extern void MMDB_close(MMDB_s *const mmdb); -extern const char *MMDB_lib_version(void); -extern int MMDB_dump_entry_data_list(FILE *const stream, - MMDB_entry_data_list_s *const entry_data_list, - int indent); -extern const char *MMDB_strerror(int error_code); - -#endif /* MAXMINDDB_H */ - -#ifdef __cplusplus -} -#endif diff --git a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/include/maxminddb_config.h.cmake.in b/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/include/maxminddb_config.h.cmake.in deleted file mode 100644 index 8b1977f86..000000000 --- a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/include/maxminddb_config.h.cmake.in +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef MAXMINDDB_CONFIG_H -#define MAXMINDDB_CONFIG_H - -#ifndef MMDB_UINT128_USING_MODE -/* Define as 1 if we use unsigned int __atribute__ ((__mode__(TI))) for uint128 values */ -#cmakedefine MMDB_UINT128_USING_MODE @MMDB_UINT128_USING_MODE@ -#endif - -#ifndef MMDB_UINT128_IS_BYTE_ARRAY -/* Define as 1 if we don't have an unsigned __int128 type */ -#cmakedefine MMDB_UINT128_IS_BYTE_ARRAY @MMDB_UINT128_IS_BYTE_ARRAY@ -#endif - -#endif /* MAXMINDDB_CONFIG_H */ diff --git a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/include/maxminddb_config.h.in b/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/include/maxminddb_config.h.in deleted file mode 100644 index 314d559d3..000000000 --- a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/include/maxminddb_config.h.in +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef MAXMINDDB_CONFIG_H -#define MAXMINDDB_CONFIG_H - -#ifndef MMDB_UINT128_USING_MODE -/* Define as 1 if we use unsigned int __atribute__ ((__mode__(TI))) for uint128 values */ -#define MMDB_UINT128_USING_MODE 0 -#endif - -#ifndef MMDB_UINT128_IS_BYTE_ARRAY -/* Define as 1 if we don't have an unsigned __int128 type */ -#undef MMDB_UINT128_IS_BYTE_ARRAY -#endif - -#endif /* MAXMINDDB_CONFIG_H */ diff --git a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/Makefile.am b/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/Makefile.am deleted file mode 100644 index 6d57acaae..000000000 --- a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/Makefile.am +++ /dev/null @@ -1,25 +0,0 @@ -include $(top_srcdir)/common.mk - -lib_LTLIBRARIES = libmaxminddb.la - -libmaxminddb_la_SOURCES = maxminddb.c maxminddb-compat-util.h \ - data-pool.c data-pool.h -libmaxminddb_la_LDFLAGS = -version-info 0:7:0 -export-symbols-regex '^MMDB_.*' -include_HEADERS = $(top_srcdir)/include/maxminddb.h - -pkgconfig_DATA = libmaxminddb.pc - -TESTS = test-data-pool - -check_PROGRAMS = test-data-pool - -test_data_pool_SOURCES = data-pool.c data-pool.h -test_data_pool_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/t -DTEST_DATA_POOL -test_data_pool_LDADD = $(top_srcdir)/t/libmmdbtest.la \ - $(top_srcdir)/t/libtap/libtap.a - -$(top_srcdir)/t/libmmdbtest.la: - $(MAKE) -C $(top_srcdir)/t libmmdbtest.la - -$(top_srcdir)/t/libtap/libtap.a: - $(MAKE) -C $(top_srcdir)/t/libtap libtap.a diff --git a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/data-pool.c b/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/data-pool.c deleted file mode 100644 index 48521b64d..000000000 --- a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/data-pool.c +++ /dev/null @@ -1,180 +0,0 @@ -#include "data-pool.h" -#include "maxminddb.h" - -#include <stdbool.h> -#include <stddef.h> -#include <stdlib.h> - -static bool can_multiply(size_t const, size_t const, size_t const); - -// Allocate an MMDB_data_pool_s. It initially has space for size -// MMDB_entry_data_list_s structs. -MMDB_data_pool_s *data_pool_new(size_t const size) -{ - MMDB_data_pool_s *const pool = calloc(1, sizeof(MMDB_data_pool_s)); - if (!pool) { - return NULL; - } - - if (size == 0 || - !can_multiply(SIZE_MAX, size, sizeof(MMDB_entry_data_list_s))) { - data_pool_destroy(pool); - return NULL; - } - pool->size = size; - pool->blocks[0] = calloc(pool->size, sizeof(MMDB_entry_data_list_s)); - if (!pool->blocks[0]) { - data_pool_destroy(pool); - return NULL; - } - pool->blocks[0]->pool = pool; - - pool->sizes[0] = size; - - pool->block = pool->blocks[0]; - - return pool; -} - -// Determine if we can multiply m*n. We can do this if the result will be below -// the given max. max will typically be SIZE_MAX. -// -// We want to know if we'll wrap around. -static bool can_multiply(size_t const max, size_t const m, size_t const n) -{ - if (m == 0) { - return false; - } - - return n <= max / m; -} - -// Clean up the data pool. -void data_pool_destroy(MMDB_data_pool_s *const pool) -{ - if (!pool) { - return; - } - - for (size_t i = 0; i <= pool->index; i++) { - free(pool->blocks[i]); - } - - free(pool); -} - -// Claim a new struct from the pool. Doing this may cause the pool's size to -// grow. -MMDB_entry_data_list_s *data_pool_alloc(MMDB_data_pool_s *const pool) -{ - if (!pool) { - return NULL; - } - - if (pool->used < pool->size) { - MMDB_entry_data_list_s *const element = pool->block + pool->used; - pool->used++; - return element; - } - - // Take it from a new block of memory. - - size_t const new_index = pool->index + 1; - if (new_index == DATA_POOL_NUM_BLOCKS) { - // See the comment about not growing this on DATA_POOL_NUM_BLOCKS. - return NULL; - } - - if (!can_multiply(SIZE_MAX, pool->size, 2)) { - return NULL; - } - size_t const new_size = pool->size * 2; - - if (!can_multiply(SIZE_MAX, new_size, sizeof(MMDB_entry_data_list_s))) { - return NULL; - } - pool->blocks[new_index] = calloc(new_size, sizeof(MMDB_entry_data_list_s)); - if (!pool->blocks[new_index]) { - return NULL; - } - - // We don't need to set this, but it's useful for introspection in tests. - pool->blocks[new_index]->pool = pool; - - pool->index = new_index; - pool->block = pool->blocks[pool->index]; - - pool->size = new_size; - pool->sizes[pool->index] = pool->size; - - MMDB_entry_data_list_s *const element = pool->block; - pool->used = 1; - return element; -} - -// Turn the structs in the array-like pool into a linked list. -// -// Before calling this function, the list isn't linked up. -MMDB_entry_data_list_s *data_pool_to_list(MMDB_data_pool_s *const pool) -{ - if (!pool) { - return NULL; - } - - if (pool->index == 0 && pool->used == 0) { - return NULL; - } - - for (size_t i = 0; i <= pool->index; i++) { - MMDB_entry_data_list_s *const block = pool->blocks[i]; - - size_t size = pool->sizes[i]; - if (i == pool->index) { - size = pool->used; - } - - for (size_t j = 0; j < size - 1; j++) { - MMDB_entry_data_list_s *const cur = block + j; - cur->next = block + j + 1; - } - - if (i < pool->index) { - MMDB_entry_data_list_s *const last = block + size - 1; - last->next = pool->blocks[i + 1]; - } - } - - return pool->blocks[0]; -} - -#ifdef TEST_DATA_POOL - -#include <libtap/tap.h> -#include <maxminddb_test_helper.h> - -static void test_can_multiply(void); - -int main(void) -{ - plan(NO_PLAN); - test_can_multiply(); - done_testing(); -} - -static void test_can_multiply(void) -{ - { - ok(can_multiply(SIZE_MAX, 1, SIZE_MAX), "1*SIZE_MAX is ok"); - } - - { - ok(!can_multiply(SIZE_MAX, 2, SIZE_MAX), "2*SIZE_MAX is not ok"); - } - - { - ok(can_multiply(SIZE_MAX, 10240, sizeof(MMDB_entry_data_list_s)), - "1024 entry_data_list_s's are okay"); - } -} - -#endif diff --git a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/data-pool.h b/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/data-pool.h deleted file mode 100644 index 25d09923e..000000000 --- a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/data-pool.h +++ /dev/null @@ -1,52 +0,0 @@ -#ifndef DATA_POOL_H -#define DATA_POOL_H - -#include "maxminddb.h" - -#include <stdbool.h> -#include <stddef.h> - -// This should be large enough that we never need to grow the array of pointers -// to blocks. 32 is enough. Even starting out of with size 1 (1 struct), the -// 32nd element alone will provide 2**32 structs as we exponentially increase -// the number in each block. Being confident that we do not have to grow the -// array lets us avoid writing code to do that. That code would be risky as it -// would rarely be hit and likely not be well tested. -#define DATA_POOL_NUM_BLOCKS 32 - -// A pool of memory for MMDB_entry_data_list_s structs. This is so we can -// allocate multiple up front rather than one at a time for performance -// reasons. -// -// The order you add elements to it (by calling data_pool_alloc()) ends up as -// the order of the list. -// -// The memory only grows. There is no support for releasing an element you take -// back to the pool. -typedef struct MMDB_data_pool_s { - // Index of the current block we're allocating out of. - size_t index; - - // The size of the current block, counting by structs. - size_t size; - - // How many used in the current block, counting by structs. - size_t used; - - // The current block we're allocating out of. - MMDB_entry_data_list_s *block; - - // The size of each block. - size_t sizes[DATA_POOL_NUM_BLOCKS]; - - // An array of pointers to blocks of memory holding space for list - // elements. - MMDB_entry_data_list_s *blocks[DATA_POOL_NUM_BLOCKS]; -} MMDB_data_pool_s; - -MMDB_data_pool_s *data_pool_new(size_t const); -void data_pool_destroy(MMDB_data_pool_s *const); -MMDB_entry_data_list_s *data_pool_alloc(MMDB_data_pool_s *const); -MMDB_entry_data_list_s *data_pool_to_list(MMDB_data_pool_s *const); - -#endif diff --git a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/libmaxminddb.pc.in b/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/libmaxminddb.pc.in deleted file mode 100644 index 00ced3ba9..000000000 --- a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/libmaxminddb.pc.in +++ /dev/null @@ -1,11 +0,0 @@ -prefix=@prefix@ -exec_prefix=@prefix@ -libdir=@libdir@ -includedir=@includedir@ - -Name: libmaxminddb -Description: C library for the MaxMind DB file format -URL: http://maxmind.github.io/libmaxminddb/ -Version: @PACKAGE_VERSION@ -Libs: -L${libdir} -lmaxminddb -Cflags: -I${includedir} diff --git a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/maxminddb-compat-util.h b/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/maxminddb-compat-util.h deleted file mode 100644 index e3f0320f2..000000000 --- a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/maxminddb-compat-util.h +++ /dev/null @@ -1,167 +0,0 @@ -#include <stdlib.h> -#include <string.h> - -/* *INDENT-OFF* */ - -/* The memmem, strdup, and strndup functions were all copied from the - * FreeBSD source, along with the relevant copyright notice. - * - * It'd be nicer to simply use the functions available on the system if they - * exist, but there doesn't seem to be a good way to detect them without also - * defining things like _GNU_SOURCE, which we want to avoid, because then we - * end up _accidentally_ using GNU features without noticing, which then - * breaks on systems like OSX. - * - * C is fun! */ - -/* Applies to memmem implementation */ -/*- - * Copyright (c) 2005 Pascal Gloor <pascal.gloor@spale.com> - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. The name of the author may not be used to endorse or promote - * products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ -static void * -mmdb_memmem(const void *l, size_t l_len, const void *s, size_t s_len) -{ - register char *cur, *last; - const char *cl = (const char *)l; - const char *cs = (const char *)s; - - /* we need something to compare */ - if (l_len == 0 || s_len == 0) - return NULL; - - /* "s" must be smaller or equal to "l" */ - if (l_len < s_len) - return NULL; - - /* special case where s_len == 1 */ - if (s_len == 1) - return memchr(l, (int)*cs, l_len); - - /* the last position where its possible to find "s" in "l" */ - last = (char *)cl + l_len - s_len; - - for (cur = (char *)cl; cur <= last; cur++) - if (cur[0] == cs[0] && memcmp(cur, cs, s_len) == 0) - return cur; - - return NULL; -} - -/* Applies to strnlen implementation */ -/*- - * Copyright (c) 2009 David Schultz <das@FreeBSD.org> - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ -static size_t -mmdb_strnlen(const char *s, size_t maxlen) -{ - size_t len; - - for (len = 0; len < maxlen; len++, s++) { - if (!*s) - break; - } - return (len); -} - -/* Applies to strdup and strndup implementation */ -/* - * Copyright (c) 1988, 1993 - * The Regents of the University of California. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the University nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ -static char * -mmdb_strdup(const char *str) -{ - size_t len; - char *copy; - - len = strlen(str) + 1; - if ((copy = malloc(len)) == NULL) - return (NULL); - memcpy(copy, str, len); - return (copy); -} - -static char * -mmdb_strndup(const char *str, size_t n) -{ - size_t len; - char *copy; - - len = mmdb_strnlen(str, n); - if ((copy = malloc(len + 1)) == NULL) - return (NULL); - memcpy(copy, str, len); - copy[len] = '\0'; - return (copy); -} -/* *INDENT-ON* */ diff --git a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/maxminddb.c b/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/maxminddb.c deleted file mode 100644 index 427f48afd..000000000 --- a/src/fluent-bit/plugins/filter_geoip2/libmaxminddb/src/maxminddb.c +++ /dev/null @@ -1,2157 +0,0 @@ -#if HAVE_CONFIG_H -#include <config.h> -#endif -#include "data-pool.h" -#include "maxminddb.h" -#include "maxminddb-compat-util.h" -#include <assert.h> -#include <errno.h> -#include <fcntl.h> -#include <inttypes.h> -#include <stdint.h> -#include <stdlib.h> -#include <string.h> -#include <sys/stat.h> - -#ifdef _WIN32 -#ifndef UNICODE -#define UNICODE -#endif -#include <windows.h> -#include <ws2ipdef.h> -#else -#include <arpa/inet.h> -#include <sys/mman.h> -#include <unistd.h> -#endif - -#define MMDB_DATA_SECTION_SEPARATOR (16) -#define MAXIMUM_DATA_STRUCTURE_DEPTH (512) - -#ifdef MMDB_DEBUG -#define LOCAL -#define DEBUG_MSG(msg) fprintf(stderr, msg "\n") -#define DEBUG_MSGF(fmt, ...) fprintf(stderr, fmt "\n", __VA_ARGS__) -#define DEBUG_BINARY(fmt, byte) \ - do { \ - char *binary = byte_to_binary(byte); \ - if (NULL == binary) { \ - fprintf(stderr, "Calloc failed in DEBUG_BINARY\n"); \ - abort(); \ - } \ - fprintf(stderr, fmt "\n", binary); \ - free(binary); \ - } while (0) -#define DEBUG_NL fprintf(stderr, "\n") -#else -#define LOCAL static -#define DEBUG_MSG(...) -#define DEBUG_MSGF(...) -#define DEBUG_BINARY(...) -#define DEBUG_NL -#endif - -#ifdef MMDB_DEBUG -char *byte_to_binary(uint8_t byte) -{ - char *bits = calloc(9, sizeof(char)); - if (NULL == bits) { - return bits; - } - - for (uint8_t i = 0; i < 8; i++) { - bits[i] = byte & (128 >> i) ? '1' : '0'; - } - bits[8] = '\0'; - - return bits; -} - -char *type_num_to_name(uint8_t num) -{ - switch (num) { - case 0: - return "extended"; - case 1: - return "pointer"; - case 2: - return "utf8_string"; - case 3: - return "double"; - case 4: - return "bytes"; - case 5: - return "uint16"; - case 6: - return "uint32"; - case 7: - return "map"; - case 8: - return "int32"; - case 9: - return "uint64"; - case 10: - return "uint128"; - case 11: - return "array"; - case 12: - return "container"; - case 13: - return "end_marker"; - case 14: - return "boolean"; - case 15: - return "float"; - default: - return "unknown type"; - } -} -#endif - -/* None of the values we check on the lhs are bigger than uint32_t, so on - * platforms where SIZE_MAX is a 64-bit integer, this would be a no-op, and it - * makes the compiler complain if we do the check anyway. */ -#if SIZE_MAX == UINT32_MAX -#define MAYBE_CHECK_SIZE_OVERFLOW(lhs, rhs, error) \ - if ((lhs) > (rhs)) { \ - return error; \ - } -#else -#define MAYBE_CHECK_SIZE_OVERFLOW(...) -#endif - -typedef struct record_info_s { - uint16_t record_length; - uint32_t (*left_record_getter)(const uint8_t *); - uint32_t (*right_record_getter)(const uint8_t *); - uint8_t right_record_offset; -} record_info_s; - -#define METADATA_MARKER "\xab\xcd\xefMaxMind.com" -/* This is 128kb */ -#define METADATA_BLOCK_MAX_SIZE 131072 - -// 64 leads us to allocating 4 KiB on a 64bit system. -#define MMDB_POOL_INIT_SIZE 64 - -LOCAL int map_file(MMDB_s *const mmdb); -LOCAL const uint8_t *find_metadata(const uint8_t *file_content, - ssize_t file_size, uint32_t *metadata_size); -LOCAL int read_metadata(MMDB_s *mmdb); -LOCAL MMDB_s make_fake_metadata_db(const MMDB_s *const mmdb); -LOCAL int value_for_key_as_uint16(MMDB_entry_s *start, char *key, - uint16_t *value); -LOCAL int value_for_key_as_uint32(MMDB_entry_s *start, char *key, - uint32_t *value); -LOCAL int value_for_key_as_uint64(MMDB_entry_s *start, char *key, - uint64_t *value); -LOCAL int value_for_key_as_string(MMDB_entry_s *start, char *key, - char const **value); -LOCAL int populate_languages_metadata(MMDB_s *mmdb, MMDB_s *metadata_db, - MMDB_entry_s *metadata_start); -LOCAL int populate_description_metadata(MMDB_s *mmdb, MMDB_s *metadata_db, - MMDB_entry_s *metadata_start); -LOCAL int resolve_any_address(const char *ipstr, struct addrinfo **addresses); -LOCAL int find_address_in_search_tree(const MMDB_s *const mmdb, - uint8_t *address, - sa_family_t address_family, - MMDB_lookup_result_s *result); -LOCAL record_info_s record_info_for_database(const MMDB_s *const mmdb); -LOCAL int find_ipv4_start_node(MMDB_s *const mmdb); -LOCAL uint8_t record_type(const MMDB_s *const mmdb, uint64_t record); -LOCAL uint32_t get_left_28_bit_record(const uint8_t *record); -LOCAL uint32_t get_right_28_bit_record(const uint8_t *record); -LOCAL uint32_t data_section_offset_for_record(const MMDB_s *const mmdb, - uint64_t record); -LOCAL int path_length(va_list va_path); -LOCAL int lookup_path_in_array(const char *path_elem, const MMDB_s *const mmdb, - MMDB_entry_data_s *entry_data); -LOCAL int lookup_path_in_map(const char *path_elem, const MMDB_s *const mmdb, - MMDB_entry_data_s *entry_data); -LOCAL int skip_map_or_array(const MMDB_s *const mmdb, - MMDB_entry_data_s *entry_data); -LOCAL int decode_one_follow(const MMDB_s *const mmdb, uint32_t offset, - MMDB_entry_data_s *entry_data); -LOCAL int decode_one(const MMDB_s *const mmdb, uint32_t offset, - MMDB_entry_data_s *entry_data); -LOCAL int get_ext_type(int raw_ext_type); -LOCAL uint32_t get_ptr_from(uint8_t ctrl, uint8_t const *const ptr, - int ptr_size); -LOCAL int get_entry_data_list(const MMDB_s *const mmdb, - uint32_t offset, - MMDB_entry_data_list_s *const entry_data_list, - MMDB_data_pool_s *const pool, - int depth); -LOCAL float get_ieee754_float(const uint8_t *restrict p); -LOCAL double get_ieee754_double(const uint8_t *restrict p); -LOCAL uint32_t get_uint32(const uint8_t *p); -LOCAL uint32_t get_uint24(const uint8_t *p); -LOCAL uint32_t get_uint16(const uint8_t *p); -LOCAL uint64_t get_uintX(const uint8_t *p, int length); -LOCAL int32_t get_sintX(const uint8_t *p, int length); -LOCAL void free_mmdb_struct(MMDB_s *const mmdb); -LOCAL void free_languages_metadata(MMDB_s *mmdb); -LOCAL void free_descriptions_metadata(MMDB_s *mmdb); -LOCAL MMDB_entry_data_list_s *dump_entry_data_list( - FILE *stream, MMDB_entry_data_list_s *entry_data_list, int indent, - int *status); -LOCAL void print_indentation(FILE *stream, int i); -LOCAL char *bytes_to_hex(uint8_t *bytes, uint32_t size); - -#define CHECKED_DECODE_ONE(mmdb, offset, entry_data) \ - do { \ - int status = decode_one(mmdb, offset, entry_data); \ - if (MMDB_SUCCESS != status) { \ - DEBUG_MSGF("CHECKED_DECODE_ONE failed." \ - " status = %d (%s)", status, MMDB_strerror(status)); \ - return status; \ - } \ - } while (0) - -#define CHECKED_DECODE_ONE_FOLLOW(mmdb, offset, entry_data) \ - do { \ - int status = decode_one_follow(mmdb, offset, entry_data); \ - if (MMDB_SUCCESS != status) { \ - DEBUG_MSGF("CHECKED_DECODE_ONE_FOLLOW failed." \ - " status = %d (%s)", status, MMDB_strerror(status)); \ - return status; \ - } \ - } while (0) - -#define FREE_AND_SET_NULL(p) { free((void *)(p)); (p) = NULL; } - -int MMDB_open(const char *const filename, uint32_t flags, MMDB_s *const mmdb) -{ - int status = MMDB_SUCCESS; - - mmdb->file_content = NULL; - mmdb->data_section = NULL; - mmdb->metadata.database_type = NULL; - mmdb->metadata.languages.count = 0; - mmdb->metadata.languages.names = NULL; - mmdb->metadata.description.count = 0; - - mmdb->filename = mmdb_strdup(filename); - if (NULL == mmdb->filename) { - status = MMDB_OUT_OF_MEMORY_ERROR; - goto cleanup; - } - - if ((flags & MMDB_MODE_MASK) == 0) { - flags |= MMDB_MODE_MMAP; - } - mmdb->flags = flags; - - if (MMDB_SUCCESS != (status = map_file(mmdb))) { - goto cleanup; - } - -#ifdef _WIN32 - WSADATA wsa; - WSAStartup(MAKEWORD(2, 2), &wsa); -#endif - - uint32_t metadata_size = 0; - const uint8_t *metadata = find_metadata(mmdb->file_content, mmdb->file_size, - &metadata_size); - if (NULL == metadata) { - status = MMDB_INVALID_METADATA_ERROR; - goto cleanup; - } - - mmdb->metadata_section = metadata; - mmdb->metadata_section_size = metadata_size; - - status = read_metadata(mmdb); - if (MMDB_SUCCESS != status) { - goto cleanup; - } - - if (mmdb->metadata.binary_format_major_version != 2) { - status = MMDB_UNKNOWN_DATABASE_FORMAT_ERROR; - goto cleanup; - } - - uint32_t search_tree_size = mmdb->metadata.node_count * - mmdb->full_record_byte_size; - - mmdb->data_section = mmdb->file_content + search_tree_size - + MMDB_DATA_SECTION_SEPARATOR; - if (search_tree_size + MMDB_DATA_SECTION_SEPARATOR > - (uint32_t)mmdb->file_size) { - status = MMDB_INVALID_METADATA_ERROR; - goto cleanup; - } - mmdb->data_section_size = (uint32_t)mmdb->file_size - search_tree_size - - MMDB_DATA_SECTION_SEPARATOR; - - // Although it is likely not possible to construct a database with valid - // valid metadata, as parsed above, and a data_section_size less than 3, - // we do this check as later we assume it is at least three when doing - // bound checks. - if (mmdb->data_section_size < 3) { - status = MMDB_INVALID_DATA_ERROR; - goto cleanup; - } - - mmdb->metadata_section = metadata; - mmdb->ipv4_start_node.node_value = 0; - mmdb->ipv4_start_node.netmask = 0; - - // We do this immediately as otherwise there is a race to set - // ipv4_start_node.node_value and ipv4_start_node.netmask. - if (mmdb->metadata.ip_version == 6) { - status = find_ipv4_start_node(mmdb); - if (status != MMDB_SUCCESS) { - goto cleanup; - } - } - - cleanup: - if (MMDB_SUCCESS != status) { - int saved_errno = errno; - free_mmdb_struct(mmdb); - errno = saved_errno; - } - return status; -} - -#ifdef _WIN32 - -LOCAL LPWSTR utf8_to_utf16(const char *utf8_str) -{ - int wide_chars = MultiByteToWideChar(CP_UTF8, 0, utf8_str, -1, NULL, 0); - wchar_t *utf16_str = (wchar_t *)calloc(wide_chars, sizeof(wchar_t)); - - if (MultiByteToWideChar(CP_UTF8, 0, utf8_str, -1, utf16_str, - wide_chars) < 1) { - free(utf16_str); - return NULL; - } - - return utf16_str; -} - -LOCAL int map_file(MMDB_s *const mmdb) -{ - DWORD size; - int status = MMDB_SUCCESS; - HANDLE mmh = NULL; - HANDLE fd = INVALID_HANDLE_VALUE; - LPWSTR utf16_filename = utf8_to_utf16(mmdb->filename); - if (!utf16_filename) { - status = MMDB_FILE_OPEN_ERROR; - goto cleanup; - } - fd = CreateFileW(utf16_filename, GENERIC_READ, FILE_SHARE_READ, NULL, - OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); - if (fd == INVALID_HANDLE_VALUE) { - status = MMDB_FILE_OPEN_ERROR; - goto cleanup; - } - size = GetFileSize(fd, NULL); - if (size == INVALID_FILE_SIZE) { - status = MMDB_FILE_OPEN_ERROR; - goto cleanup; - } - mmh = CreateFileMapping(fd, NULL, PAGE_READONLY, 0, size, NULL); - /* Microsoft documentation for CreateFileMapping indicates this returns - NULL not INVALID_HANDLE_VALUE on error */ - if (NULL == mmh) { - status = MMDB_IO_ERROR; - goto cleanup; - } - uint8_t *file_content = - (uint8_t *)MapViewOfFile(mmh, FILE_MAP_READ, 0, 0, 0); - if (file_content == NULL) { - status = MMDB_IO_ERROR; - goto cleanup; - } - - mmdb->file_size = size; - mmdb->file_content = file_content; - - cleanup:; - int saved_errno = errno; - if (INVALID_HANDLE_VALUE != fd) { - CloseHandle(fd); - } - if (NULL != mmh) { - CloseHandle(mmh); - } - errno = saved_errno; - free(utf16_filename); - - return status; -} - -#else // _WIN32 - -LOCAL int map_file(MMDB_s *const mmdb) -{ - ssize_t size; - int status = MMDB_SUCCESS; - - int flags = O_RDONLY; -#ifdef O_CLOEXEC - flags |= O_CLOEXEC; -#endif - int fd = open(mmdb->filename, flags); - struct stat s; - if (fd < 0 || fstat(fd, &s)) { - status = MMDB_FILE_OPEN_ERROR; - goto cleanup; - } - - size = s.st_size; - if (size < 0 || size != s.st_size) { - status = MMDB_OUT_OF_MEMORY_ERROR; - goto cleanup; - } - - uint8_t *file_content = - (uint8_t *)mmap(NULL, size, PROT_READ, MAP_SHARED, fd, 0); - if (MAP_FAILED == file_content) { - if (ENOMEM == errno) { - status = MMDB_OUT_OF_MEMORY_ERROR; - } else { - status = MMDB_IO_ERROR; - } - goto cleanup; - } - - mmdb->file_size = size; - mmdb->file_content = file_content; - - cleanup:; - int saved_errno = errno; - if (fd >= 0) { - close(fd); - } - errno = saved_errno; - - return status; -} - -#endif // _WIN32 - -LOCAL const uint8_t *find_metadata(const uint8_t *file_content, - ssize_t file_size, uint32_t *metadata_size) -{ - const ssize_t marker_len = sizeof(METADATA_MARKER) - 1; - ssize_t max_size = file_size > - METADATA_BLOCK_MAX_SIZE ? METADATA_BLOCK_MAX_SIZE : - file_size; - - uint8_t *search_area = (uint8_t *)(file_content + (file_size - max_size)); - uint8_t *start = search_area; - uint8_t *tmp; - do { - tmp = mmdb_memmem(search_area, max_size, - METADATA_MARKER, marker_len); - - if (NULL != tmp) { - max_size -= tmp - search_area; - search_area = tmp; - - /* Continue searching just after the marker we just read, in case - * there are multiple markers in the same file. This would be odd - * but is certainly not impossible. */ - max_size -= marker_len; - search_area += marker_len; - } - } while (NULL != tmp); - - if (search_area == start) { - return NULL; - } - - *metadata_size = (uint32_t)max_size; - - return search_area; -} - -LOCAL int read_metadata(MMDB_s *mmdb) -{ - /* We need to create a fake MMDB_s struct in order to decode values from - the metadata. The metadata is basically just like the data section, so we - want to use the same functions we use for the data section to get metadata - values. */ - MMDB_s metadata_db = make_fake_metadata_db(mmdb); - - MMDB_entry_s metadata_start = { - .mmdb = &metadata_db, - .offset = 0 - }; - - int status = - value_for_key_as_uint32(&metadata_start, "node_count", - &mmdb->metadata.node_count); - if (MMDB_SUCCESS != status) { - return status; - } - if (!mmdb->metadata.node_count) { - DEBUG_MSG("could not find node_count value in metadata"); - return MMDB_INVALID_METADATA_ERROR; - } - - status = value_for_key_as_uint16(&metadata_start, "record_size", - &mmdb->metadata.record_size); - if (MMDB_SUCCESS != status) { - return status; - } - if (!mmdb->metadata.record_size) { - DEBUG_MSG("could not find record_size value in metadata"); - return MMDB_INVALID_METADATA_ERROR; - } - - if (mmdb->metadata.record_size != 24 && mmdb->metadata.record_size != 28 - && mmdb->metadata.record_size != 32) { - DEBUG_MSGF("bad record size in metadata: %i", - mmdb->metadata.record_size); - return MMDB_UNKNOWN_DATABASE_FORMAT_ERROR; - } - - status = value_for_key_as_uint16(&metadata_start, "ip_version", - &mmdb->metadata.ip_version); - if (MMDB_SUCCESS != status) { - return status; - } - if (!mmdb->metadata.ip_version) { - DEBUG_MSG("could not find ip_version value in metadata"); - return MMDB_INVALID_METADATA_ERROR; - } - if (!(mmdb->metadata.ip_version == 4 || mmdb->metadata.ip_version == 6)) { - DEBUG_MSGF("ip_version value in metadata is not 4 or 6 - it was %i", - mmdb->metadata.ip_version); - return MMDB_INVALID_METADATA_ERROR; - } - - status = value_for_key_as_string(&metadata_start, "database_type", - &mmdb->metadata.database_type); - if (MMDB_SUCCESS != status) { - DEBUG_MSG("error finding database_type value in metadata"); - return status; - } - - status = - populate_languages_metadata(mmdb, &metadata_db, &metadata_start); - if (MMDB_SUCCESS != status) { - DEBUG_MSG("could not populate languages from metadata"); - return status; - } - - status = value_for_key_as_uint16( - &metadata_start, "binary_format_major_version", - &mmdb->metadata.binary_format_major_version); - if (MMDB_SUCCESS != status) { - return status; - } - if (!mmdb->metadata.binary_format_major_version) { - DEBUG_MSG( - "could not find binary_format_major_version value in metadata"); - return MMDB_INVALID_METADATA_ERROR; - } - - status = value_for_key_as_uint16( - &metadata_start, "binary_format_minor_version", - &mmdb->metadata.binary_format_minor_version); - if (MMDB_SUCCESS != status) { - return status; - } - - status = value_for_key_as_uint64(&metadata_start, "build_epoch", - &mmdb->metadata.build_epoch); - if (MMDB_SUCCESS != status) { - return status; - } - if (!mmdb->metadata.build_epoch) { - DEBUG_MSG("could not find build_epoch value in metadata"); - return MMDB_INVALID_METADATA_ERROR; - } - - status = populate_description_metadata(mmdb, &metadata_db, &metadata_start); - if (MMDB_SUCCESS != status) { - DEBUG_MSG("could not populate description from metadata"); - return status; - } - - mmdb->full_record_byte_size = mmdb->metadata.record_size * 2 / 8U; - - mmdb->depth = mmdb->metadata.ip_version == 4 ? 32 : 128; - - return MMDB_SUCCESS; -} - -LOCAL MMDB_s make_fake_metadata_db(const MMDB_s *const mmdb) -{ - MMDB_s fake_metadata_db = { - .data_section = mmdb->metadata_section, - .data_section_size = mmdb->metadata_section_size - }; - - return fake_metadata_db; -} - -LOCAL int value_for_key_as_uint16(MMDB_entry_s *start, char *key, - uint16_t *value) -{ - MMDB_entry_data_s entry_data; - const char *path[] = { key, NULL }; - int status = MMDB_aget_value(start, &entry_data, path); - if (MMDB_SUCCESS != status) { - return status; - } - if (MMDB_DATA_TYPE_UINT16 != entry_data.type) { - DEBUG_MSGF("expect uint16 for %s but received %s", key, - type_num_to_name( - entry_data.type)); - return MMDB_INVALID_METADATA_ERROR; - } - *value = entry_data.uint16; - return MMDB_SUCCESS; -} - -LOCAL int value_for_key_as_uint32(MMDB_entry_s *start, char *key, - uint32_t *value) -{ - MMDB_entry_data_s entry_data; - const char *path[] = { key, NULL }; - int status = MMDB_aget_value(start, &entry_data, path); - if (MMDB_SUCCESS != status) { - return status; - } - if (MMDB_DATA_TYPE_UINT32 != entry_data.type) { - DEBUG_MSGF("expect uint32 for %s but received %s", key, - type_num_to_name( - entry_data.type)); - return MMDB_INVALID_METADATA_ERROR; - } - *value = entry_data.uint32; - return MMDB_SUCCESS; -} - -LOCAL int value_for_key_as_uint64(MMDB_entry_s *start, char *key, - uint64_t *value) -{ - MMDB_entry_data_s entry_data; - const char *path[] = { key, NULL }; - int status = MMDB_aget_value(start, &entry_data, path); - if (MMDB_SUCCESS != status) { - return status; - } - if (MMDB_DATA_TYPE_UINT64 != entry_data.type) { - DEBUG_MSGF("expect uint64 for %s but received %s", key, - type_num_to_name( - entry_data.type)); - return MMDB_INVALID_METADATA_ERROR; - } - *value = entry_data.uint64; - return MMDB_SUCCESS; -} - -LOCAL int value_for_key_as_string(MMDB_entry_s *start, char *key, - char const **value) -{ - MMDB_entry_data_s entry_data; - const char *path[] = { key, NULL }; - int status = MMDB_aget_value(start, &entry_data, path); - if (MMDB_SUCCESS != status) { - return status; - } - if (MMDB_DATA_TYPE_UTF8_STRING != entry_data.type) { - DEBUG_MSGF("expect string for %s but received %s", key, - type_num_to_name( - entry_data.type)); - return MMDB_INVALID_METADATA_ERROR; - } - *value = mmdb_strndup((char *)entry_data.utf8_string, entry_data.data_size); - if (NULL == *value) { - return MMDB_OUT_OF_MEMORY_ERROR; - } - return MMDB_SUCCESS; -} - -LOCAL int populate_languages_metadata(MMDB_s *mmdb, MMDB_s *metadata_db, - MMDB_entry_s *metadata_start) -{ - MMDB_entry_data_s entry_data; - - const char *path[] = { "languages", NULL }; - int status = MMDB_aget_value(metadata_start, &entry_data, path); - if (MMDB_SUCCESS != status) { - return status; - } - if (MMDB_DATA_TYPE_ARRAY != entry_data.type) { - return MMDB_INVALID_METADATA_ERROR; - } - - MMDB_entry_s array_start = { - .mmdb = metadata_db, - .offset = entry_data.offset - }; - - MMDB_entry_data_list_s *member; - status = MMDB_get_entry_data_list(&array_start, &member); - if (MMDB_SUCCESS != status) { - return status; - } - - MMDB_entry_data_list_s *first_member = member; - - uint32_t array_size = member->entry_data.data_size; - MAYBE_CHECK_SIZE_OVERFLOW(array_size, SIZE_MAX / sizeof(char *), - MMDB_INVALID_METADATA_ERROR); - - mmdb->metadata.languages.count = 0; - mmdb->metadata.languages.names = calloc(array_size, sizeof(char *)); - if (NULL == mmdb->metadata.languages.names) { - return MMDB_OUT_OF_MEMORY_ERROR; - } - - for (uint32_t i = 0; i < array_size; i++) { - member = member->next; - if (MMDB_DATA_TYPE_UTF8_STRING != member->entry_data.type) { - return MMDB_INVALID_METADATA_ERROR; - } - - mmdb->metadata.languages.names[i] = - mmdb_strndup((char *)member->entry_data.utf8_string, - member->entry_data.data_size); - - if (NULL == mmdb->metadata.languages.names[i]) { - return MMDB_OUT_OF_MEMORY_ERROR; - } - // We assign this as we go so that if we fail a calloc and need to - // free it, the count is right. - mmdb->metadata.languages.count = i + 1; - } - - MMDB_free_entry_data_list(first_member); - - return MMDB_SUCCESS; -} - -LOCAL int populate_description_metadata(MMDB_s *mmdb, MMDB_s *metadata_db, - MMDB_entry_s *metadata_start) -{ - MMDB_entry_data_s entry_data; - - const char *path[] = { "description", NULL }; - int status = MMDB_aget_value(metadata_start, &entry_data, path); - if (MMDB_SUCCESS != status) { - return status; - } - - if (MMDB_DATA_TYPE_MAP != entry_data.type) { - DEBUG_MSGF("Unexpected entry_data type: %d", entry_data.type); - return MMDB_INVALID_METADATA_ERROR; - } - - MMDB_entry_s map_start = { - .mmdb = metadata_db, - .offset = entry_data.offset - }; - - MMDB_entry_data_list_s *member; - status = MMDB_get_entry_data_list(&map_start, &member); - if (MMDB_SUCCESS != status) { - DEBUG_MSGF( - "MMDB_get_entry_data_list failed while populating description." - " status = %d (%s)", status, MMDB_strerror(status)); - return status; - } - - MMDB_entry_data_list_s *first_member = member; - - uint32_t map_size = member->entry_data.data_size; - mmdb->metadata.description.count = 0; - if (0 == map_size) { - mmdb->metadata.description.descriptions = NULL; - goto cleanup; - } - MAYBE_CHECK_SIZE_OVERFLOW(map_size, SIZE_MAX / sizeof(MMDB_description_s *), - MMDB_INVALID_METADATA_ERROR); - - mmdb->metadata.description.descriptions = - calloc(map_size, sizeof(MMDB_description_s *)); - if (NULL == mmdb->metadata.description.descriptions) { - status = MMDB_OUT_OF_MEMORY_ERROR; - goto cleanup; - } - - for (uint32_t i = 0; i < map_size; i++) { - mmdb->metadata.description.descriptions[i] = - calloc(1, sizeof(MMDB_description_s)); - if (NULL == mmdb->metadata.description.descriptions[i]) { - status = MMDB_OUT_OF_MEMORY_ERROR; - goto cleanup; - } - - mmdb->metadata.description.count = i + 1; - mmdb->metadata.description.descriptions[i]->language = NULL; - mmdb->metadata.description.descriptions[i]->description = NULL; - - member = member->next; - - if (MMDB_DATA_TYPE_UTF8_STRING != member->entry_data.type) { - status = MMDB_INVALID_METADATA_ERROR; - goto cleanup; - } - - mmdb->metadata.description.descriptions[i]->language = - mmdb_strndup((char *)member->entry_data.utf8_string, - member->entry_data.data_size); - - if (NULL == mmdb->metadata.description.descriptions[i]->language) { - status = MMDB_OUT_OF_MEMORY_ERROR; - goto cleanup; - } - - member = member->next; - - if (MMDB_DATA_TYPE_UTF8_STRING != member->entry_data.type) { - status = MMDB_INVALID_METADATA_ERROR; - goto cleanup; - } - - mmdb->metadata.description.descriptions[i]->description = - mmdb_strndup((char *)member->entry_data.utf8_string, - member->entry_data.data_size); - - if (NULL == mmdb->metadata.description.descriptions[i]->description) { - status = MMDB_OUT_OF_MEMORY_ERROR; - goto cleanup; - } - } - - cleanup: - MMDB_free_entry_data_list(first_member); - - return status; -} - -MMDB_lookup_result_s MMDB_lookup_string(const MMDB_s *const mmdb, - const char *const ipstr, - int *const gai_error, - int *const mmdb_error) -{ - MMDB_lookup_result_s result = { - .found_entry = false, - .netmask = 0, - .entry = { - .mmdb = mmdb, - .offset = 0 - } - }; - - struct addrinfo *addresses = NULL; - *gai_error = resolve_any_address(ipstr, &addresses); - - if (!*gai_error) { - result = MMDB_lookup_sockaddr(mmdb, addresses->ai_addr, mmdb_error); - } - - if (NULL != addresses) { - freeaddrinfo(addresses); - } - - return result; -} - -LOCAL int resolve_any_address(const char *ipstr, struct addrinfo **addresses) -{ - struct addrinfo hints = { - .ai_family = AF_UNSPEC, - .ai_flags = AI_NUMERICHOST, - // We set ai_socktype so that we only get one result back - .ai_socktype = SOCK_STREAM - }; - - int gai_status = getaddrinfo(ipstr, NULL, &hints, addresses); - if (gai_status) { - return gai_status; - } - - return 0; -} - -MMDB_lookup_result_s MMDB_lookup_sockaddr( - const MMDB_s *const mmdb, - const struct sockaddr *const sockaddr, - int *const mmdb_error) -{ - MMDB_lookup_result_s result = { - .found_entry = false, - .netmask = 0, - .entry = { - .mmdb = mmdb, - .offset = 0 - } - }; - - uint8_t mapped_address[16], *address; - if (mmdb->metadata.ip_version == 4) { - if (sockaddr->sa_family == AF_INET6) { - *mmdb_error = MMDB_IPV6_LOOKUP_IN_IPV4_DATABASE_ERROR; - return result; - } - address = (uint8_t *)&((struct sockaddr_in *)sockaddr)->sin_addr.s_addr; - } else { - if (sockaddr->sa_family == AF_INET6) { - address = - (uint8_t *)&((struct sockaddr_in6 *)sockaddr)->sin6_addr. - s6_addr; - } else { - address = mapped_address; - memset(address, 0, 12); - memcpy(address + 12, - &((struct sockaddr_in *)sockaddr)->sin_addr.s_addr, 4); - } - } - - *mmdb_error = - find_address_in_search_tree(mmdb, address, sockaddr->sa_family, - &result); - - return result; -} - -LOCAL int find_address_in_search_tree(const MMDB_s *const mmdb, - uint8_t *address, - sa_family_t address_family, - MMDB_lookup_result_s *result) -{ - record_info_s record_info = record_info_for_database(mmdb); - if (0 == record_info.right_record_offset) { - return MMDB_UNKNOWN_DATABASE_FORMAT_ERROR; - } - - uint32_t value = 0; - uint16_t current_bit = 0; - if (mmdb->metadata.ip_version == 6 && address_family == AF_INET) { - value = mmdb->ipv4_start_node.node_value; - current_bit = mmdb->ipv4_start_node.netmask; - } - - uint32_t node_count = mmdb->metadata.node_count; - const uint8_t *search_tree = mmdb->file_content; - const uint8_t *record_pointer; - for (; current_bit < mmdb->depth && value < node_count; current_bit++) { - uint8_t bit = 1U & - (address[current_bit >> 3] >> (7 - (current_bit % 8))); - - record_pointer = &search_tree[value * record_info.record_length]; - if (record_pointer + record_info.record_length > mmdb->data_section) { - return MMDB_CORRUPT_SEARCH_TREE_ERROR; - } - if (bit) { - record_pointer += record_info.right_record_offset; - value = record_info.right_record_getter(record_pointer); - } else { - value = record_info.left_record_getter(record_pointer); - } - } - - result->netmask = current_bit; - - if (value >= node_count + mmdb->data_section_size) { - // The pointer points off the end of the database. - return MMDB_CORRUPT_SEARCH_TREE_ERROR; - } - - if (value == node_count) { - // record is empty - result->found_entry = false; - return MMDB_SUCCESS; - } - result->found_entry = true; - result->entry.offset = data_section_offset_for_record(mmdb, value); - - return MMDB_SUCCESS; -} - -LOCAL record_info_s record_info_for_database(const MMDB_s *const mmdb) -{ - record_info_s record_info = { - .record_length = mmdb->full_record_byte_size, - .right_record_offset = 0 - }; - - if (record_info.record_length == 6) { - record_info.left_record_getter = &get_uint24; - record_info.right_record_getter = &get_uint24; - record_info.right_record_offset = 3; - } else if (record_info.record_length == 7) { - record_info.left_record_getter = &get_left_28_bit_record; - record_info.right_record_getter = &get_right_28_bit_record; - record_info.right_record_offset = 3; - } else if (record_info.record_length == 8) { - record_info.left_record_getter = &get_uint32; - record_info.right_record_getter = &get_uint32; - record_info.right_record_offset = 4; - } else { - assert(false); - } - - return record_info; -} - -LOCAL int find_ipv4_start_node(MMDB_s *const mmdb) -{ - /* In a pathological case of a database with a single node search tree, - * this check will be true even after we've found the IPv4 start node, but - * that doesn't seem worth trying to fix. */ - if (mmdb->ipv4_start_node.node_value != 0) { - return MMDB_SUCCESS; - } - - record_info_s record_info = record_info_for_database(mmdb); - - const uint8_t *search_tree = mmdb->file_content; - uint32_t node_value = 0; - const uint8_t *record_pointer; - uint16_t netmask; - uint32_t node_count = mmdb->metadata.node_count; - - for (netmask = 0; netmask < 96 && node_value < node_count; netmask++) { - record_pointer = &search_tree[node_value * record_info.record_length]; - if (record_pointer + record_info.record_length > mmdb->data_section) { - return MMDB_CORRUPT_SEARCH_TREE_ERROR; - } - node_value = record_info.left_record_getter(record_pointer); - } - - mmdb->ipv4_start_node.node_value = node_value; - mmdb->ipv4_start_node.netmask = netmask; - - return MMDB_SUCCESS; -} - -LOCAL uint8_t record_type(const MMDB_s *const mmdb, uint64_t record) -{ - uint32_t node_count = mmdb->metadata.node_count; - - /* Ideally we'd check to make sure that a record never points to a - * previously seen value, but that's more complicated. For now, we can - * at least check that we don't end up at the top of the tree again. */ - if (record == 0) { - DEBUG_MSG("record has a value of 0"); - return MMDB_RECORD_TYPE_INVALID; - } - - if (record < node_count) { - return MMDB_RECORD_TYPE_SEARCH_NODE; - } - - if (record == node_count) { - return MMDB_RECORD_TYPE_EMPTY; - } - - if (record - node_count < mmdb->data_section_size) { - return MMDB_RECORD_TYPE_DATA; - } - - DEBUG_MSG("record has a value that points outside of the database"); - return MMDB_RECORD_TYPE_INVALID; -} - -LOCAL uint32_t get_left_28_bit_record(const uint8_t *record) -{ - return record[0] * 65536 + record[1] * 256 + record[2] + - ((record[3] & 0xf0) << 20); -} - -LOCAL uint32_t get_right_28_bit_record(const uint8_t *record) -{ - uint32_t value = get_uint32(record); - return value & 0xfffffff; -} - -int MMDB_read_node(const MMDB_s *const mmdb, uint32_t node_number, - MMDB_search_node_s *const node) -{ - record_info_s record_info = record_info_for_database(mmdb); - if (0 == record_info.right_record_offset) { - return MMDB_UNKNOWN_DATABASE_FORMAT_ERROR; - } - - if (node_number > mmdb->metadata.node_count) { - return MMDB_INVALID_NODE_NUMBER_ERROR; - } - - const uint8_t *search_tree = mmdb->file_content; - const uint8_t *record_pointer = - &search_tree[node_number * record_info.record_length]; - node->left_record = record_info.left_record_getter(record_pointer); - record_pointer += record_info.right_record_offset; - node->right_record = record_info.right_record_getter(record_pointer); - - node->left_record_type = record_type(mmdb, node->left_record); - node->right_record_type = record_type(mmdb, node->right_record); - - // Note that offset will be invalid if the record type is not - // MMDB_RECORD_TYPE_DATA, but that's ok. Any use of the record entry - // for other data types is a programming error. - node->left_record_entry = (struct MMDB_entry_s) { - .mmdb = mmdb, - .offset = data_section_offset_for_record(mmdb, node->left_record), - }; - node->right_record_entry = (struct MMDB_entry_s) { - .mmdb = mmdb, - .offset = data_section_offset_for_record(mmdb, node->right_record), - }; - - return MMDB_SUCCESS; -} - -LOCAL uint32_t data_section_offset_for_record(const MMDB_s *const mmdb, - uint64_t record) -{ - return (uint32_t)record - mmdb->metadata.node_count - - MMDB_DATA_SECTION_SEPARATOR; -} - -int MMDB_get_value(MMDB_entry_s *const start, - MMDB_entry_data_s *const entry_data, - ...) -{ - va_list path; - va_start(path, entry_data); - int status = MMDB_vget_value(start, entry_data, path); - va_end(path); - return status; -} - -int MMDB_vget_value(MMDB_entry_s *const start, - MMDB_entry_data_s *const entry_data, - va_list va_path) -{ - int length = path_length(va_path); - const char *path_elem; - int i = 0; - - MAYBE_CHECK_SIZE_OVERFLOW(length, SIZE_MAX / sizeof(const char *) - 1, - MMDB_INVALID_METADATA_ERROR); - - const char **path = calloc(length + 1, sizeof(const char *)); - if (NULL == path) { - return MMDB_OUT_OF_MEMORY_ERROR; - } - - while (NULL != (path_elem = va_arg(va_path, char *))) { - path[i] = path_elem; - i++; - } - path[i] = NULL; - - int status = MMDB_aget_value(start, entry_data, path); - - free((char **)path); - - return status; -} - -LOCAL int path_length(va_list va_path) -{ - int i = 0; - const char *ignore; - va_list path_copy; - va_copy(path_copy, va_path); - - while (NULL != (ignore = va_arg(path_copy, char *))) { - i++; - } - - va_end(path_copy); - - return i; -} - -int MMDB_aget_value(MMDB_entry_s *const start, - MMDB_entry_data_s *const entry_data, - const char *const *const path) -{ - const MMDB_s *const mmdb = start->mmdb; - uint32_t offset = start->offset; - - memset(entry_data, 0, sizeof(MMDB_entry_data_s)); - DEBUG_NL; - DEBUG_MSG("looking up value by path"); - - CHECKED_DECODE_ONE_FOLLOW(mmdb, offset, entry_data); - - DEBUG_NL; - DEBUG_MSGF("top level element is a %s", type_num_to_name(entry_data->type)); - - /* Can this happen? It'd probably represent a pathological case under - * normal use, but there's nothing preventing someone from passing an - * invalid MMDB_entry_s struct to this function */ - if (!entry_data->has_data) { - return MMDB_INVALID_LOOKUP_PATH_ERROR; - } - - const char *path_elem; - int i = 0; - while (NULL != (path_elem = path[i++])) { - DEBUG_NL; - DEBUG_MSGF("path elem = %s", path_elem); - - /* XXX - it'd be good to find a quicker way to skip through these - entries that doesn't involve decoding them - completely. Basically we need to just use the size from the - control byte to advance our pointer rather than calling - decode_one(). */ - if (entry_data->type == MMDB_DATA_TYPE_ARRAY) { - int status = lookup_path_in_array(path_elem, mmdb, entry_data); - if (MMDB_SUCCESS != status) { - memset(entry_data, 0, sizeof(MMDB_entry_data_s)); - return status; - } - } else if (entry_data->type == MMDB_DATA_TYPE_MAP) { - int status = lookup_path_in_map(path_elem, mmdb, entry_data); - if (MMDB_SUCCESS != status) { - memset(entry_data, 0, sizeof(MMDB_entry_data_s)); - return status; - } - } else { - /* Once we make the code traverse maps & arrays without calling - * decode_one() we can get rid of this. */ - memset(entry_data, 0, sizeof(MMDB_entry_data_s)); - return MMDB_LOOKUP_PATH_DOES_NOT_MATCH_DATA_ERROR; - } - } - - return MMDB_SUCCESS; -} - -LOCAL int lookup_path_in_array(const char *path_elem, - const MMDB_s *const mmdb, - MMDB_entry_data_s *entry_data) -{ - uint32_t size = entry_data->data_size; - char *first_invalid; - - int saved_errno = errno; - errno = 0; - int array_index = strtol(path_elem, &first_invalid, 10); - if (ERANGE == errno) { - errno = saved_errno; - return MMDB_INVALID_LOOKUP_PATH_ERROR; - } - errno = saved_errno; - - if (array_index < 0) { - array_index += size; - - if (array_index < 0) { - return MMDB_LOOKUP_PATH_DOES_NOT_MATCH_DATA_ERROR; - } - } - - if (*first_invalid || (uint32_t)array_index >= size) { - return MMDB_LOOKUP_PATH_DOES_NOT_MATCH_DATA_ERROR; - } - - for (int i = 0; i < array_index; i++) { - /* We don't want to follow a pointer here. If the next element is a - * pointer we simply skip it and keep going */ - CHECKED_DECODE_ONE(mmdb, entry_data->offset_to_next, entry_data); - int status = skip_map_or_array(mmdb, entry_data); - if (MMDB_SUCCESS != status) { - return status; - } - } - - MMDB_entry_data_s value; - CHECKED_DECODE_ONE_FOLLOW(mmdb, entry_data->offset_to_next, &value); - memcpy(entry_data, &value, sizeof(MMDB_entry_data_s)); - - return MMDB_SUCCESS; -} - -LOCAL int lookup_path_in_map(const char *path_elem, - const MMDB_s *const mmdb, - MMDB_entry_data_s *entry_data) -{ - uint32_t size = entry_data->data_size; - uint32_t offset = entry_data->offset_to_next; - size_t path_elem_len = strlen(path_elem); - - while (size-- > 0) { - MMDB_entry_data_s key, value; - CHECKED_DECODE_ONE_FOLLOW(mmdb, offset, &key); - - uint32_t offset_to_value = key.offset_to_next; - - if (MMDB_DATA_TYPE_UTF8_STRING != key.type) { - return MMDB_INVALID_DATA_ERROR; - } - - if (key.data_size == path_elem_len && - !memcmp(path_elem, key.utf8_string, path_elem_len)) { - - DEBUG_MSG("found key matching path elem"); - - CHECKED_DECODE_ONE_FOLLOW(mmdb, offset_to_value, &value); - memcpy(entry_data, &value, sizeof(MMDB_entry_data_s)); - return MMDB_SUCCESS; - } else { - /* We don't want to follow a pointer here. If the next element is - * a pointer we simply skip it and keep going */ - CHECKED_DECODE_ONE(mmdb, offset_to_value, &value); - int status = skip_map_or_array(mmdb, &value); - if (MMDB_SUCCESS != status) { - return status; - } - offset = value.offset_to_next; - } - } - - memset(entry_data, 0, sizeof(MMDB_entry_data_s)); - return MMDB_LOOKUP_PATH_DOES_NOT_MATCH_DATA_ERROR; -} - -LOCAL int skip_map_or_array(const MMDB_s *const mmdb, - MMDB_entry_data_s *entry_data) -{ - if (entry_data->type == MMDB_DATA_TYPE_MAP) { - uint32_t size = entry_data->data_size; - while (size-- > 0) { - CHECKED_DECODE_ONE(mmdb, entry_data->offset_to_next, entry_data); // key - CHECKED_DECODE_ONE(mmdb, entry_data->offset_to_next, entry_data); // value - int status = skip_map_or_array(mmdb, entry_data); - if (MMDB_SUCCESS != status) { - return status; - } - } - } else if (entry_data->type == MMDB_DATA_TYPE_ARRAY) { - uint32_t size = entry_data->data_size; - while (size-- > 0) { - CHECKED_DECODE_ONE(mmdb, entry_data->offset_to_next, entry_data); // value - int status = skip_map_or_array(mmdb, entry_data); - if (MMDB_SUCCESS != status) { - return status; - } - } - } - - return MMDB_SUCCESS; -} - -LOCAL int decode_one_follow(const MMDB_s *const mmdb, uint32_t offset, - MMDB_entry_data_s *entry_data) -{ - CHECKED_DECODE_ONE(mmdb, offset, entry_data); - if (entry_data->type == MMDB_DATA_TYPE_POINTER) { - uint32_t next = entry_data->offset_to_next; - CHECKED_DECODE_ONE(mmdb, entry_data->pointer, entry_data); - /* Pointers to pointers are illegal under the spec */ - if (entry_data->type == MMDB_DATA_TYPE_POINTER) { - DEBUG_MSG("pointer points to another pointer"); - return MMDB_INVALID_DATA_ERROR; - } - - /* The pointer could point to any part of the data section but the - * next entry for this particular offset may be the one after the - * pointer, not the one after whatever the pointer points to. This - * depends on whether the pointer points to something that is a simple - * value or a compound value. For a compound value, the next one is - * the one after the pointer result, not the one after the pointer. */ - if (entry_data->type != MMDB_DATA_TYPE_MAP - && entry_data->type != MMDB_DATA_TYPE_ARRAY) { - - entry_data->offset_to_next = next; - } - } - - return MMDB_SUCCESS; -} - -#if !MMDB_UINT128_IS_BYTE_ARRAY -LOCAL mmdb_uint128_t get_uint128(const uint8_t *p, int length) -{ - mmdb_uint128_t value = 0; - while (length-- > 0) { - value <<= 8; - value += *p++; - } - return value; -} -#endif - -LOCAL int decode_one(const MMDB_s *const mmdb, uint32_t offset, - MMDB_entry_data_s *entry_data) -{ - const uint8_t *mem = mmdb->data_section; - - // We subtract rather than add as it possible that offset + 1 - // could overflow for a corrupt database while an underflow - // from data_section_size - 1 should not be possible. - if (offset > mmdb->data_section_size - 1) { - DEBUG_MSGF("Offset (%d) past data section (%d)", offset, - mmdb->data_section_size); - return MMDB_INVALID_DATA_ERROR; - } - - entry_data->offset = offset; - entry_data->has_data = true; - - DEBUG_NL; - DEBUG_MSGF("Offset: %i", offset); - - uint8_t ctrl = mem[offset++]; - DEBUG_BINARY("Control byte: %s", ctrl); - - int type = (ctrl >> 5) & 7; - DEBUG_MSGF("Type: %i (%s)", type, type_num_to_name(type)); - - if (type == MMDB_DATA_TYPE_EXTENDED) { - // Subtracting 1 to avoid possible overflow on offset + 1 - if (offset > mmdb->data_section_size - 1) { - DEBUG_MSGF("Extended type offset (%d) past data section (%d)", - offset, - mmdb->data_section_size); - return MMDB_INVALID_DATA_ERROR; - } - type = get_ext_type(mem[offset++]); - DEBUG_MSGF("Extended type: %i (%s)", type, type_num_to_name(type)); - } - - entry_data->type = type; - - if (type == MMDB_DATA_TYPE_POINTER) { - uint8_t psize = ((ctrl >> 3) & 3) + 1; - DEBUG_MSGF("Pointer size: %i", psize); - - // We check that the offset does not extend past the end of the - // database and that the subtraction of psize did not underflow. - if (offset > mmdb->data_section_size - psize || - mmdb->data_section_size < psize) { - DEBUG_MSGF("Pointer offset (%d) past data section (%d)", offset + - psize, - mmdb->data_section_size); - return MMDB_INVALID_DATA_ERROR; - } - entry_data->pointer = get_ptr_from(ctrl, &mem[offset], psize); - DEBUG_MSGF("Pointer to: %i", entry_data->pointer); - - entry_data->data_size = psize; - entry_data->offset_to_next = offset + psize; - return MMDB_SUCCESS; - } - - uint32_t size = ctrl & 31; - switch (size) { - case 29: - // We subtract when checking offset to avoid possible overflow - if (offset > mmdb->data_section_size - 1) { - DEBUG_MSGF("String end (%d, case 29) past data section (%d)", - offset, - mmdb->data_section_size); - return MMDB_INVALID_DATA_ERROR; - } - size = 29 + mem[offset++]; - break; - case 30: - // We subtract when checking offset to avoid possible overflow - if (offset > mmdb->data_section_size - 2) { - DEBUG_MSGF("String end (%d, case 30) past data section (%d)", - offset, - mmdb->data_section_size); - return MMDB_INVALID_DATA_ERROR; - } - size = 285 + get_uint16(&mem[offset]); - offset += 2; - break; - case 31: - // We subtract when checking offset to avoid possible overflow - if (offset > mmdb->data_section_size - 3) { - DEBUG_MSGF("String end (%d, case 31) past data section (%d)", - offset, - mmdb->data_section_size); - return MMDB_INVALID_DATA_ERROR; - } - size = 65821 + get_uint24(&mem[offset]); - offset += 3; - default: - break; - } - - DEBUG_MSGF("Size: %i", size); - - if (type == MMDB_DATA_TYPE_MAP || type == MMDB_DATA_TYPE_ARRAY) { - entry_data->data_size = size; - entry_data->offset_to_next = offset; - return MMDB_SUCCESS; - } - - if (type == MMDB_DATA_TYPE_BOOLEAN) { - entry_data->boolean = size ? true : false; - entry_data->data_size = 0; - entry_data->offset_to_next = offset; - DEBUG_MSGF("boolean value: %s", entry_data->boolean ? "true" : "false"); - return MMDB_SUCCESS; - } - - // Check that the data doesn't extend past the end of the memory - // buffer and that the calculation in doing this did not underflow. - if (offset > mmdb->data_section_size - size || - mmdb->data_section_size < size) { - DEBUG_MSGF("Data end (%d) past data section (%d)", offset + size, - mmdb->data_section_size); - return MMDB_INVALID_DATA_ERROR; - } - - if (type == MMDB_DATA_TYPE_UINT16) { - if (size > 2) { - DEBUG_MSGF("uint16 of size %d", size); - return MMDB_INVALID_DATA_ERROR; - } - entry_data->uint16 = (uint16_t)get_uintX(&mem[offset], size); - DEBUG_MSGF("uint16 value: %u", entry_data->uint16); - } else if (type == MMDB_DATA_TYPE_UINT32) { - if (size > 4) { - DEBUG_MSGF("uint32 of size %d", size); - return MMDB_INVALID_DATA_ERROR; - } - entry_data->uint32 = (uint32_t)get_uintX(&mem[offset], size); - DEBUG_MSGF("uint32 value: %u", entry_data->uint32); - } else if (type == MMDB_DATA_TYPE_INT32) { - if (size > 4) { - DEBUG_MSGF("int32 of size %d", size); - return MMDB_INVALID_DATA_ERROR; - } - entry_data->int32 = get_sintX(&mem[offset], size); - DEBUG_MSGF("int32 value: %i", entry_data->int32); - } else if (type == MMDB_DATA_TYPE_UINT64) { - if (size > 8) { - DEBUG_MSGF("uint64 of size %d", size); - return MMDB_INVALID_DATA_ERROR; - } - entry_data->uint64 = get_uintX(&mem[offset], size); - DEBUG_MSGF("uint64 value: %" PRIu64, entry_data->uint64); - } else if (type == MMDB_DATA_TYPE_UINT128) { - if (size > 16) { - DEBUG_MSGF("uint128 of size %d", size); - return MMDB_INVALID_DATA_ERROR; - } -#if MMDB_UINT128_IS_BYTE_ARRAY - memset(entry_data->uint128, 0, 16); - if (size > 0) { - memcpy(entry_data->uint128 + 16 - size, &mem[offset], size); - } -#else - entry_data->uint128 = get_uint128(&mem[offset], size); -#endif - } else if (type == MMDB_DATA_TYPE_FLOAT) { - if (size != 4) { - DEBUG_MSGF("float of size %d", size); - return MMDB_INVALID_DATA_ERROR; - } - size = 4; - entry_data->float_value = get_ieee754_float(&mem[offset]); - DEBUG_MSGF("float value: %f", entry_data->float_value); - } else if (type == MMDB_DATA_TYPE_DOUBLE) { - if (size != 8) { - DEBUG_MSGF("double of size %d", size); - return MMDB_INVALID_DATA_ERROR; - } - size = 8; - entry_data->double_value = get_ieee754_double(&mem[offset]); - DEBUG_MSGF("double value: %f", entry_data->double_value); - } else if (type == MMDB_DATA_TYPE_UTF8_STRING) { - entry_data->utf8_string = size == 0 ? "" : (char *)&mem[offset]; - entry_data->data_size = size; -#ifdef MMDB_DEBUG - char *string = mmdb_strndup(entry_data->utf8_string, - size > 50 ? 50 : size); - if (NULL == string) { - abort(); - } - DEBUG_MSGF("string value: %s", string); - free(string); -#endif - } else if (type == MMDB_DATA_TYPE_BYTES) { - entry_data->bytes = &mem[offset]; - entry_data->data_size = size; - } - - entry_data->offset_to_next = offset + size; - - return MMDB_SUCCESS; -} - -LOCAL int get_ext_type(int raw_ext_type) -{ - return 7 + raw_ext_type; -} - -LOCAL uint32_t get_ptr_from(uint8_t ctrl, uint8_t const *const ptr, - int ptr_size) -{ - uint32_t new_offset; - switch (ptr_size) { - case 1: - new_offset = ( (ctrl & 7) << 8) + ptr[0]; - break; - case 2: - new_offset = 2048 + ( (ctrl & 7) << 16 ) + ( ptr[0] << 8) + ptr[1]; - break; - case 3: - new_offset = 2048 + 524288 + ( (ctrl & 7) << 24 ) + get_uint24(ptr); - break; - case 4: - default: - new_offset = get_uint32(ptr); - break; - } - return new_offset; -} - -int MMDB_get_metadata_as_entry_data_list( - const MMDB_s *const mmdb, MMDB_entry_data_list_s **const entry_data_list) -{ - MMDB_s metadata_db = make_fake_metadata_db(mmdb); - - MMDB_entry_s metadata_start = { - .mmdb = &metadata_db, - .offset = 0 - }; - - return MMDB_get_entry_data_list(&metadata_start, entry_data_list); -} - -int MMDB_get_entry_data_list( - MMDB_entry_s *start, MMDB_entry_data_list_s **const entry_data_list) -{ - MMDB_data_pool_s *const pool = data_pool_new(MMDB_POOL_INIT_SIZE); - if (!pool) { - return MMDB_OUT_OF_MEMORY_ERROR; - } - - MMDB_entry_data_list_s *const list = data_pool_alloc(pool); - if (!list) { - data_pool_destroy(pool); - return MMDB_OUT_OF_MEMORY_ERROR; - } - - int const status = get_entry_data_list(start->mmdb, start->offset, list, - pool, 0); - - *entry_data_list = data_pool_to_list(pool); - if (!*entry_data_list) { - data_pool_destroy(pool); - return MMDB_OUT_OF_MEMORY_ERROR; - } - - return status; -} - -LOCAL int get_entry_data_list(const MMDB_s *const mmdb, - uint32_t offset, - MMDB_entry_data_list_s *const entry_data_list, - MMDB_data_pool_s *const pool, - int depth) -{ - if (depth >= MAXIMUM_DATA_STRUCTURE_DEPTH) { - DEBUG_MSG("reached the maximum data structure depth"); - return MMDB_INVALID_DATA_ERROR; - } - depth++; - CHECKED_DECODE_ONE(mmdb, offset, &entry_data_list->entry_data); - - switch (entry_data_list->entry_data.type) { - case MMDB_DATA_TYPE_POINTER: - { - uint32_t next_offset = entry_data_list->entry_data.offset_to_next; - uint32_t last_offset; - CHECKED_DECODE_ONE(mmdb, last_offset = - entry_data_list->entry_data.pointer, - &entry_data_list->entry_data); - - /* Pointers to pointers are illegal under the spec */ - if (entry_data_list->entry_data.type == MMDB_DATA_TYPE_POINTER) { - DEBUG_MSG("pointer points to another pointer"); - return MMDB_INVALID_DATA_ERROR; - } - - if (entry_data_list->entry_data.type == MMDB_DATA_TYPE_ARRAY - || entry_data_list->entry_data.type == MMDB_DATA_TYPE_MAP) { - - int status = - get_entry_data_list(mmdb, last_offset, entry_data_list, - pool, depth); - if (MMDB_SUCCESS != status) { - DEBUG_MSG("get_entry_data_list on pointer failed."); - return status; - } - } - entry_data_list->entry_data.offset_to_next = next_offset; - } - break; - case MMDB_DATA_TYPE_ARRAY: - { - uint32_t array_size = entry_data_list->entry_data.data_size; - uint32_t array_offset = entry_data_list->entry_data.offset_to_next; - while (array_size-- > 0) { - MMDB_entry_data_list_s *entry_data_list_to = - data_pool_alloc(pool); - if (!entry_data_list_to) { - return MMDB_OUT_OF_MEMORY_ERROR; - } - - int status = - get_entry_data_list(mmdb, array_offset, entry_data_list_to, - pool, depth); - if (MMDB_SUCCESS != status) { - DEBUG_MSG("get_entry_data_list on array element failed."); - return status; - } - - array_offset = entry_data_list_to->entry_data.offset_to_next; - } - entry_data_list->entry_data.offset_to_next = array_offset; - - } - break; - case MMDB_DATA_TYPE_MAP: - { - uint32_t size = entry_data_list->entry_data.data_size; - - offset = entry_data_list->entry_data.offset_to_next; - while (size-- > 0) { - MMDB_entry_data_list_s *list_key = data_pool_alloc(pool); - if (!list_key) { - return MMDB_OUT_OF_MEMORY_ERROR; - } - - int status = - get_entry_data_list(mmdb, offset, list_key, pool, depth); - if (MMDB_SUCCESS != status) { - DEBUG_MSG("get_entry_data_list on map key failed."); - return status; - } - - offset = list_key->entry_data.offset_to_next; - - MMDB_entry_data_list_s *list_value = data_pool_alloc(pool); - if (!list_value) { - return MMDB_OUT_OF_MEMORY_ERROR; - } - - status = get_entry_data_list(mmdb, offset, list_value, pool, - depth); - if (MMDB_SUCCESS != status) { - DEBUG_MSG("get_entry_data_list on map element failed."); - return status; - } - offset = list_value->entry_data.offset_to_next; - } - entry_data_list->entry_data.offset_to_next = offset; - } - break; - default: - break; - } - - return MMDB_SUCCESS; -} - -LOCAL float get_ieee754_float(const uint8_t *restrict p) -{ - volatile float f; - uint8_t *q = (void *)&f; -/* Windows builds don't use autoconf but we can assume they're all - * little-endian. */ -#if MMDB_LITTLE_ENDIAN || _WIN32 - q[3] = p[0]; - q[2] = p[1]; - q[1] = p[2]; - q[0] = p[3]; -#else - memcpy(q, p, 4); -#endif - return f; -} - -LOCAL double get_ieee754_double(const uint8_t *restrict p) -{ - volatile double d; - uint8_t *q = (void *)&d; -#if MMDB_LITTLE_ENDIAN || _WIN32 - q[7] = p[0]; - q[6] = p[1]; - q[5] = p[2]; - q[4] = p[3]; - q[3] = p[4]; - q[2] = p[5]; - q[1] = p[6]; - q[0] = p[7]; -#else - memcpy(q, p, 8); -#endif - - return d; -} - -LOCAL uint32_t get_uint32(const uint8_t *p) -{ - return p[0] * 16777216U + p[1] * 65536 + p[2] * 256 + p[3]; -} - -LOCAL uint32_t get_uint24(const uint8_t *p) -{ - return p[0] * 65536U + p[1] * 256 + p[2]; -} - -LOCAL uint32_t get_uint16(const uint8_t *p) -{ - return p[0] * 256U + p[1]; -} - -LOCAL uint64_t get_uintX(const uint8_t *p, int length) -{ - uint64_t value = 0; - while (length-- > 0) { - value <<= 8; - value += *p++; - } - return value; -} - -LOCAL int32_t get_sintX(const uint8_t *p, int length) -{ - return (int32_t)get_uintX(p, length); -} - -void MMDB_free_entry_data_list(MMDB_entry_data_list_s *const entry_data_list) -{ - if (entry_data_list == NULL) { - return; - } - data_pool_destroy(entry_data_list->pool); -} - -void MMDB_close(MMDB_s *const mmdb) -{ - free_mmdb_struct(mmdb); -} - -LOCAL void free_mmdb_struct(MMDB_s *const mmdb) -{ - if (!mmdb) { - return; - } - - if (NULL != mmdb->filename) { - FREE_AND_SET_NULL(mmdb->filename); - } - if (NULL != mmdb->file_content) { -#ifdef _WIN32 - UnmapViewOfFile(mmdb->file_content); - /* Winsock is only initialized if open was successful so we only have - * to cleanup then. */ - WSACleanup(); -#else - munmap((void *)mmdb->file_content, mmdb->file_size); -#endif - } - - if (NULL != mmdb->metadata.database_type) { - FREE_AND_SET_NULL(mmdb->metadata.database_type); - } - - free_languages_metadata(mmdb); - free_descriptions_metadata(mmdb); -} - -LOCAL void free_languages_metadata(MMDB_s *mmdb) -{ - if (!mmdb->metadata.languages.names) { - return; - } - - for (size_t i = 0; i < mmdb->metadata.languages.count; i++) { - FREE_AND_SET_NULL(mmdb->metadata.languages.names[i]); - } - FREE_AND_SET_NULL(mmdb->metadata.languages.names); -} - -LOCAL void free_descriptions_metadata(MMDB_s *mmdb) -{ - if (!mmdb->metadata.description.count) { - return; - } - - for (size_t i = 0; i < mmdb->metadata.description.count; i++) { - if (NULL != mmdb->metadata.description.descriptions[i]) { - if (NULL != - mmdb->metadata.description.descriptions[i]->language) { - FREE_AND_SET_NULL( - mmdb->metadata.description.descriptions[i]->language); - } - - if (NULL != - mmdb->metadata.description.descriptions[i]->description) { - FREE_AND_SET_NULL( - mmdb->metadata.description.descriptions[i]->description); - } - FREE_AND_SET_NULL(mmdb->metadata.description.descriptions[i]); - } - } - - FREE_AND_SET_NULL(mmdb->metadata.description.descriptions); -} - -const char *MMDB_lib_version(void) -{ - return PACKAGE_VERSION; -} - -int MMDB_dump_entry_data_list(FILE *const stream, - MMDB_entry_data_list_s *const entry_data_list, - int indent) -{ - int status; - dump_entry_data_list(stream, entry_data_list, indent, &status); - return status; -} - -LOCAL MMDB_entry_data_list_s *dump_entry_data_list( - FILE *stream, MMDB_entry_data_list_s *entry_data_list, int indent, - int *status) -{ - switch (entry_data_list->entry_data.type) { - case MMDB_DATA_TYPE_MAP: - { - uint32_t size = entry_data_list->entry_data.data_size; - - print_indentation(stream, indent); - fprintf(stream, "{\n"); - indent += 2; - - for (entry_data_list = entry_data_list->next; - size && entry_data_list; size--) { - - if (MMDB_DATA_TYPE_UTF8_STRING != - entry_data_list->entry_data.type) { - *status = MMDB_INVALID_DATA_ERROR; - return NULL; - } - char *key = - mmdb_strndup( - (char *)entry_data_list->entry_data.utf8_string, - entry_data_list->entry_data.data_size); - if (NULL == key) { - *status = MMDB_OUT_OF_MEMORY_ERROR; - return NULL; - } - - print_indentation(stream, indent); - fprintf(stream, "\"%s\": \n", key); - free(key); - - entry_data_list = entry_data_list->next; - entry_data_list = - dump_entry_data_list(stream, entry_data_list, indent + 2, - status); - - if (MMDB_SUCCESS != *status) { - return NULL; - } - } - - indent -= 2; - print_indentation(stream, indent); - fprintf(stream, "}\n"); - } - break; - case MMDB_DATA_TYPE_ARRAY: - { - uint32_t size = entry_data_list->entry_data.data_size; - - print_indentation(stream, indent); - fprintf(stream, "[\n"); - indent += 2; - - for (entry_data_list = entry_data_list->next; - size && entry_data_list; size--) { - entry_data_list = - dump_entry_data_list(stream, entry_data_list, indent, - status); - if (MMDB_SUCCESS != *status) { - return NULL; - } - } - - indent -= 2; - print_indentation(stream, indent); - fprintf(stream, "]\n"); - } - break; - case MMDB_DATA_TYPE_UTF8_STRING: - { - char *string = - mmdb_strndup((char *)entry_data_list->entry_data.utf8_string, - entry_data_list->entry_data.data_size); - if (NULL == string) { - *status = MMDB_OUT_OF_MEMORY_ERROR; - return NULL; - } - print_indentation(stream, indent); - fprintf(stream, "\"%s\" <utf8_string>\n", string); - free(string); - entry_data_list = entry_data_list->next; - } - break; - case MMDB_DATA_TYPE_BYTES: - { - char *hex_string = - bytes_to_hex((uint8_t *)entry_data_list->entry_data.bytes, - entry_data_list->entry_data.data_size); - - if (NULL == hex_string) { - *status = MMDB_OUT_OF_MEMORY_ERROR; - return NULL; - } - - print_indentation(stream, indent); - fprintf(stream, "%s <bytes>\n", hex_string); - free(hex_string); - - entry_data_list = entry_data_list->next; - } - break; - case MMDB_DATA_TYPE_DOUBLE: - print_indentation(stream, indent); - fprintf(stream, "%f <double>\n", - entry_data_list->entry_data.double_value); - entry_data_list = entry_data_list->next; - break; - case MMDB_DATA_TYPE_FLOAT: - print_indentation(stream, indent); - fprintf(stream, "%f <float>\n", - entry_data_list->entry_data.float_value); - entry_data_list = entry_data_list->next; - break; - case MMDB_DATA_TYPE_UINT16: - print_indentation(stream, indent); - fprintf(stream, "%u <uint16>\n", entry_data_list->entry_data.uint16); - entry_data_list = entry_data_list->next; - break; - case MMDB_DATA_TYPE_UINT32: - print_indentation(stream, indent); - fprintf(stream, "%u <uint32>\n", entry_data_list->entry_data.uint32); - entry_data_list = entry_data_list->next; - break; - case MMDB_DATA_TYPE_BOOLEAN: - print_indentation(stream, indent); - fprintf(stream, "%s <boolean>\n", - entry_data_list->entry_data.boolean ? "true" : "false"); - entry_data_list = entry_data_list->next; - break; - case MMDB_DATA_TYPE_UINT64: - print_indentation(stream, indent); - fprintf(stream, "%" PRIu64 " <uint64>\n", - entry_data_list->entry_data.uint64); - entry_data_list = entry_data_list->next; - break; - case MMDB_DATA_TYPE_UINT128: - print_indentation(stream, indent); -#if MMDB_UINT128_IS_BYTE_ARRAY - char *hex_string = - bytes_to_hex((uint8_t *)entry_data_list->entry_data.uint128, 16); - if (NULL == hex_string) { - *status = MMDB_OUT_OF_MEMORY_ERROR; - return NULL; - } - fprintf(stream, "0x%s <uint128>\n", hex_string); - free(hex_string); -#else - uint64_t high = entry_data_list->entry_data.uint128 >> 64; - uint64_t low = (uint64_t)entry_data_list->entry_data.uint128; - fprintf(stream, "0x%016" PRIX64 "%016" PRIX64 " <uint128>\n", high, - low); -#endif - entry_data_list = entry_data_list->next; - break; - case MMDB_DATA_TYPE_INT32: - print_indentation(stream, indent); - fprintf(stream, "%d <int32>\n", entry_data_list->entry_data.int32); - entry_data_list = entry_data_list->next; - break; - default: - *status = MMDB_INVALID_DATA_ERROR; - return NULL; - } - - *status = MMDB_SUCCESS; - return entry_data_list; -} - -LOCAL void print_indentation(FILE *stream, int i) -{ - char buffer[1024]; - int size = i >= 1024 ? 1023 : i; - memset(buffer, 32, size); - buffer[size] = '\0'; - fputs(buffer, stream); -} - -LOCAL char *bytes_to_hex(uint8_t *bytes, uint32_t size) -{ - char *hex_string; - MAYBE_CHECK_SIZE_OVERFLOW(size, SIZE_MAX / 2 - 1, NULL); - - hex_string = calloc((size * 2) + 1, sizeof(char)); - if (NULL == hex_string) { - return NULL; - } - - for (uint32_t i = 0; i < size; i++) { - sprintf(hex_string + (2 * i), "%02X", bytes[i]); - } - - - - return hex_string; -} - -const char *MMDB_strerror(int error_code) -{ - switch (error_code) { - case MMDB_SUCCESS: - return "Success (not an error)"; - case MMDB_FILE_OPEN_ERROR: - return "Error opening the specified MaxMind DB file"; - case MMDB_CORRUPT_SEARCH_TREE_ERROR: - return "The MaxMind DB file's search tree is corrupt"; - case MMDB_INVALID_METADATA_ERROR: - return "The MaxMind DB file contains invalid metadata"; - case MMDB_IO_ERROR: - return "An attempt to read data from the MaxMind DB file failed"; - case MMDB_OUT_OF_MEMORY_ERROR: - return "A memory allocation call failed"; - case MMDB_UNKNOWN_DATABASE_FORMAT_ERROR: - return - "The MaxMind DB file is in a format this library can't handle (unknown record size or binary format version)"; - case MMDB_INVALID_DATA_ERROR: - return - "The MaxMind DB file's data section contains bad data (unknown data type or corrupt data)"; - case MMDB_INVALID_LOOKUP_PATH_ERROR: - return - "The lookup path contained an invalid value (like a negative integer for an array index)"; - case MMDB_LOOKUP_PATH_DOES_NOT_MATCH_DATA_ERROR: - return - "The lookup path does not match the data (key that doesn't exist, array index bigger than the array, expected array or map where none exists)"; - case MMDB_INVALID_NODE_NUMBER_ERROR: - return - "The MMDB_read_node function was called with a node number that does not exist in the search tree"; - case MMDB_IPV6_LOOKUP_IN_IPV4_DATABASE_ERROR: - return - "You attempted to look up an IPv6 address in an IPv4-only database"; - default: - return "Unknown error code"; - } -} |