diff options
| author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-03-09 13:19:22 +0000 |
|---|---|---|
| committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-03-09 13:19:22 +0000 |
| commit | c21c3b0befeb46a51b6bf3758ffa30813bea0ff0 (patch) | |
| tree | 9754ff1ca740f6346cf8483ec915d4054bc5da2d /fluent-bit/lib/avro | |
| parent | Adding upstream version 1.43.2. (diff) | |
| download | netdata-c21c3b0befeb46a51b6bf3758ffa30813bea0ff0.tar.xz netdata-c21c3b0befeb46a51b6bf3758ffa30813bea0ff0.zip | |
Adding upstream version 1.44.3.upstream/1.44.3
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'fluent-bit/lib/avro')
167 files changed, 37991 insertions, 0 deletions
diff --git a/fluent-bit/lib/avro/.gitignore b/fluent-bit/lib/avro/.gitignore new file mode 100644 index 00000000..827a3dfe --- /dev/null +++ b/fluent-bit/lib/avro/.gitignore @@ -0,0 +1,3 @@ +build +cscope.* +VERSION.txt diff --git a/fluent-bit/lib/avro/AUTHORS b/fluent-bit/lib/avro/AUTHORS new file mode 100644 index 00000000..3ebe7c3f --- /dev/null +++ b/fluent-bit/lib/avro/AUTHORS @@ -0,0 +1,4 @@ + +See https://avro.apache.org/ for a list of authors + + diff --git a/fluent-bit/lib/avro/CMakeLists.txt b/fluent-bit/lib/avro/CMakeLists.txt new file mode 100644 index 00000000..36b0a028 --- /dev/null +++ b/fluent-bit/lib/avro/CMakeLists.txt @@ -0,0 +1,203 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +cmake_minimum_required(VERSION 3.1) +project(AvroC C) +enable_testing() + +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_SOURCE_DIR}) + +# Eliminates warning about linker paths when linking both zlib and +# liblzma. +cmake_policy(SET CMP0003 NEW) + +#----------------------------------------------------------------------- +# Retrieve the current version number + +if (UNIX) + execute_process(COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/version.sh project + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + RESULT_VARIABLE AVRO_VERSION_RESULT + OUTPUT_VARIABLE AVRO_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE) + if(AVRO_VERSION_RESULT) + message(FATAL_ERROR "Cannot determine Avro version number") + endif(AVRO_VERSION_RESULT) + + execute_process(COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/version.sh libtool + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + RESULT_VARIABLE LIBAVRO_VERSION_RESULT + OUTPUT_VARIABLE LIBAVRO_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE) + if(LIBAVRO_VERSION_RESULT) + message(FATAL_ERROR "Cannot determine libavro version number") + endif(LIBAVRO_VERSION_RESULT) + + execute_process(COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/version.sh libcurrent + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + RESULT_VARIABLE LIBAVRO_SOVERSION_RESULT + OUTPUT_VARIABLE LIBAVRO_SOVERSION + OUTPUT_STRIP_TRAILING_WHITESPACE) + if(LIBAVRO_SOVERSION_RESULT) + message(FATAL_ERROR "Cannot determine libavro version number") + endif(LIBAVRO_SOVERSION_RESULT) + +else(UNIX) + # Hard code for win32 -- need to figure out how to port version.sh for + # Windows. + set(LIBAVRO_VERSION "22:0:0") +endif(UNIX) + + +#----------------------------------------------------------------------- +# Extract major.minor.patch from version number + +if (UNIX) + string(REGEX REPLACE "([0-9]+)\\..*" + "\\1" + AVRO_MAJOR_VERSION + ${AVRO_VERSION} + ) + string(REGEX REPLACE ".*\\.([0-9]+)\\..*" + "\\1" + AVRO_MINOR_VERSION + ${AVRO_VERSION} + ) + string(REGEX REPLACE ".*\\..*\\.([0-9]+).*" + "\\1" + AVRO_PATCH_VERSION + ${AVRO_VERSION} + ) + string(REGEX REPLACE ".*\\..*\\.[0-9]+(.*)" + "\\1" + AVRO_VERSION_EXTENSION + ${AVRO_VERSION} + ) +endif(UNIX) + +#----------------------------------------------------------------------- +# Source package support + +include(CPackConfig.txt) +include(CheckLibraryExists) + + +if(APPLE) + if (NOT CMAKE_INSTALL_NAME_DIR) + set(CMAKE_INSTALL_NAME_DIR "${CMAKE_INSTALL_PREFIX}/lib") + endif (NOT CMAKE_INSTALL_NAME_DIR) +endif(APPLE) + +if(CMAKE_COMPILER_IS_GNUCC) + add_definitions(-W -Wall) +endif(CMAKE_COMPILER_IS_GNUCC) + +if (WIN32) + # Compile win32 in C++ to allow declarations after statements + add_definitions(/TP) +endif(WIN32) + +# Uncomment to allow missing fields in the resolved-writer +# add_definitions(-DAVRO_ALLOW_MISSING_FIELDS_IN_RESOLVED_WRITER) + +# Uncomment to allow non-atomic increment/decrement of reference count +# add_definitions(-DAVRO_ALLOW_NON_ATOMIC_REFCOUNT) + +# Thread support (only for *nix with pthreads) +set(THREADS_LIBRARIES) +if(UNIX AND THREADSAFE AND CMAKE_COMPILER_IS_GNUCC) + set(CMAKE_THREAD_PREFER_PTHREAD) + find_package(Threads) + + if(NOT CMAKE_USE_PTHREADS_INIT) + message(FATAL_ERROR "pthreads not found") + endif(NOT CMAKE_USE_PTHREADS_INIT) + + add_definitions(-DTHREADSAFE -D_REENTRANT) + set(THREADS_LIBRARIES ${CMAKE_THREAD_LIBS_INIT}) +endif(UNIX AND THREADSAFE AND CMAKE_COMPILER_IS_GNUCC) + +include_directories(${AvroC_SOURCE_DIR}/src) + + +# Enable codecs + +find_package(ZLIB) +if (ZLIB_FOUND) + set(ZLIB_PKG zlib) + add_definitions(-DDEFLATE_CODEC) + include_directories(${ZLIB_INCLUDE_DIRS}) + message("Enabled deflate codec") +else (ZLIB_FOUND) + set(ZLIB_PKG "") + set(ZLIB_LIBRARIES "") + message("Disabled deflate codec. zlib not found.") +endif (ZLIB_FOUND) + +find_package(Snappy) +if (SNAPPY_FOUND AND ZLIB_FOUND) # Snappy borrows crc32 from zlib + set(SNAPPY_PKG libsnappy) + add_definitions(-DSNAPPY_CODEC) + include_directories(${SNAPPY_INCLUDE_DIRS}) + message("Enabled snappy codec") +else (SNAPPY_FOUND AND ZLIB_FOUND) + set(SNAPPY_PKG "") + set(SNAPPY_LIBRARIES "") + message("Disabled snappy codec. libsnappy not found or zlib not found.") +endif (SNAPPY_FOUND AND ZLIB_FOUND) + +find_package(PkgConfig) +pkg_check_modules(LZMA liblzma) +if (LZMA_FOUND) + set(LZMA_PKG liblzma) + add_definitions(-DLZMA_CODEC) + include_directories(${LZMA_INCLUDE_DIRS}) + link_directories(${LZMA_LIBRARY_DIRS}) + message("Enabled lzma codec") +else (LZMA_FOUND) + set(LZMA_PKG "") + set(LZMA_LIBRARIES "") + message("Disabled lzma codec. liblzma not found.") +endif (LZMA_FOUND) + +set(CODEC_LIBRARIES ${ZLIB_LIBRARIES} ${LZMA_LIBRARIES} ${SNAPPY_LIBRARIES}) +set(CODEC_PKG "@ZLIB_PKG@ @LZMA_PKG@ @SNAPPY_PKG@") + +# updated by edsiper, avro version 77692d8c1 +# commented out by mcqueen because jansson is now embedded here 07OCT2020 +# Jansson JSON library +#pkg_check_modules(JANSSON jansson>=2.3) +#if (JANSSON_FOUND) +# set(JANSSON_PKG libjansson) +# include_directories(${JANSSON_INCLUDE_DIRS}) +# link_directories(${JANSSON_LIBRARY_DIRS}) +#else (JANSSON_FOUND) +# message(FATAL_ERROR "libjansson >=2.3 not found") +#endif (JANSSON_FOUND) + +# updated by edsiper, avro version 77692d8c1 +# added by mcqueen to get the headers into the fluent-bit build root 07OCT2020 +include_directories(${CMAKE_BINARY_DIR}/lib/jansson/include/) + +add_subdirectory(src) +#add_subdirectory(examples) +#add_subdirectory(tests) +#add_subdirectory(docs) + +add_custom_target(pretty + "${CMAKE_COMMAND}" -P "${CMAKE_CURRENT_SOURCE_DIR}/cmake_pretty.cmake") diff --git a/fluent-bit/lib/avro/CPackConfig.txt b/fluent-bit/lib/avro/CPackConfig.txt new file mode 100644 index 00000000..b27cbed8 --- /dev/null +++ b/fluent-bit/lib/avro/CPackConfig.txt @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +set(CPACK_PACKAGE_NAME "avro-c") +set(CPACK_PACKAGE_DESCRIPTION "C bindings for Avro data serialization framework") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "C bindings for Avro data serialization framework") +set(CPACK_PACKAGE_VENDOR "Apache Software Foundation") +set(CPACK_PACKAGE_DESCRIPTION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/README") +set(CPACK_PACKAGE_VERSION_MAJOR "${AVRO_MAJOR_VERSION}") +set(CPACK_PACKAGE_VERSION_MINOR "${AVRO_MINOR_VERSION}") +set(CPACK_PACKAGE_VERSION_PATCH "${AVRO_PATCH_VERSION}${AVRO_VERSION_EXTENSION}") +set(CPACK_PACKAGE_VERSION "${AVRO_VERSION}") +set(CPACK_PACKAGE_CONTACT "avro-dev@apache.org") +set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE") +set(CPACK_STRIP_FILES true) + +set(CPACK_SOURCE_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}") +set(CPACK_PACKAGE_INSTALL_DIRECTORY "/usr") +set(CPACK_GENERATOR "TGZ") +set(CPACK_SOURCE_GENERATOR "TGZ") + +set(CPACK_SOURCE_IGNORE_FILES "^${CMAKE_BINARY_DIR};/\\\\.gitignore;/\\\\.svn;\\\\.swp$;\\\\.#;/#;.*~") + +include(CPack) diff --git a/fluent-bit/lib/avro/ChangeLog b/fluent-bit/lib/avro/ChangeLog new file mode 100644 index 00000000..fd40910d --- /dev/null +++ b/fluent-bit/lib/avro/ChangeLog @@ -0,0 +1,4 @@ + + + + diff --git a/fluent-bit/lib/avro/FindSnappy.cmake b/fluent-bit/lib/avro/FindSnappy.cmake new file mode 100644 index 00000000..5f158fdc --- /dev/null +++ b/fluent-bit/lib/avro/FindSnappy.cmake @@ -0,0 +1,54 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Tries to find Snappy headers and libraries. +# +# Usage of this module as follows: +# +# find_package(Snappy) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# SNAPPY_ROOT_DIR Set this variable to the root installation of +# Snappy if the module has problems finding +# the proper installation path. +# +# Variables defined by this module: +# +# SNAPPY_FOUND System has Snappy libs/headers +# SNAPPY_LIBRARIES The Snappy libraries +# SNAPPY_INCLUDE_DIR The location of Snappy headers + +find_path(SNAPPY_INCLUDE_DIR + NAMES snappy.h + HINTS ${SNAPPY_ROOT_DIR}/include) + +find_library(SNAPPY_LIBRARIES + NAMES snappy + HINTS ${SNAPPY_ROOT_DIR}/lib) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Snappy DEFAULT_MSG + SNAPPY_LIBRARIES + SNAPPY_INCLUDE_DIR) + +mark_as_advanced( + SNAPPY_ROOT_DIR + SNAPPY_LIBRARIES + SNAPPY_INCLUDE_DIR) diff --git a/fluent-bit/lib/avro/INSTALL b/fluent-bit/lib/avro/INSTALL new file mode 100644 index 00000000..6cffb4b4 --- /dev/null +++ b/fluent-bit/lib/avro/INSTALL @@ -0,0 +1,69 @@ +Installation instructions +========================= + +The Avro library is written in ANSI C. It uses CMake [1] as its build +manager. + +[1] https://www.cmake.org/ + + +Prerequisites +------------- + +Avro uses CMake [1] as its build manager. You must have this installed, +along with a C compiler (such as gcc) to build the library. + +Avro depends on the Jansson JSON parser [2], version 2.3 or higher. On +many operating systems this library is available through your package +manager (for example, `apt-get install libjansson-dev` on Ubuntu/Debian, +and `brew install jansson` on Mac OS). If not, please download and +install it from source [2]. + +To build the documentation you need asciidoc [3] and source-highlight [4] +installed. The build scripts will automatically detect whether these +tools are installed, and will skip the documentation if they're not. + +[1] https://www.cmake.org/ +[2] http://www.digip.org/jansson/ +[3] https://www.methods.co.nz/asciidoc +[4] https://www.gnu.org/software/src-highlite/ + + +Building from source +-------------------- + +The Avro C library uses CMake as its build manager. In most cases, you +should be able to build the source code by running the following: + + $ mkdir build + $ cd build + $ cmake .. \ + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo + $ make + $ make test + $ make install + +You might have to run the last command using sudo, if you need +administrative privileges to write to the $PREFIX directory. + +The "RelWithDebInfo" build type will build an optimized copy of the +library, including debugging symbols. Use the "Release" build type if +you don't want debugging symbols. Use the "Debug" build type if you +want a non-optimized library, with debugging symbols. + +On Mac OS X, you can build a universal binary by setting the +CMAKE_OSX_ARCHITECTURES option when running cmake. Just add something +like the following: + + -DCMAKE_OSX_ARCHITECTURES=i386;x86_64 + +to the cmake command given above. The set of architectures that you can +choose from differs depending on which version of OS X you're running; +common possibilities include "i386", "x86_64", "ppc", and "ppc64". + +On Unix, you can request thread-safe versions of the Avro library's +global functions by defining the THREADSAFE cmake variable. Just add +the following to your cmake invokation: + + -DTHREADSAFE=true diff --git a/fluent-bit/lib/avro/LICENSE b/fluent-bit/lib/avro/LICENSE new file mode 100644 index 00000000..a7cd64e8 --- /dev/null +++ b/fluent-bit/lib/avro/LICENSE @@ -0,0 +1,266 @@ + + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License for msinttypes.h and msstdint.h used in the C implementation: +Source from: +https://code.google.com/p/msinttypes/downloads/detail?name=msinttypes-r26.zip + +Copyright (c) 2006-2008 Alexander Chemeris + +| Redistribution and use in source and binary forms, with or without +| modification, are permitted provided that the following conditions are met: +| +| 1. Redistributions of source code must retain the above copyright notice, +| this list of conditions and the following disclaimer. +| +| 2. Redistributions in binary form must reproduce the above copyright +| notice, this list of conditions and the following disclaimer in the +| documentation and/or other materials provided with the distribution. +| +| 3. The name of the author may be used to endorse or promote products +| derived from this software without specific prior written permission. +| +| THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED +| WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +| MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +| EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +| SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +| PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +| OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +| WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +| OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +| ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License for st.c and st.h used in the C implementation: + +| This is a public domain general purpose hash table package written by +| Peter Moore @ UCB. + +---------------------------------------------------------------------- +License for Dirent API for Microsoft Visual Studio used in the C implementation: +Source from: +http://www.softagalleria.net/download/dirent/dirent-1.11.zip + +Copyright (C) 2006 Toni Ronkko + +| Permission is hereby granted, free of charge, to any person obtaining +| a copy of this software and associated documentation files (the +| ``Software''), to deal in the Software without restriction, including +| without limitation the rights to use, copy, modify, merge, publish, +| distribute, sublicense, and/or sell copies of the Software, and to +| permit persons to whom the Software is furnished to do so, subject to +| the following conditions: +| +| The above copyright notice and this permission notice shall be included +| in all copies or substantial portions of the Software. +| +| THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS +| OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +| MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +| IN NO EVENT SHALL TONI RONKKO BE LIABLE FOR ANY CLAIM, DAMAGES OR +| OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +| ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +| OTHER DEALINGS IN THE SOFTWARE. + diff --git a/fluent-bit/lib/avro/NEWS b/fluent-bit/lib/avro/NEWS new file mode 100644 index 00000000..415280b0 --- /dev/null +++ b/fluent-bit/lib/avro/NEWS @@ -0,0 +1,5 @@ + +For news, visit the Avro web site at +https://avro.apache.org/ + + diff --git a/fluent-bit/lib/avro/NOTICE b/fluent-bit/lib/avro/NOTICE new file mode 100644 index 00000000..5562db84 --- /dev/null +++ b/fluent-bit/lib/avro/NOTICE @@ -0,0 +1,6 @@ +Apache Avro +Copyright 2010 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (https://www.apache.org/). + diff --git a/fluent-bit/lib/avro/README b/fluent-bit/lib/avro/README new file mode 100644 index 00000000..442415ea --- /dev/null +++ b/fluent-bit/lib/avro/README @@ -0,0 +1,10 @@ + ___ ______ + / |_ ___________ / ____/ + / /| | | / / ___/ __ \ / / + / ___ | |/ / / / /_/ / / /___ +/_/ |_|___/_/ \____/ \____/ + +====================================================== +Please see the INSTALL file for installation instructions, and the +documentation in the "docs" directory for more details about the +library. diff --git a/fluent-bit/lib/avro/README.maintaining_win32.txt b/fluent-bit/lib/avro/README.maintaining_win32.txt new file mode 100644 index 00000000..b603b1ec --- /dev/null +++ b/fluent-bit/lib/avro/README.maintaining_win32.txt @@ -0,0 +1,166 @@ +Win32 C++ builds of Avro-C +***************************** + +April 2, 2012 + +These instructions describe some of the changes required to allow +Avro-C to compile under the Microsoft Visual C++ 2008 compiler, as +well as some limitations of the Windows build. + +Status of the Windows Build: +**************************** + +The Windows build of Avro-C compiles under Microsoft Visual C++ +2008. You can use C-Make to create the solution file (AvroC.sln) for +the build. The solution file contains projects for the build as well +as projects for the tests. All the tests are run and pass when the +project RUN_TESTS is built from within MS Visual C++ 2008. + +Limitations of Windows Build: +****************************** + +1. The Windows build of Avro-C is compiled using Microsoft's C++ + compiler and not Microsoft's C compiler. This is done using the /TP + flag in the MSVC++ compiler. This flag is automatically set using + CMakeLists.txt + + The reason to compile Win32 under C++ instead of C is that there + are lots of places where variables are declared after statements, + the Microsoft C compiler does not support declarations after + statements. It is possible, that if all the declarations after + statements were removed, that Avro-C would compile under + Microsoft's C compiler also. I have not tried this. + +2. The shared library, i.e. DLL, for avro has not been built. There + are instructions on how to build DLLs using CMake at + https://www.cmake.org/Wiki/BuildingWinDLL + +3. Currently avropipe.c and avromod.c do not compile under Windows. In + order for them to compile we would have to either port getopt() to + Windows, or remove their dependency on getopt(). + +4. Windows cannot run version.sh to generate the version + number. Currently, LIBAVRO_VERSION is hard coded to "22:0:0" for + the Windows build, in the top level CMakeLists.txt. + +5. Found two bugs related to improper return values under error + conditions in Avro. These bugs were marked with #warnings in the + code. + + +Instructions for Maintenance +***************************** + +1. Instructions to check name mangling in Visual C++: + + In a DOS prompt go to "C:\Program Files(x86)\Microsoft Visual Studio 9.0\VC\" + Run the program vcvarsall.bat . This will set up environment variables. + + Now go to the avro_c\build_win32\src\Debug\ directory. + Run the command + + dumpbin /ALL avro.lib > tmp.txt + + View tmp.txt in your favorite editor. This will allow you to see + which names are mangled and which names are not mangled. + + Every header file should start with + + #ifndef HEADER_FILE_NAME_H + #define HEADER_FILE_NAME_H + #ifdef __cplusplus + extern "C" { + #define CLOSE_EXTERN } + #else + #define CLOSE_EXTERN + #endif + + and end with + + CLOSE_EXTERN + #endif /* HEADER_FILE_NAME_H */ + + This will ensure that all exported (public) functions are mangled + using C name mangling instead of C++ name mangling. + +2. All file I/O operations should have "b" for binary in the fopen + statements. Otherwise Windows will replace LF with CRLF in binary + data. + +3. Windows does not allow writing to a file with exclusive access + using the mode "wbx". Use the non-exclusive mode "wb" instead. + +4. If the hashtable from st.c is used, the functions in the struct + st_hash_type should be cast to HASH_FUNCTION_CAST. + +5. Do not use "%zu" to print size_t. Use '"%" PRIsz' without the + single quotes, instead. + +6. MS Visual C++ 2008 does not properly expand variadic preprocessor + macros by default. It is possible to "trick" MSVC++ to properly + expand variadic preprocessor macros by using an extra (dummy) + preprocessor macro, whose only purpose is to properly expand its + input. This method is described here: + + https://stackoverflow.com/questions/2575864/the-problem-about-different-treatment-to-va-args-when-using-vs-2008-and-gcc + See the solution described by monkeyman. + + This method is used in the macro expand_args(...) in test_avro_values.c. + +7. Web site describing how to debug macro expansion in Visual C++: + http://fneep.fiffa.net/?p=66 + +8. Sometimes it is necessary to declare a struct at the top of a file + and define it at the bottom of a file. An example is + AVRO_DATUM_VALUE_CLASS in src/datum_value.c. A C++ compiler will + complain that the struct is defined twice. To avoid this, declare + the struct with the modifier "extern" at the top of the file, and + then define it at the bottom of the file. Note that it cannot be + defined as "static" because Win32 does not like an extern struct + mapping to a static struct. + +9. Use __FUNCTION__ instead of __func__ for generating the function + name. + +10. All variables need to be explicitly cast when calling functions + with differing function signatures + +11. All pointers need to be explicitly cast when assigning to other + pointers of a different type. + +12. Do not perform pointer arithmetic on void * pointers. Cast the + pointers to char * before performing pointer arithmetic. + +13. The executable names of any examples and tests need to be set + explicitly to include the "Debug/" directory in the path, and the + ".exe" ending. See the CMakeLists.txt in the examples and the + tests directory to see how this is done. + +14. Do not include the headers inttypes.h or unistd.h or + stdint.h. Instead include avro/platform.h in your C files. + +15. Do not include dirent.h in your tests. When _WIN32 is defined + include msdirent.h. See example in test_avro_schema.c. + +16. If _WIN32 is defined, define snprintf() to_snprintf(), which MS + Visual C++ recognizes. + +17. MSVC++ does not recognize strtoll(). Define it to _strtoi64() + instead. + +18. Old-style C function declarations are not allowed in C++. See the + changes in st.c and st.h -- which were converted to new-style + function declarations. + +19. Structures cannot be initialized using the .element notation for + Win32. For example if we have a struct test_t: + typedef struct + { + int a; + int b; + } test_t; + Then we can initialize the struct using the syntax: + test_t t1 = { 0, 0 }; + But we cannot use the syntax: + test_t t2 = { .a = 0, . b = 0 }; + because Win32 does not support it. diff --git a/fluent-bit/lib/avro/build.sh b/fluent-bit/lib/avro/build.sh new file mode 100644 index 00000000..6753e778 --- /dev/null +++ b/fluent-bit/lib/avro/build.sh @@ -0,0 +1,103 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -e # exit on error + +root_dir=$(pwd) +build_dir="../../build/c" +dist_dir="../../dist/c" +version=$(./version.sh project) +tarball="avro-c-$version.tar.gz" +doc_dir="../../build/avro-doc-$version/api/c" + +function prepare_build { + clean + mkdir -p $build_dir + (cd $build_dir && cmake $root_dir -DCMAKE_BUILD_TYPE=RelWithDebInfo) +} + +function clean { + if [ -d $build_dir ]; then + find $build_dir | xargs chmod 755 + rm -rf $build_dir + fi + rm -f VERSION.txt + rm -f examples/quickstop.db +} + +for target in "$@" +do + + case "$target" in + + interop-data-generate) + prepare_build + make -C $build_dir + $build_dir/tests/generate_interop_data "../../share/test/schemas/interop.avsc" "../../build/interop/data" + ;; + + interop-data-test) + prepare_build + make -C $build_dir + $build_dir/tests/test_interop_data "../../build/interop/data" + ;; + + lint) + echo 'This is a stub where someone can provide linting.' + ;; + + test) + prepare_build + make -C $build_dir + make -C $build_dir test + ;; + + dist) + prepare_build + cp ../../share/VERSION.txt $root_dir + make -C $build_dir docs + # This is a hack to force the built documentation to be included + # in the source package. + cp $build_dir/docs/*.html $root_dir/docs + make -C $build_dir package_source + rm $root_dir/docs/*.html + if [ ! -d $dist_dir ]; then + mkdir -p $dist_dir + fi + if [ ! -d $doc_dir ]; then + mkdir -p $doc_dir + fi + mv $build_dir/$tarball $dist_dir + cp $build_dir/docs/*.html $doc_dir + clean + ;; + + clean) + clean + ;; + + *) + echo "Usage: $0 {interop-data-generate|interop-data-test|lint|test|dist|clean}" + exit 1 + esac + +done + +exit 0 diff --git a/fluent-bit/lib/avro/cmake_avrolib.bat b/fluent-bit/lib/avro/cmake_avrolib.bat new file mode 100644 index 00000000..76934bca --- /dev/null +++ b/fluent-bit/lib/avro/cmake_avrolib.bat @@ -0,0 +1,48 @@ +REM Licensed to the Apache Software Foundation (ASF) under one
+REM or more contributor license agreements. See the NOTICE file
+REM distributed with this work for additional information
+REM regarding copyright ownership. The ASF licenses this file
+REM to you under the Apache License, Version 2.0 (the
+REM "License"); you may not use this file except in compliance
+REM with the License. You may obtain a copy of the License at
+REM
+REM https://www.apache.org/licenses/LICENSE-2.0
+REM
+REM Unless required by applicable law or agreed to in writing,
+REM software distributed under the License is distributed on an
+REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+REM KIND, either express or implied. See the License for the
+REM specific language governing permissions and limitations
+REM under the License.
+
+echo off
+
+REM Set up the solution file in Windows.
+
+set my_cmake_path="put_your_cmake_path_here"
+set cmake_path_win7="C:\Program Files (x86)\CMake 2.8\bin\cmake.exe"
+set cmake_path_xp="C:\Program Files\CMake 2.8\bin\cmake.exe"
+
+if exist %my_cmake_path% (
+ set cmake_path=%my_cmake_path%
+ goto RUN_CMAKE
+)
+
+if exist %cmake_path_win7% (
+ set cmake_path=%cmake_path_win7%
+ goto RUN_CMAKE
+)
+
+if exist %cmake_path_xp% (
+ set cmake_path=%cmake_path_xp%
+ goto RUN_CMAKE
+)
+
+echo "Set the proper cmake path in the variable 'my_cmake_path' in cmake_windows.bat, and re-run"
+goto EXIT_ERROR
+
+:RUN_CMAKE
+%cmake_path% -G"Visual Studio 9 2008" -H. -Bbuild_win32
+
+
+:EXIT_ERROR
diff --git a/fluent-bit/lib/avro/cmake_avrolib.sh b/fluent-bit/lib/avro/cmake_avrolib.sh new file mode 100644 index 00000000..13209581 --- /dev/null +++ b/fluent-bit/lib/avro/cmake_avrolib.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +mkdir -p build +cd build +cmake .. -DCMAKE_INSTALL_PREFIX=avrolib -DCMAKE_BUILD_TYPE=Debug +make +make test +make install +mkdir -p avrolib/lib/static +cp avrolib/lib/libavro.a avrolib/lib/static/libavro.a diff --git a/fluent-bit/lib/avro/cmake_pretty.cmake b/fluent-bit/lib/avro/cmake_pretty.cmake new file mode 100644 index 00000000..9f328f8e --- /dev/null +++ b/fluent-bit/lib/avro/cmake_pretty.cmake @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Linux kernel source indent format options +set(INDENT_OPTS -nbad -bap -nbc -bbo -hnl -br -brs -c33 -cd33 -ncdb -ce -ci4 + -cli0 -d0 -di1 -nfc1 -i8 -ip0 -l80 -lp -npcs -nprs -npsl -sai + -saf -saw -ncs -nsc -sob -nfca -cp33 -ss -ts8) + +foreach($dir src tests examples) + exec_program(indent + ARGS ${INDENT_OPTS} ${CMAKE_CURRENT_SOURCE_DIR}/${dir}/*.[c,h] + OUTPUT_VARIABLE indent_output + RETURN_VALUE ret) + message(STATUS ${indent_output}) + # TODO: mv ${CMAKE_CURRENT_SOURCE_DIR}/${dir}/*~ /tmp; \ +endforeach() + diff --git a/fluent-bit/lib/avro/docs/CMakeLists.txt b/fluent-bit/lib/avro/docs/CMakeLists.txt new file mode 100644 index 00000000..98e770c9 --- /dev/null +++ b/fluent-bit/lib/avro/docs/CMakeLists.txt @@ -0,0 +1,53 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set (AVRO_DOC_SRC + index.txt +) + +# TODO(dln): Use FindAsciidoc script instead. +message(STATUS "Searching for asciidoc...") +find_program(ASCIIDOC_EXECUTABLE asciidoc) +find_program(SOURCE_HIGHLIGHT_EXECUTABLE source-highlight) + +if (ASCIIDOC_EXECUTABLE AND SOURCE_HIGHLIGHT_EXECUTABLE) + foreach(_file ${AVRO_DOC_SRC}) + get_filename_component(_file_we ${_file} NAME_WE) + set(_file_path "${CMAKE_CURRENT_SOURCE_DIR}/${_file}") + set(_html_out "${CMAKE_CURRENT_BINARY_DIR}/${_file_we}.html") + add_custom_command( + OUTPUT "${_html_out}" + COMMAND ${ASCIIDOC_EXECUTABLE} + -a avro_version=${AVRO_VERSION} + -a libavro_version=${LIBAVRO_VERSION} + -a toc + --unsafe -n -o "${_html_out}" "${_file_path}" + DEPENDS "${_file_path}" + COMMENT "asciidoc ${_file}" + ) + install(FILES "${_html_out}" DESTINATION share/doc/avro-c) + add_custom_target("${_file_we}_html" ALL echo -n + DEPENDS "${_file}" "${_html_out}" + ) + add_custom_target(docs DEPENDS "${_html_out}") + endforeach(_file) +else(ASCIIDOC_EXECUTABLE AND SOURCE_HIGHLIGHT_EXECUTABLE) + message(WARNING "asciidoc and/or source-highlight not found. HTML documentation will *NOT* be built.") +endif(ASCIIDOC_EXECUTABLE AND SOURCE_HIGHLIGHT_EXECUTABLE) + diff --git a/fluent-bit/lib/avro/docs/index.txt b/fluent-bit/lib/avro/docs/index.txt new file mode 100644 index 00000000..a439a052 --- /dev/null +++ b/fluent-bit/lib/avro/docs/index.txt @@ -0,0 +1,758 @@ +Avro C +====== + +The current version of Avro is +{avro_version}+. The current version of +libavro+ is +{libavro_version}+. +This document was created +{docdate}+. + +== Introduction to Avro + +Avro is a data serialization system. + +Avro provides: + +* Rich data structures. +* A compact, fast, binary data format. +* A container file, to store persistent data. +* Remote procedure call (RPC). + +This document will focus on the C implementation of Avro. To learn more about +Avro in general, https://avro.apache.org/[visit the Avro website]. + +== Introduction to Avro C + +.... + ___ ______ + / |_ ___________ / ____/ + / /| | | / / ___/ __ \ / / + / ___ | |/ / / / /_/ / / /___ +/_/ |_|___/_/ \____/ \____/ + +.... + +[quote,Waldi Ravens,(walra%moacs11 @ nl.net) 94/03/18] +____ +A C program is like a fast dance on a newly waxed dance floor by people carrying razors. +____ + +The C implementation has been tested on +MacOSX+ and +Linux+ but, over +time, the number of support OSes should grow. Please let us know if +you're using +Avro C+ on other systems. + +Avro depends on the http://www.digip.org/jansson/[Jansson JSON parser], +version 2.3 or higher. On many operating systems this library is +available through your package manager (for example, ++apt-get install libjansson-dev+ on Ubuntu/Debian, and ++brew install jansson+ on Mac OS). If not, please download and install +it from source. + +The C implementation supports: + +* binary encoding/decoding of all primitive and complex data types +* storage to an Avro Object Container File +* schema resolution, promotion and projection +* validating and non-validating mode for writing Avro data + +The C implementation is lacking: + +* RPC + +To learn about the API, take a look at the examples and reference files +later in this document. + +We're always looking for contributions so, if you're a C hacker, please +feel free to https://avro.apache.org/[submit patches to the project]. + + +== Error reporting + +Most functions in the Avro C library return a single +int+ status code. +Following the POSIX _errno.h_ convention, a status code of 0 indicates +success. Non-zero codes indicate an error condition. Some functions +return a pointer value instead of an +int+ status code; for these +functions, a +NULL+ pointer indicates an error. + +You can retrieve +a string description of the most recent error using the +avro_strerror+ +function: + +[source,c] +---- +avro_schema_t schema = avro_schema_string(); +if (schema == NULL) { + fprintf(stderr, "Error was %s\n", avro_strerror()); +} +---- + + +== Avro values + +Starting with version 1.6.0, the Avro C library has a new API for +handling Avro data. To help distinguish between the two APIs, we refer +to the old one as the _legacy_ or _datum_ API, and the new one as the +_value_ API. (These names come from the names of the C types used to +represent Avro data in the corresponding API — +avro_datum_t+ and ++avro_value_t+.) The legacy API is still present, but it's deprecated — +you shouldn't use the +avro_datum_t+ type or the +avro_datum_*+ +functions in new code. + +One main benefit of the new value API is that you can treat any existing +C type as an Avro value; you just have to provide a custom +implementation of the value interface. In addition, we provide a +_generic_ value implementation; “generic”, in this sense, meaning that +this single implementation works for instances of any Avro schema type. +Finally, we also provide a wrapper implementation for the deprecated ++avro_datum_t+ type, which lets you gradually transition to the new +value API. + + +=== Avro value interface + +You interact with Avro values using the _value interface_, which defines +methods for setting and retrieving the contents of an Avro value. An +individual value is represented by an instance of the +avro_value_t+ +type. + +This section provides an overview of the methods that you can call on an ++avro_value_t+ instance. There are quite a few methods in the value +interface, but not all of them make sense for all Avro schema types. +For instance, you won't be able to call +avro_value_set_boolean+ on an +Avro array value. If you try to call an inappropriate method, we'll +return an +EINVAL+ error code. + +Note that the functions in this section apply to _all_ Avro values, +regardless of which value implementation is used under the covers. This +section doesn't describe how to _create_ value instances, since those +constructors will be specific to a particular value implementation. + + +==== Common methods + +There are a handful of methods that can be used with any value, +regardless of which Avro schema it's an instance of: + +[source,c] +---- +#include <stdint.h> +#include <avro.h> + +avro_type_t avro_value_get_type(const avro_value_t *value); +avro_schema_t avro_value_get_schema(const avro_value_t *value); + +int avro_value_equal(const avro_value_t *v1, const avro_value_t *v2); +int avro_value_equal_fast(const avro_value_t *v1, const avro_value_t *v2); + +int avro_value_copy(avro_value_t *dest, const avro_value_t *src); +int avro_value_copy_fast(avro_value_t *dest, const avro_value_t *src); + +uint32_t avro_value_hash(avro_value_t *value); + +int avro_value_reset(avro_value_t *value); +---- + +The +get_type+ and +get_schema+ methods can be used to get information +about what kind of Avro value a given +avro_value_t+ instance +represents. (For +get_schema+, you borrow the value's reference to the +schema; if you need to save it and ensure that it outlives the value, +you need to call +avro_schema_incref+ on it.) + +The +equal+ and +equal_fast+ methods compare two values for equality. +The two values do _not_ have to have the same value implementations, but +they _do_ have to be instances of the same schema. (Not _equivalent_ +schemas; the _same_ schema.) The +equal+ method checks that the schemas +match; the +equal_fast+ method assumes that they do. + +The +copy+ and +copy_fast+ methods copy the contents of one Avro value +into another. (Where possible, this is done without copying the actual +content of a +bytes+, +string+, or +fixed+ value, using the ++avro_wrapped_buffer_t+ functions described in the next section.) Like ++equal+, the two values must have the same schema; +copy+ checks this, +while +copy_fast+ assumes it. + +The +hash+ method returns a hash value for the given Avro value. This +can be used to construct hash tables that use Avro values as keys. The +function works correctly even with maps; it produces a hash that doesn't +depend on the ordering of the elements of the map. Hash values are only +meaningful for comparing values of exactly the same schema. Hash values +are _not_ guaranteed to be consistent across different platforms, or +different versions of the Avro library. That means that it's really +only safe to use these hash values internally within the context of a +single execution of a single application. + +The +reset+ method “clears out” an +avro_value_t instance, making sure +that it's ready to accept the contents of a new value. For scalars, +this is usually a no-op, since the new value will just overwrite the old +one. For arrays and maps, this removes any existing elements from the +container, so that we can append the elements of the new value. For +records and unions, this just recursively resets the fields or current +branch. + + +==== Scalar values + +The simplest case is handling instances of the scalar Avro schema types. +In Avro, the scalars are all of the primitive schema types, as well as ++enum+ and +fixed+ — i.e., anything that can't contain another Avro +value. Note that we use standard C99 types to represent the primitive +contents of an Avro scalar. + +To retrieve the contents of an Avro scalar, you can use one of the +_getter_ methods: + +[source,c] +---- +#include <stdint.h> +#include <stdlib.h> +#include <avro.h> + +int avro_value_get_boolean(const avro_value_t *value, int *dest); +int avro_value_get_bytes(const avro_value_t *value, + const void **dest, size_t *size); +int avro_value_get_double(const avro_value_t *value, double *dest); +int avro_value_get_float(const avro_value_t *value, float *dest); +int avro_value_get_int(const avro_value_t *value, int32_t *dest); +int avro_value_get_long(const avro_value_t *value, int64_t *dest); +int avro_value_get_null(const avro_value_t *value); +int avro_value_get_string(const avro_value_t *value, + const char **dest, size_t *size); +int avro_value_get_enum(const avro_value_t *value, int *dest); +int avro_value_get_fixed(const avro_value_t *value, + const void **dest, size_t *size); +---- + +For the most part, these should be self-explanatory. For +bytes+, ++string+, and +fixed+ values, the pointer to the underlying content is ++const+ — you aren't allowed to modify the contents directly. We +guarantee that the content of a +string+ will be NUL-terminated, so you +can use it as a C string as you'd expect. The +size+ returned for a ++string+ object will include the NUL terminator; it will be one more +than you'd get from calling +strlen+ on the content. + +Also, for +bytes+, +string+, and +fixed+, the +dest+ and +size+ +parameters are optional; if you only want to determine the length of a ++bytes+ value, you can use: + +[source,c] +---- +avro_value_t *value = /* from somewhere */; +size_t size; +avro_value_get_bytes(value, NULL, &size); +---- + +To set the contents of an Avro scalar, you can use one of the _setter_ +methods: + +[source,c] +---- +#include <stdint.h> +#include <stdlib.h> +#include <avro.h> + +int avro_value_set_boolean(avro_value_t *value, int src); +int avro_value_set_bytes(avro_value_t *value, + void *buf, size_t size); +int avro_value_set_double(avro_value_t *value, double src); +int avro_value_set_float(avro_value_t *value, float src); +int avro_value_set_int(avro_value_t *value, int32_t src); +int avro_value_set_long(avro_value_t *value, int64_t src); +int avro_value_set_null(avro_value_t *value); +int avro_value_set_string(avro_value_t *value, const char *src); +int avro_value_set_string_len(avro_value_t *value, + const char *src, size_t size); +int avro_value_set_enum(avro_value_t *value, int src); +int avro_value_set_fixed(avro_value_t *value, + void *buf, size_t size); +---- + +These are also straightforward. For +bytes+, +string+, and +fixed+ +values, the +set+ methods will make a copy of the underlying data. For ++string+ values, the content must be NUL-terminated. You can use ++set_string_len+ if you already know the length of the string content; +the length you pass in should include the NUL terminator. If you call ++set_string+, then we'll use +strlen+ to calculate the length. + +For +fixed+ values, the +size+ must match what's expected by the value's +underlying +fixed+ schema; if the sizes don't match, you'll get an error +code. + +If you don't want to copy the contents of a +bytes+, +string+, or ++fixed+ value, you can use the _giver_ and _grabber_ functions: + +[source,c] +---- +#include <stdint.h> +#include <stdlib.h> +#include <avro.h> + +typedef void +(*avro_buf_free_t)(void *ptr, size_t sz, void *user_data); + +int avro_value_give_bytes(avro_value_t *value, avro_wrapped_buffer_t *src); +int avro_value_give_string_len(avro_value_t *value, avro_wrapped_buffer_t *src); +int avro_value_give_fixed(avro_value_t *value, avro_wrapped_buffer_t *src); + +int avro_value_grab_bytes(const avro_value_t *value, avro_wrapped_buffer_t *dest); +int avro_value_grab_string(const avro_value_t *value, avro_wrapped_buffer_t *dest); +int avro_value_grab_fixed(const avro_value_t *value, avro_wrapped_buffer_t *dest); + +typedef struct avro_wrapped_buffer { + const void *buf; + size_t size; + void (*free)(avro_wrapped_buffer_t *self); + int (*copy)(avro_wrapped_buffer_t *dest, + const avro_wrapped_buffer_t *src, + size_t offset, size_t length); + int (*slice)(avro_wrapped_buffer_t *self, + size_t offset, size_t length); +} avro_wrapped_buffer_t; + +void +avro_wrapped_buffer_free(avro_wrapped_buffer_t *buf); + +int +avro_wrapped_buffer_copy(avro_wrapped_buffer_t *dest, + const avro_wrapped_buffer_t *src, + size_t offset, size_t length); + +int +avro_wrapped_buffer_slice(avro_wrapped_buffer_t *self, + size_t offset, size_t length); +---- + +The +give+ functions give control of an existing buffer to the value. +(You should *not* try to free the +src+ wrapped buffer after calling +this method.) The +grab+ function fills in a wrapped buffer with a +pointer to the contents of an Avro value. (You *should* free the +dest+ +wrapped buffer when you're done with it.) + +The +avro_wrapped_buffer_t+ struct encapsulates the location and size of +the existing buffer. It also includes several methods. The +free+ +method will be called when the content of the buffer is no longer +needed. The +slice+ method will be called when the wrapped buffer needs +to be updated to point at a subset of what it pointed at before. (This +doesn't create a new wrapped buffer; it updates an existing one.) The ++copy+ method will be called if the content needs to be copied. Note +that if you're wrapping a buffer with nice reference counting features, +you don't need to perform an actual copy; you just need to ensure that +the +free+ function can be called on both the original and the copy, and +not have things blow up. + +The “generic” value implementation takes advantage of this feature; if +you pass in a wrapped buffer with a +give+ method, and then retrieve it +later with a +grab+ method, then we'll use the wrapped buffer's +copy+ +method to fill in the +dest+ parameter. If your wrapped buffer +implements a +slice+ method that updates reference counts instead of +actually copying, then you've got nice zero-copy access to the contents +of an Avro value. + + +==== Compound values + +The following sections describe the getter and setter methods for +handling compound Avro values. All of the compound values are +responsible for the storage of their children; this means that there +isn't a method, for instance, that lets you add an existing ++avro_value_t+ to an array. Instead, there's a method that creates a +new, empty +avro_value_t+ of the appropriate type, adds it to the array, +and returns it for you to fill in as needed. + +You also shouldn't try to free the child elements that are created this +way; the container value is responsible for their life cycle. The child +element is guaranteed to be valid for as long as the container value +is. You'll usually define an +avro_value_t+ in the stack, and let it +fall out of scope when you're done with it: + +[source,c] +---- +avro_value_t *array = /* from somewhere else */; + +{ + avro_value_t child; + avro_value_get_by_index(array, 0, &child, NULL); + /* do something interesting with the array element */ +} +---- + + +==== Arrays + +There are three methods that can be used with array values: + +[source,c] +---- +#include <stdlib.h> +#include <avro.h> + +int avro_value_get_size(const avro_value_t *array, size_t *size); +int avro_value_get_by_index(const avro_value_t *array, size_t index, + avro_value_t *element, const char **unused); +int avro_value_append(avro_value_t *array, avro_value_t *element, + size_t *new_index); +---- + +The +get_size+ method returns the number of elements currently in the +array. The +get_by_index+ method fills in +element+ to point at the +array element with the given index. (You should use +NULL+ for the ++unused+ parameter; it's ignored for array values.) + +The +append+ method creates a new value, appends it to the array, and +returns it in +element+. If +new_index+ is given, then it will be +filled in with the index of the new element. + + +==== Maps + +There are four methods that can be used with map values: + +[source,c] +---- +#include <stdlib.h> +#include <avro.h> + +int avro_value_get_size(const avro_value_t *map, size_t *size); +int avro_value_get_by_name(const avro_value_t *map, const char *key, + avro_value_t *element, size_t *index); +int avro_value_get_by_index(const avro_value_t *map, size_t index, + avro_value_t *element, const char **key); +int avro_value_add(avro_value_t *map, + const char *key, avro_value_t *element, + size_t *index, int *is_new); +---- + +The +get_size+ method returns the number of elements currently in the +map. Map elements can be retrieved either by their key (+get_by_name+) +or by their numeric index (+get_by_index+). (Numeric indices in a map +are based on the order that the elements were added to the map.) In +either case, the method takes in an optional output parameter that let +you retrieve the index associated with a key, and vice versa. + +The +add+ method will add a new value to the map, if the given key isn't +already present. If the key is present, then the existing value with be +returned. The +index+ parameter, if given, will be filled in the +element's index. The +is_new+ parameter, if given, can be used to +determine whether the mapped value is new or not. + + +==== Records + +There are three methods that can be used with record values: + +[source,c] +---- +#include <stdlib.h> +#include <avro.h> + +int avro_value_get_size(const avro_value_t *record, size_t *size); +int avro_value_get_by_index(const avro_value_t *record, size_t index, + avro_value_t *element, const char **field_name); +int avro_value_get_by_name(const avro_value_t *record, const char *field_name, + avro_value_t *element, size_t *index); +---- + +The +get_size+ method returns the number of fields in the record. (You +can also get this by querying the value's schema, but for some +implementations, this method can be faster.) + +The +get_by_index+ and +get_by_name+ functions can be used to retrieve +one of the fields in the record, either by its ordinal position within +the record, or by the name of the underlying field. Like with maps, the +methods take in an additional parameter that let you retrieve the index +associated with a field name, and vice versa. + +When possible, it's recommended that you access record fields by their +numeric index, rather than by their field name. For most +implementations, this will be more efficient. + + +==== Unions + +There are three methods that can be used with union values: + +[source,c] +---- +#include <avro.h> + +int avro_value_get_discriminant(const avro_value_t *union_val, int *disc); +int avro_value_get_current_branch(const avro_value_t *union_val, avro_value_t *branch); +int avro_value_set_branch(avro_value_t *union_val, + int discriminant, avro_value_t *branch); +---- + +The +get_discriminant+ and +get_current_branch+ methods return the +current state of the union value, without modifying which branch is +currently selected. The +set_branch+ method can be used to choose the +active branch, filling in the +branch+ value to point at the branch's +value instance. (Most implementations will be smart enough to detect +when the desired branch is already selected, so you should always call +this method unless you can _guarantee_ that the right branch is already +current.) + + +=== Creating value instances + +Okay, so we've described how to interact with a value that you already +have a pointer to, but how do you create one in the first place? Each +implementation of the value interface must provide its own functions for +creating +avro_value_t+ instances for that class. The 10,000-foot view +is to: + +1. Get an _implementation struct_ for the value implementation that you + want to use. (This is represented by an +avro_value_iface_t+ + pointer.) + +2. Use the implementation's constructor function to allocate instances + of that value implementation. + +3. Do whatever you need to the value (using the +avro_value_t+ methods + described in the previous section). + +4. Free the value instance, if necessary, using the implementation's + destructor function. + +5. Free the implementation struct when you're done creating value + instances. + +These steps use the following functions: + +[source,c] +---- +#include <avro.h> + +avro_value_iface_t *avro_value_iface_incref(avro_value_iface_t *iface); +void avro_value_iface_decref(avro_value_iface_t *iface); +---- + +Note that for most value implementations, it's fine to reuse a single ++avro_value_t+ instance for multiple values, using the ++avro_value_reset+ function before filling in the instance for each +value. (This helps reduce the number of +malloc+ and +free+ calls that +your application will make.) + +We provide a “generic” value implementation that will work (efficiently) +for any Avro schema. + + +For most applications, you won't need to write your own value +implementation; the Avro C library provides an efficient “generic” +implementation, which supports the full range of Avro schema types. +There's a good chance that you just want to use this implementation, +rather than rolling your own. (The primary reason for rolling your own +would be if you want to access the elements of a compound value using C +syntax — for instance, translating an Avro record into a C struct.) You +can use the following functions to create and work with a generic value +implementation for a particular schema: + +[source,c] +---- +#include <avro.h> + +avro_value_iface_t *avro_generic_class_from_schema(avro_schema_t schema); +int avro_generic_value_new(const avro_value_iface_t *iface, avro_value_t *dest); +void avro_generic_value_free(avro_value_t *self); +---- + +Combining all of this together, you might have the following snippet of +code: + +[source,c] +---- +avro_schema_t schema = avro_schema_long(); +avro_value_iface_t *iface = avro_generic_class_from_schema(schema); + +avro_value_t val; +avro_generic_value_new(iface, &val); + +/* Generate Avro longs from 0-499 */ +int i; +for (i = 0; i < 500; i++) { + avro_value_reset(&val); + avro_value_set_long(&val, i); + /* do something with the value */ +} + +avro_generic_value_free(&val); +avro_value_iface_decref(iface); +avro_schema_decref(schema); +---- + + +== Reference Counting + ++Avro C+ does reference counting for all schema and data objects. +When the number of references drops to zero, the memory is freed. + +For example, to create and free a string, you would use: +---- +avro_datum_t string = avro_string("This is my string"); + +... +avro_datum_decref(string); +---- + +Things get a little more complicated when you consider more elaborate +schema and data structures. + +For example, let's say that you create a record with a single +string field: +---- +avro_datum_t example = avro_record("Example"); +avro_datum_t solo_field = avro_string("Example field value"); + +avro_record_set(example, "solo", solo_field); + +... +avro_datum_decref(example); +---- + +In this example, the +solo_field+ datum would *not* be freed since it +has two references: the original reference and a reference inside +the +Example+ record. The +avro_datum_decref(example)+ call drops +the number of reference to one. If you are finished with the +solo_field+ +schema, then you need to +avro_schema_decref(solo_field)+ to +completely dereference the +solo_field+ datum and free it. + +== Wrap It and Give It + +You'll notice that some datatypes can be "wrapped" and "given". This +allows C programmers the freedom to decide who is responsible for +the memory. Let's take strings for example. + +To create a string datum, you have three different methods: +---- +avro_datum_t avro_string(const char *str); +avro_datum_t avro_wrapstring(const char *str); +avro_datum_t avro_givestring(const char *str); +---- + +If you use, +avro_string+ then +Avro C+ will make a copy of your +string and free it when the datum is dereferenced. In some cases, +especially when dealing with large amounts of data, you want +to avoid this memory copy. That's where +avro_wrapstring+ and ++avro_givestring+ can help. + +If you use, +avro_wrapstring+ then +Avro C+ will do no memory +management at all. It will just save a pointer to your data and +it's your responsibility to free the string. + +WARNING: When using +avro_wrapstring+, do not free the string +before you dereference the string datum with +avro_datum_decref()+. + +Lastly, if you use +avro_givestring+ then +Avro C+ will free the +string later when the datum is dereferenced. In a sense, you +are "giving" responsibility for freeing the string to +Avro C+. + +[WARNING] +=============================== +Don't "give" +Avro C+ a string that you haven't allocated from the heap with e.g. +malloc+ or +strdup+. + +For example, *don't* do this: +---- +avro_datum_t bad_idea = avro_givestring("This isn't allocated on the heap"); +---- +=============================== + +== Schema Validation + +If you want to write a datum, you would use the following function + +[source,c] +---- +int avro_write_data(avro_writer_t writer, + avro_schema_t writers_schema, avro_datum_t datum); +---- + +If you pass in a +writers_schema+, then you +datum+ will be validated *before* +it is sent to the +writer+. This check ensures that your data has the +correct format. If you are certain your datum is correct, you can pass +a +NULL+ value for +writers_schema+ and +Avro C+ will not validate before +writing. + +NOTE: Data written to an Avro File Object Container is always validated. + +== Examples + +[quote,Dante Hicks] +____ +I'm not even supposed to be here today! +____ + +Imagine you're a free-lance hacker in Leonardo, New Jersey and you've +been approached by the owner of the local *Quick Stop Convenience* store. +He wants you to create a contact database case he needs to call employees +to work on their day off. + +You might build a simple contact system using Avro C like the following... + +[source,c] +---- +include::../examples/quickstop.c[] +---- + +When you compile and run this program, you should get the following output + +---- +Successfully added Hicks, Dante id=1 +Successfully added Graves, Randal id=2 +Successfully added Loughran, Veronica id=3 +Successfully added Bree, Caitlin id=4 +Successfully added Silent, Bob id=5 +Successfully added ???, Jay id=6 + +Avro is compact. Here is the data for all 6 people. +| 02 0A 44 61 6E 74 65 0A | 48 69 63 6B 73 1C 28 35 | ..Dante.Hicks.(5 +| 35 35 29 20 31 32 33 2D | 34 35 36 37 40 04 0C 52 | 55) 123-4567@..R +| 61 6E 64 61 6C 0C 47 72 | 61 76 65 73 1C 28 35 35 | andal.Graves.(55 +| 35 29 20 31 32 33 2D 35 | 36 37 38 3C 06 10 56 65 | 5) 123-5678<..Ve +| 72 6F 6E 69 63 61 10 4C | 6F 75 67 68 72 61 6E 1C | ronica.Loughran. +| 28 35 35 35 29 20 31 32 | 33 2D 30 39 38 37 38 08 | (555) 123-09878. +| 0E 43 61 69 74 6C 69 6E | 08 42 72 65 65 1C 28 35 | .Caitlin.Bree.(5 +| 35 35 29 20 31 32 33 2D | 32 33 32 33 36 0A 06 42 | 55) 123-23236..B +| 6F 62 0C 53 69 6C 65 6E | 74 1C 28 35 35 35 29 20 | ob.Silent.(555) +| 31 32 33 2D 36 34 32 32 | 3A 0C 06 4A 61 79 06 3F | 123-6422:..Jay.? +| 3F 3F 1C 28 35 35 35 29 | 20 31 32 33 2D 39 31 38 | ??.(555) 123-918 +| 32 34 .. .. .. .. .. .. | .. .. .. .. .. .. .. .. | 24.............. + +Now let's read all the records back out +1 | Dante | Hicks | (555) 123-4567 | 32 +2 | Randal | Graves | (555) 123-5678 | 30 +3 | Veronica | Loughran | (555) 123-0987 | 28 +4 | Caitlin | Bree | (555) 123-2323 | 27 +5 | Bob | Silent | (555) 123-6422 | 29 +6 | Jay | ??? | (555) 123-9182 | 26 + + +Use projection to print only the First name and phone numbers + Dante | (555) 123-4567 | + Randal | (555) 123-5678 | + Veronica | (555) 123-0987 | + Caitlin | (555) 123-2323 | + Bob | (555) 123-6422 | + Jay | (555) 123-9182 | +---- + +The *Quick Stop* owner was so pleased, he asked you to create a +movie database for his *RST Video* store. + +== Reference files + +=== avro.h + +The +avro.h+ header file contains the complete public API +for +Avro C+. The documentation is rather sparse right now +but we'll be adding more information soon. + +[source,c] +---- +include::../src/avro.h[] +---- + +=== test_avro_data.c + +Another good way to learn how to encode/decode data in +Avro C+ is +to look at the +test_avro_data.c+ unit test. This simple unit test +checks that all the avro types can be encoded/decoded correctly. + +[source,c] +---- +include::../tests/test_avro_data.c[] +---- + diff --git a/fluent-bit/lib/avro/examples/.gitignore b/fluent-bit/lib/avro/examples/.gitignore new file mode 100644 index 00000000..8505a325 --- /dev/null +++ b/fluent-bit/lib/avro/examples/.gitignore @@ -0,0 +1,2 @@ +quickstop +quickstop.db diff --git a/fluent-bit/lib/avro/examples/CMakeLists.txt b/fluent-bit/lib/avro/examples/CMakeLists.txt new file mode 100644 index 00000000..1c96b71c --- /dev/null +++ b/fluent-bit/lib/avro/examples/CMakeLists.txt @@ -0,0 +1,32 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +add_executable(quickstop quickstop.c) +target_link_libraries(quickstop avro-static) + +if (WIN32) + set(exec_name ${CMAKE_CURRENT_BINARY_DIR}/Debug/quickstop.exe) +else (WIN32) + set(exec_name ${CMAKE_CURRENT_BINARY_DIR}/quickstop) +endif (WIN32) + +add_test(quickstop + ${CMAKE_COMMAND} -E chdir ${AvroC_SOURCE_DIR}/examples + ${exec_name} +) diff --git a/fluent-bit/lib/avro/examples/quickstop.c b/fluent-bit/lib/avro/examples/quickstop.c new file mode 100644 index 00000000..ff9e9700 --- /dev/null +++ b/fluent-bit/lib/avro/examples/quickstop.c @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <avro.h> +#include <stdio.h> +#include <stdlib.h> + +#ifdef DEFLATE_CODEC +#define QUICKSTOP_CODEC "deflate" +#else +#define QUICKSTOP_CODEC "null" +#endif + +avro_schema_t person_schema; +int64_t id = 0; + +/* A simple schema for our tutorial */ +const char PERSON_SCHEMA[] = +"{\"type\":\"record\",\ + \"name\":\"Person\",\ + \"fields\":[\ + {\"name\": \"ID\", \"type\": \"long\"},\ + {\"name\": \"First\", \"type\": \"string\"},\ + {\"name\": \"Last\", \"type\": \"string\"},\ + {\"name\": \"Phone\", \"type\": \"string\"},\ + {\"name\": \"Age\", \"type\": \"int\"}]}"; + +/* Parse schema into a schema data structure */ +void init_schema(void) +{ + if (avro_schema_from_json_literal(PERSON_SCHEMA, &person_schema)) { + fprintf(stderr, "Unable to parse person schema\n"); + exit(EXIT_FAILURE); + } +} + +/* Create a value to match the person schema and save it */ +void +add_person(avro_file_writer_t db, const char *first, const char *last, + const char *phone, int32_t age) +{ + avro_value_iface_t *person_class = + avro_generic_class_from_schema(person_schema); + + avro_value_t person; + avro_generic_value_new(person_class, &person); + + avro_value_t id_value; + avro_value_t first_value; + avro_value_t last_value; + avro_value_t age_value; + avro_value_t phone_value; + + if (avro_value_get_by_name(&person, "ID", &id_value, NULL) == 0) { + avro_value_set_long(&id_value, ++id); + } + if (avro_value_get_by_name(&person, "First", &first_value, NULL) == 0) { + avro_value_set_string(&first_value, first); + } + if (avro_value_get_by_name(&person, "Last", &last_value, NULL) == 0) { + avro_value_set_string(&last_value, last); + } + if (avro_value_get_by_name(&person, "Age", &age_value, NULL) == 0) { + avro_value_set_int(&age_value, age); + } + if (avro_value_get_by_name(&person, "Phone", &phone_value, NULL) == 0) { + avro_value_set_string(&phone_value, phone); + } + + if (avro_file_writer_append_value(db, &person)) { + fprintf(stderr, + "Unable to write Person value to memory buffer\nMessage: %s\n", avro_strerror()); + exit(EXIT_FAILURE); + } + + /* Decrement all our references to prevent memory from leaking */ + avro_value_decref(&person); + avro_value_iface_decref(person_class); +} + +int print_person(avro_file_reader_t db, avro_schema_t reader_schema) +{ + + avro_value_iface_t *person_class = + avro_generic_class_from_schema(person_schema); + + avro_value_t person; + avro_generic_value_new(person_class, &person); + + int rval; + + rval = avro_file_reader_read_value(db, &person); + if (rval == 0) { + int64_t id; + int32_t age; + int32_t *p; + size_t size; + avro_value_t id_value; + avro_value_t first_value; + avro_value_t last_value; + avro_value_t age_value; + avro_value_t phone_value; + + if (avro_value_get_by_name(&person, "ID", &id_value, NULL) == 0) { + avro_value_get_long(&id_value, &id); + fprintf(stdout, "%"PRId64" | ", id); + } + if (avro_value_get_by_name(&person, "First", &first_value, NULL) == 0) { + avro_value_get_string(&first_value, &p, &size); + fprintf(stdout, "%15s | ", p); + } + if (avro_value_get_by_name(&person, "Last", &last_value, NULL) == 0) { + avro_value_get_string(&last_value, &p, &size); + fprintf(stdout, "%15s | ", p); + } + if (avro_value_get_by_name(&person, "Phone", &phone_value, NULL) == 0) { + avro_value_get_string(&phone_value, &p, &size); + fprintf(stdout, "%15s | ", p); + } + if (avro_value_get_by_name(&person, "Age", &age_value, NULL) == 0) { + avro_value_get_int(&age_value, &age); + fprintf(stdout, "%"PRId32" | ", age); + } + fprintf(stdout, "\n"); + + /* We no longer need this memory */ + avro_value_decref(&person); + avro_value_iface_decref(person_class); + } + return rval; +} + +int main(void) +{ + int rval; + avro_file_reader_t dbreader; + avro_file_writer_t db; + avro_schema_t projection_schema, first_name_schema, phone_schema; + int64_t i; + const char *dbname = "quickstop.db"; + char number[15] = {0}; + + /* Initialize the schema structure from JSON */ + init_schema(); + + /* Delete the database if it exists */ + remove(dbname); + /* Create a new database */ + rval = avro_file_writer_create_with_codec + (dbname, person_schema, &db, QUICKSTOP_CODEC, 0); + if (rval) { + fprintf(stderr, "There was an error creating %s\n", dbname); + fprintf(stderr, " error message: %s\n", avro_strerror()); + exit(EXIT_FAILURE); + } + + /* Add lots of people to the database */ + for (i = 0; i < 1000; i++) + { + sprintf(number, "(%d)", (int)i); + add_person(db, "Dante", "Hicks", number, 32); + add_person(db, "Randal", "Graves", "(555) 123-5678", 30); + add_person(db, "Veronica", "Loughran", "(555) 123-0987", 28); + add_person(db, "Caitlin", "Bree", "(555) 123-2323", 27); + add_person(db, "Bob", "Silent", "(555) 123-6422", 29); + add_person(db, "Jay", "???", number, 26); + } + + /* Close the block and open a new one */ + avro_file_writer_flush(db); + add_person(db, "Super", "Man", "123456", 31); + + avro_file_writer_close(db); + + fprintf(stdout, "\nNow let's read all the records back out\n"); + + /* Read all the records and print them */ + if (avro_file_reader(dbname, &dbreader)) { + fprintf(stderr, "Error opening file: %s\n", avro_strerror()); + exit(EXIT_FAILURE); + } + for (i = 0; i < id; i++) { + if (print_person(dbreader, NULL)) { + fprintf(stderr, "Error printing person\nMessage: %s\n", avro_strerror()); + exit(EXIT_FAILURE); + } + } + avro_file_reader_close(dbreader); + + /* You can also use projection, to only decode only the data you are + interested in. This is particularly useful when you have + huge data sets and you'll only interest in particular fields + e.g. your contacts First name and phone number */ + projection_schema = avro_schema_record("Person", NULL); + first_name_schema = avro_schema_string(); + phone_schema = avro_schema_string(); + avro_schema_record_field_append(projection_schema, "First", + first_name_schema); + avro_schema_record_field_append(projection_schema, "Phone", + phone_schema); + + /* Read only the record you're interested in */ + fprintf(stdout, + "\n\nUse projection to print only the First name and phone numbers\n"); + if (avro_file_reader(dbname, &dbreader)) { + fprintf(stderr, "Error opening file: %s\n", avro_strerror()); + exit(EXIT_FAILURE); + } + for (i = 0; i < id; i++) { + if (print_person(dbreader, projection_schema)) { + fprintf(stderr, "Error printing person: %s\n", + avro_strerror()); + exit(EXIT_FAILURE); + } + } + avro_file_reader_close(dbreader); + avro_schema_decref(first_name_schema); + avro_schema_decref(phone_schema); + avro_schema_decref(projection_schema); + + /* We don't need this schema anymore */ + avro_schema_decref(person_schema); + return 0; +} diff --git a/fluent-bit/lib/avro/quickstop.db b/fluent-bit/lib/avro/quickstop.db new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/fluent-bit/lib/avro/quickstop.db diff --git a/fluent-bit/lib/avro/src/.gitignore b/fluent-bit/lib/avro/src/.gitignore new file mode 100644 index 00000000..e278a8b9 --- /dev/null +++ b/fluent-bit/lib/avro/src/.gitignore @@ -0,0 +1,2 @@ +avro-c.pc +avropipe diff --git a/fluent-bit/lib/avro/src/CMakeLists.txt b/fluent-bit/lib/avro/src/CMakeLists.txt new file mode 100644 index 00000000..a911115f --- /dev/null +++ b/fluent-bit/lib/avro/src/CMakeLists.txt @@ -0,0 +1,158 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set(AVRO_SRC + allocation.c + array.c + avro.h + avro/allocation.h + avro/basics.h + avro/consumer.h + avro/data.h + avro/errors.h + avro/generic.h + avro/io.h + avro/legacy.h + avro/refcount.h + avro/resolver.h + avro/schema.h + avro/value.h + avro_generic_internal.h + avro_private.h + codec.c + codec.h + consumer.c + consume-binary.c + datafile.c + datum.c + datum.h + datum_equal.c + datum_read.c + datum_size.c + datum_skip.c + datum_validate.c + datum_value.c + datum_write.c + dump.c + dump.h + encoding.h + encoding_binary.c + errors.c + generic.c + io.c + map.c + memoize.c + resolved-reader.c + resolved-writer.c + resolver.c + schema.c + schema.h + schema_equal.c + st.c + st.h + string.c + value.c + value-hash.c + value-json.c + value-read.c + value-sizeof.c + value-write.c + wrapped-buffer.c +) + + +source_group(Avro FILES ${AVRO_SRC}) + +# The version.sh script gives us a VERSION that uses colon as a +# separator; we need periods. + +string(REPLACE ":" "." LIBAVRO_DOT_VERSION ${LIBAVRO_VERSION}) + +add_library(avro-static STATIC ${AVRO_SRC}) +target_link_libraries(avro-static ${JANSSON_LIBRARIES} ${CODEC_LIBRARIES} ${THREADS_LIBRARIES}) +set_target_properties(avro-static PROPERTIES OUTPUT_NAME avro) + +if (NOT WIN32) +# TODO: Create Windows DLLs. See https://www.cmake.org/Wiki/BuildingWinDLL +add_library(avro-shared SHARED ${AVRO_SRC}) +target_link_libraries(avro-shared ${JANSSON_LIBRARIES} ${CODEC_LIBRARIES} ${THREADS_LIBRARIES}) +set_target_properties(avro-shared PROPERTIES + OUTPUT_NAME avro + VERSION ${LIBAVRO_DOT_VERSION} + SOVERSION ${LIBAVRO_SOVERSION}) +endif(NOT WIN32) + +install(FILES + ${CMAKE_CURRENT_SOURCE_DIR}/avro.h + DESTINATION include) +install(DIRECTORY + ${CMAKE_CURRENT_SOURCE_DIR}/avro + DESTINATION include + FILES_MATCHING PATTERN "*.h") + +# updated by edsiper, avro version 77692d8c1 +# added by mcqueen to get the headers into the fluent-bit build root 07OCT2020 +file (COPY ${CMAKE_CURRENT_SOURCE_DIR}/avro.h + DESTINATION ${CMAKE_BINARY_DIR}/include/) +file (COPY ${CMAKE_CURRENT_SOURCE_DIR}/avro + DESTINATION ${CMAKE_BINARY_DIR}/include/ + FILES_MATCHING PATTERN "*.h") + +include(GNUInstallDirs) + +if (WIN32) +install(TARGETS avro-static + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + ) +else(WIN32) +install(TARGETS avro-static avro-shared + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + ) +endif(WIN32) + +# Install pkg-config file + +set(prefix ${CMAKE_INSTALL_PREFIX}) +set(VERSION ${AVRO_VERSION}) +configure_file(avro-c.pc.in avro-c.pc) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/avro-c.pc + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) + +add_executable(avrocat avrocat.c) +target_link_libraries(avrocat avro-static) +install(TARGETS avrocat RUNTIME DESTINATION bin) + +add_executable(avroappend avroappend.c) +target_link_libraries(avroappend avro-static) +install(TARGETS avroappend RUNTIME DESTINATION bin) + +if (NOT WIN32) +#TODO: Port getopt() to Windows to compile avropipe.c and avromod.c +add_executable(avropipe avropipe.c) +target_link_libraries(avropipe avro-static) +install(TARGETS avropipe RUNTIME DESTINATION bin) + +add_executable(avromod avromod.c) +target_link_libraries(avromod avro-static) +install(TARGETS avromod RUNTIME DESTINATION bin) +endif(NOT WIN32) diff --git a/fluent-bit/lib/avro/src/allocation.c b/fluent-bit/lib/avro/src/allocation.c new file mode 100644 index 00000000..2059f92a --- /dev/null +++ b/fluent-bit/lib/avro/src/allocation.c @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <stdlib.h> +#include <string.h> + +#include "avro_private.h" +#include "avro/allocation.h" +#include "avro/data.h" +#include "avro/legacy.h" + +static void * +avro_default_allocator(void *ud, void *ptr, size_t osize, size_t nsize) +{ + AVRO_UNUSED(ud); + AVRO_UNUSED(osize); + + if (nsize == 0) { + free(ptr); + return NULL; + } else { + return realloc(ptr, nsize); + } +} + +struct avro_allocator_state AVRO_CURRENT_ALLOCATOR = { + avro_default_allocator, + NULL +}; + +void avro_set_allocator(avro_allocator_t alloc, void *user_data) +{ + AVRO_CURRENT_ALLOCATOR.alloc = alloc; + AVRO_CURRENT_ALLOCATOR.user_data = user_data; +} + +void *avro_calloc(size_t count, size_t size) +{ + void *ptr = avro_malloc(count * size); + if (ptr != NULL) { + memset(ptr, 0, count * size); + } + return ptr; +} + +char *avro_str_alloc(size_t str_size) +{ + size_t buf_size = str_size + sizeof(size_t); + + void *buf = avro_malloc(buf_size); + if (buf == NULL) { + return NULL; + } + + size_t *size = (size_t *) buf; + char *new_str = (char *) (size + 1); + + *size = buf_size; + + return new_str; +} + +char *avro_strdup(const char *str) +{ + if (str == NULL) { + return NULL; + } + + size_t str_size = strlen(str)+1; + char *new_str = avro_str_alloc(str_size); + memcpy(new_str, str, str_size); + + //fprintf(stderr, "--- new %" PRIsz " %p %s\n", *size, new_str, new_str); + return new_str; +} + +char *avro_strndup(const char *str, size_t size) +{ + if (str == NULL) { + return NULL; + } + + char *new_str = avro_str_alloc(size + 1); + memcpy(new_str, str, size); + new_str[size] = '\0'; + + return new_str; +} + +void avro_str_free(char *str) +{ + size_t *size = ((size_t *) str) - 1; + //fprintf(stderr, "--- free %" PRIsz " %p %s\n", *size, str, str); + avro_free(size, *size); +} + + +void +avro_alloc_free_func(void *ptr, size_t sz) +{ + avro_free(ptr, sz); +} diff --git a/fluent-bit/lib/avro/src/array.c b/fluent-bit/lib/avro/src/array.c new file mode 100644 index 00000000..94dce295 --- /dev/null +++ b/fluent-bit/lib/avro/src/array.c @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <errno.h> +#include <stdlib.h> +#include <string.h> + +#include "avro/allocation.h" +#include "avro/data.h" +#include "avro/errors.h" +#include "avro_private.h" + + +void avro_raw_array_init(avro_raw_array_t *array, size_t element_size) +{ + memset(array, 0, sizeof(avro_raw_array_t)); + array->element_size = element_size; +} + + +void avro_raw_array_done(avro_raw_array_t *array) +{ + if (array->data) { + avro_free(array->data, array->allocated_size); + } + memset(array, 0, sizeof(avro_raw_array_t)); +} + + +void avro_raw_array_clear(avro_raw_array_t *array) +{ + array->element_count = 0; +} + + +int +avro_raw_array_ensure_size(avro_raw_array_t *array, size_t desired_count) +{ + size_t required_size = array->element_size * desired_count; + if (array->allocated_size >= required_size) { + return 0; + } + + /* + * Double the old size when reallocating. + */ + + size_t new_size; + if (array->allocated_size == 0) { + /* + * Start with an arbitrary 10 items. + */ + + new_size = 10 * array->element_size; + } else { + new_size = array->allocated_size * 2; + } + + if (required_size > new_size) { + new_size = required_size; + } + + array->data = avro_realloc(array->data, array->allocated_size, new_size); + if (array->data == NULL) { + avro_set_error("Cannot allocate space in array for %" PRIsz " elements", + desired_count); + return ENOMEM; + } + array->allocated_size = new_size; + + return 0; +} + + +int +avro_raw_array_ensure_size0(avro_raw_array_t *array, size_t desired_count) +{ + int rval; + size_t old_allocated_size = array->allocated_size; + check(rval, avro_raw_array_ensure_size(array, desired_count)); + + if (array->allocated_size > old_allocated_size) { + size_t extra_space = array->allocated_size - old_allocated_size; + void *buf = array->data; + memset((char *)buf + old_allocated_size, 0, extra_space); + } + + return 0; +} + + +void *avro_raw_array_append(avro_raw_array_t *array) +{ + int rval; + + rval = avro_raw_array_ensure_size(array, array->element_count + 1); + if (rval) { + return NULL; + } + + size_t offset = array->element_size * array->element_count; + array->element_count++; + return (char *)array->data + offset; +} diff --git a/fluent-bit/lib/avro/src/avro-c.pc.in b/fluent-bit/lib/avro/src/avro-c.pc.in new file mode 100644 index 00000000..013afe4d --- /dev/null +++ b/fluent-bit/lib/avro/src/avro-c.pc.in @@ -0,0 +1,7 @@ +Name: avro-c +Description: C library for parsing Avro data +Version: @VERSION@ +URL: https://avro.apache.org/ +Libs: -L@prefix@/lib -lavro +Cflags: -I@prefix@/include +Requires: @CODEC_PKG@ diff --git a/fluent-bit/lib/avro/src/avro.h b/fluent-bit/lib/avro/src/avro.h new file mode 100644 index 00000000..af32a32d --- /dev/null +++ b/fluent-bit/lib/avro/src/avro.h @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +#ifndef AVRO_H +#define AVRO_H +#ifdef __cplusplus +extern "C" { +#define CLOSE_EXTERN } +#else +#define CLOSE_EXTERN +#endif + +#include <avro/allocation.h> +#include <avro/basics.h> +#include <avro/consumer.h> +#include <avro/data.h> +#include <avro/errors.h> +#include <avro/generic.h> +#include <avro/io.h> +#include <avro/legacy.h> +#include <avro/platform.h> +#include <avro/resolver.h> +#include <avro/schema.h> +#include <avro/value.h> + +CLOSE_EXTERN +#endif diff --git a/fluent-bit/lib/avro/src/avro/allocation.h b/fluent-bit/lib/avro/src/avro/allocation.h new file mode 100644 index 00000000..0ed412b9 --- /dev/null +++ b/fluent-bit/lib/avro/src/avro/allocation.h @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#ifndef AVRO_ALLOCATION_H +#define AVRO_ALLOCATION_H +#ifdef __cplusplus +extern "C" { +#define CLOSE_EXTERN } +#else +#define CLOSE_EXTERN +#endif + +#include <stdlib.h> + +/* + * Allocation interface. You can provide a custom allocator for the + * library, should you wish. The allocator is provided as a single + * generic function, which can emulate the standard malloc, realloc, and + * free functions. The design of this allocation interface is inspired + * by the implementation of the Lua interpreter. + * + * The ptr parameter will be the location of any existing memory + * buffer. The osize parameter will be the size of this existing + * buffer. If ptr is NULL, then osize will be 0. The nsize parameter + * will be the size of the new buffer, or 0 if the new buffer should be + * freed. + * + * If nsize is 0, then the allocation function must return NULL. If + * nsize is not 0, then it should return NULL if the allocation fails. + */ + +typedef void * +(*avro_allocator_t)(void *user_data, void *ptr, size_t osize, size_t nsize); + +void avro_set_allocator(avro_allocator_t alloc, void *user_data); + +struct avro_allocator_state { + avro_allocator_t alloc; + void *user_data; +}; + +extern struct avro_allocator_state AVRO_CURRENT_ALLOCATOR; + +#define avro_realloc(ptr, osz, nsz) \ + (AVRO_CURRENT_ALLOCATOR.alloc \ + (AVRO_CURRENT_ALLOCATOR.user_data, \ + (ptr), (osz), (nsz))) + +#define avro_malloc(sz) (avro_realloc(NULL, 0, (sz))) +#define avro_free(ptr, osz) (avro_realloc((ptr), (osz), 0)) + +#define avro_new(type) (avro_realloc(NULL, 0, sizeof(type))) +#define avro_freet(type, ptr) (avro_realloc((ptr), sizeof(type), 0)) + +void *avro_calloc(size_t count, size_t size); + +/* + * This is probably too clever for our own good, but when we duplicate a + * string, we actually store its size in the same allocated memory + * buffer. That lets us free the string later, without having to call + * strlen to get its size, and without the containing struct having to + * manually store the strings length. + * + * This means that any string return by avro_strdup MUST be freed using + * avro_str_free, and the only thing that can be passed into + * avro_str_free is a string created via avro_strdup. + */ + +char *avro_str_alloc(size_t str_size); +char *avro_strdup(const char *str); +char *avro_strndup(const char *str, size_t size); +void avro_str_free(char *str); + +CLOSE_EXTERN +#endif diff --git a/fluent-bit/lib/avro/src/avro/basics.h b/fluent-bit/lib/avro/src/avro/basics.h new file mode 100644 index 00000000..368509b9 --- /dev/null +++ b/fluent-bit/lib/avro/src/avro/basics.h @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#ifndef AVRO_BASICS_H +#define AVRO_BASICS_H +#ifdef __cplusplus +extern "C" { +#define CLOSE_EXTERN } +#else +#define CLOSE_EXTERN +#endif + + +enum avro_type_t { + AVRO_STRING, + AVRO_BYTES, + AVRO_INT32, + AVRO_INT64, + AVRO_FLOAT, + AVRO_DOUBLE, + AVRO_BOOLEAN, + AVRO_NULL, + AVRO_RECORD, + AVRO_ENUM, + AVRO_FIXED, + AVRO_MAP, + AVRO_ARRAY, + AVRO_UNION, + AVRO_LINK +}; +typedef enum avro_type_t avro_type_t; + +enum avro_class_t { + AVRO_SCHEMA, + AVRO_DATUM +}; +typedef enum avro_class_t avro_class_t; + +struct avro_obj_t { + avro_type_t type; + avro_class_t class_type; + volatile int refcount; +}; + +#define avro_classof(obj) ((obj)->class_type) +#define is_avro_schema(obj) (obj && avro_classof(obj) == AVRO_SCHEMA) +#define is_avro_datum(obj) (obj && avro_classof(obj) == AVRO_DATUM) + +#define avro_typeof(obj) ((obj)->type) +#define is_avro_string(obj) (obj && avro_typeof(obj) == AVRO_STRING) +#define is_avro_bytes(obj) (obj && avro_typeof(obj) == AVRO_BYTES) +#define is_avro_int32(obj) (obj && avro_typeof(obj) == AVRO_INT32) +#define is_avro_int64(obj) (obj && avro_typeof(obj) == AVRO_INT64) +#define is_avro_float(obj) (obj && avro_typeof(obj) == AVRO_FLOAT) +#define is_avro_double(obj) (obj && avro_typeof(obj) == AVRO_DOUBLE) +#define is_avro_boolean(obj) (obj && avro_typeof(obj) == AVRO_BOOLEAN) +#define is_avro_null(obj) (obj && avro_typeof(obj) == AVRO_NULL) +#define is_avro_primitive(obj)(is_avro_string(obj) \ + ||is_avro_bytes(obj) \ + ||is_avro_int32(obj) \ + ||is_avro_int64(obj) \ + ||is_avro_float(obj) \ + ||is_avro_double(obj) \ + ||is_avro_boolean(obj) \ + ||is_avro_null(obj)) +#define is_avro_record(obj) (obj && avro_typeof(obj) == AVRO_RECORD) +#define is_avro_enum(obj) (obj && avro_typeof(obj) == AVRO_ENUM) +#define is_avro_fixed(obj) (obj && avro_typeof(obj) == AVRO_FIXED) +#define is_avro_named_type(obj)(is_avro_record(obj) \ + ||is_avro_enum(obj) \ + ||is_avro_fixed(obj)) +#define is_avro_map(obj) (obj && avro_typeof(obj) == AVRO_MAP) +#define is_avro_array(obj) (obj && avro_typeof(obj) == AVRO_ARRAY) +#define is_avro_union(obj) (obj && avro_typeof(obj) == AVRO_UNION) +#define is_avro_complex_type(obj) (!(is_avro_primitive(obj)) +#define is_avro_link(obj) (obj && avro_typeof(obj) == AVRO_LINK) + + + +CLOSE_EXTERN +#endif diff --git a/fluent-bit/lib/avro/src/avro/consumer.h b/fluent-bit/lib/avro/src/avro/consumer.h new file mode 100644 index 00000000..4128eef1 --- /dev/null +++ b/fluent-bit/lib/avro/src/avro/consumer.h @@ -0,0 +1,317 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#ifndef AVRO_CONSUMER_H +#define AVRO_CONSUMER_H +#ifdef __cplusplus +extern "C" { +#define CLOSE_EXTERN } +#else +#define CLOSE_EXTERN +#endif + +#include <avro/platform.h> +#include <stdlib.h> + +#include <avro/io.h> +#include <avro/schema.h> + + +/*--------------------------------------------------------------------- + * Consumers + */ + +/** + * A <i>consumer</i> is an object that knows how to process Avro data. + * There are consumer methods for each type of Avro data. The + * <code>avro_consumer_t</code> struct is an abstract superclass, which + * you don't instantiate directly. Later in this file, we define + * several consumer classes that know how to process Avro data in + * specific ways. + * + * For compound Avro values (records, arrays, maps, and unions), the + * consumer callbacks provide a nested consumer that should be used to + * process subvalues. Each consumer instance, including these + * "subconsumers", contains a reference to the schema of the data that + * it expects to process. This means that the functions that produce + * Avro data (such as avro_consume_binary) don't need to maintain their + * own references to any schemas, since they'll be encapsulated in the + * consumer that they pass their data off to. + */ + +typedef struct avro_consumer_t avro_consumer_t; + +struct avro_consumer_t { + /** + * The schema of the data that this consumer expects to process. + */ + + avro_schema_t schema; + + /** + * Called when this consumer is freed. This function should + * free any additional resources acquired by a consumer + * subclass. + */ + + void (*free)(avro_consumer_t *consumer); + + /* PRIMITIVE VALUES */ + + /** + * Called when a boolean value is encountered. + */ + + int (*boolean_value)(avro_consumer_t *consumer, + int value, + void *user_data); + + /** + * Called when a bytes value is encountered. The @ref value + * pointer is only guaranteed to be valid for the duration of + * the callback function. If you need to save the data for + * processing later, you must copy it into another buffer. + */ + + int (*bytes_value)(avro_consumer_t *consumer, + const void *value, size_t value_len, + void *user_data); + + /** + * Called when a double value is encountered. + */ + + int (*double_value)(avro_consumer_t *consumer, + double value, + void *user_data); + + /** + * Called when a float value is encountered. + */ + + int (*float_value)(avro_consumer_t *consumer, + float value, + void *user_data); + + /** + * Called when an int value is encountered. + */ + + int (*int_value)(avro_consumer_t *consumer, + int32_t value, + void *user_data); + + /** + * Called when a long value is encountered. + */ + + int (*long_value)(avro_consumer_t *consumer, + int64_t value, + void *user_data); + + /** + * Called when a null value is encountered. + */ + + int (*null_value)(avro_consumer_t *consumer, void *user_data); + + /** + * Called when a string value is encountered. The @ref value + * pointer will point at UTF-8 encoded data. (If the data + * you're representing isn't a UTF-8 Unicode string, you + * should use the bytes type.) The @ref value_len parameter + * gives the length of the data in bytes, not in Unicode + * characters. The @ref value pointer is only guaranteed to + * be valid for the duration of the callback function. If you + * need to save the data for processing later, you must copy + * it into another buffer. + */ + + int (*string_value)(avro_consumer_t *consumer, + const void *value, size_t value_len, + void *user_data); + + /* COMPOUND VALUES */ + + /** + * Called when the beginning of an array block is encountered. + * The @ref block_count parameter will contain the number of + * elements in this block. + */ + + int (*array_start_block)(avro_consumer_t *consumer, + int is_first_block, + unsigned int block_count, + void *user_data); + + /** + * Called before each individual element of an array is + * processed. The index of the current element is passed into + * the callback. The callback should fill in @ref + * element_consumer and @ref element_user_data with the consumer + * and <code>user_data</code> pointer to use to process the + * element. + */ + + int (*array_element)(avro_consumer_t *consumer, + unsigned int index, + avro_consumer_t **element_consumer, + void **element_user_data, + void *user_data); + + /** + * Called when an enum value is encountered. + */ + + int (*enum_value)(avro_consumer_t *consumer, int value, + void *user_data); + + /** + * Called when a fixed value is encountered. The @ref value + * pointer is only guaranteed to be valid for the duration of + * the callback function. If you need to save the data for + * processing later, you must copy it into another buffer. + */ + + int (*fixed_value)(avro_consumer_t *consumer, + const void *value, size_t value_len, + void *user_data); + + /** + * Called when the beginning of a map block is encountered. + * The @ref block_count parameter will contain the number of + * elements in this block. + */ + + int (*map_start_block)(avro_consumer_t *consumer, + int is_first_block, + unsigned int block_count, + void *user_data); + + /** + * Called before each individual element of a map is + * processed. The index and key of the current element is + * passed into the callback. The key is only guaranteed to be + * valid for the duration of the map_element_start callback, + * and the map's subschema callback. If you need to save it for + * later use, you must copy the key into another memory + * location. The callback should fill in @ref value_consumer + * and @ref value_user_data with the consumer and + * <code>user_data</code> pointer to use to process the value. + */ + + int (*map_element)(avro_consumer_t *consumer, + unsigned int index, + const char *key, + avro_consumer_t **value_consumer, + void **value_user_data, + void *user_data); + + /** + * Called when the beginning of a record is encountered. + */ + + int (*record_start)(avro_consumer_t *consumer, + void *user_data); + + /** + * Called before each individual field of a record is + * processed. The index and name of the current field is + * passed into the callback. The name is only guaranteed to + * be valid for the duration of the record_field_start + * callback, and the field's subschema callback. If you need to + * save it for later use, you must copy the key into another + * memory location. The callback should fill in @ref + * field_consumer and @ref field_user_data with the consumer + * <code>user_data</code> pointer to use to process the field. + */ + + int (*record_field)(avro_consumer_t *consumer, + unsigned int index, + avro_consumer_t **field_consumer, + void **field_user_data, + void *user_data); + + /** + * Called when a union value is encountered. The callback + * should fill in @ref branch_consumer and @ref branch_user_data + * with the consumer <code>user_data</code> pointer to use to + * process the branch. + */ + + int (*union_branch)(avro_consumer_t *consumer, + unsigned int discriminant, + avro_consumer_t **branch_consumer, + void **branch_user_data, + void *user_data); +}; + + +/** + * Calls the given callback in consumer, if it's present. If the + * callback is NULL, it just returns a success code. + */ + +#define avro_consumer_call(consumer, callback, ...) \ + (((consumer)->callback == NULL)? 0: \ + (consumer)->callback((consumer), __VA_ARGS__)) + + +/** + * Frees an @ref avro_consumer_t instance. (This function works on + * consumer subclasses, too.) + */ + +void avro_consumer_free(avro_consumer_t *consumer); + + +/*--------------------------------------------------------------------- + * Resolvers + */ + +/** + * A <i>resolver</i> is a special kind of consumer that knows how to + * implement Avro's schema resolution rules to translate between a + * writer schema and a reader schema. The consumer callbacks line up + * with the writer schema; as each element of the datum is produced, the + * resolver fills in the contents of an @ref avro_datum_t instance. + * (The datum is provided as the user_data when you use the consumer.) + */ + +avro_consumer_t * +avro_resolver_new(avro_schema_t writer_schema, + avro_schema_t reader_schema); + + +/*--------------------------------------------------------------------- + * Binary encoding + */ + +/** + * Reads an Avro datum from the given @ref avro_reader_t. As the + * datum is read, each portion of it is passed off to the appropriate + * callback in @ref consumer. + */ + +int +avro_consume_binary(avro_reader_t reader, + avro_consumer_t *consumer, + void *ud); + + +CLOSE_EXTERN +#endif diff --git a/fluent-bit/lib/avro/src/avro/data.h b/fluent-bit/lib/avro/src/avro/data.h new file mode 100644 index 00000000..5b1d429b --- /dev/null +++ b/fluent-bit/lib/avro/src/avro/data.h @@ -0,0 +1,526 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#ifndef AVRO_DATA_H +#define AVRO_DATA_H +#ifdef __cplusplus +extern "C" { +#define CLOSE_EXTERN } +#else +#define CLOSE_EXTERN +#endif + +#include <stdlib.h> +#include <string.h> + +/* + * This file defines some helper data structures that are used within + * Avro, and in the schema-specific types created by avrocc. + */ + + +/*--------------------------------------------------------------------- + * Arrays + */ + +/** + * A resizeable array of fixed-size elements. + */ + +typedef struct avro_raw_array { + size_t element_size; + size_t element_count; + size_t allocated_size; + void *data; +} avro_raw_array_t; + +/** + * Initializes a new avro_raw_array_t that you've allocated yourself. + */ + +void avro_raw_array_init(avro_raw_array_t *array, size_t element_size); + +/** + * Finalizes an avro_raw_array_t. + */ + +void avro_raw_array_done(avro_raw_array_t *array); + +/** + * Clears an avro_raw_array_t. This does not deallocate any space; this + * allows us to reuse the underlying array buffer as we start to re-add + * elements to the array. + */ + +void avro_raw_array_clear(avro_raw_array_t *array); + +/** + * Ensures that there is enough allocated space to store the given + * number of elements in an avro_raw_array_t. If we can't allocate that + * much space, we return ENOMEM. + */ + +int +avro_raw_array_ensure_size(avro_raw_array_t *array, size_t desired_count); + +/** + * Ensures that there is enough allocated space to store the given + * number of elements in an avro_raw_array_t. If the array grows as a + * result of this operation, we will fill in any newly allocated space + * with 0 bytes. If we can't allocate that much space, we return + * ENOMEM. + */ + +int +avro_raw_array_ensure_size0(avro_raw_array_t *array, size_t desired_count); + +/** + * Returns the number of elements in an avro_raw_array_t. + */ + +#define avro_raw_array_size(array) ((array)->element_count) + +/** + * Returns the given element of an avro_raw_array_t as a <code>void + * *</code>. + */ + +#define avro_raw_array_get_raw(array, index) \ + ((char *) (array)->data + (array)->element_size * index) + +/** + * Returns the given element of an avro_raw_array_t, using element_type + * as the type of the elements. The result is *not* a pointer to the + * element; you get the element itself. + */ + +#define avro_raw_array_get(array, element_type, index) \ + (((element_type *) (array)->data)[index]) + +/** + * Appends a new element to an avro_raw_array_t, expanding it if + * necessary. Returns a pointer to the new element, or NULL if we + * needed to reallocate the array and couldn't. + */ + +void *avro_raw_array_append(avro_raw_array_t *array); + + +/*--------------------------------------------------------------------- + * Maps + */ + +/** + * The type of the elements in a map's elements array. + */ + +typedef struct avro_raw_map_entry { + const char *key; +} avro_raw_map_entry_t; + +/** + * A string-indexed map of fixed-size elements. + */ + +typedef struct avro_raw_map { + avro_raw_array_t elements; + void *indices_by_key; +} avro_raw_map_t; + +/** + * Initializes a new avro_raw_map_t that you've allocated yourself. + */ + +void avro_raw_map_init(avro_raw_map_t *map, size_t element_size); + +/** + * Finalizes an avro_raw_map_t. + */ + +void avro_raw_map_done(avro_raw_map_t *map); + +/** + * Clears an avro_raw_map_t. + */ + +void avro_raw_map_clear(avro_raw_map_t *map); + +/** + * Ensures that there is enough allocated space to store the given + * number of elements in an avro_raw_map_t. If we can't allocate that + * much space, we return ENOMEM. + */ + +int +avro_raw_map_ensure_size(avro_raw_map_t *map, size_t desired_count); + +/** + * Returns the number of elements in an avro_raw_map_t. + */ + +#define avro_raw_map_size(map) avro_raw_array_size(&((map)->elements)) + +/** + * Returns the avro_raw_map_entry_t for a given index. + */ + +#define avro_raw_get_entry(map, index) \ + ((avro_raw_map_entry_t *) \ + avro_raw_array_get_raw(&(map)->elements, index)) + +/** + * Returns the given element of an avro_raw_array_t as a <code>void + * *</code>. The indexes are assigned based on the order that the + * elements are added to the map. + */ + +#define avro_raw_map_get_raw(map, index) \ + (avro_raw_array_get_raw(&(map)->elements, index) + \ + sizeof(avro_raw_map_entry_t)) + +/** + * Returns the element of an avro_raw_map_t with the given numeric + * index. The indexes are assigned based on the order that the elements + * are added to the map. + */ + +#define avro_raw_map_get_by_index(map, element_type, index) \ + (*((element_type *) avro_raw_map_get_raw(map, index))) + +/** + * Returns the key of the element with the given numeric index. + */ + +#define avro_raw_map_get_key(map, index) \ + (avro_raw_get_entry(map, index)->key) + +/** + * Returns the element of an avro_raw_map_t with the given string key. + * If the given element doesn't exist, returns NULL. If @ref index + * isn't NULL, it will be filled in with the index of the element. + */ + +void *avro_raw_map_get(const avro_raw_map_t *map, const char *key, + size_t *index); + +/** + * Retrieves the element of an avro_raw_map_t with the given string key, + * creating it if necessary. A pointer to the element is placed into + * @ref element. If @ref index isn't NULL, it will be filled in with + * the index of the element. We return 1 if the element is new; 0 if + * it's not, and a negative error code if there was some problem. + */ + +int avro_raw_map_get_or_create(avro_raw_map_t *map, const char *key, + void **element, size_t *index); + + +/*--------------------------------------------------------------------- + * Wrapped buffers + */ + +/** + * A pointer to an unmodifiable external memory region, along with + * functions for freeing that buffer when it's no longer needed, and + * copying it. + */ + +typedef struct avro_wrapped_buffer avro_wrapped_buffer_t; + +struct avro_wrapped_buffer { + /** A pointer to the memory region */ + const void *buf; + + /** The size of the memory region */ + size_t size; + + /** Additional data needed by the methods below */ + void *user_data; + + /** + * A function that will be called when the memory region is no + * longer needed. This pointer can be NULL if nothing special + * needs to be done to free the buffer. + */ + void + (*free)(avro_wrapped_buffer_t *self); + + /** + * A function that makes a copy of a portion of a wrapped + * buffer. This doesn't have to involve duplicating the memory + * region, but it should ensure that the free method can be + * safely called on both copies without producing any errors or + * memory corruption. If this function is NULL, then we'll use + * a default implementation that calls @ref + * avro_wrapped_buffer_new_copy. + */ + int + (*copy)(avro_wrapped_buffer_t *dest, const avro_wrapped_buffer_t *src, + size_t offset, size_t length); + + /** + * A function that "slices" a wrapped buffer, causing it to + * point at a subset of the existing buffer. Usually, this just + * requires * updating the @ref buf and @ref size fields. If + * you don't need to do anything other than this, this function + * pointer can be left @c NULL. The function can assume that + * the @a offset and @a length parameters point to a valid + * subset of the existing wrapped buffer. + */ + int + (*slice)(avro_wrapped_buffer_t *self, size_t offset, size_t length); +}; + +/** + * Free a wrapped buffer. + */ + +#define avro_wrapped_buffer_free(self) \ + do { \ + if ((self)->free != NULL) { \ + (self)->free((self)); \ + } \ + } while (0) + +/** + * A static initializer for an empty wrapped buffer. + */ + +#define AVRO_WRAPPED_BUFFER_EMPTY { NULL, 0, NULL, NULL, NULL, NULL } + +/** + * Moves a wrapped buffer. After returning, @a dest will wrap the + * buffer that @a src used to point at, and @a src will be empty. + */ + +void +avro_wrapped_buffer_move(avro_wrapped_buffer_t *dest, + avro_wrapped_buffer_t *src); + +/** + * Copies a buffer. + */ + +int +avro_wrapped_buffer_copy(avro_wrapped_buffer_t *dest, + const avro_wrapped_buffer_t *src, + size_t offset, size_t length); + +/** + * Slices a buffer. + */ + +int +avro_wrapped_buffer_slice(avro_wrapped_buffer_t *self, + size_t offset, size_t length); + +/** + * Creates a new wrapped buffer wrapping the given memory region. You + * have to ensure that buf stays around for as long as you need to new + * wrapped buffer. If you copy the wrapped buffer (using + * avro_wrapped_buffer_copy), this will create a copy of the data. + * Additional copies will reuse this new copy. + */ + +int +avro_wrapped_buffer_new(avro_wrapped_buffer_t *dest, + const void *buf, size_t length); + +/** + * Creates a new wrapped buffer wrapping the given C string. + */ + +#define avro_wrapped_buffer_new_string(dest, str) \ + (avro_wrapped_buffer_new((dest), (str), strlen((str))+1)) + +/** + * Creates a new wrapped buffer containing a copy of the given memory + * region. This new copy will be reference counted; if you copy it + * further (using avro_wrapped_buffer_copy), the new copies will share a + * single underlying buffer. + */ + +int +avro_wrapped_buffer_new_copy(avro_wrapped_buffer_t *dest, + const void *buf, size_t length); + +/** + * Creates a new wrapped buffer containing a copy of the given C string. + */ + +#define avro_wrapped_buffer_new_string_copy(dest, str) \ + (avro_wrapped_buffer_new_copy((dest), (str), strlen((str))+1)) + + +/*--------------------------------------------------------------------- + * Strings + */ + +/** + * A resizable buffer for storing strings and bytes values. + */ + +typedef struct avro_raw_string { + avro_wrapped_buffer_t wrapped; +} avro_raw_string_t; + +/** + * Initializes an avro_raw_string_t that you've allocated yourself. + */ + +void avro_raw_string_init(avro_raw_string_t *str); + +/** + * Finalizes an avro_raw_string_t. + */ + +void avro_raw_string_done(avro_raw_string_t *str); + +/** + * Returns the length of the data stored in an avro_raw_string_t. If + * the buffer contains a C string, this length includes the NUL + * terminator. + */ + +#define avro_raw_string_length(str) ((str)->wrapped.size) + +/** + * Returns a pointer to the data stored in an avro_raw_string_t. + */ + +#define avro_raw_string_get(str) ((str)->wrapped.buf) + +/** + * Fills an avro_raw_string_t with a copy of the given buffer. + */ + +void avro_raw_string_set_length(avro_raw_string_t *str, + const void *src, + size_t length); + +/** + * Fills an avro_raw_string_t with a copy of the given C string. + */ + +void avro_raw_string_set(avro_raw_string_t *str, const char *src); + +/** + * Appends the given C string to an avro_raw_string_t. + */ + +void avro_raw_string_append(avro_raw_string_t *str, const char *src); + +/** + * Appends the given C string to an avro_raw_string_t, using the + * provided length instead of calling strlen(src). + */ + +void avro_raw_string_append_length(avro_raw_string_t *str, + const void *src, + size_t length); +/** + * Gives control of a buffer to an avro_raw_string_t. + */ + +void +avro_raw_string_give(avro_raw_string_t *str, + avro_wrapped_buffer_t *src); + +/** + * Returns an avro_wrapped_buffer_t for the content of the string, + * ideally without copying it. + */ + +int +avro_raw_string_grab(const avro_raw_string_t *str, + avro_wrapped_buffer_t *dest); + +/** + * Clears an avro_raw_string_t. + */ + +void avro_raw_string_clear(avro_raw_string_t *str); + + +/** + * Tests two avro_raw_string_t instances for equality. + */ + +int avro_raw_string_equals(const avro_raw_string_t *str1, + const avro_raw_string_t *str2); + + +/*--------------------------------------------------------------------- + * Memoization + */ + +/** + * A specialized map that can be used to memoize the results of a + * function. The API allows you to use two keys as the memoization + * keys; if you only need one key, just use NULL for the second key. + * The result of the function should be a single pointer, or an integer + * that can be cast into a pointer (i.e., an intptr_t). + */ + +typedef struct avro_memoize { + void *cache; +} avro_memoize_t; + +/** + * Initialize an avro_memoize_t that you've allocated for yourself. + */ + +void +avro_memoize_init(avro_memoize_t *mem); + +/** + * Finalizes an avro_memoize_t. + */ + +void +avro_memoize_done(avro_memoize_t *mem); + +/** + * Search for a cached value in an avro_memoize_t. Returns a boolean + * indicating whether there's a value in the cache for the given keys. + * If there is, the cached result is placed into @ref result. + */ + +int +avro_memoize_get(avro_memoize_t *mem, + void *key1, void *key2, + void **result); + +/** + * Stores a new cached value into an avro_memoize_t, overwriting it if + * necessary. + */ + +void +avro_memoize_set(avro_memoize_t *mem, + void *key1, void *key2, + void *result); + +/** + * Removes any cached value for the given key from an avro_memoize_t. + */ + +void +avro_memoize_delete(avro_memoize_t *mem, void *key1, void *key2); + +CLOSE_EXTERN +#endif diff --git a/fluent-bit/lib/avro/src/avro/errors.h b/fluent-bit/lib/avro/src/avro/errors.h new file mode 100644 index 00000000..c7991344 --- /dev/null +++ b/fluent-bit/lib/avro/src/avro/errors.h @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#ifndef AVRO_ERRORS_H +#define AVRO_ERRORS_H +#ifdef __cplusplus +extern "C" { +#define CLOSE_EXTERN } +#else +#define CLOSE_EXTERN +#endif + +/* + * Returns a textual description of the last error condition returned by + * an Avro function. + */ + +const char *avro_strerror(void); + +void +avro_set_error(const char *fmt, ...); + +void +avro_prefix_error(const char *fmt, ...); + +CLOSE_EXTERN +#endif diff --git a/fluent-bit/lib/avro/src/avro/generic.h b/fluent-bit/lib/avro/src/avro/generic.h new file mode 100644 index 00000000..5e6047fb --- /dev/null +++ b/fluent-bit/lib/avro/src/avro/generic.h @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#ifndef AVRO_GENERIC_H +#define AVRO_GENERIC_H +#ifdef __cplusplus +extern "C" { +#define CLOSE_EXTERN } +#else +#define CLOSE_EXTERN +#endif + +#include <avro/platform.h> +#include <stdlib.h> + +#include <avro/schema.h> +#include <avro/value.h> + +/* + * This file contains an avro_value_t implementation that can store + * values of any Avro schema. It replaces the old avro_datum_t class. + */ + + +/** + * Return a generic avro_value_iface_t implementation for the given + * schema, regardless of what type it is. + */ + +avro_value_iface_t * +avro_generic_class_from_schema(avro_schema_t schema); + +/** + * Allocate a new instance of the given generic value class. @a iface + * must have been created by @ref avro_generic_class_from_schema. + */ + +int +avro_generic_value_new(avro_value_iface_t *iface, avro_value_t *dest); + + +/* + * These functions return an avro_value_iface_t implementation for each + * primitive schema type. (For enum, fixed, and the compound types, you + * must use the @ref avro_generic_class_from_schema function.) + */ + +avro_value_iface_t *avro_generic_boolean_class(void); +avro_value_iface_t *avro_generic_bytes_class(void); +avro_value_iface_t *avro_generic_double_class(void); +avro_value_iface_t *avro_generic_float_class(void); +avro_value_iface_t *avro_generic_int_class(void); +avro_value_iface_t *avro_generic_long_class(void); +avro_value_iface_t *avro_generic_null_class(void); +avro_value_iface_t *avro_generic_string_class(void); + + +/* + * These functions instantiate a new generic primitive value. + */ + +int avro_generic_boolean_new(avro_value_t *value, int val); +int avro_generic_bytes_new(avro_value_t *value, void *buf, size_t size); +int avro_generic_double_new(avro_value_t *value, double val); +int avro_generic_float_new(avro_value_t *value, float val); +int avro_generic_int_new(avro_value_t *value, int32_t val); +int avro_generic_long_new(avro_value_t *value, int64_t val); +int avro_generic_null_new(avro_value_t *value); +int avro_generic_string_new(avro_value_t *value, const char *val); +int avro_generic_string_new_length(avro_value_t *value, const char *val, size_t size); + + +CLOSE_EXTERN +#endif diff --git a/fluent-bit/lib/avro/src/avro/io.h b/fluent-bit/lib/avro/src/avro/io.h new file mode 100644 index 00000000..ffbb68dc --- /dev/null +++ b/fluent-bit/lib/avro/src/avro/io.h @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#ifndef AVRO_IO_H +#define AVRO_IO_H +#ifdef __cplusplus +extern "C" { +#define CLOSE_EXTERN } +#else +#define CLOSE_EXTERN +#endif + +#include <avro/platform.h> +#include <stdio.h> + +#include <avro/basics.h> +#include <avro/legacy.h> +#include <avro/schema.h> +#include <avro/value.h> + +typedef struct avro_reader_t_ *avro_reader_t; +typedef struct avro_writer_t_ *avro_writer_t; + +/* + * io + */ + +avro_reader_t avro_reader_file(FILE * fp); +avro_reader_t avro_reader_file_fp(FILE * fp, int should_close); +avro_writer_t avro_writer_file(FILE * fp); +avro_writer_t avro_writer_file_fp(FILE * fp, int should_close); +avro_reader_t avro_reader_memory(const char *buf, int64_t len); +avro_writer_t avro_writer_memory(const char *buf, int64_t len); + +void +avro_reader_memory_set_source(avro_reader_t reader, const char *buf, int64_t len); + +void +avro_writer_memory_set_dest(avro_writer_t writer, const char *buf, int64_t len); + +int avro_read(avro_reader_t reader, void *buf, int64_t len); +int avro_skip(avro_reader_t reader, int64_t len); +int avro_write(avro_writer_t writer, void *buf, int64_t len); + +void avro_reader_reset(avro_reader_t reader); + +void avro_writer_reset(avro_writer_t writer); +int64_t avro_writer_tell(avro_writer_t writer); +void avro_writer_flush(avro_writer_t writer); + +void avro_writer_dump(avro_writer_t writer, FILE * fp); +void avro_reader_dump(avro_reader_t reader, FILE * fp); + +int avro_reader_is_eof(avro_reader_t reader); + +void avro_reader_free(avro_reader_t reader); +void avro_writer_free(avro_writer_t writer); + +int avro_schema_to_json(const avro_schema_t schema, avro_writer_t out); + +/* + * Reads a binary-encoded Avro value from the given reader object, + * storing the result into dest. + */ + +int +avro_value_read(avro_reader_t reader, avro_value_t *dest); + +/* + * Writes a binary-encoded Avro value to the given writer object. + */ + +int +avro_value_write(avro_writer_t writer, avro_value_t *src); + +/* + * Returns the size of the binary encoding of the given Avro value. + */ + +int +avro_value_sizeof(avro_value_t *src, size_t *size); + + +/* File object container */ +typedef struct avro_file_reader_t_ *avro_file_reader_t; +typedef struct avro_file_writer_t_ *avro_file_writer_t; + +int avro_file_writer_create(const char *path, avro_schema_t schema, + avro_file_writer_t * writer); +int avro_file_writer_create_fp(FILE *fp, const char *path, int should_close, + avro_schema_t schema, avro_file_writer_t * writer); +int avro_file_writer_create_with_codec(const char *path, + avro_schema_t schema, avro_file_writer_t * writer, + const char *codec, size_t block_size); +int avro_file_writer_create_with_codec_fp(FILE *fp, const char *path, int should_close, + avro_schema_t schema, avro_file_writer_t * writer, + const char *codec, size_t block_size); +int avro_file_writer_open(const char *path, avro_file_writer_t * writer); +int avro_file_writer_open_bs(const char *path, avro_file_writer_t * writer, size_t block_size); +int avro_file_reader(const char *path, avro_file_reader_t * reader); +int avro_file_reader_fp(FILE *fp, const char *path, int should_close, + avro_file_reader_t * reader); + +avro_schema_t +avro_file_reader_get_writer_schema(avro_file_reader_t reader); + +int avro_file_writer_sync(avro_file_writer_t writer); +int avro_file_writer_flush(avro_file_writer_t writer); +int avro_file_writer_close(avro_file_writer_t writer); + +int avro_file_reader_close(avro_file_reader_t reader); + +int +avro_file_reader_read_value(avro_file_reader_t reader, avro_value_t *dest); + +int +avro_file_writer_append_value(avro_file_writer_t writer, avro_value_t *src); + +int +avro_file_writer_append_encoded(avro_file_writer_t writer, + const void *buf, int64_t len); + +/* + * Legacy avro_datum_t API + */ + +int avro_read_data(avro_reader_t reader, + avro_schema_t writer_schema, + avro_schema_t reader_schema, avro_datum_t * datum); +int avro_skip_data(avro_reader_t reader, avro_schema_t writer_schema); +int avro_write_data(avro_writer_t writer, + avro_schema_t writer_schema, avro_datum_t datum); +int64_t avro_size_data(avro_writer_t writer, + avro_schema_t writer_schema, avro_datum_t datum); + +int avro_file_writer_append(avro_file_writer_t writer, avro_datum_t datum); + +int avro_file_reader_read(avro_file_reader_t reader, + avro_schema_t readers_schema, avro_datum_t * datum); + +CLOSE_EXTERN +#endif diff --git a/fluent-bit/lib/avro/src/avro/legacy.h b/fluent-bit/lib/avro/src/avro/legacy.h new file mode 100644 index 00000000..ba8f29ba --- /dev/null +++ b/fluent-bit/lib/avro/src/avro/legacy.h @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#ifndef AVRO_LEGACY_H +#define AVRO_LEGACY_H +#ifdef __cplusplus +extern "C" { +#define CLOSE_EXTERN } +#else +#define CLOSE_EXTERN +#endif + +#include <avro/platform.h> +#include <stdio.h> + +#include <avro/basics.h> +#include <avro/data.h> +#include <avro/schema.h> +#include <avro/value.h> + +/* + * This file defines the deprecated interface for handling Avro values. + * It's here solely for backwards compatibility. New code should use + * the avro_value_t interface (defined in avro/value.h). The + * avro_datum_t type has been replaced by the “generic” implementation + * of the value interface, which is defined in avro/generic.h. You can + * also use your own application-specific types as Avro values by + * defining your own avro_value_t implementation for them. + */ + +/** + * A function used to free a bytes, string, or fixed buffer once it is + * no longer needed by the datum that wraps it. + */ + +typedef void +(*avro_free_func_t)(void *ptr, size_t sz); + +/** + * An avro_free_func_t that frees the buffer using the custom allocator + * provided to avro_set_allocator. + */ + +void +avro_alloc_free_func(void *ptr, size_t sz); + +/* + * Datum constructors. Each datum stores a reference to the schema that + * the datum is an instance of. The primitive datum constructors don't + * need to take in an explicit avro_schema_t parameter, since there's + * only one schema that they could be an instance of. The complex + * constructors do need an explicit schema parameter. + */ + +typedef struct avro_obj_t *avro_datum_t; +avro_datum_t avro_string(const char *str); +avro_datum_t avro_givestring(const char *str, + avro_free_func_t free); +avro_datum_t avro_bytes(const char *buf, int64_t len); +avro_datum_t avro_givebytes(const char *buf, int64_t len, + avro_free_func_t free); +avro_datum_t avro_int32(int32_t i); +avro_datum_t avro_int64(int64_t l); +avro_datum_t avro_float(float f); +avro_datum_t avro_double(double d); +avro_datum_t avro_boolean(int8_t i); +avro_datum_t avro_null(void); +avro_datum_t avro_record(avro_schema_t schema); +avro_datum_t avro_enum(avro_schema_t schema, int i); +avro_datum_t avro_fixed(avro_schema_t schema, + const char *bytes, const int64_t size); +avro_datum_t avro_givefixed(avro_schema_t schema, + const char *bytes, const int64_t size, + avro_free_func_t free); +avro_datum_t avro_map(avro_schema_t schema); +avro_datum_t avro_array(avro_schema_t schema); +avro_datum_t avro_union(avro_schema_t schema, + int64_t discriminant, const avro_datum_t datum); + +/** + * Returns the schema that the datum is an instance of. + */ + +avro_schema_t avro_datum_get_schema(const avro_datum_t datum); + +/* + * Constructs a new avro_datum_t instance that's appropriate for holding + * values of the given schema. + */ + +avro_datum_t avro_datum_from_schema(const avro_schema_t schema); + +/* getters */ +int avro_string_get(avro_datum_t datum, char **p); +int avro_bytes_get(avro_datum_t datum, char **bytes, int64_t * size); +int avro_int32_get(avro_datum_t datum, int32_t * i); +int avro_int64_get(avro_datum_t datum, int64_t * l); +int avro_float_get(avro_datum_t datum, float *f); +int avro_double_get(avro_datum_t datum, double *d); +int avro_boolean_get(avro_datum_t datum, int8_t * i); + +int avro_enum_get(const avro_datum_t datum); +const char *avro_enum_get_name(const avro_datum_t datum); +int avro_fixed_get(avro_datum_t datum, char **bytes, int64_t * size); +int avro_record_get(const avro_datum_t record, const char *field_name, + avro_datum_t * value); + +/* + * A helper macro that extracts the value of the given field of a + * record. + */ + +#define avro_record_get_field_value(rc, rec, typ, fname, ...) \ + do { \ + avro_datum_t field = NULL; \ + (rc) = avro_record_get((rec), (fname), &field); \ + if (rc) break; \ + (rc) = avro_##typ##_get(field, __VA_ARGS__); \ + } while (0) + + +int avro_map_get(const avro_datum_t datum, const char *key, + avro_datum_t * value); +/* + * For maps, the "index" for each entry is based on the order that they + * were added to the map. + */ +int avro_map_get_key(const avro_datum_t datum, int index, + const char **key); +int avro_map_get_index(const avro_datum_t datum, const char *key, + int *index); +size_t avro_map_size(const avro_datum_t datum); +int avro_array_get(const avro_datum_t datum, int64_t index, avro_datum_t * value); +size_t avro_array_size(const avro_datum_t datum); + +/* + * These accessors allow you to query the current branch of a union + * value, returning either the branch's discriminant value or the + * avro_datum_t of the branch. A union value can be uninitialized, in + * which case the discriminant will be -1 and the datum NULL. + */ + +int64_t avro_union_discriminant(const avro_datum_t datum); +avro_datum_t avro_union_current_branch(avro_datum_t datum); + +/* setters */ +int avro_string_set(avro_datum_t datum, const char *p); +int avro_givestring_set(avro_datum_t datum, const char *p, + avro_free_func_t free); + +int avro_bytes_set(avro_datum_t datum, const char *bytes, const int64_t size); +int avro_givebytes_set(avro_datum_t datum, const char *bytes, + const int64_t size, + avro_free_func_t free); + +int avro_int32_set(avro_datum_t datum, const int32_t i); +int avro_int64_set(avro_datum_t datum, const int64_t l); +int avro_float_set(avro_datum_t datum, const float f); +int avro_double_set(avro_datum_t datum, const double d); +int avro_boolean_set(avro_datum_t datum, const int8_t i); + +int avro_enum_set(avro_datum_t datum, const int symbol_value); +int avro_enum_set_name(avro_datum_t datum, const char *symbol_name); +int avro_fixed_set(avro_datum_t datum, const char *bytes, const int64_t size); +int avro_givefixed_set(avro_datum_t datum, const char *bytes, + const int64_t size, + avro_free_func_t free); + +int avro_record_set(avro_datum_t record, const char *field_name, + avro_datum_t value); + +/* + * A helper macro that sets the value of the given field of a record. + */ + +#define avro_record_set_field_value(rc, rec, typ, fname, ...) \ + do { \ + avro_datum_t field = NULL; \ + (rc) = avro_record_get((rec), (fname), &field); \ + if (rc) break; \ + (rc) = avro_##typ##_set(field, __VA_ARGS__); \ + } while (0) + +int avro_map_set(avro_datum_t map, const char *key, + avro_datum_t value); +int avro_array_append_datum(avro_datum_t array_datum, + avro_datum_t datum); + +/* + * This function selects the active branch of a union value, and can be + * safely called on an existing union to change the current branch. If + * the branch changes, we'll automatically construct a new avro_datum_t + * for the new branch's schema type. If the desired branch is already + * the active branch of the union, we'll leave the existing datum + * instance as-is. The branch datum will be placed into the "branch" + * parameter, regardless of whether we have to create a new datum + * instance or not. + */ + +int avro_union_set_discriminant(avro_datum_t unionp, + int discriminant, + avro_datum_t *branch); + +/** + * Resets a datum instance. For arrays and maps, this frees all + * elements and clears the container. For records and unions, this + * recursively resets any child datum instances. + */ + +int +avro_datum_reset(avro_datum_t value); + +/* reference counting */ +avro_datum_t avro_datum_incref(avro_datum_t value); +void avro_datum_decref(avro_datum_t value); + +void avro_datum_print(avro_datum_t value, FILE * fp); + +int avro_datum_equal(avro_datum_t a, avro_datum_t b); + +/* + * Returns a string containing the JSON encoding of an Avro value. You + * must free this string when you're done with it, using the standard + * free() function. (*Not* using the custom Avro allocator.) + */ + +int avro_datum_to_json(const avro_datum_t datum, + int one_line, char **json_str); + + +int avro_schema_datum_validate(avro_schema_t + expected_schema, avro_datum_t datum); + +/* + * An avro_value_t implementation for avro_datum_t objects. + */ + +avro_value_iface_t * +avro_datum_class(void); + +/* + * Creates a new avro_value_t instance for the given datum. + */ + +int +avro_datum_as_value(avro_value_t *value, avro_datum_t src); + + +CLOSE_EXTERN +#endif diff --git a/fluent-bit/lib/avro/src/avro/msinttypes.h b/fluent-bit/lib/avro/src/avro/msinttypes.h new file mode 100644 index 00000000..29be14b9 --- /dev/null +++ b/fluent-bit/lib/avro/src/avro/msinttypes.h @@ -0,0 +1,315 @@ +// ISO C9x compliant inttypes.h for Microsoft Visual Studio +// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 +// +// Copyright (c) 2006 Alexander Chemeris +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. The name of the author may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _MSC_VER // [ +#error "Use this header only with Microsoft Visual C++ compilers!" +#endif // _MSC_VER ] + +#ifndef _MSC_INTTYPES_H_ // [ +#define _MSC_INTTYPES_H_ + +#if _MSC_VER > 1000 +#pragma once +#endif + +////////////////////////////////////// +// Start AVRO specific modifications +////////////////////////////////////// +#include "msstdint.h" + +// Modification for AVRO of inttypes.h +#define __STDC_FORMAT_MACROS (1) + +////////////////////////////////////// +// End AVRO specific modifications +////////////////////////////////////// + +// 7.8 Format conversion of integer types + +typedef struct { + intmax_t quot; + intmax_t rem; +} imaxdiv_t; + +// 7.8.1 Macros for format specifiers + +#if !defined(__cplusplus) || defined(__STDC_FORMAT_MACROS) // [ See footnote 185 at page 198 + +// The fprintf macros for signed integers are: +#define PRId8 "d" +#define PRIi8 "i" +#define PRIdLEAST8 "d" +#define PRIiLEAST8 "i" +#define PRIdFAST8 "d" +#define PRIiFAST8 "i" + +#define PRId16 "hd" +#define PRIi16 "hi" +#define PRIdLEAST16 "hd" +#define PRIiLEAST16 "hi" +#define PRIdFAST16 "hd" +#define PRIiFAST16 "hi" + +#define PRId32 "I32d" +#define PRIi32 "I32i" +#define PRIdLEAST32 "I32d" +#define PRIiLEAST32 "I32i" +#define PRIdFAST32 "I32d" +#define PRIiFAST32 "I32i" + +#define PRId64 "I64d" +#define PRIi64 "I64i" +#define PRIdLEAST64 "I64d" +#define PRIiLEAST64 "I64i" +#define PRIdFAST64 "I64d" +#define PRIiFAST64 "I64i" + +#define PRIdMAX "I64d" +#define PRIiMAX "I64i" + +#define PRIdPTR "Id" +#define PRIiPTR "Ii" + +// The fprintf macros for unsigned integers are: +#define PRIo8 "o" +#define PRIu8 "u" +#define PRIx8 "x" +#define PRIX8 "X" +#define PRIoLEAST8 "o" +#define PRIuLEAST8 "u" +#define PRIxLEAST8 "x" +#define PRIXLEAST8 "X" +#define PRIoFAST8 "o" +#define PRIuFAST8 "u" +#define PRIxFAST8 "x" +#define PRIXFAST8 "X" + +#define PRIo16 "ho" +#define PRIu16 "hu" +#define PRIx16 "hx" +#define PRIX16 "hX" +#define PRIoLEAST16 "ho" +#define PRIuLEAST16 "hu" +#define PRIxLEAST16 "hx" +#define PRIXLEAST16 "hX" +#define PRIoFAST16 "ho" +#define PRIuFAST16 "hu" +#define PRIxFAST16 "hx" +#define PRIXFAST16 "hX" + +#define PRIo32 "I32o" +#define PRIu32 "I32u" +#define PRIx32 "I32x" +#define PRIX32 "I32X" +#define PRIoLEAST32 "I32o" +#define PRIuLEAST32 "I32u" +#define PRIxLEAST32 "I32x" +#define PRIXLEAST32 "I32X" +#define PRIoFAST32 "I32o" +#define PRIuFAST32 "I32u" +#define PRIxFAST32 "I32x" +#define PRIXFAST32 "I32X" + +#define PRIo64 "I64o" +#define PRIu64 "I64u" +#define PRIx64 "I64x" +#define PRIX64 "I64X" +#define PRIoLEAST64 "I64o" +#define PRIuLEAST64 "I64u" +#define PRIxLEAST64 "I64x" +#define PRIXLEAST64 "I64X" +#define PRIoFAST64 "I64o" +#define PRIuFAST64 "I64u" +#define PRIxFAST64 "I64x" +#define PRIXFAST64 "I64X" + +#define PRIoMAX "I64o" +#define PRIuMAX "I64u" +#define PRIxMAX "I64x" +#define PRIXMAX "I64X" + +#define PRIoPTR "Io" +#define PRIuPTR "Iu" +#define PRIxPTR "Ix" +#define PRIXPTR "IX" + +// The fscanf macros for signed integers are: +#define SCNd8 "d" +#define SCNi8 "i" +#define SCNdLEAST8 "d" +#define SCNiLEAST8 "i" +#define SCNdFAST8 "d" +#define SCNiFAST8 "i" + +#define SCNd16 "hd" +#define SCNi16 "hi" +#define SCNdLEAST16 "hd" +#define SCNiLEAST16 "hi" +#define SCNdFAST16 "hd" +#define SCNiFAST16 "hi" + +#define SCNd32 "ld" +#define SCNi32 "li" +#define SCNdLEAST32 "ld" +#define SCNiLEAST32 "li" +#define SCNdFAST32 "ld" +#define SCNiFAST32 "li" + +#define SCNd64 "I64d" +#define SCNi64 "I64i" +#define SCNdLEAST64 "I64d" +#define SCNiLEAST64 "I64i" +#define SCNdFAST64 "I64d" +#define SCNiFAST64 "I64i" + +#define SCNdMAX "I64d" +#define SCNiMAX "I64i" + +#ifdef _WIN64 // [ +# define SCNdPTR "I64d" +# define SCNiPTR "I64i" +#else // _WIN64 ][ +# define SCNdPTR "ld" +# define SCNiPTR "li" +#endif // _WIN64 ] + +// The fscanf macros for unsigned integers are: +#define SCNo8 "o" +#define SCNu8 "u" +#define SCNx8 "x" +#define SCNX8 "X" +#define SCNoLEAST8 "o" +#define SCNuLEAST8 "u" +#define SCNxLEAST8 "x" +#define SCNXLEAST8 "X" +#define SCNoFAST8 "o" +#define SCNuFAST8 "u" +#define SCNxFAST8 "x" +#define SCNXFAST8 "X" + +#define SCNo16 "ho" +#define SCNu16 "hu" +#define SCNx16 "hx" +#define SCNX16 "hX" +#define SCNoLEAST16 "ho" +#define SCNuLEAST16 "hu" +#define SCNxLEAST16 "hx" +#define SCNXLEAST16 "hX" +#define SCNoFAST16 "ho" +#define SCNuFAST16 "hu" +#define SCNxFAST16 "hx" +#define SCNXFAST16 "hX" + +#define SCNo32 "lo" +#define SCNu32 "lu" +#define SCNx32 "lx" +#define SCNX32 "lX" +#define SCNoLEAST32 "lo" +#define SCNuLEAST32 "lu" +#define SCNxLEAST32 "lx" +#define SCNXLEAST32 "lX" +#define SCNoFAST32 "lo" +#define SCNuFAST32 "lu" +#define SCNxFAST32 "lx" +#define SCNXFAST32 "lX" + +#define SCNo64 "I64o" +#define SCNu64 "I64u" +#define SCNx64 "I64x" +#define SCNX64 "I64X" +#define SCNoLEAST64 "I64o" +#define SCNuLEAST64 "I64u" +#define SCNxLEAST64 "I64x" +#define SCNXLEAST64 "I64X" +#define SCNoFAST64 "I64o" +#define SCNuFAST64 "I64u" +#define SCNxFAST64 "I64x" +#define SCNXFAST64 "I64X" + +#define SCNoMAX "I64o" +#define SCNuMAX "I64u" +#define SCNxMAX "I64x" +#define SCNXMAX "I64X" + +#ifdef _WIN64 // [ +# define SCNoPTR "I64o" +# define SCNuPTR "I64u" +# define SCNxPTR "I64x" +# define SCNXPTR "I64X" +#else // _WIN64 ][ +# define SCNoPTR "lo" +# define SCNuPTR "lu" +# define SCNxPTR "lx" +# define SCNXPTR "lX" +#endif // _WIN64 ] + +#endif // __STDC_FORMAT_MACROS ] + +// 7.8.2 Functions for greatest-width integer types + +// 7.8.2.1 The imaxabs function +#define imaxabs _abs64 + +// 7.8.2.2 The imaxdiv function + +// This is modified version of div() function from Microsoft's div.c found +// in %MSVC.NET%\crt\src\div.c +#ifdef STATIC_IMAXDIV // [ +static +#else // STATIC_IMAXDIV ][ +_inline +#endif // STATIC_IMAXDIV ] +imaxdiv_t __cdecl imaxdiv(intmax_t numer, intmax_t denom) +{ + imaxdiv_t result; + + result.quot = numer / denom; + result.rem = numer % denom; + + if (numer < 0 && result.rem > 0) { + // did division wrong; must fix up + ++result.quot; + result.rem -= denom; + } + + return result; +} + +// 7.8.2.3 The strtoimax and strtoumax functions +#define strtoimax _strtoi64 +#define strtoumax _strtoui64 + +// 7.8.2.4 The wcstoimax and wcstoumax functions +#define wcstoimax _wcstoi64 +#define wcstoumax _wcstoui64 + + +#endif // _MSC_INTTYPES_H_ ] diff --git a/fluent-bit/lib/avro/src/avro/msstdint.h b/fluent-bit/lib/avro/src/avro/msstdint.h new file mode 100644 index 00000000..d02608a5 --- /dev/null +++ b/fluent-bit/lib/avro/src/avro/msstdint.h @@ -0,0 +1,247 @@ +// ISO C9x compliant stdint.h for Microsoft Visual Studio +// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 +// +// Copyright (c) 2006-2008 Alexander Chemeris +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. The name of the author may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _MSC_VER // [ +#error "Use this header only with Microsoft Visual C++ compilers!" +#endif // _MSC_VER ] + +#ifndef _MSC_STDINT_H_ // [ +#define _MSC_STDINT_H_ + +#if _MSC_VER > 1000 +#pragma once +#endif + +#include <limits.h> + +// For Visual Studio 6 in C++ mode and for many Visual Studio versions when +// compiling for ARM we should wrap <wchar.h> include with 'extern "C++" {}' +// or compiler give many errors like this: +// error C2733: second C linkage of overloaded function 'wmemchr' not allowed +#ifdef __cplusplus +extern "C" { +#endif +# include <wchar.h> +#ifdef __cplusplus +} +#endif + +// Define _W64 macros to mark types changing their size, like intptr_t. +#ifndef _W64 +# if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) && _MSC_VER >= 1300 +# define _W64 __w64 +# else +# define _W64 +# endif +#endif + + +// 7.18.1 Integer types + +// 7.18.1.1 Exact-width integer types + +// Visual Studio 6 and Embedded Visual C++ 4 doesn't +// realize that, e.g. char has the same size as __int8 +// so we give up on __intX for them. +#if (_MSC_VER < 1300) + typedef signed char int8_t; + typedef signed short int16_t; + typedef signed int int32_t; + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; +#else + typedef signed __int8 int8_t; + typedef signed __int16 int16_t; + typedef signed __int32 int32_t; + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; +#endif +typedef signed __int64 int64_t; +typedef unsigned __int64 uint64_t; + + +// 7.18.1.2 Minimum-width integer types +typedef int8_t int_least8_t; +typedef int16_t int_least16_t; +typedef int32_t int_least32_t; +typedef int64_t int_least64_t; +typedef uint8_t uint_least8_t; +typedef uint16_t uint_least16_t; +typedef uint32_t uint_least32_t; +typedef uint64_t uint_least64_t; + +// 7.18.1.3 Fastest minimum-width integer types +typedef int8_t int_fast8_t; +typedef int16_t int_fast16_t; +typedef int32_t int_fast32_t; +typedef int64_t int_fast64_t; +typedef uint8_t uint_fast8_t; +typedef uint16_t uint_fast16_t; +typedef uint32_t uint_fast32_t; +typedef uint64_t uint_fast64_t; + +// 7.18.1.4 Integer types capable of holding object pointers +#ifdef _WIN64 // [ + typedef signed __int64 intptr_t; + typedef unsigned __int64 uintptr_t; +#else // _WIN64 ][ + typedef _W64 signed int intptr_t; + typedef _W64 unsigned int uintptr_t; +#endif // _WIN64 ] + +// 7.18.1.5 Greatest-width integer types +typedef int64_t intmax_t; +typedef uint64_t uintmax_t; + + +// 7.18.2 Limits of specified-width integer types + +#if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) // [ See footnote 220 at page 257 and footnote 221 at page 259 + +// 7.18.2.1 Limits of exact-width integer types +#define INT8_MIN ((int8_t)_I8_MIN) +#define INT8_MAX _I8_MAX +#define INT16_MIN ((int16_t)_I16_MIN) +#define INT16_MAX _I16_MAX +#define INT32_MIN ((int32_t)_I32_MIN) +#define INT32_MAX _I32_MAX +#define INT64_MIN ((int64_t)_I64_MIN) +#define INT64_MAX _I64_MAX +#define UINT8_MAX _UI8_MAX +#define UINT16_MAX _UI16_MAX +#define UINT32_MAX _UI32_MAX +#define UINT64_MAX _UI64_MAX + +// 7.18.2.2 Limits of minimum-width integer types +#define INT_LEAST8_MIN INT8_MIN +#define INT_LEAST8_MAX INT8_MAX +#define INT_LEAST16_MIN INT16_MIN +#define INT_LEAST16_MAX INT16_MAX +#define INT_LEAST32_MIN INT32_MIN +#define INT_LEAST32_MAX INT32_MAX +#define INT_LEAST64_MIN INT64_MIN +#define INT_LEAST64_MAX INT64_MAX +#define UINT_LEAST8_MAX UINT8_MAX +#define UINT_LEAST16_MAX UINT16_MAX +#define UINT_LEAST32_MAX UINT32_MAX +#define UINT_LEAST64_MAX UINT64_MAX + +// 7.18.2.3 Limits of fastest minimum-width integer types +#define INT_FAST8_MIN INT8_MIN +#define INT_FAST8_MAX INT8_MAX +#define INT_FAST16_MIN INT16_MIN +#define INT_FAST16_MAX INT16_MAX +#define INT_FAST32_MIN INT32_MIN +#define INT_FAST32_MAX INT32_MAX +#define INT_FAST64_MIN INT64_MIN +#define INT_FAST64_MAX INT64_MAX +#define UINT_FAST8_MAX UINT8_MAX +#define UINT_FAST16_MAX UINT16_MAX +#define UINT_FAST32_MAX UINT32_MAX +#define UINT_FAST64_MAX UINT64_MAX + +// 7.18.2.4 Limits of integer types capable of holding object pointers +#ifdef _WIN64 // [ +# define INTPTR_MIN INT64_MIN +# define INTPTR_MAX INT64_MAX +# define UINTPTR_MAX UINT64_MAX +#else // _WIN64 ][ +# define INTPTR_MIN INT32_MIN +# define INTPTR_MAX INT32_MAX +# define UINTPTR_MAX UINT32_MAX +#endif // _WIN64 ] + +// 7.18.2.5 Limits of greatest-width integer types +#define INTMAX_MIN INT64_MIN +#define INTMAX_MAX INT64_MAX +#define UINTMAX_MAX UINT64_MAX + +// 7.18.3 Limits of other integer types + +#ifdef _WIN64 // [ +# define PTRDIFF_MIN _I64_MIN +# define PTRDIFF_MAX _I64_MAX +#else // _WIN64 ][ +# define PTRDIFF_MIN _I32_MIN +# define PTRDIFF_MAX _I32_MAX +#endif // _WIN64 ] + +#define SIG_ATOMIC_MIN INT_MIN +#define SIG_ATOMIC_MAX INT_MAX + +#ifndef SIZE_MAX // [ +# ifdef _WIN64 // [ +# define SIZE_MAX _UI64_MAX +# else // _WIN64 ][ +# define SIZE_MAX _UI32_MAX +# endif // _WIN64 ] +#endif // SIZE_MAX ] + +// WCHAR_MIN and WCHAR_MAX are also defined in <wchar.h> +#ifndef WCHAR_MIN // [ +# define WCHAR_MIN 0 +#endif // WCHAR_MIN ] +#ifndef WCHAR_MAX // [ +# define WCHAR_MAX _UI16_MAX +#endif // WCHAR_MAX ] + +#define WINT_MIN 0 +#define WINT_MAX _UI16_MAX + +#endif // __STDC_LIMIT_MACROS ] + + +// 7.18.4 Limits of other integer types + +#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 + +// 7.18.4.1 Macros for minimum-width integer constants + +#define INT8_C(val) val##i8 +#define INT16_C(val) val##i16 +#define INT32_C(val) val##i32 +#define INT64_C(val) val##i64 + +#define UINT8_C(val) val##ui8 +#define UINT16_C(val) val##ui16 +#define UINT32_C(val) val##ui32 +#define UINT64_C(val) val##ui64 + +// 7.18.4.2 Macros for greatest-width integer constants +#define INTMAX_C INT64_C +#define UINTMAX_C UINT64_C + +#endif // __STDC_CONSTANT_MACROS ] + + +#endif // _MSC_STDINT_H_ ] diff --git a/fluent-bit/lib/avro/src/avro/platform.h b/fluent-bit/lib/avro/src/avro/platform.h new file mode 100644 index 00000000..92930550 --- /dev/null +++ b/fluent-bit/lib/avro/src/avro/platform.h @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#ifndef AVRO_PLATFORM_H +#define AVRO_PLATFORM_H +#ifdef __cplusplus +extern "C" { +#define CLOSE_EXTERN } +#else +#define CLOSE_EXTERN +#endif + +/* Use this header file to include platform specific definitions */ + +#ifdef _WIN32 + #include <avro/msinttypes.h> +#else + #include <inttypes.h> +#endif + +// Defines for printing size_t. +#if defined(_WIN64) + #define PRIsz PRIu64 +#elif defined(_WIN32) + #define PRIsz PRIu32 +#else // GCC + #define PRIsz "zu" +#endif + +CLOSE_EXTERN +#endif diff --git a/fluent-bit/lib/avro/src/avro/refcount.h b/fluent-bit/lib/avro/src/avro/refcount.h new file mode 100644 index 00000000..27369900 --- /dev/null +++ b/fluent-bit/lib/avro/src/avro/refcount.h @@ -0,0 +1,306 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#ifndef AVRO_REFCOUNT_H +#define AVRO_REFCOUNT_H + +#if defined(_WIN32) && defined(__cplusplus) +/* Include the C++ file <intrin.h> outside the scope of extern "C" */ +#include <intrin.h> +#endif + +#ifdef __cplusplus +extern "C" { +#define CLOSE_EXTERN } +#else +#define CLOSE_EXTERN +#endif + +/** + * Atomically sets the value of a reference count. + */ + +static inline void +avro_refcount_set(volatile int *refcount, int value); + +/** + * Increments a reference count, ensuring that its value doesn't + * overflow. + */ + +static inline void +avro_refcount_inc(volatile int *refcount); + +/** + * Decrements a reference count, and returns whether the resulting + * (decremented) value is 0. + */ + +static inline int +avro_refcount_dec(volatile int *refcount); + + +/*----------------------------------------------------------------------- + * Non-Atomic Reference Count + */ +#if defined(AVRO_NON_ATOMIC_REFCOUNT) +static inline void +avro_refcount_set(volatile int *refcount, int value) +{ + *refcount = value; +} + +static inline void +avro_refcount_inc(volatile int *refcount) +{ + if (*refcount != (int) -1) { + *refcount += 1; + } +} + +static inline int +avro_refcount_dec(volatile int *refcount) +{ + if (*refcount != (int) -1) { + *refcount -= 1; + return (*refcount == 0); + } + return 0; +} + +/*----------------------------------------------------------------------- + * Mac OS X + */ + +#elif __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1050 + +#include <libkern/OSAtomic.h> + +static inline void +avro_refcount_set(volatile int *refcount, int value) +{ + *refcount = value; +} + +static inline void +avro_refcount_inc(volatile int *refcount) +{ + if (*refcount != (int) -1) { + OSAtomicIncrement32(refcount); + } +} + +static inline int +avro_refcount_dec(volatile int *refcount) +{ + if (*refcount != (int) -1) { + return (OSAtomicDecrement32(refcount) == 0); + } + return 0; +} + + +/*----------------------------------------------------------------------- + * GCC intrinsics + */ + +#elif (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40500 \ +|| defined(__clang__) + +static inline void +avro_refcount_set(volatile int *refcount, int value) +{ + *refcount = value; +} + +static inline void +avro_refcount_inc(volatile int *refcount) +{ + if (*refcount != (int) -1) { + __sync_add_and_fetch(refcount, 1); + } +} + +static inline int +avro_refcount_dec(volatile int *refcount) +{ + if (*refcount != (int) -1) { + return (__sync_sub_and_fetch(refcount, 1) == 0); + } + return 0; +} + + +/*----------------------------------------------------------------------- + * Raw x86 assembly + */ + +#elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) + +/* determine the size of int */ + +#include <limits.h> +#include <avro/platform.h> +#if INT_MAX == INT32_MAX +#define REFCOUNT_SS "l" +#elif INT_MAX == INT64_MAX +#define REFCOUNT_SS "q" +#else +#error "Unknown int size" +#endif + +static inline void +avro_refcount_set(volatile int *refcount, int value) +{ + *refcount = value; +} + +static inline void +avro_refcount_inc(volatile int *refcount) +{ + if (*refcount != (int) -1) { + __asm__ __volatile__ ("lock ; inc"REFCOUNT_SS" %0" + :"=m" (*refcount) + :"m" (*refcount)); + } +} + +static inline int +avro_refcount_dec(volatile int *refcount) +{ + if (*refcount != (int) -1) { + char result; + __asm__ __volatile__ ("lock ; dec"REFCOUNT_SS" %0; setz %1" + :"=m" (*refcount), "=q" (result) + :"m" (*refcount)); + return result; + } + return 0; +} + +#undef REFCOUNT_SS + + +/*----------------------------------------------------------------------- + * Raw PPC assembly + */ + +#elif defined(__GNUC__) && defined(__ppc__) + +static inline int +avro_refcount_LL_int(volatile int *ptr) +{ + int val; + __asm__ __volatile__ ("lwarx %[val],0,%[ptr]" + : [val] "=r" (val) + : [ptr] "r" (&ptr) + : "cc"); + + return val; +} + +/* Returns non-zero if the store was successful, zero otherwise. */ +static inline int +avro_refcount_SC_int(volatile int *ptr, int val) +{ + int ret = 1; /* init to non-zero, will be reset to 0 if SC was successful */ + __asm__ __volatile__ ("stwcx. %[val],0,%[ptr];\n" + "beq 1f;\n" + "li %[ret], 0;\n" + "1: ;\n" + : [ret] "=r" (ret) + : [ptr] "r" (&ptr), [val] "r" (val), "0" (ret) + : "cc", "memory"); + return ret; +} + +static inline void +avro_refcount_set(volatile int *refcount, int value) +{ + *refcount = value; +} + +static inline void +avro_refcount_inc(volatile int *refcount) +{ + int prev; + do { + prev = avro_refcount_LL_int(refcount); + if (prev == (int) -1) { + return; + } + } while (!avro_refcount_SC_int(refcount, prev + 1)); +} + +static inline int +avro_refcount_dec(volatile int *refcount) +{ + int prev; + do { + prev = avro_refcount_LL_int(refcount); + if (prev == (int) -1) { + return 0; + } + } while (!avro_refcount_SC_int(refcount, prev - 1)); + return prev == 1; +} + + +/*----------------------------------------------------------------------- + * Windows intrinsics + */ +#elif defined(_WIN32) + +#ifdef __cplusplus +// Note: <intrin.h> included outside the extern "C" wrappers above +#else +#include <windows.h> +#include <intrin.h> +#endif // __cplusplus + +static inline void +avro_refcount_set(volatile int *refcount, int value) +{ + *refcount = value; +} + +static inline void +avro_refcount_inc(volatile int *refcount) +{ + if (*refcount != (int) -1) { + _InterlockedIncrement((volatile long *) refcount); + } +} + +static inline int +avro_refcount_dec(volatile int *refcount) +{ + if (*refcount != (int) -1) { + return (_InterlockedDecrement((volatile long *) refcount) == 0); + } + return 0; +} + +/*----------------------------------------------------------------------- + * Fallback + */ +#else +#error "No atomic implementation!" +#endif + +CLOSE_EXTERN +#endif diff --git a/fluent-bit/lib/avro/src/avro/resolver.h b/fluent-bit/lib/avro/src/avro/resolver.h new file mode 100644 index 00000000..472e0f95 --- /dev/null +++ b/fluent-bit/lib/avro/src/avro/resolver.h @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#ifndef AVRO_RESOLVER_H +#define AVRO_RESOLVER_H +#ifdef __cplusplus +extern "C" { +#define CLOSE_EXTERN } +#else +#define CLOSE_EXTERN +#endif + +#include <avro/schema.h> +#include <avro/value.h> + +/* + * A <i>resolved value</i> is a special kind of value that knows how to + * implement Avro's schema resolution rules to translate between a + * writer schema and a reader schema. A resolved value doesn't store or + * process data itself; instead, it wraps an existing value instance. + * + * There are two resolved value classes. In the first (@ref + * avro_resolved_writer_t), the resolved value is an instance of the + * writer schema, and wraps an instance of the reader schema. This is + * used, for instance, when reading from an Avro data file; you want the + * end result to be a reader schema value, and the resolved value allows + * the file reader to ignore the schema resolution and simply fill in + * the values of the writer schema. You can only set the values of a + * resolved writer; you must use the original wrapped value to read. + * + * With other class (@ref avro_resolved_reader_t), the resolved value is + * an instance of the reader schema, and wraps an instance of the writer + * schema. This is used when resolving an existing Avro value to + * another schema; you've already got the value in the original (writer) + * schema, and want to transparently treat it as if it were an instance + * of the new (reader) schema. You can only read the values of a + * resolved reader; you must use the original wrapped value to write. + * + * For both classes, the “self” pointer of the resolved value is an + * avro_value_t pointer, which points at the wrapped value. + */ + + +/** + * Create a new resolved writer implementation for the given writer and + * reader schemas. + */ + +avro_value_iface_t * +avro_resolved_writer_new(avro_schema_t writer_schema, + avro_schema_t reader_schema); + +/** + * Creates a new resolved writer value. + */ + +int +avro_resolved_writer_new_value(avro_value_iface_t *iface, + avro_value_t *value); + +/** + * Sets the wrapped value for a resolved writer. This must be an + * instance of the reader schema. We create our own reference to the + * destination value. + */ + +void +avro_resolved_writer_set_dest(avro_value_t *resolved, + avro_value_t *dest); + + +/** + * Clears the wrapped value for a resolved writer. + */ + +void +avro_resolved_writer_clear_dest(avro_value_t *resolved); + + +/** + * Create a new resolved reader implementation for the given writer and + * reader schemas. + */ + +avro_value_iface_t * +avro_resolved_reader_new(avro_schema_t writer_schema, + avro_schema_t reader_schema); + +/** + * Creates a new resolved reader value. + */ + +int +avro_resolved_reader_new_value(avro_value_iface_t *iface, + avro_value_t *value); + +/** + * Sets the wrapped value for a resolved reader. This must be an + * instance of the reader schema. We create our own reference to the + * source value. + */ + +void +avro_resolved_reader_set_source(avro_value_t *resolved, + avro_value_t *dest); + + +/** + * Clears the wrapped value for a resolved reader. + */ + +void +avro_resolved_reader_clear_source(avro_value_t *resolved); + +CLOSE_EXTERN +#endif diff --git a/fluent-bit/lib/avro/src/avro/schema.h b/fluent-bit/lib/avro/src/avro/schema.h new file mode 100644 index 00000000..51d45615 --- /dev/null +++ b/fluent-bit/lib/avro/src/avro/schema.h @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#ifndef AVRO_SCHEMA_H +#define AVRO_SCHEMA_H +#ifdef __cplusplus +extern "C" { +#define CLOSE_EXTERN } +#else +#define CLOSE_EXTERN +#endif + +#include <avro/platform.h> +#include <stdlib.h> + +#include <avro/basics.h> + +typedef struct avro_obj_t *avro_schema_t; + +avro_schema_t avro_schema_string(void); +avro_schema_t avro_schema_bytes(void); +avro_schema_t avro_schema_int(void); +avro_schema_t avro_schema_long(void); +avro_schema_t avro_schema_float(void); +avro_schema_t avro_schema_double(void); +avro_schema_t avro_schema_boolean(void); +avro_schema_t avro_schema_null(void); + +avro_schema_t avro_schema_record(const char *name, const char *space); +avro_schema_t avro_schema_record_field_get(const avro_schema_t + record, const char *field_name); +const char *avro_schema_record_field_name(const avro_schema_t schema, int index); +int avro_schema_record_field_get_index(const avro_schema_t schema, + const char *field_name); +avro_schema_t avro_schema_record_field_get_by_index +(const avro_schema_t record, int index); +int avro_schema_record_field_append(const avro_schema_t record, + const char *field_name, + const avro_schema_t type); +size_t avro_schema_record_size(const avro_schema_t record); + +avro_schema_t avro_schema_enum(const char *name); +avro_schema_t avro_schema_enum_ns(const char *name, const char *space); +const char *avro_schema_enum_get(const avro_schema_t enump, + int index); +int avro_schema_enum_get_by_name(const avro_schema_t enump, + const char *symbol_name); +int avro_schema_enum_symbol_append(const avro_schema_t + enump, const char *symbol); +int avro_schema_enum_number_of_symbols(const avro_schema_t enump); + +avro_schema_t avro_schema_fixed(const char *name, const int64_t len); +avro_schema_t avro_schema_fixed_ns(const char *name, const char *space, + const int64_t len); +int64_t avro_schema_fixed_size(const avro_schema_t fixed); + +avro_schema_t avro_schema_map(const avro_schema_t values); +avro_schema_t avro_schema_map_values(avro_schema_t map); + +avro_schema_t avro_schema_array(const avro_schema_t items); +avro_schema_t avro_schema_array_items(avro_schema_t array); + +avro_schema_t avro_schema_union(void); +size_t avro_schema_union_size(const avro_schema_t union_schema); +int avro_schema_union_append(const avro_schema_t + union_schema, const avro_schema_t schema); +avro_schema_t avro_schema_union_branch(avro_schema_t union_schema, + int branch_index); +avro_schema_t avro_schema_union_branch_by_name +(avro_schema_t union_schema, int *branch_index, const char *name); + +avro_schema_t avro_schema_link(avro_schema_t schema); +avro_schema_t avro_schema_link_target(avro_schema_t schema); + +typedef struct avro_schema_error_t_ *avro_schema_error_t; + +int avro_schema_from_json(const char *jsontext, int32_t unused1, + avro_schema_t *schema, avro_schema_error_t *unused2); + +/* jsontext does not need to be NUL terminated. length must *NOT* + * include the NUL terminator, if one is present. */ +int avro_schema_from_json_length(const char *jsontext, size_t length, + avro_schema_t *schema); + +/* A helper macro for loading a schema from a string literal. The + * literal must be declared as a char[], not a char *, since we use the + * sizeof operator to determine its length. */ +#define avro_schema_from_json_literal(json, schema) \ + (avro_schema_from_json_length((json), sizeof((json))-1, (schema))) + +int avro_schema_to_specific(avro_schema_t schema, const char *prefix); + +avro_schema_t avro_schema_get_subschema(const avro_schema_t schema, + const char *name); +const char *avro_schema_name(const avro_schema_t schema); +const char *avro_schema_namespace(const avro_schema_t schema); +const char *avro_schema_type_name(const avro_schema_t schema); +avro_schema_t avro_schema_copy(avro_schema_t schema); +int avro_schema_equal(avro_schema_t a, avro_schema_t b); + +avro_schema_t avro_schema_incref(avro_schema_t schema); +int avro_schema_decref(avro_schema_t schema); + +int avro_schema_match(avro_schema_t writers_schema, + avro_schema_t readers_schema); + +CLOSE_EXTERN +#endif diff --git a/fluent-bit/lib/avro/src/avro/value.h b/fluent-bit/lib/avro/src/avro/value.h new file mode 100644 index 00000000..c5619e7b --- /dev/null +++ b/fluent-bit/lib/avro/src/avro/value.h @@ -0,0 +1,498 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#ifndef AVRO_VALUE_H +#define AVRO_VALUE_H +#ifdef __cplusplus +extern "C" { +#define CLOSE_EXTERN } +#else +#define CLOSE_EXTERN +#endif + +#include <errno.h> +#include <avro/platform.h> +#include <stdlib.h> + +#include <avro/data.h> +#include <avro/schema.h> + +/* + * This file defines an interface struct for Avro data. Most of the + * interesting parts of this library will work with Avro data values + * that are expressed in whatever C type you want, as long as you can + * provide an implementation of this interface for that type. + */ + +typedef struct avro_value_iface avro_value_iface_t; + +typedef struct avro_value { + avro_value_iface_t *iface; + void *self; +} avro_value_t; + +struct avro_value_iface { + /*------------------------------------------------------------- + * "class" methods + */ + + /** + * Increment the reference count of the interface struct. This + * should be a no-op for static structs, since they don't need + * reference counts. + */ + avro_value_iface_t * + (*incref_iface)(avro_value_iface_t *iface); + + /** + * Decrement the reference count of the interface struct. If + * the count falls to 0, free the struct. This should be a + * no-op for static structs, since they don't need reference + * counts. + */ + void + (*decref_iface)(avro_value_iface_t *iface); + + /*------------------------------------------------------------- + * General "instance" methods + */ + + /** + * Increments the reference count of a value. + */ + + void + (*incref)(avro_value_t *value); + + /** + * Decrements the reference count of a value, and frees the + * value if the reference count drops to 0. After calling this + * method, your value instance is undefined, and cannot be used + * anymore. + */ + + void + (*decref)(avro_value_t *value); + + /** + * Reset the instance to its "empty", default value. You don't + * have to free the underlying storage, if you want to keep it + * around for later values. + */ + int + (*reset)(const avro_value_iface_t *iface, void *self); + + /** + * Return the general Avro type of a value instance. + */ + avro_type_t + (*get_type)(const avro_value_iface_t *iface, const void *self); + + /** + * Return the Avro schema that a value is an instance of. + */ + avro_schema_t + (*get_schema)(const avro_value_iface_t *iface, const void *self); + + /*------------------------------------------------------------- + * Primitive value getters + */ + int (*get_boolean)(const avro_value_iface_t *iface, + const void *self, int *out); + int (*get_bytes)(const avro_value_iface_t *iface, + const void *self, const void **buf, size_t *size); + int (*grab_bytes)(const avro_value_iface_t *iface, + const void *self, avro_wrapped_buffer_t *dest); + int (*get_double)(const avro_value_iface_t *iface, + const void *self, double *out); + int (*get_float)(const avro_value_iface_t *iface, + const void *self, float *out); + int (*get_int)(const avro_value_iface_t *iface, + const void *self, int32_t *out); + int (*get_long)(const avro_value_iface_t *iface, + const void *self, int64_t *out); + int (*get_null)(const avro_value_iface_t *iface, + const void *self); + /* The result will be NUL-terminated; the size will INCLUDE the + * NUL terminator. str will never be NULL unless there's an + * error. */ + int (*get_string)(const avro_value_iface_t *iface, + const void *self, const char **str, size_t *size); + int (*grab_string)(const avro_value_iface_t *iface, + const void *self, avro_wrapped_buffer_t *dest); + + int (*get_enum)(const avro_value_iface_t *iface, + const void *self, int *out); + int (*get_fixed)(const avro_value_iface_t *iface, + const void *self, const void **buf, size_t *size); + int (*grab_fixed)(const avro_value_iface_t *iface, + const void *self, avro_wrapped_buffer_t *dest); + + /*------------------------------------------------------------- + * Primitive value setters + */ + + /* + * The "give" setters can be used to give control of an existing + * buffer to a bytes, fixed, or string value. The free function + * will be called when the buffer is no longer needed. (It's + * okay for free to be NULL; that just means that nothing + * special needs to be done to free the buffer. That's useful + * for a static string, for instance.) + * + * If your class can't take control of an existing buffer, then + * your give functions should pass the buffer into the + * corresponding "set" method and then immediately free the + * buffer. + * + * Note that for strings, the free function will be called with + * a size that *includes* the NUL terminator, even though you + * provide a size that does *not*. + */ + + int (*set_boolean)(const avro_value_iface_t *iface, + void *self, int val); + int (*set_bytes)(const avro_value_iface_t *iface, + void *self, void *buf, size_t size); + int (*give_bytes)(const avro_value_iface_t *iface, + void *self, avro_wrapped_buffer_t *buf); + int (*set_double)(const avro_value_iface_t *iface, + void *self, double val); + int (*set_float)(const avro_value_iface_t *iface, + void *self, float val); + int (*set_int)(const avro_value_iface_t *iface, + void *self, int32_t val); + int (*set_long)(const avro_value_iface_t *iface, + void *self, int64_t val); + int (*set_null)(const avro_value_iface_t *iface, void *self); + /* The input must be NUL-terminated */ + int (*set_string)(const avro_value_iface_t *iface, + void *self, const char *str); + /* and size must INCLUDE the NUL terminator */ + int (*set_string_len)(const avro_value_iface_t *iface, + void *self, const char *str, size_t size); + int (*give_string_len)(const avro_value_iface_t *iface, + void *self, avro_wrapped_buffer_t *buf); + + int (*set_enum)(const avro_value_iface_t *iface, + void *self, int val); + int (*set_fixed)(const avro_value_iface_t *iface, + void *self, void *buf, size_t size); + int (*give_fixed)(const avro_value_iface_t *iface, + void *self, avro_wrapped_buffer_t *buf); + + /*------------------------------------------------------------- + * Compound value getters + */ + + /* Number of elements in array/map, or the number of fields in a + * record. */ + int (*get_size)(const avro_value_iface_t *iface, + const void *self, size_t *size); + + /* + * For arrays and maps, returns the element with the given + * index. (For maps, the "index" is based on the order that the + * keys were added to the map.) For records, returns the field + * with that index in the schema. + * + * For maps and records, the name parameter (if given) will be + * filled in with the key or field name of the returned value. + * For arrays, the name parameter will always be ignored. + */ + int (*get_by_index)(const avro_value_iface_t *iface, + const void *self, size_t index, + avro_value_t *child, const char **name); + + /* + * For maps, returns the element with the given key. For + * records, returns the element with the given field name. If + * index is given, it will be filled in with the numeric index + * of the returned value. + */ + int (*get_by_name)(const avro_value_iface_t *iface, + const void *self, const char *name, + avro_value_t *child, size_t *index); + + /* Discriminant of current union value */ + int (*get_discriminant)(const avro_value_iface_t *iface, + const void *self, int *out); + /* Current union value */ + int (*get_current_branch)(const avro_value_iface_t *iface, + const void *self, avro_value_t *branch); + + /*------------------------------------------------------------- + * Compound value setters + */ + + /* + * For all of these, the value class should know which class to + * use for its children. + */ + + /* Creates a new array element. */ + int (*append)(const avro_value_iface_t *iface, + void *self, avro_value_t *child_out, size_t *new_index); + + /* Creates a new map element, or returns an existing one. */ + int (*add)(const avro_value_iface_t *iface, + void *self, const char *key, + avro_value_t *child, size_t *index, int *is_new); + + /* Select a union branch. */ + int (*set_branch)(const avro_value_iface_t *iface, + void *self, int discriminant, + avro_value_t *branch); +}; + + +/** + * Increments the reference count of a value instance. Normally you + * don't need to call this directly; you'll have a reference whenever + * you create the value, and @ref avro_value_copy and @ref + * avro_value_move update the reference counts correctly for you. + */ + +void +avro_value_incref(avro_value_t *value); + +/** + * Decremenets the reference count of a value instance, freeing it if + * its reference count drops to 0. + */ + +void +avro_value_decref(avro_value_t *value); + +/** + * Copies a reference to a value. This does not copy any of the data + * in the value; you get two avro_value_t references that point at the + * same underlying value instance. + */ + +void +avro_value_copy_ref(avro_value_t *dest, const avro_value_t *src); + +/** + * Moves a reference to a value. This does not copy any of the data in + * the value. The @ref src value is invalidated by this function; its + * equivalent to the following: + * + * <code> + * avro_value_copy_ref(dest, src); + * avro_value_decref(src); + * </code> + */ + +void +avro_value_move_ref(avro_value_t *dest, avro_value_t *src); + +/** + * Compares two values for equality. The two values don't need to have + * the same implementation of the value interface, but they do need to + * represent Avro values of the same schema. This function ensures that + * the schemas match; if you want to skip this check, use + * avro_value_equal_fast. + */ + +int +avro_value_equal(avro_value_t *val1, avro_value_t *val2); + +/** + * Compares two values for equality. The two values don't need to have + * the same implementation of the value interface, but they do need to + * represent Avro values of the same schema. This function assumes that + * the schemas match; if you can't guarantee this, you should use + * avro_value_equal, which compares the schemas before comparing the + * values. + */ + +int +avro_value_equal_fast(avro_value_t *val1, avro_value_t *val2); + +/** + * Compares two values using the sort order defined in the Avro + * specification. The two values don't need to have the same + * implementation of the value interface, but they do need to represent + * Avro values of the same schema. This function ensures that the + * schemas match; if you want to skip this check, use + * avro_value_cmp_fast. + */ + +int +avro_value_cmp(avro_value_t *val1, avro_value_t *val2); + +/** + * Compares two values using the sort order defined in the Avro + * specification. The two values don't need to have the same + * implementation of the value interface, but they do need to represent + * Avro values of the same schema. This function assumes that the + * schemas match; if you can't guarantee this, you should use + * avro_value_cmp, which compares the schemas before comparing the + * values. + */ + +int +avro_value_cmp_fast(avro_value_t *val1, avro_value_t *val2); + + + +/** + * Copies the contents of src into dest. The two values don't need to + * have the same implementation of the value interface, but they do need + * to represent Avro values of the same schema. This function ensures + * that the schemas match; if you want to skip this check, use + * avro_value_copy_fast. + */ + +int +avro_value_copy(avro_value_t *dest, const avro_value_t *src); + +/** + * Copies the contents of src into dest. The two values don't need to + * have the same implementation of the value interface, but they do need + * to represent Avro values of the same schema. This function assumes + * that the schemas match; if you can't guarantee this, you should use + * avro_value_copy, which compares the schemas before comparing the + * values. + */ + +int +avro_value_copy_fast(avro_value_t *dest, const avro_value_t *src); + +/** + * Returns a hash value for a given Avro value. + */ + +uint32_t +avro_value_hash(avro_value_t *value); + +/* + * Returns a string containing the JSON encoding of an Avro value. You + * must free this string when you're done with it, using the standard + * free() function. (*Not* using the custom Avro allocator.) + */ + +int +avro_value_to_json(const avro_value_t *value, + int one_line, char **json_str); + + +/** + * A helper macro for calling a given method in a value instance, if + * it's present. If the value's class doesn't implement the given + * method, we return dflt. You usually won't call this directly; it's + * just here to implement the macros below. + */ + +#define avro_value_call0(value, method, dflt) \ + ((value)->iface->method == NULL? (dflt): \ + (value)->iface->method((value)->iface, (value)->self)) + +#define avro_value_call(value, method, dflt, ...) \ + ((value)->iface->method == NULL? (dflt): \ + (value)->iface->method((value)->iface, (value)->self, __VA_ARGS__)) + + +#define avro_value_iface_incref(cls) \ + ((cls)->incref_iface == NULL? (cls): (cls)->incref_iface((cls))) +#define avro_value_iface_decref(cls) \ + ((cls)->decref_iface == NULL? (void) 0: (cls)->decref_iface((cls))) + +#define avro_value_reset(value) \ + avro_value_call0(value, reset, EINVAL) +#define avro_value_get_type(value) \ + avro_value_call0(value, get_type, (avro_type_t) -1) +#define avro_value_get_schema(value) \ + avro_value_call0(value, get_schema, NULL) + +#define avro_value_get_boolean(value, out) \ + avro_value_call(value, get_boolean, EINVAL, out) +#define avro_value_get_bytes(value, buf, size) \ + avro_value_call(value, get_bytes, EINVAL, buf, size) +#define avro_value_grab_bytes(value, dest) \ + avro_value_call(value, grab_bytes, EINVAL, dest) +#define avro_value_get_double(value, out) \ + avro_value_call(value, get_double, EINVAL, out) +#define avro_value_get_float(value, out) \ + avro_value_call(value, get_float, EINVAL, out) +#define avro_value_get_int(value, out) \ + avro_value_call(value, get_int, EINVAL, out) +#define avro_value_get_long(value, out) \ + avro_value_call(value, get_long, EINVAL, out) +#define avro_value_get_null(value) \ + avro_value_call0(value, get_null, EINVAL) +#define avro_value_get_string(value, str, size) \ + avro_value_call(value, get_string, EINVAL, str, size) +#define avro_value_grab_string(value, dest) \ + avro_value_call(value, grab_string, EINVAL, dest) +#define avro_value_get_enum(value, out) \ + avro_value_call(value, get_enum, EINVAL, out) +#define avro_value_get_fixed(value, buf, size) \ + avro_value_call(value, get_fixed, EINVAL, buf, size) +#define avro_value_grab_fixed(value, dest) \ + avro_value_call(value, grab_fixed, EINVAL, dest) + +#define avro_value_set_boolean(value, val) \ + avro_value_call(value, set_boolean, EINVAL, val) +#define avro_value_set_bytes(value, buf, size) \ + avro_value_call(value, set_bytes, EINVAL, buf, size) +#define avro_value_give_bytes(value, buf) \ + avro_value_call(value, give_bytes, EINVAL, buf) +#define avro_value_set_double(value, val) \ + avro_value_call(value, set_double, EINVAL, val) +#define avro_value_set_float(value, val) \ + avro_value_call(value, set_float, EINVAL, val) +#define avro_value_set_int(value, val) \ + avro_value_call(value, set_int, EINVAL, val) +#define avro_value_set_long(value, val) \ + avro_value_call(value, set_long, EINVAL, val) +#define avro_value_set_null(value) \ + avro_value_call0(value, set_null, EINVAL) +#define avro_value_set_string(value, str) \ + avro_value_call(value, set_string, EINVAL, str) +#define avro_value_set_string_len(value, str, size) \ + avro_value_call(value, set_string_len, EINVAL, str, size) +#define avro_value_give_string_len(value, buf) \ + avro_value_call(value, give_string_len, EINVAL, buf) +#define avro_value_set_enum(value, val) \ + avro_value_call(value, set_enum, EINVAL, val) +#define avro_value_set_fixed(value, buf, size) \ + avro_value_call(value, set_fixed, EINVAL, buf, size) +#define avro_value_give_fixed(value, buf) \ + avro_value_call(value, give_fixed, EINVAL, buf) + +#define avro_value_get_size(value, size) \ + avro_value_call(value, get_size, EINVAL, size) +#define avro_value_get_by_index(value, idx, child, name) \ + avro_value_call(value, get_by_index, EINVAL, idx, child, name) +#define avro_value_get_by_name(value, name, child, index) \ + avro_value_call(value, get_by_name, EINVAL, name, child, index) +#define avro_value_get_discriminant(value, out) \ + avro_value_call(value, get_discriminant, EINVAL, out) +#define avro_value_get_current_branch(value, branch) \ + avro_value_call(value, get_current_branch, EINVAL, branch) + +#define avro_value_append(value, child, new_index) \ + avro_value_call(value, append, EINVAL, child, new_index) +#define avro_value_add(value, key, child, index, is_new) \ + avro_value_call(value, add, EINVAL, key, child, index, is_new) +#define avro_value_set_branch(value, discriminant, branch) \ + avro_value_call(value, set_branch, EINVAL, discriminant, branch) + +CLOSE_EXTERN +#endif diff --git a/fluent-bit/lib/avro/src/avro_generic_internal.h b/fluent-bit/lib/avro/src/avro_generic_internal.h new file mode 100644 index 00000000..9843ed65 --- /dev/null +++ b/fluent-bit/lib/avro/src/avro_generic_internal.h @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#ifndef AVRO_GENERIC_INTERNAL_H +#define AVRO_GENERIC_INTERNAL_H +#ifdef __cplusplus +extern "C" { +#define CLOSE_EXTERN } +#else +#define CLOSE_EXTERN +#endif + +#include <sys/types.h> + +#include "avro/generic.h" +#include "avro/schema.h" +#include "avro/value.h" + +/* + * Each generic value implementation struct defines a couple of extra + * methods that we use to control the lifecycle of the value objects. + */ + +typedef struct avro_generic_value_iface { + avro_value_iface_t parent; + + /** + * Return the size of an instance of this value type. If this + * returns 0, then this value type can't be used with any + * function or type (like avro_value_new) that expects to + * allocate space for the value itself. + */ + size_t + (*instance_size)(const avro_value_iface_t *iface); + + /** + * Initialize a new value instance. + */ + int + (*init)(const avro_value_iface_t *iface, void *self); + + /** + * Finalize a value instance. + */ + void + (*done)(const avro_value_iface_t *iface, void *self); +} avro_generic_value_iface_t; + + +#define avro_value_instance_size(gcls) \ + ((gcls)->instance_size == NULL ? (ssize_t)-1 : (ssize_t)(gcls)->instance_size(&(gcls)->parent)) +#define avro_value_init(gcls, self) \ + ((gcls)->init == NULL? EINVAL: (gcls)->init(&(gcls)->parent, (self))) +#define avro_value_done(gcls, self) \ + ((gcls)->done == NULL? (void) 0: (gcls)->done(&(gcls)->parent, (self))) + + +CLOSE_EXTERN +#endif diff --git a/fluent-bit/lib/avro/src/avro_private.h b/fluent-bit/lib/avro/src/avro_private.h new file mode 100644 index 00000000..f97ef6b5 --- /dev/null +++ b/fluent-bit/lib/avro/src/avro_private.h @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +#ifndef AVRO_PRIVATE_H +#define AVRO_PRIVATE_H +#ifdef __cplusplus +extern "C" { +#define CLOSE_EXTERN } +#else +#define CLOSE_EXTERN +#endif + +#include <errno.h> + +#include "avro/errors.h" +#include "avro/platform.h" + +#ifdef HAVE_CONFIG_H +/* This is only true for now in the autotools build */ +#include "config.h" +#endif + +#ifdef _WIN32 +#define snprintf _snprintf +#endif + +/* Note that AVRO_PLATFORM_IS_BIG_ENDIAN is *always* defined. It is + * either TRUE (1) or FALSE (0). + */ +#ifdef _WIN32 + #define AVRO_PLATFORM_IS_BIG_ENDIAN (0) +#else // UNIX + #include <sys/param.h> + #if BYTE_ORDER == BIG_ENDIAN + #define AVRO_PLATFORM_IS_BIG_ENDIAN (1) + #else + #define AVRO_PLATFORM_IS_BIG_ENDIAN (0) + #endif +#endif + +/* Add definition of EILSEQ if it is not defined in errno.h. */ +#include <errno.h> +#ifndef EILSEQ +#define EILSEQ 138 +#endif + + +#define check(rval, call) { rval = call; if(rval) return rval; } + +#define check_set(rval, call, ...) \ + { \ + rval = call; \ + if (rval) { \ + avro_set_error(__VA_ARGS__); \ + return rval; \ + } \ + } + +#define check_prefix(rval, call, ...) \ + { \ + rval = call; \ + if (rval) { \ + avro_prefix_error(__VA_ARGS__); \ + return rval; \ + } \ + } + +#define check_param(result, test, name) \ + { \ + if (!(test)) { \ + avro_set_error("Invalid " name " in %s", \ + __FUNCTION__); \ + return result; \ + } \ + } + +#define AVRO_UNUSED(var) (void)var; + +#define container_of(ptr_, type_, member_) \ + ((type_ *)((char *)ptr_ - (size_t)&((type_ *)0)->member_)) + +#define nullstrcmp(s1, s2) \ + (((s1) && (s2)) ? strcmp(s1, s2) : ((s1) || (s2))) + +CLOSE_EXTERN +#endif diff --git a/fluent-bit/lib/avro/src/avroappend.c b/fluent-bit/lib/avro/src/avroappend.c new file mode 100644 index 00000000..7243c600 --- /dev/null +++ b/fluent-bit/lib/avro/src/avroappend.c @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <sys/types.h> +#include <sys/stat.h> +#ifdef _WIN32 +#include <unistd.h> +#endif + +#include "avro.h" + +int process_file(const char *in_filename, const char *out_filename) +{ + avro_file_reader_t reader; + avro_file_writer_t writer; + + if (in_filename == NULL) { + if (avro_file_reader_fp(stdin, "<stdin>", 0, &reader)) { + fprintf(stderr, "Error opening <stdin>:\n %s\n", + avro_strerror()); + return 1; + } + } else { + if (avro_file_reader(in_filename, &reader)) { + fprintf(stderr, "Error opening %s:\n %s\n", + in_filename, avro_strerror()); + return 1; + } + } + + avro_schema_t wschema; + wschema = avro_file_reader_get_writer_schema(reader); + + /* Check that the reader schema is the same as the writer schema */ + { + avro_schema_t oschema; + avro_file_reader_t oreader; + + if (avro_file_reader(out_filename, &oreader)) { + fprintf(stderr, "Error opening %s:\n %s\n", + out_filename, avro_strerror()); + avro_file_reader_close(reader); + return 1; + } + + oschema = avro_file_reader_get_writer_schema(oreader); + + if (avro_schema_equal(oschema, wschema) == 0) { + fprintf(stderr, "Error: reader and writer schema are not equal.\n"); + avro_file_reader_close(oreader); + avro_file_reader_close(reader); + return 1; + } + + avro_file_reader_close(oreader); + avro_schema_decref(oschema); + } + + if (avro_file_writer_open(out_filename, &writer)) { + fprintf(stderr, "Error opening %s:\n %s\n", + out_filename, avro_strerror()); + avro_file_reader_close(reader); + return 1; + } + + avro_value_iface_t *iface; + avro_value_t value; + + iface = avro_generic_class_from_schema(wschema); + avro_generic_value_new(iface, &value); + + while (avro_file_reader_read_value(reader, &value) == 0) { + if (avro_file_writer_append_value(writer, &value)) { + fprintf(stderr, "Error writing to %s:\n %s\n", + out_filename, avro_strerror()); + return 1; + } + avro_value_reset(&value); + } + + avro_file_reader_close(reader); + avro_file_writer_close(writer); + avro_value_decref(&value); + avro_value_iface_decref(iface); + avro_schema_decref(wschema); + + return 0; +} + +static void usage(void) +{ + fprintf(stderr, + "Usage: avroappend [<input avro file>] <output avro file>\n"); +} + +static int check_filenames(const char *in_filename, const char *out_filename) +{ + if (in_filename == NULL) { + return 0; + } + + struct stat in_stat; + struct stat out_stat; + + if (stat(in_filename, &in_stat) == -1) { + fprintf(stderr, "stat error on %s: %s\n", in_filename, strerror(errno)); + return 2; + } + + if (stat(out_filename, &out_stat) == -1) { + fprintf(stderr, "stat error on %s: %s\n", out_filename, strerror(errno)); + return 2; + } + + if (in_stat.st_dev == out_stat.st_dev && in_stat.st_ino == out_stat.st_ino) { + return 1; + } + + return 0; +} + +int main(int argc, char **argv) +{ + char *in_filename; + char *out_filename; + + argc--; + argv++; + + if (argc == 2) { + in_filename = argv[0]; + out_filename = argv[1]; + } else if (argc == 1) { + in_filename = NULL; + out_filename = argv[0]; + } else { + fprintf(stderr, "Not enough arguments\n\n"); + usage(); + exit(1); + } + + int ret = check_filenames(in_filename, out_filename); + + if (ret == 1) { + fprintf(stderr, "Files are the same.\n"); + } + + if (ret > 0) { + exit(1); + } + + exit(process_file(in_filename, out_filename)); +} diff --git a/fluent-bit/lib/avro/src/avrocat.c b/fluent-bit/lib/avro/src/avrocat.c new file mode 100644 index 00000000..134ea8e6 --- /dev/null +++ b/fluent-bit/lib/avro/src/avrocat.c @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "avro.h" +#include "avro_private.h" + + +/*-- PROCESSING A FILE --*/ + +static void +process_file(const char *filename) +{ + avro_file_reader_t reader; + FILE *fp; + int should_close; + + if (filename == NULL) { + fp = stdin; + filename = "<stdin>"; + should_close = 0; + } else { + fp = fopen(filename, "rb"); + should_close = 1; + + if (fp == NULL) { + fprintf(stderr, "Error opening %s:\n %s\n", + filename, strerror(errno)); + exit(1); + } + } + + if (avro_file_reader_fp(fp, filename, 0, &reader)) { + fprintf(stderr, "Error opening %s:\n %s\n", + filename, avro_strerror()); + if (should_close) { + fclose(fp); + } + exit(1); + } + + avro_schema_t wschema; + avro_value_iface_t *iface; + avro_value_t value; + + wschema = avro_file_reader_get_writer_schema(reader); + iface = avro_generic_class_from_schema(wschema); + avro_generic_value_new(iface, &value); + + int rval; + + while ((rval = avro_file_reader_read_value(reader, &value)) == 0) { + char *json; + + if (avro_value_to_json(&value, 1, &json)) { + fprintf(stderr, "Error converting value to JSON: %s\n", + avro_strerror()); + } else { + printf("%s\n", json); + free(json); + } + + avro_value_reset(&value); + } + + // If it was not an EOF that caused it to fail, + // print the error. + if (rval != EOF) { + fprintf(stderr, "Error: %s\n", avro_strerror()); + } + + avro_file_reader_close(reader); + avro_value_decref(&value); + avro_value_iface_decref(iface); + avro_schema_decref(wschema); + + if (should_close) { + fclose(fp); + } +} + + +/*-- MAIN PROGRAM --*/ + +static void usage(void) +{ + fprintf(stderr, + "Usage: avrocat <avro data file>\n"); +} + + +int main(int argc, char **argv) +{ + char *data_filename; + + if (argc == 2) { + data_filename = argv[1]; + } else if (argc == 1) { + data_filename = NULL; + } else { + fprintf(stderr, "Can't read from multiple input files.\n"); + usage(); + exit(1); + } + + /* Process the data file */ + process_file(data_filename); + return 0; +} diff --git a/fluent-bit/lib/avro/src/avromod.c b/fluent-bit/lib/avro/src/avromod.c new file mode 100644 index 00000000..7415d6e2 --- /dev/null +++ b/fluent-bit/lib/avro/src/avromod.c @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <errno.h> +#include <getopt.h> +#include <limits.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "avro.h" +#include "avro_private.h" + + +/* The compression codec to use. */ +static const char *codec = "null"; + +/* The block size to use. */ +static size_t block_size = 0; + +/*-- PROCESSING A FILE --*/ + +static void +process_file(const char *in_filename, const char *out_filename) +{ + avro_file_reader_t reader; + avro_file_writer_t writer; + + if (in_filename == NULL) { + if (avro_file_reader_fp(stdin, "<stdin>", 0, &reader)) { + fprintf(stderr, "Error opening <stdin>:\n %s\n", + avro_strerror()); + exit(1); + } + } else { + if (avro_file_reader(in_filename, &reader)) { + fprintf(stderr, "Error opening %s:\n %s\n", + in_filename, avro_strerror()); + exit(1); + } + } + + avro_schema_t wschema; + avro_value_iface_t *iface; + avro_value_t value; + int rval; + + wschema = avro_file_reader_get_writer_schema(reader); + iface = avro_generic_class_from_schema(wschema); + avro_generic_value_new(iface, &value); + + if (avro_file_writer_create_with_codec + (out_filename, wschema, &writer, codec, block_size)) { + fprintf(stderr, "Error creating %s:\n %s\n", + out_filename, avro_strerror()); + exit(1); + } + + while ((rval = avro_file_reader_read_value(reader, &value)) == 0) { + if (avro_file_writer_append_value(writer, &value)) { + fprintf(stderr, "Error writing to %s:\n %s\n", + out_filename, avro_strerror()); + exit(1); + } + avro_value_reset(&value); + } + + if (rval != EOF) { + fprintf(stderr, "Error reading value: %s", avro_strerror()); + } + + avro_file_reader_close(reader); + avro_file_writer_close(writer); + avro_value_decref(&value); + avro_value_iface_decref(iface); + avro_schema_decref(wschema); +} + + +/*-- MAIN PROGRAM --*/ + +static struct option longopts[] = { + { "block-size", required_argument, NULL, 'b' }, + { "codec", required_argument, NULL, 'c' }, + { NULL, 0, NULL, 0 } +}; + +static void usage(void) +{ + fprintf(stderr, + "Usage: avromod [--codec=<compression codec>]\n" + " [--block-size=<block size>]\n" + " [<input avro file>]\n" + " <output avro file>\n"); +} + +static void +parse_block_size(const char *optarg) +{ + unsigned long ul; + char *end; + + ul = strtoul(optarg, &end, 10); + if ((ul == 0 && end == optarg) || + (ul == ULONG_MAX && errno == ERANGE)) { + fprintf(stderr, "Invalid block size: %s\n\n", optarg); + usage(); + exit(1); + } + block_size = ul; +} + + +int main(int argc, char **argv) +{ + char *in_filename; + char *out_filename; + + int ch; + while ((ch = getopt_long(argc, argv, "b:c:", longopts, NULL)) != -1) { + switch (ch) { + case 'b': + parse_block_size(optarg); + break; + + case 'c': + codec = optarg; + break; + + default: + usage(); + exit(1); + } + } + + argc -= optind; + argv += optind; + + if (argc == 2) { + in_filename = argv[0]; + out_filename = argv[1]; + } else if (argc == 1) { + in_filename = NULL; + out_filename = argv[0]; + } else { + fprintf(stderr, "Can't read from multiple input files.\n"); + usage(); + exit(1); + } + + /* Process the data file */ + process_file(in_filename, out_filename); + return 0; +} diff --git a/fluent-bit/lib/avro/src/avropipe.c b/fluent-bit/lib/avro/src/avropipe.c new file mode 100644 index 00000000..7bda1253 --- /dev/null +++ b/fluent-bit/lib/avro/src/avropipe.c @@ -0,0 +1,432 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <ctype.h> +#include <errno.h> +#include <getopt.h> +#include <avro/platform.h> +#include <avro/platform.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "avro.h" +#include "avro_private.h" + + +/* The path separator to use in the JSON output. */ + +static const char *separator = "/"; + + +/*-- PROCESSING A FILE --*/ + +/** + * Fills in a raw string with the path to an element of an array. + */ + +static void +create_array_prefix(avro_raw_string_t *dest, const char *prefix, size_t index) +{ + static char buf[100]; + snprintf(buf, sizeof(buf), "%" PRIsz, index); + avro_raw_string_set(dest, prefix); + avro_raw_string_append(dest, separator); + avro_raw_string_append(dest, buf); +} + +static void +create_object_prefix(avro_raw_string_t *dest, const char *prefix, const char *key) +{ + /* + * Make sure that the key doesn't contain the separator + * character. + */ + + if (strstr(key, separator) != NULL) { + fprintf(stderr, + "Error: Element \"%s\" in object %s " + "contains the separator character.\n" + "Please use the --separator option to choose another.\n", + key, prefix); + exit(1); + } + + avro_raw_string_set(dest, prefix); + avro_raw_string_append(dest, separator); + avro_raw_string_append(dest, key); +} + +static void +print_bytes_value(const char *buf, size_t size) +{ + size_t i; + printf("\""); + for (i = 0; i < size; i++) + { + if (buf[i] == '"') { + printf("\\\""); + } else if (buf[i] == '\\') { + printf("\\\\"); + } else if (buf[i] == '\b') { + printf("\\b"); + } else if (buf[i] == '\f') { + printf("\\f"); + } else if (buf[i] == '\n') { + printf("\\n"); + } else if (buf[i] == '\r') { + printf("\\r"); + } else if (buf[i] == '\t') { + printf("\\t"); + } else if (isprint(buf[i])) { + printf("%c", (int) buf[i]); + } else { + printf("\\u00%02x", (unsigned int) (unsigned char) buf[i]); + } + } + printf("\""); +} + +static void +process_value(const char *prefix, avro_value_t *value); + +static void +process_array(const char *prefix, avro_value_t *value) +{ + printf("%s\t[]\n", prefix); + size_t element_count; + avro_value_get_size(value, &element_count); + + avro_raw_string_t element_prefix; + avro_raw_string_init(&element_prefix); + + size_t i; + for (i = 0; i < element_count; i++) { + avro_value_t element_value; + avro_value_get_by_index(value, i, &element_value, NULL); + + create_array_prefix(&element_prefix, prefix, i); + process_value((const char *) avro_raw_string_get(&element_prefix), &element_value); + } + + avro_raw_string_done(&element_prefix); +} + +static void +process_enum(const char *prefix, avro_value_t *value) +{ + int val; + const char *symbol_name; + + avro_schema_t schema = avro_value_get_schema(value); + avro_value_get_enum(value, &val); + symbol_name = avro_schema_enum_get(schema, val); + printf("%s\t", prefix); + print_bytes_value(symbol_name, strlen(symbol_name)); + printf("\n"); +} + +static void +process_map(const char *prefix, avro_value_t *value) +{ + printf("%s\t{}\n", prefix); + size_t element_count; + avro_value_get_size(value, &element_count); + + avro_raw_string_t element_prefix; + avro_raw_string_init(&element_prefix); + + size_t i; + for (i = 0; i < element_count; i++) { + const char *key; + avro_value_t element_value; + avro_value_get_by_index(value, i, &element_value, &key); + + create_object_prefix(&element_prefix, prefix, key); + process_value((const char *) avro_raw_string_get(&element_prefix), &element_value); + } + + avro_raw_string_done(&element_prefix); +} + +static void +process_record(const char *prefix, avro_value_t *value) +{ + printf("%s\t{}\n", prefix); + size_t field_count; + avro_value_get_size(value, &field_count); + + avro_raw_string_t field_prefix; + avro_raw_string_init(&field_prefix); + + size_t i; + for (i = 0; i < field_count; i++) { + avro_value_t field_value; + const char *field_name; + avro_value_get_by_index(value, i, &field_value, &field_name); + + create_object_prefix(&field_prefix, prefix, field_name); + process_value((const char *) avro_raw_string_get(&field_prefix), &field_value); + } + + avro_raw_string_done(&field_prefix); +} + +static void +process_union(const char *prefix, avro_value_t *value) +{ + avro_value_t branch_value; + avro_value_get_current_branch(value, &branch_value); + + /* nulls in a union aren't wrapped in a JSON object */ + if (avro_value_get_type(&branch_value) == AVRO_NULL) { + printf("%s\tnull\n", prefix); + return; + } + + int discriminant; + avro_value_get_discriminant(value, &discriminant); + + avro_schema_t schema = avro_value_get_schema(value); + avro_schema_t branch_schema = avro_schema_union_branch(schema, discriminant); + const char *branch_name = avro_schema_type_name(branch_schema); + + avro_raw_string_t branch_prefix; + avro_raw_string_init(&branch_prefix); + create_object_prefix(&branch_prefix, prefix, branch_name); + + printf("%s\t{}\n", prefix); + process_value((const char *) avro_raw_string_get(&branch_prefix), &branch_value); + + avro_raw_string_done(&branch_prefix); +} + +static void +process_value(const char *prefix, avro_value_t *value) +{ + avro_type_t type = avro_value_get_type(value); + switch (type) { + case AVRO_BOOLEAN: + { + int val; + avro_value_get_boolean(value, &val); + printf("%s\t%s\n", prefix, val? "true": "false"); + return; + } + + case AVRO_BYTES: + { + const void *buf; + size_t size; + avro_value_get_bytes(value, &buf, &size); + printf("%s\t", prefix); + print_bytes_value((const char *) buf, size); + printf("\n"); + return; + } + + case AVRO_DOUBLE: + { + double val; + avro_value_get_double(value, &val); + printf("%s\t%lf\n", prefix, val); + return; + } + + case AVRO_FLOAT: + { + float val; + avro_value_get_float(value, &val); + printf("%s\t%f\n", prefix, val); + return; + } + + case AVRO_INT32: + { + int32_t val; + avro_value_get_int(value, &val); + printf("%s\t%" PRId32 "\n", prefix, val); + return; + } + + case AVRO_INT64: + { + int64_t val; + avro_value_get_long(value, &val); + printf("%s\t%" PRId64 "\n", prefix, val); + return; + } + + case AVRO_NULL: + { + avro_value_get_null(value); + printf("%s\tnull\n", prefix); + return; + } + + case AVRO_STRING: + { + /* TODO: Convert the UTF-8 to the current + * locale's character set */ + const char *buf; + size_t size; + avro_value_get_string(value, &buf, &size); + printf("%s\t", prefix); + /* For strings, size includes the NUL terminator. */ + print_bytes_value(buf, size-1); + printf("\n"); + return; + } + + case AVRO_ARRAY: + process_array(prefix, value); + return; + + case AVRO_ENUM: + process_enum(prefix, value); + return; + + case AVRO_FIXED: + { + const void *buf; + size_t size; + avro_value_get_fixed(value, &buf, &size); + printf("%s\t", prefix); + print_bytes_value((const char *) buf, size); + printf("\n"); + return; + } + + case AVRO_MAP: + process_map(prefix, value); + return; + + case AVRO_RECORD: + process_record(prefix, value); + return; + + case AVRO_UNION: + process_union(prefix, value); + return; + + default: + { + fprintf(stderr, "Unknown schema type\n"); + exit(1); + } + } +} + +static void +process_file(const char *filename) +{ + avro_file_reader_t reader; + + if (filename == NULL) { + if (avro_file_reader_fp(stdin, "<stdin>", 0, &reader)) { + fprintf(stderr, "Error opening <stdin>:\n %s\n", + avro_strerror()); + exit(1); + } + } else { + if (avro_file_reader(filename, &reader)) { + fprintf(stderr, "Error opening %s:\n %s\n", + filename, avro_strerror()); + exit(1); + } + } + + /* The JSON root is an array */ + printf("%s\t[]\n", separator); + + avro_raw_string_t prefix; + avro_raw_string_init(&prefix); + + avro_schema_t wschema = avro_file_reader_get_writer_schema(reader); + avro_value_iface_t *iface = avro_generic_class_from_schema(wschema); + avro_value_t value; + avro_generic_value_new(iface, &value); + + size_t record_number = 0; + int rval; + + for (; (rval = avro_file_reader_read_value(reader, &value)) == 0; record_number++) { + create_array_prefix(&prefix, "", record_number); + process_value((const char *) avro_raw_string_get(&prefix), &value); + avro_value_reset(&value); + } + + if (rval != EOF) { + fprintf(stderr, "Error reading value: %s", avro_strerror()); + } + + avro_raw_string_done(&prefix); + avro_value_decref(&value); + avro_value_iface_decref(iface); + avro_file_reader_close(reader); + avro_schema_decref(wschema); +} + + +/*-- MAIN PROGRAM --*/ +static struct option longopts[] = { + { "separator", required_argument, NULL, 's' }, + { NULL, 0, NULL, 0 } +}; + +static void usage(void) +{ + fprintf(stderr, + "Usage: avropipe [--separator=<separator>]\n" + " <avro data file>\n"); +} + + +int main(int argc, char **argv) +{ + char *data_filename; + + int ch; + while ((ch = getopt_long(argc, argv, "s:", longopts, NULL) ) != -1) { + switch (ch) { + case 's': + separator = optarg; + break; + + default: + usage(); + exit(1); + } + } + + argc -= optind; + argv += optind; + + if (argc == 1) { + data_filename = argv[0]; + } else if (argc == 0) { + data_filename = NULL; + } else { + fprintf(stderr, "Can't read from multiple input files.\n"); + usage(); + exit(1); + } + + /* Process the data file */ + process_file(data_filename); + return 0; +} diff --git a/fluent-bit/lib/avro/src/codec.c b/fluent-bit/lib/avro/src/codec.c new file mode 100644 index 00000000..613a9143 --- /dev/null +++ b/fluent-bit/lib/avro/src/codec.c @@ -0,0 +1,620 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <string.h> +#ifdef SNAPPY_CODEC +#include <snappy-c.h> +# if defined(__APPLE__) +# include <libkern/OSByteOrder.h> +# define __bswap_32 OSSwapInt32 +# elif defined(__FreeBSD__) +# include <sys/endian.h> +# define __bswap_32 bswap32 +# elif defined(_WIN32) +# include <stdlib.h> +# define __bswap_32 _byteswap_ulong +# else +# include <byteswap.h> +# endif +#endif +#ifdef DEFLATE_CODEC +#include <zlib.h> +#endif +#ifdef LZMA_CODEC +#include <lzma.h> +#endif +#include "avro/errors.h" +#include "avro/allocation.h" +#include "codec.h" + +#define DEFAULT_BLOCK_SIZE (16 * 1024) + +/* NULL codec */ + +static int +codec_null(avro_codec_t codec) +{ + codec->name = "null"; + codec->type = AVRO_CODEC_NULL; + codec->block_size = 0; + codec->used_size = 0; + codec->block_data = NULL; + codec->codec_data = NULL; + + return 0; +} + +static int encode_null(avro_codec_t c, void * data, int64_t len) +{ + c->block_data = data; + c->block_size = len; + c->used_size = len; + + return 0; +} + +static int decode_null(avro_codec_t c, void * data, int64_t len) +{ + c->block_data = data; + c->block_size = len; + c->used_size = len; + + return 0; +} + +static int reset_null(avro_codec_t c) +{ + c->block_data = NULL; + c->block_size = 0; + c->used_size = 0; + c->codec_data = NULL; + + return 0; +} + +/* Snappy codec */ + +#ifdef SNAPPY_CODEC + +static int +codec_snappy(avro_codec_t codec) +{ + codec->name = "snappy"; + codec->type = AVRO_CODEC_SNAPPY; + codec->block_size = 0; + codec->used_size = 0; + codec->block_data = NULL; + codec->codec_data = NULL; + + return 0; +} + +static int encode_snappy(avro_codec_t c, void * data, int64_t len) +{ + uint32_t crc; + size_t outlen = snappy_max_compressed_length(len); + + if (!c->block_data) { + c->block_data = avro_malloc(outlen+4); + c->block_size = outlen+4; + } else if (c->block_size < (int64_t) (outlen+4)) { + c->block_data = avro_realloc(c->block_data, c->block_size, (outlen+4)); + c->block_size = outlen+4; + } + + if (!c->block_data) { + avro_set_error("Cannot allocate memory for snappy"); + return 1; + } + + if (snappy_compress((const char *)data, len, (char*)c->block_data, &outlen) != SNAPPY_OK) + { + avro_set_error("Error compressing block with Snappy"); + return 1; + } + + crc = __bswap_32(crc32(0, (const Bytef *)data, len)); + memcpy((char*)c->block_data+outlen, &crc, 4); + c->used_size = outlen+4; + + return 0; +} + +static int decode_snappy(avro_codec_t c, void * data, int64_t len) +{ + uint32_t crc; + size_t outlen; + + if (snappy_uncompressed_length((const char*)data, len-4, &outlen) != SNAPPY_OK) { + avro_set_error("Uncompressed length error in snappy"); + return 1; + } + + if (!c->block_data) { + c->block_data = avro_malloc(outlen); + c->block_size = outlen; + } else if ( (size_t)c->block_size < outlen) { + c->block_data = avro_realloc(c->block_data, c->block_size, outlen); + c->block_size = outlen; + } + + if (!c->block_data) + { + avro_set_error("Cannot allocate memory for snappy"); + return 1; + } + + if (snappy_uncompress((const char*)data, len-4, (char*)c->block_data, &outlen) != SNAPPY_OK) + { + avro_set_error("Error uncompressing block with Snappy"); + return 1; + } + + crc = __bswap_32(crc32(0, (const Bytef *)c->block_data, outlen)); + if (memcmp(&crc, (char*)data+len-4, 4)) + { + avro_set_error("CRC32 check failure uncompressing block with Snappy"); + return 1; + } + + c->used_size = outlen; + + return 0; +} + +static int reset_snappy(avro_codec_t c) +{ + if (c->block_data) { + avro_free(c->block_data, c->block_size); + } + + c->block_data = NULL; + c->block_size = 0; + c->used_size = 0; + c->codec_data = NULL; + + return 0; +} + +#endif // SNAPPY_CODEC + +/* Deflate codec */ + +#ifdef DEFLATE_CODEC + +struct codec_data_deflate { + z_stream deflate; + z_stream inflate; +}; +#define codec_data_deflate_stream(cd) &((struct codec_data_deflate *)cd)->deflate +#define codec_data_inflate_stream(cd) &((struct codec_data_deflate *)cd)->inflate + + +static int +codec_deflate(avro_codec_t codec) +{ + codec->name = "deflate"; + codec->type = AVRO_CODEC_DEFLATE; + codec->block_size = 0; + codec->used_size = 0; + codec->block_data = NULL; + codec->codec_data = avro_new(struct codec_data_deflate); + + if (!codec->codec_data) { + avro_set_error("Cannot allocate memory for zlib"); + return 1; + } + + z_stream *ds = codec_data_deflate_stream(codec->codec_data); + z_stream *is = codec_data_inflate_stream(codec->codec_data); + + memset(ds, 0, sizeof(z_stream)); + memset(is, 0, sizeof(z_stream)); + + ds->zalloc = is->zalloc = Z_NULL; + ds->zfree = is->zfree = Z_NULL; + ds->opaque = is->opaque = Z_NULL; + + if (deflateInit2(ds, Z_BEST_COMPRESSION, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY) != Z_OK) { + avro_freet(struct codec_data_deflate, codec->codec_data); + codec->codec_data = NULL; + avro_set_error("Cannot initialize zlib deflate"); + return 1; + } + + if (inflateInit2(is, -15) != Z_OK) { + avro_freet(struct codec_data_deflate, codec->codec_data); + codec->codec_data = NULL; + avro_set_error("Cannot initialize zlib inflate"); + return 1; + } + + return 0; +} + +static int encode_deflate(avro_codec_t c, void * data, int64_t len) +{ + int err; + int64_t defl_len = compressBound((uLong)len * 1.2); + + if (!c->block_data) { + c->block_data = avro_malloc(defl_len); + c->block_size = defl_len; + } else if ( c->block_size < defl_len) { + c->block_data = avro_realloc(c->block_data, c->block_size, defl_len); + c->block_size = defl_len; + } + + if (!c->block_data) + { + avro_set_error("Cannot allocate memory for deflate"); + return 1; + } + + c->used_size = 0; + + z_stream *s = codec_data_deflate_stream(c->codec_data); + + s->next_in = (Bytef*)data; + s->avail_in = (uInt)len; + + s->next_out = c->block_data; + s->avail_out = (uInt)c->block_size; + + s->total_out = 0; + + err = deflate(s, Z_FINISH); + if (err != Z_STREAM_END) { + deflateEnd(s); + if (err != Z_OK) { + avro_set_error("Error compressing block with deflate (%i)", err); + return 1; + } + return 0; + } + + // zlib resizes the buffer? + c->block_size = s->total_out; + c->used_size = s->total_out; + + if (deflateReset(s) != Z_OK) { + return 1; + } + + return 0; +} + +static int decode_deflate(avro_codec_t c, void * data, int64_t len) +{ + int err; + z_stream *s = codec_data_inflate_stream(c->codec_data); + + if (!c->block_data) { + c->block_data = avro_malloc(DEFAULT_BLOCK_SIZE); + c->block_size = DEFAULT_BLOCK_SIZE; + } + + if (!c->block_data) + { + avro_set_error("Cannot allocate memory for deflate"); + return 1; + } + + c->used_size = 0; + + s->next_in = data; + s->avail_in = len; + + s->next_out = c->block_data; + s->avail_out = c->block_size; + + s->total_out = 0; + + do + { + err = inflate(s, Z_FINISH); + + // Apparently if there is yet available space in the output then something + // has gone wrong in decompressing the data (according to cpython zlibmodule.c) + if (err == Z_BUF_ERROR && s->avail_out > 0) { + inflateEnd(s); + avro_set_error("Error decompressing block with deflate, possible data error"); + return 1; + } + + // The buffer was not big enough. resize it. + if (err == Z_BUF_ERROR) + { + c->block_data = avro_realloc(c->block_data, c->block_size, c->block_size * 2); + s->next_out = c->block_data + s->total_out; + s->avail_out += c->block_size; + c->block_size = c->block_size * 2; + } + } while (err == Z_BUF_ERROR); + + if (err != Z_STREAM_END) { + inflateEnd(s); + if (err != Z_OK) { + avro_set_error("Error decompressing block with deflate (%i)", err); + return 1; + } + return 0; + } + + c->used_size = s->total_out; + + if (inflateReset(s) != Z_OK) { + avro_set_error("Error resetting deflate decompression"); + return 1; + } + + return 0; +} + +static int reset_deflate(avro_codec_t c) +{ + if (c->block_data) { + avro_free(c->block_data, c->block_size); + } + if (c->codec_data) { + deflateEnd(codec_data_deflate_stream(c->codec_data)); + inflateEnd(codec_data_inflate_stream(c->codec_data)); + avro_freet(struct codec_data_deflate, c->codec_data); + } + + c->block_data = NULL; + c->block_size = 0; + c->used_size = 0; + c->codec_data = NULL; + + return 0; +} + +#endif // DEFLATE_CODEC + +/* LZMA codec */ + +#ifdef LZMA_CODEC + +struct codec_data_lzma { + lzma_filter filters[2]; + lzma_options_lzma options; +}; +#define codec_data_lzma_filters(cd) ((struct codec_data_lzma *)cd)->filters +#define codec_data_lzma_options(cd) &((struct codec_data_lzma *)cd)->options + +static int +codec_lzma(avro_codec_t codec) +{ + codec->name = "lzma"; + codec->type = AVRO_CODEC_LZMA; + codec->block_size = 0; + codec->used_size = 0; + codec->block_data = NULL; + codec->codec_data = avro_new(struct codec_data_lzma); + + if (!codec->codec_data) { + avro_set_error("Cannot allocate memory for lzma"); + return 1; + } + + lzma_options_lzma* opt = codec_data_lzma_options(codec->codec_data); + lzma_lzma_preset(opt, LZMA_PRESET_DEFAULT); + + lzma_filter* filters = codec_data_lzma_filters(codec->codec_data); + filters[0].id = LZMA_FILTER_LZMA2; + filters[0].options = opt; + filters[1].id = LZMA_VLI_UNKNOWN; + filters[1].options = NULL; + + return 0; +} + +static int encode_lzma(avro_codec_t codec, void * data, int64_t len) +{ + lzma_ret ret; + size_t written = 0; + lzma_filter* filters = codec_data_lzma_filters(codec->codec_data); + + int64_t buff_len = len + lzma_raw_encoder_memusage(filters); + + if (!codec->block_data) { + codec->block_data = avro_malloc(buff_len); + codec->block_size = buff_len; + } + + if (!codec->block_data) + { + avro_set_error("Cannot allocate memory for lzma encoder"); + return 1; + } + + ret = lzma_raw_buffer_encode(filters, NULL, data, len, codec->block_data, &written, codec->block_size); + + codec->used_size = written; + + if (ret != LZMA_OK) { + avro_set_error("Error in lzma encoder"); + return 1; + } + + return 0; +} + +static int decode_lzma(avro_codec_t codec, void * data, int64_t len) +{ + size_t read_pos = 0; + size_t write_pos = 0; + lzma_ret ret; + lzma_filter* filters = codec_data_lzma_filters(codec->codec_data); + + if (!codec->block_data) { + codec->block_data = avro_malloc(DEFAULT_BLOCK_SIZE); + codec->block_size = DEFAULT_BLOCK_SIZE; + } + + if (!codec->block_data) { + avro_set_error("Cannot allocate memory for lzma decoder"); + return 1; + } + + do + { + ret = lzma_raw_buffer_decode(filters, NULL, data, + &read_pos, len, codec->block_data, &write_pos, + codec->block_size); + + codec->used_size = write_pos; + + // If it ran out of space to decode, give it more!! + // It will continue where it left off because of read_pos and write_pos. + if (ret == LZMA_BUF_ERROR) { + codec->block_data = avro_realloc(codec->block_data, codec->block_size, codec->block_size * 2); + codec->block_size = codec->block_size * 2; + } + + } while (ret == LZMA_BUF_ERROR); + + if (ret != LZMA_OK) { + avro_set_error("Error in lzma decoder"); + return 1; + } + + return 0; +} + +static int reset_lzma(avro_codec_t c) +{ + if (c->block_data) { + avro_free(c->block_data, c->block_size); + } + if (c->codec_data) { + avro_freet(struct codec_data_lzma, c->codec_data); + } + + c->block_data = NULL; + c->block_size = 0; + c->used_size = 0; + c->codec_data = NULL; + + return 0; +} + +#endif // LZMA_CODEC + +/* Common interface */ + +int avro_codec(avro_codec_t codec, const char *type) +{ + if (type == NULL) { + return codec_null(codec); + } + +#ifdef SNAPPY_CODEC + if (strcmp("snappy", type) == 0) { + return codec_snappy(codec); + } +#endif + +#ifdef DEFLATE_CODEC + if (strcmp("deflate", type) == 0) { + return codec_deflate(codec); + } +#endif + +#ifdef LZMA_CODEC + if (strcmp("lzma", type) == 0) { + return codec_lzma(codec); + } +#endif + + if (strcmp("null", type) == 0) { + return codec_null(codec); + } + + avro_set_error("Unknown codec %s", type); + return 1; +} + +int avro_codec_encode(avro_codec_t c, void * data, int64_t len) +{ + switch(c->type) + { + case AVRO_CODEC_NULL: + return encode_null(c, data, len); +#ifdef SNAPPY_CODEC + case AVRO_CODEC_SNAPPY: + return encode_snappy(c, data, len); +#endif +#ifdef DEFLATE_CODEC + case AVRO_CODEC_DEFLATE: + return encode_deflate(c, data, len); +#endif +#ifdef LZMA_CODEC + case AVRO_CODEC_LZMA: + return encode_lzma(c, data, len); +#endif + default: + return 1; + } +} + +int avro_codec_decode(avro_codec_t c, void * data, int64_t len) +{ + switch(c->type) + { + case AVRO_CODEC_NULL: + return decode_null(c, data, len); +#ifdef SNAPPY_CODEC + case AVRO_CODEC_SNAPPY: + return decode_snappy(c, data, len); +#endif +#ifdef DEFLATE_CODEC + case AVRO_CODEC_DEFLATE: + return decode_deflate(c, data, len); +#endif +#ifdef LZMA_CODEC + case AVRO_CODEC_LZMA: + return decode_lzma(c, data, len); +#endif + default: + return 1; + } +} + +int avro_codec_reset(avro_codec_t c) +{ + switch(c->type) + { + case AVRO_CODEC_NULL: + return reset_null(c); +#ifdef SNAPPY_CODEC + case AVRO_CODEC_SNAPPY: + return reset_snappy(c); +#endif +#ifdef DEFLATE_CODEC + case AVRO_CODEC_DEFLATE: + return reset_deflate(c); +#endif +#ifdef LZMA_CODEC + case AVRO_CODEC_LZMA: + return reset_lzma(c); +#endif + default: + return 1; + } +} diff --git a/fluent-bit/lib/avro/src/codec.h b/fluent-bit/lib/avro/src/codec.h new file mode 100644 index 00000000..68912243 --- /dev/null +++ b/fluent-bit/lib/avro/src/codec.h @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#ifndef AVRO_CODEC_H +#define AVRO_CODEC_H +#ifdef __cplusplus +extern "C" { +#define CLOSE_EXTERN } +#else +#define CLOSE_EXTERN +#endif + +#include <avro/platform.h> + +enum avro_codec_type_t { + AVRO_CODEC_NULL, + AVRO_CODEC_DEFLATE, + AVRO_CODEC_LZMA, + AVRO_CODEC_SNAPPY +}; +typedef enum avro_codec_type_t avro_codec_type_t; + +struct avro_codec_t_ { + const char * name; + avro_codec_type_t type; + int64_t block_size; + int64_t used_size; + void * block_data; + void * codec_data; +}; +typedef struct avro_codec_t_* avro_codec_t; + +int avro_codec(avro_codec_t c, const char *type); +int avro_codec_reset(avro_codec_t c); +int avro_codec_encode(avro_codec_t c, void * data, int64_t len); +int avro_codec_decode(avro_codec_t c, void * data, int64_t len); + +CLOSE_EXTERN +#endif diff --git a/fluent-bit/lib/avro/src/consume-binary.c b/fluent-bit/lib/avro/src/consume-binary.c new file mode 100644 index 00000000..9f92799d --- /dev/null +++ b/fluent-bit/lib/avro/src/consume-binary.c @@ -0,0 +1,328 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include "avro_private.h" +#include "avro/allocation.h" +#include "avro/consumer.h" +#include "avro/errors.h" +#include "avro/resolver.h" +#include "avro/value.h" +#include <stdlib.h> +#include <errno.h> +#include <string.h> +#include "encoding.h" +#include "schema.h" +#include "datum.h" + + +static int +read_enum(avro_reader_t reader, const avro_encoding_t * enc, + avro_consumer_t *consumer, void *ud) +{ + int rval; + int64_t index; + + check_prefix(rval, enc->read_long(reader, &index), + "Cannot read enum value: "); + return avro_consumer_call(consumer, enum_value, index, ud); +} + +static int +read_array(avro_reader_t reader, const avro_encoding_t * enc, + avro_consumer_t *consumer, void *ud) +{ + int rval; + int64_t i; /* index within the current block */ + int64_t index = 0; /* index within the entire array */ + int64_t block_count; + int64_t block_size; + + check_prefix(rval, enc->read_long(reader, &block_count), + "Cannot read array block count: "); + check(rval, avro_consumer_call(consumer, array_start_block, + 1, block_count, ud)); + + while (block_count != 0) { + if (block_count < 0) { + block_count = block_count * -1; + check_prefix(rval, enc->read_long(reader, &block_size), + "Cannot read array block size: "); + } + + for (i = 0; i < block_count; i++, index++) { + avro_consumer_t *element_consumer = NULL; + void *element_ud = NULL; + + check(rval, + avro_consumer_call(consumer, array_element, + index, &element_consumer, &element_ud, + ud)); + + check(rval, avro_consume_binary(reader, element_consumer, element_ud)); + } + + check_prefix(rval, enc->read_long(reader, &block_count), + "Cannot read array block count: "); + check(rval, avro_consumer_call(consumer, array_start_block, + 0, block_count, ud)); + } + + return 0; +} + +static int +read_map(avro_reader_t reader, const avro_encoding_t * enc, + avro_consumer_t *consumer, void *ud) +{ + int rval; + int64_t i; /* index within the current block */ + int64_t index = 0; /* index within the entire array */ + int64_t block_count; + int64_t block_size; + + check_prefix(rval, enc->read_long(reader, &block_count), + "Cannot read map block count: "); + check(rval, avro_consumer_call(consumer, map_start_block, + 1, block_count, ud)); + + while (block_count != 0) { + if (block_count < 0) { + block_count = block_count * -1; + check_prefix(rval, enc->read_long(reader, &block_size), + "Cannot read map block size: "); + } + + for (i = 0; i < block_count; i++, index++) { + char *key; + int64_t key_size; + avro_consumer_t *element_consumer = NULL; + void *element_ud = NULL; + + check_prefix(rval, enc->read_string(reader, &key, &key_size), + "Cannot read map key: "); + + rval = avro_consumer_call(consumer, map_element, + index, key, + &element_consumer, &element_ud, + ud); + if (rval) { + avro_free(key, key_size); + return rval; + } + + rval = avro_consume_binary(reader, element_consumer, element_ud); + if (rval) { + avro_free(key, key_size); + return rval; + } + + avro_free(key, key_size); + } + + check_prefix(rval, enc->read_long(reader, &block_count), + "Cannot read map block count: "); + check(rval, avro_consumer_call(consumer, map_start_block, + 0, block_count, ud)); + } + + return 0; +} + +static int +read_union(avro_reader_t reader, const avro_encoding_t * enc, + avro_consumer_t *consumer, void *ud) +{ + int rval; + int64_t discriminant; + avro_consumer_t *branch_consumer = NULL; + void *branch_ud = NULL; + + check_prefix(rval, enc->read_long(reader, &discriminant), + "Cannot read union discriminant: "); + check(rval, avro_consumer_call(consumer, union_branch, + discriminant, + &branch_consumer, &branch_ud, ud)); + return avro_consume_binary(reader, branch_consumer, branch_ud); +} + +static int +read_record(avro_reader_t reader, const avro_encoding_t * enc, + avro_consumer_t *consumer, void *ud) +{ + int rval; + size_t num_fields; + unsigned int i; + + AVRO_UNUSED(enc); + + check(rval, avro_consumer_call(consumer, record_start, ud)); + + num_fields = avro_schema_record_size(consumer->schema); + for (i = 0; i < num_fields; i++) { + avro_consumer_t *field_consumer = NULL; + void *field_ud = NULL; + + check(rval, avro_consumer_call(consumer, record_field, + i, &field_consumer, &field_ud, + ud)); + + if (field_consumer) { + check(rval, avro_consume_binary(reader, field_consumer, field_ud)); + } else { + avro_schema_t field_schema = + avro_schema_record_field_get_by_index(consumer->schema, i); + check(rval, avro_skip_data(reader, field_schema)); + } + } + + return 0; +} + +int +avro_consume_binary(avro_reader_t reader, avro_consumer_t *consumer, void *ud) +{ + int rval; + const avro_encoding_t *enc = &avro_binary_encoding; + + check_param(EINVAL, reader, "reader"); + check_param(EINVAL, consumer, "consumer"); + + switch (avro_typeof(consumer->schema)) { + case AVRO_NULL: + check_prefix(rval, enc->read_null(reader), + "Cannot read null value: "); + check(rval, avro_consumer_call(consumer, null_value, ud)); + break; + + case AVRO_BOOLEAN: + { + int8_t b; + check_prefix(rval, enc->read_boolean(reader, &b), + "Cannot read boolean value: "); + check(rval, avro_consumer_call(consumer, boolean_value, b, ud)); + } + break; + + case AVRO_STRING: + { + int64_t len; + char *s; + check_prefix(rval, enc->read_string(reader, &s, &len), + "Cannot read string value: "); + check(rval, avro_consumer_call(consumer, string_value, s, len, ud)); + } + break; + + case AVRO_INT32: + { + int32_t i; + check_prefix(rval, enc->read_int(reader, &i), + "Cannot read int value: "); + check(rval, avro_consumer_call(consumer, int_value, i, ud)); + } + break; + + case AVRO_INT64: + { + int64_t l; + check_prefix(rval, enc->read_long(reader, &l), + "Cannot read long value: "); + check(rval, avro_consumer_call(consumer, long_value, l, ud)); + } + break; + + case AVRO_FLOAT: + { + float f; + check_prefix(rval, enc->read_float(reader, &f), + "Cannot read float value: "); + check(rval, avro_consumer_call(consumer, float_value, f, ud)); + } + break; + + case AVRO_DOUBLE: + { + double d; + check_prefix(rval, enc->read_double(reader, &d), + "Cannot read double value: "); + check(rval, avro_consumer_call(consumer, double_value, d, ud)); + } + break; + + case AVRO_BYTES: + { + char *bytes; + int64_t len; + check_prefix(rval, enc->read_bytes(reader, &bytes, &len), + "Cannot read bytes value: "); + check(rval, avro_consumer_call(consumer, bytes_value, bytes, len, ud)); + } + break; + + case AVRO_FIXED: + { + char *bytes; + int64_t size = + avro_schema_to_fixed(consumer->schema)->size; + + bytes = (char *) avro_malloc(size); + if (!bytes) { + avro_prefix_error("Cannot allocate new fixed value"); + return ENOMEM; + } + rval = avro_read(reader, bytes, size); + if (rval) { + avro_prefix_error("Cannot read fixed value: "); + avro_free(bytes, size); + return rval; + } + + rval = avro_consumer_call(consumer, fixed_value, bytes, size, ud); + if (rval) { + avro_free(bytes, size); + return rval; + } + } + break; + + case AVRO_ENUM: + check(rval, read_enum(reader, enc, consumer, ud)); + break; + + case AVRO_ARRAY: + check(rval, read_array(reader, enc, consumer, ud)); + break; + + case AVRO_MAP: + check(rval, read_map(reader, enc, consumer, ud)); + break; + + case AVRO_UNION: + check(rval, read_union(reader, enc, consumer, ud)); + break; + + case AVRO_RECORD: + check(rval, read_record(reader, enc, consumer, ud)); + break; + + case AVRO_LINK: + avro_set_error("Consumer can't consume a link schema directly"); + return EINVAL; + } + + return 0; +} diff --git a/fluent-bit/lib/avro/src/consumer.c b/fluent-bit/lib/avro/src/consumer.c new file mode 100644 index 00000000..e01958c8 --- /dev/null +++ b/fluent-bit/lib/avro/src/consumer.c @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include "avro/consumer.h" + +void avro_consumer_free(avro_consumer_t *consumer) +{ + consumer->free(consumer); +} diff --git a/fluent-bit/lib/avro/src/datafile.c b/fluent-bit/lib/avro/src/datafile.c new file mode 100644 index 00000000..c9d4dfeb --- /dev/null +++ b/fluent-bit/lib/avro/src/datafile.c @@ -0,0 +1,766 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include "avro_private.h" +#include "avro/allocation.h" +#include "avro/generic.h" +#include "avro/errors.h" +#include "avro/value.h" +#include "encoding.h" +#include "codec.h" +#include <stdio.h> +#include <stdlib.h> +#include <errno.h> +#include <fcntl.h> +#include <time.h> +#include <string.h> + +struct avro_file_reader_t_ { + avro_schema_t writers_schema; + avro_reader_t reader; + avro_reader_t block_reader; + avro_codec_t codec; + char sync[16]; + int64_t blocks_read; + int64_t blocks_total; + int64_t current_blocklen; + char * current_blockdata; +}; + +struct avro_file_writer_t_ { + avro_schema_t writers_schema; + avro_writer_t writer; + avro_codec_t codec; + char sync[16]; + int block_count; + size_t block_size; + avro_writer_t datum_writer; + char* datum_buffer; + size_t datum_buffer_size; + char schema_buf[64 * 1024]; +}; + +#define DEFAULT_BLOCK_SIZE 16 * 1024 + +/* Note: We should not just read /dev/random here, because it may not + * exist on all platforms e.g. Win32. + */ +static void generate_sync(avro_file_writer_t w) +{ + unsigned int i; + srand(time(NULL)); + for (i = 0; i < sizeof(w->sync); i++) { + w->sync[i] = ((double)rand() / (RAND_MAX + 1.0)) * 255; + } +} + +static int write_sync(avro_file_writer_t w) +{ + return avro_write(w->writer, w->sync, sizeof(w->sync)); +} + +static int write_header(avro_file_writer_t w) +{ + int rval; + uint8_t version = 1; + /* TODO: remove this static buffer */ + avro_writer_t schema_writer; + const avro_encoding_t *enc = &avro_binary_encoding; + int64_t schema_len; + + /* Generate random sync */ + generate_sync(w); + + check(rval, avro_write(w->writer, "Obj", 3)); + check(rval, avro_write(w->writer, &version, 1)); + + check(rval, enc->write_long(w->writer, 2)); + check(rval, enc->write_string(w->writer, "avro.codec")); + check(rval, enc->write_bytes(w->writer, w->codec->name, strlen(w->codec->name))); + check(rval, enc->write_string(w->writer, "avro.schema")); + schema_writer = + avro_writer_memory(&w->schema_buf[0], sizeof(w->schema_buf)); + rval = avro_schema_to_json(w->writers_schema, schema_writer); + if (rval) { + avro_writer_free(schema_writer); + return rval; + } + schema_len = avro_writer_tell(schema_writer); + avro_writer_free(schema_writer); + check(rval, + enc->write_bytes(w->writer, w->schema_buf, schema_len)); + check(rval, enc->write_long(w->writer, 0)); + return write_sync(w); +} + +static int +file_writer_init_fp(FILE *fp, const char *path, int should_close, const char *mode, avro_file_writer_t w) +{ + if (!fp) { + fp = fopen(path, mode); + } + + if (!fp) { + avro_set_error("Cannot open file for %s", path); + return ENOMEM; + } + w->writer = avro_writer_file_fp(fp, should_close); + if (!w->writer) { + if (should_close) { + fclose(fp); + } + avro_set_error("Cannot create file writer for %s", path); + return ENOMEM; + } + return 0; +} + +/* Exclusive file writing is supported by GCC using the mode + * "wx". Win32 does not support exclusive file writing, so for win32 + * fall back to the non-exclusive file writing. + */ +#ifdef _WIN32 + #define EXCLUSIVE_WRITE_MODE "wb" +#else + #define EXCLUSIVE_WRITE_MODE "wbx" +#endif + +static int +file_writer_create(FILE *fp, const char *path, int should_close, avro_schema_t schema, avro_file_writer_t w, size_t block_size) +{ + int rval; + + w->block_count = 0; + rval = file_writer_init_fp(fp, path, should_close, EXCLUSIVE_WRITE_MODE, w); + if (rval) { + check(rval, file_writer_init_fp(fp, path, should_close, "wb", w)); + } + + w->datum_buffer_size = block_size; + w->datum_buffer = (char *) avro_malloc(w->datum_buffer_size); + + if(!w->datum_buffer) { + avro_set_error("Could not allocate datum buffer\n"); + avro_writer_free(w->writer); + return ENOMEM; + } + + w->datum_writer = + avro_writer_memory(w->datum_buffer, w->datum_buffer_size); + if (!w->datum_writer) { + avro_set_error("Cannot create datum writer for file %s", path); + avro_writer_free(w->writer); + avro_free(w->datum_buffer, w->datum_buffer_size); + return ENOMEM; + } + + w->writers_schema = avro_schema_incref(schema); + return write_header(w); +} + +int +avro_file_writer_create(const char *path, avro_schema_t schema, + avro_file_writer_t * writer) +{ + return avro_file_writer_create_with_codec_fp(NULL, path, 1, schema, writer, "null", 0); +} + +int +avro_file_writer_create_fp(FILE *fp, const char *path, int should_close, avro_schema_t schema, + avro_file_writer_t * writer) +{ + return avro_file_writer_create_with_codec_fp(fp, path, should_close, schema, writer, "null", 0); +} + +int avro_file_writer_create_with_codec(const char *path, + avro_schema_t schema, avro_file_writer_t * writer, + const char *codec, size_t block_size) +{ + return avro_file_writer_create_with_codec_fp(NULL, path, 1, schema, writer, codec, block_size); +} + +int avro_file_writer_create_with_codec_fp(FILE *fp, const char *path, int should_close, + avro_schema_t schema, avro_file_writer_t * writer, + const char *codec, size_t block_size) +{ + avro_file_writer_t w; + int rval; + check_param(EINVAL, path, "path"); + check_param(EINVAL, is_avro_schema(schema), "schema"); + check_param(EINVAL, writer, "writer"); + check_param(EINVAL, codec, "codec"); + + if (block_size == 0) { + block_size = DEFAULT_BLOCK_SIZE; + } + + w = (avro_file_writer_t) avro_new(struct avro_file_writer_t_); + if (!w) { + avro_set_error("Cannot allocate new file writer"); + return ENOMEM; + } + w->codec = (avro_codec_t) avro_new(struct avro_codec_t_); + if (!w->codec) { + avro_set_error("Cannot allocate new codec"); + avro_freet(struct avro_file_writer_t_, w); + return ENOMEM; + } + rval = avro_codec(w->codec, codec); + if (rval) { + avro_codec_reset(w->codec); + avro_freet(struct avro_codec_t_, w->codec); + avro_freet(struct avro_file_writer_t_, w); + return rval; + } + rval = file_writer_create(fp, path, should_close, schema, w, block_size); + if (rval) { + avro_codec_reset(w->codec); + avro_freet(struct avro_codec_t_, w->codec); + avro_freet(struct avro_file_writer_t_, w); + return rval; + } + *writer = w; + + return 0; +} + +static int file_read_header(avro_reader_t reader, + avro_schema_t * writers_schema, avro_codec_t codec, + char *sync, int synclen) +{ + int rval; + avro_schema_t meta_schema; + avro_schema_t meta_values_schema; + avro_value_iface_t *meta_iface; + avro_value_t meta; + char magic[4]; + avro_value_t codec_val; + avro_value_t schema_bytes; + const void *p; + size_t len; + + check(rval, avro_read(reader, magic, sizeof(magic))); + if (magic[0] != 'O' || magic[1] != 'b' || magic[2] != 'j' + || magic[3] != 1) { + avro_set_error("Incorrect Avro container file magic number"); + return EILSEQ; + } + + meta_values_schema = avro_schema_bytes(); + meta_schema = avro_schema_map(meta_values_schema); + meta_iface = avro_generic_class_from_schema(meta_schema); + if (meta_iface == NULL) { + return EILSEQ; + } + check(rval, avro_generic_value_new(meta_iface, &meta)); + rval = avro_value_read(reader, &meta); + if (rval) { + avro_prefix_error("Cannot read file header: "); + return EILSEQ; + } + avro_schema_decref(meta_schema); + + rval = avro_value_get_by_name(&meta, "avro.codec", &codec_val, NULL); + if (rval) { + if (avro_codec(codec, NULL) != 0) { + avro_set_error("Codec not specified in header and unable to set 'null' codec"); + avro_value_decref(&meta); + return EILSEQ; + } + } else { + const void *buf; + size_t size; + char codec_name[11]; + + avro_type_t type = avro_value_get_type(&codec_val); + + if (type != AVRO_BYTES) { + avro_set_error("Value type of codec is unexpected"); + avro_value_decref(&meta); + return EILSEQ; + } + + avro_value_get_bytes(&codec_val, &buf, &size); + memset(codec_name, 0, sizeof(codec_name)); + strncpy(codec_name, (const char *) buf, size < 10 ? size : 10); + + if (avro_codec(codec, codec_name) != 0) { + avro_set_error("File header contains an unknown codec"); + avro_value_decref(&meta); + return EILSEQ; + } + } + + rval = avro_value_get_by_name(&meta, "avro.schema", &schema_bytes, NULL); + if (rval) { + avro_set_error("File header doesn't contain a schema"); + avro_value_decref(&meta); + return EILSEQ; + } + + avro_value_get_bytes(&schema_bytes, &p, &len); + rval = avro_schema_from_json_length((const char *) p, len, writers_schema); + if (rval) { + avro_prefix_error("Cannot parse file header: "); + avro_value_decref(&meta); + return rval; + } + + avro_value_decref(&meta); + avro_value_iface_decref(meta_iface); + return avro_read(reader, sync, synclen); +} + +static int +file_writer_open(const char *path, avro_file_writer_t w, size_t block_size) +{ + int rval; + FILE *fp; + avro_reader_t reader; + + /* Open for read AND write */ + fp = fopen(path, "r+b"); + if (!fp) { + avro_set_error("Error opening file: %s", + strerror(errno)); + return errno; + } + + /* Don`t close the underlying file descriptor, logrotate can + * vanish it from sight. */ + reader = avro_reader_file_fp(fp, 0); + if (!reader) { + fclose(fp); + avro_set_error("Cannot create file reader for %s", path); + return ENOMEM; + } + rval = + file_read_header(reader, &w->writers_schema, w->codec, w->sync, + sizeof(w->sync)); + + avro_reader_free(reader); + if (rval) { + fclose(fp); + return rval; + } + + w->block_count = 0; + + /* Position to end of file and get ready to write */ + fseek(fp, 0, SEEK_END); + + w->writer = avro_writer_file(fp); + if (!w->writer) { + fclose(fp); + avro_set_error("Cannot create file writer for %s", path); + return ENOMEM; + } + + if (block_size == 0) { + block_size = DEFAULT_BLOCK_SIZE; + } + + w->datum_buffer_size = block_size; + w->datum_buffer = (char *) avro_malloc(w->datum_buffer_size); + + if(!w->datum_buffer) { + avro_set_error("Could not allocate datum buffer\n"); + avro_writer_free(w->writer); + return ENOMEM; + } + + w->datum_writer = + avro_writer_memory(w->datum_buffer, w->datum_buffer_size); + if (!w->datum_writer) { + avro_set_error("Cannot create datum writer for file %s", path); + avro_writer_free(w->writer); + avro_free(w->datum_buffer, w->datum_buffer_size); + return ENOMEM; + } + + return 0; +} + +int +avro_file_writer_open_bs(const char *path, avro_file_writer_t * writer, + size_t block_size) +{ + avro_file_writer_t w; + int rval; + check_param(EINVAL, path, "path"); + check_param(EINVAL, writer, "writer"); + + w = (avro_file_writer_t) avro_new(struct avro_file_writer_t_); + if (!w) { + avro_set_error("Cannot create new file writer for %s", path); + return ENOMEM; + } + w->codec = (avro_codec_t) avro_new(struct avro_codec_t_); + if (!w->codec) { + avro_set_error("Cannot allocate new codec"); + avro_freet(struct avro_file_writer_t_, w); + return ENOMEM; + } + avro_codec(w->codec, NULL); + rval = file_writer_open(path, w, block_size); + if (rval) { + avro_codec_reset(w->codec); + avro_freet(struct avro_codec_t_, w->codec); + avro_freet(struct avro_file_writer_t_, w); + return rval; + } + + *writer = w; + return 0; +} + +int +avro_file_writer_open(const char *path, avro_file_writer_t * writer) +{ + return avro_file_writer_open_bs(path, writer, 0); +} + +static int file_read_block_count(avro_file_reader_t r) +{ + int rval; + int64_t len; + const avro_encoding_t *enc = &avro_binary_encoding; + + /* For a correctly formatted file, EOF will occur here */ + rval = enc->read_long(r->reader, &r->blocks_total); + + if (rval == EILSEQ && avro_reader_is_eof(r->reader)) { + return EOF; + } + + check_prefix(rval, rval, + "Cannot read file block count: "); + check_prefix(rval, enc->read_long(r->reader, &len), + "Cannot read file block size: "); + + if (r->current_blockdata && len > r->current_blocklen) { + r->current_blockdata = (char *) avro_realloc(r->current_blockdata, r->current_blocklen, len); + r->current_blocklen = len; + } else if (!r->current_blockdata) { + r->current_blockdata = (char *) avro_malloc(len); + r->current_blocklen = len; + } + + if (len > 0) { + check_prefix(rval, avro_read(r->reader, r->current_blockdata, len), + "Cannot read file block: "); + + check_prefix(rval, avro_codec_decode(r->codec, r->current_blockdata, len), + "Cannot decode file block: "); + } + + avro_reader_memory_set_source(r->block_reader, (const char *) r->codec->block_data, r->codec->used_size); + + r->blocks_read = 0; + return 0; +} + +int avro_file_reader_fp(FILE *fp, const char *path, int should_close, + avro_file_reader_t * reader) +{ + int rval; + avro_file_reader_t r = (avro_file_reader_t) avro_new(struct avro_file_reader_t_); + if (!r) { + if (should_close) { + fclose(fp); + } + avro_set_error("Cannot allocate file reader for %s", path); + return ENOMEM; + } + + r->reader = avro_reader_file_fp(fp, should_close); + if (!r->reader) { + if (should_close) { + fclose(fp); + } + avro_set_error("Cannot allocate reader for file %s", path); + avro_freet(struct avro_file_reader_t_, r); + return ENOMEM; + } + r->block_reader = avro_reader_memory(0, 0); + if (!r->block_reader) { + avro_set_error("Cannot allocate block reader for file %s", path); + avro_reader_free(r->reader); + avro_freet(struct avro_file_reader_t_, r); + return ENOMEM; + } + + r->codec = (avro_codec_t) avro_new(struct avro_codec_t_); + if (!r->codec) { + avro_set_error("Could not allocate codec for file %s", path); + avro_reader_free(r->reader); + avro_freet(struct avro_file_reader_t_, r); + return ENOMEM; + } + avro_codec(r->codec, NULL); + + rval = file_read_header(r->reader, &r->writers_schema, r->codec, + r->sync, sizeof(r->sync)); + if (rval) { + avro_reader_free(r->reader); + avro_codec_reset(r->codec); + avro_freet(struct avro_codec_t_, r->codec); + avro_freet(struct avro_file_reader_t_, r); + return rval; + } + + r->current_blockdata = NULL; + r->current_blocklen = 0; + + rval = file_read_block_count(r); + if (rval == EOF) { + r->blocks_total = 0; + } else if (rval) { + avro_reader_free(r->reader); + avro_codec_reset(r->codec); + avro_freet(struct avro_codec_t_, r->codec); + avro_freet(struct avro_file_reader_t_, r); + return rval; + } + + *reader = r; + return 0; +} + +int avro_file_reader(const char *path, avro_file_reader_t * reader) +{ + FILE *fp; + + fp = fopen(path, "rb"); + if (!fp) { + return errno; + } + + return avro_file_reader_fp(fp, path, 1, reader); +} + +avro_schema_t +avro_file_reader_get_writer_schema(avro_file_reader_t r) +{ + check_param(NULL, r, "reader"); + return avro_schema_incref(r->writers_schema); +} + +static int file_write_block(avro_file_writer_t w) +{ + const avro_encoding_t *enc = &avro_binary_encoding; + int rval; + + if (w->block_count) { + /* Write the block count */ + check_prefix(rval, enc->write_long(w->writer, w->block_count), + "Cannot write file block count: "); + /* Encode the block */ + check_prefix(rval, avro_codec_encode(w->codec, w->datum_buffer, w->block_size), + "Cannot encode file block: "); + /* Write the block length */ + check_prefix(rval, enc->write_long(w->writer, w->codec->used_size), + "Cannot write file block size: "); + /* Write the block */ + check_prefix(rval, avro_write(w->writer, w->codec->block_data, w->codec->used_size), + "Cannot write file block: "); + /* Write the sync marker */ + check_prefix(rval, write_sync(w), + "Cannot write sync marker: "); + /* Reset the datum writer */ + avro_writer_reset(w->datum_writer); + w->block_count = 0; + w->block_size = 0; + } + return 0; +} + +int avro_file_writer_append(avro_file_writer_t w, avro_datum_t datum) +{ + int rval; + check_param(EINVAL, w, "writer"); + check_param(EINVAL, datum, "datum"); + + rval = avro_write_data(w->datum_writer, w->writers_schema, datum); + if (rval) { + check(rval, file_write_block(w)); + rval = + avro_write_data(w->datum_writer, w->writers_schema, datum); + if (rval) { + avro_set_error("Datum too large for file block size"); + /* TODO: if the datum encoder larger than our buffer, + just write a single large datum */ + return rval; + } + } + w->block_count++; + w->block_size = avro_writer_tell(w->datum_writer); + return 0; +} + +int +avro_file_writer_append_value(avro_file_writer_t w, avro_value_t *value) +{ + int rval; + check_param(EINVAL, w, "writer"); + check_param(EINVAL, value, "value"); + + rval = avro_value_write(w->datum_writer, value); + if (rval) { + check(rval, file_write_block(w)); + rval = avro_value_write(w->datum_writer, value); + if (rval) { + avro_set_error("Value too large for file block size"); + /* TODO: if the value encoder larger than our buffer, + just write a single large datum */ + return rval; + } + } + w->block_count++; + w->block_size = avro_writer_tell(w->datum_writer); + return 0; +} + +int +avro_file_writer_append_encoded(avro_file_writer_t w, + const void *buf, int64_t len) +{ + int rval; + check_param(EINVAL, w, "writer"); + + rval = avro_write(w->datum_writer, (void *) buf, len); + if (rval) { + check(rval, file_write_block(w)); + rval = avro_write(w->datum_writer, (void *) buf, len); + if (rval) { + avro_set_error("Value too large for file block size"); + /* TODO: if the value encoder larger than our buffer, + just write a single large datum */ + return rval; + } + } + w->block_count++; + w->block_size = avro_writer_tell(w->datum_writer); + return 0; +} + +int avro_file_writer_sync(avro_file_writer_t w) +{ + return file_write_block(w); +} + +int avro_file_writer_flush(avro_file_writer_t w) +{ + int rval; + check(rval, file_write_block(w)); + avro_writer_flush(w->writer); + return 0; +} + +int avro_file_writer_close(avro_file_writer_t w) +{ + int rval; + check(rval, avro_file_writer_flush(w)); + avro_schema_decref(w->writers_schema); + avro_writer_free(w->datum_writer); + avro_writer_free(w->writer); + avro_free(w->datum_buffer, w->datum_buffer_size); + avro_codec_reset(w->codec); + avro_freet(struct avro_codec_t_, w->codec); + avro_freet(struct avro_file_writer_t_, w); + return 0; +} + +int avro_file_reader_read(avro_file_reader_t r, avro_schema_t readers_schema, + avro_datum_t * datum) +{ + int rval; + char sync[16]; + + check_param(EINVAL, r, "reader"); + check_param(EINVAL, datum, "datum"); + + /* This will be set to zero when an empty file is opened. + * Return EOF here when the user attempts to read. */ + if (r->blocks_total == 0) { + return EOF; + } + + if (r->blocks_read == r->blocks_total) { + check(rval, avro_read(r->reader, sync, sizeof(sync))); + if (memcmp(r->sync, sync, sizeof(r->sync)) != 0) { + /* wrong sync bytes */ + avro_set_error("Incorrect sync bytes"); + return EILSEQ; + } + check(rval, file_read_block_count(r)); + } + + check(rval, + avro_read_data(r->block_reader, r->writers_schema, readers_schema, + datum)); + r->blocks_read++; + + return 0; +} + +int +avro_file_reader_read_value(avro_file_reader_t r, avro_value_t *value) +{ + int rval; + char sync[16]; + + check_param(EINVAL, r, "reader"); + check_param(EINVAL, value, "value"); + + /* This will be set to zero when an empty file is opened. + * Return EOF here when the user attempts to read. */ + if (r->blocks_total == 0) { + return EOF; + } + + if (r->blocks_read == r->blocks_total) { + /* reads sync bytes and buffers further bytes */ + check(rval, avro_read(r->reader, sync, sizeof(sync))); + if (memcmp(r->sync, sync, sizeof(r->sync)) != 0) { + /* wrong sync bytes */ + avro_set_error("Incorrect sync bytes"); + return EILSEQ; + } + + check(rval, file_read_block_count(r)); + } + + check(rval, avro_value_read(r->block_reader, value)); + r->blocks_read++; + + return 0; +} + +int avro_file_reader_close(avro_file_reader_t reader) +{ + avro_schema_decref(reader->writers_schema); + avro_reader_free(reader->reader); + avro_reader_free(reader->block_reader); + avro_codec_reset(reader->codec); + avro_freet(struct avro_codec_t_, reader->codec); + if (reader->current_blockdata) { + avro_free(reader->current_blockdata, reader->current_blocklen); + } + avro_freet(struct avro_file_reader_t_, reader); + return 0; +} diff --git a/fluent-bit/lib/avro/src/datum.c b/fluent-bit/lib/avro/src/datum.c new file mode 100644 index 00000000..2c427809 --- /dev/null +++ b/fluent-bit/lib/avro/src/datum.c @@ -0,0 +1,1255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include "avro/allocation.h" +#include "avro/basics.h" +#include "avro/errors.h" +#include "avro/legacy.h" +#include "avro/refcount.h" +#include "avro/schema.h" +#include "avro_private.h" +#include <stdlib.h> +#include <string.h> +#include <errno.h> +#include "datum.h" +#include "schema.h" +#include "encoding.h" + +#define DEFAULT_TABLE_SIZE 32 + +static void avro_datum_init(avro_datum_t datum, avro_type_t type) +{ + datum->type = type; + datum->class_type = AVRO_DATUM; + avro_refcount_set(&datum->refcount, 1); +} + +static void +avro_str_free_wrapper(void *ptr, size_t sz) +{ + // don't need sz, since the size is stored in the string buffer + AVRO_UNUSED(sz); + avro_str_free((char *)ptr); +} + +static avro_datum_t avro_string_private(char *str, int64_t size, + avro_free_func_t string_free) +{ + struct avro_string_datum_t *datum = + (struct avro_string_datum_t *) avro_new(struct avro_string_datum_t); + if (!datum) { + avro_set_error("Cannot create new string datum"); + return NULL; + } + datum->s = str; + datum->size = size; + datum->free = string_free; + + avro_datum_init(&datum->obj, AVRO_STRING); + return &datum->obj; +} + +avro_datum_t avro_string(const char *str) +{ + char *p = avro_strdup(str); + if (!p) { + avro_set_error("Cannot copy string content"); + return NULL; + } + avro_datum_t s_datum = avro_string_private(p, 0, avro_str_free_wrapper); + if (!s_datum) { + avro_str_free(p); + } + + return s_datum; +} + +avro_datum_t avro_givestring(const char *str, + avro_free_func_t free) +{ + int64_t sz = strlen(str)+1; + return avro_string_private((char *)str, sz, free); +} + +int avro_string_get(avro_datum_t datum, char **p) +{ + check_param(EINVAL, is_avro_datum(datum), "datum"); + check_param(EINVAL, is_avro_string(datum), "string datum"); + check_param(EINVAL, p, "string buffer"); + + *p = avro_datum_to_string(datum)->s; + return 0; +} + +static int avro_string_set_private(avro_datum_t datum, + const char *p, int64_t size, + avro_free_func_t string_free) +{ + check_param(EINVAL, is_avro_datum(datum), "datum"); + check_param(EINVAL, is_avro_string(datum), "string datum"); + check_param(EINVAL, p, "string content"); + + struct avro_string_datum_t *string = avro_datum_to_string(datum); + if (string->free) { + string->free(string->s, string->size); + } + string->free = string_free; + string->s = (char *)p; + string->size = size; + return 0; +} + +int avro_string_set(avro_datum_t datum, const char *p) +{ + char *string_copy = avro_strdup(p); + int rval; + if (!string_copy) { + avro_set_error("Cannot copy string content"); + return ENOMEM; + } + rval = avro_string_set_private(datum, string_copy, 0, + avro_str_free_wrapper); + if (rval) { + avro_str_free(string_copy); + } + return rval; +} + +int avro_givestring_set(avro_datum_t datum, const char *p, + avro_free_func_t free) +{ + int64_t size = strlen(p)+1; + return avro_string_set_private(datum, p, size, free); +} + +static avro_datum_t avro_bytes_private(char *bytes, int64_t size, + avro_free_func_t bytes_free) +{ + struct avro_bytes_datum_t *datum; + datum = (struct avro_bytes_datum_t *) avro_new(struct avro_bytes_datum_t); + if (!datum) { + avro_set_error("Cannot create new bytes datum"); + return NULL; + } + datum->bytes = bytes; + datum->size = size; + datum->free = bytes_free; + + avro_datum_init(&datum->obj, AVRO_BYTES); + return &datum->obj; +} + +avro_datum_t avro_bytes(const char *bytes, int64_t size) +{ + char *bytes_copy = (char *) avro_malloc(size); + if (!bytes_copy) { + avro_set_error("Cannot copy bytes content"); + return NULL; + } + memcpy(bytes_copy, bytes, size); + avro_datum_t result = + avro_bytes_private(bytes_copy, size, avro_alloc_free_func); + if (result == NULL) { + avro_free(bytes_copy, size); + } + return result; +} + +avro_datum_t avro_givebytes(const char *bytes, int64_t size, + avro_free_func_t free) +{ + return avro_bytes_private((char *)bytes, size, free); +} + +static int avro_bytes_set_private(avro_datum_t datum, const char *bytes, + const int64_t size, + avro_free_func_t bytes_free) +{ + check_param(EINVAL, is_avro_datum(datum), "datum"); + check_param(EINVAL, is_avro_bytes(datum), "bytes datum"); + + struct avro_bytes_datum_t *b = avro_datum_to_bytes(datum); + if (b->free) { + b->free(b->bytes, b->size); + } + + b->free = bytes_free; + b->bytes = (char *)bytes; + b->size = size; + return 0; +} + +int avro_bytes_set(avro_datum_t datum, const char *bytes, const int64_t size) +{ + int rval; + char *bytes_copy = (char *) avro_malloc(size); + if (!bytes_copy) { + avro_set_error("Cannot copy bytes content"); + return ENOMEM; + } + memcpy(bytes_copy, bytes, size); + rval = avro_bytes_set_private(datum, bytes_copy, size, avro_alloc_free_func); + if (rval) { + avro_free(bytes_copy, size); + } + return rval; +} + +int avro_givebytes_set(avro_datum_t datum, const char *bytes, + const int64_t size, avro_free_func_t free) +{ + return avro_bytes_set_private(datum, bytes, size, free); +} + +int avro_bytes_get(avro_datum_t datum, char **bytes, int64_t * size) +{ + check_param(EINVAL, is_avro_datum(datum), "datum"); + check_param(EINVAL, is_avro_bytes(datum), "bytes datum"); + check_param(EINVAL, bytes, "bytes"); + check_param(EINVAL, size, "size"); + + *bytes = avro_datum_to_bytes(datum)->bytes; + *size = avro_datum_to_bytes(datum)->size; + return 0; +} + +avro_datum_t avro_int32(int32_t i) +{ + struct avro_int32_datum_t *datum = + (struct avro_int32_datum_t *) avro_new(struct avro_int32_datum_t); + if (!datum) { + avro_set_error("Cannot create new int datum"); + return NULL; + } + datum->i32 = i; + + avro_datum_init(&datum->obj, AVRO_INT32); + return &datum->obj; +} + +int avro_int32_get(avro_datum_t datum, int32_t * i) +{ + check_param(EINVAL, is_avro_datum(datum), "datum"); + check_param(EINVAL, is_avro_int32(datum), "int datum"); + check_param(EINVAL, i, "value pointer"); + + *i = avro_datum_to_int32(datum)->i32; + return 0; +} + +int avro_int32_set(avro_datum_t datum, const int32_t i) +{ + check_param(EINVAL, is_avro_datum(datum), "datum"); + check_param(EINVAL, is_avro_int32(datum), "int datum"); + + avro_datum_to_int32(datum)->i32 = i; + return 0; +} + +avro_datum_t avro_int64(int64_t l) +{ + struct avro_int64_datum_t *datum = + (struct avro_int64_datum_t *) avro_new(struct avro_int64_datum_t); + if (!datum) { + avro_set_error("Cannot create new long datum"); + return NULL; + } + datum->i64 = l; + + avro_datum_init(&datum->obj, AVRO_INT64); + return &datum->obj; +} + +int avro_int64_get(avro_datum_t datum, int64_t * l) +{ + check_param(EINVAL, is_avro_datum(datum), "datum"); + check_param(EINVAL, is_avro_int64(datum), "long datum"); + check_param(EINVAL, l, "value pointer"); + + *l = avro_datum_to_int64(datum)->i64; + return 0; +} + +int avro_int64_set(avro_datum_t datum, const int64_t l) +{ + check_param(EINVAL, is_avro_datum(datum), "datum"); + check_param(EINVAL, is_avro_int64(datum), "long datum"); + + avro_datum_to_int64(datum)->i64 = l; + return 0; +} + +avro_datum_t avro_float(float f) +{ + struct avro_float_datum_t *datum = + (struct avro_float_datum_t *) avro_new(struct avro_float_datum_t); + if (!datum) { + avro_set_error("Cannot create new float datum"); + return NULL; + } + datum->f = f; + + avro_datum_init(&datum->obj, AVRO_FLOAT); + return &datum->obj; +} + +int avro_float_set(avro_datum_t datum, const float f) +{ + check_param(EINVAL, is_avro_datum(datum), "datum"); + check_param(EINVAL, is_avro_float(datum), "float datum"); + + avro_datum_to_float(datum)->f = f; + return 0; +} + +int avro_float_get(avro_datum_t datum, float *f) +{ + check_param(EINVAL, is_avro_datum(datum), "datum"); + check_param(EINVAL, is_avro_float(datum), "float datum"); + check_param(EINVAL, f, "value pointer"); + + *f = avro_datum_to_float(datum)->f; + return 0; +} + +avro_datum_t avro_double(double d) +{ + struct avro_double_datum_t *datum = + (struct avro_double_datum_t *) avro_new(struct avro_double_datum_t); + if (!datum) { + avro_set_error("Cannot create new double atom"); + return NULL; + } + datum->d = d; + + avro_datum_init(&datum->obj, AVRO_DOUBLE); + return &datum->obj; +} + +int avro_double_set(avro_datum_t datum, const double d) +{ + check_param(EINVAL, is_avro_datum(datum), "datum"); + check_param(EINVAL, is_avro_double(datum), "double datum"); + + avro_datum_to_double(datum)->d = d; + return 0; +} + +int avro_double_get(avro_datum_t datum, double *d) +{ + check_param(EINVAL, is_avro_datum(datum), "datum"); + check_param(EINVAL, is_avro_double(datum), "double datum"); + check_param(EINVAL, d, "value pointer"); + + *d = avro_datum_to_double(datum)->d; + return 0; +} + +avro_datum_t avro_boolean(int8_t i) +{ + struct avro_boolean_datum_t *datum = + (struct avro_boolean_datum_t *) avro_new(struct avro_boolean_datum_t); + if (!datum) { + avro_set_error("Cannot create new boolean datum"); + return NULL; + } + datum->i = i; + avro_datum_init(&datum->obj, AVRO_BOOLEAN); + return &datum->obj; +} + +int avro_boolean_set(avro_datum_t datum, const int8_t i) +{ + check_param(EINVAL, is_avro_datum(datum), "datum"); + check_param(EINVAL, is_avro_boolean(datum), "boolean datum"); + + avro_datum_to_boolean(datum)->i = i; + return 0; +} + +int avro_boolean_get(avro_datum_t datum, int8_t * i) +{ + check_param(EINVAL, is_avro_datum(datum), "datum"); + check_param(EINVAL, is_avro_boolean(datum), "boolean datum"); + check_param(EINVAL, i, "value pointer"); + + *i = avro_datum_to_boolean(datum)->i; + return 0; +} + +avro_datum_t avro_null(void) +{ + static struct avro_obj_t obj = { + AVRO_NULL, + AVRO_DATUM, + 1 + }; + return avro_datum_incref(&obj); +} + +avro_datum_t avro_union(avro_schema_t schema, + int64_t discriminant, avro_datum_t value) +{ + check_param(NULL, is_avro_schema(schema), "schema"); + + struct avro_union_datum_t *datum = + (struct avro_union_datum_t *) avro_new(struct avro_union_datum_t); + if (!datum) { + avro_set_error("Cannot create new union datum"); + return NULL; + } + datum->schema = avro_schema_incref(schema); + datum->discriminant = discriminant; + datum->value = avro_datum_incref(value); + + avro_datum_init(&datum->obj, AVRO_UNION); + return &datum->obj; +} + +int64_t avro_union_discriminant(const avro_datum_t datum) +{ + return avro_datum_to_union(datum)->discriminant; +} + +avro_datum_t avro_union_current_branch(avro_datum_t datum) +{ + return avro_datum_to_union(datum)->value; +} + +int avro_union_set_discriminant(avro_datum_t datum, + int discriminant, + avro_datum_t *branch) +{ + check_param(EINVAL, is_avro_datum(datum), "datum"); + check_param(EINVAL, is_avro_union(datum), "union datum"); + + struct avro_union_datum_t *unionp = avro_datum_to_union(datum); + + avro_schema_t schema = unionp->schema; + avro_schema_t branch_schema = + avro_schema_union_branch(schema, discriminant); + + if (branch_schema == NULL) { + // That branch doesn't exist! + avro_set_error("Branch %d doesn't exist", discriminant); + return EINVAL; + } + + if (unionp->discriminant != discriminant) { + // If we're changing the branch, throw away any old + // branch value. + if (unionp->value != NULL) { + avro_datum_decref(unionp->value); + unionp->value = NULL; + } + + unionp->discriminant = discriminant; + } + + // Create a new branch value, if there isn't one already. + if (unionp->value == NULL) { + unionp->value = avro_datum_from_schema(branch_schema); + } + + if (branch != NULL) { + *branch = unionp->value; + } + + return 0; +} + +avro_datum_t avro_record(avro_schema_t schema) +{ + check_param(NULL, is_avro_schema(schema), "schema"); + + struct avro_record_datum_t *datum = + (struct avro_record_datum_t *) avro_new(struct avro_record_datum_t); + if (!datum) { + avro_set_error("Cannot create new record datum"); + return NULL; + } + datum->field_order = st_init_numtable_with_size(DEFAULT_TABLE_SIZE); + if (!datum->field_order) { + avro_set_error("Cannot create new record datum"); + avro_freet(struct avro_record_datum_t, datum); + return NULL; + } + datum->fields_byname = st_init_strtable_with_size(DEFAULT_TABLE_SIZE); + if (!datum->fields_byname) { + avro_set_error("Cannot create new record datum"); + st_free_table(datum->field_order); + avro_freet(struct avro_record_datum_t, datum); + return NULL; + } + + datum->schema = avro_schema_incref(schema); + avro_datum_init(&datum->obj, AVRO_RECORD); + return &datum->obj; +} + +int +avro_record_get(const avro_datum_t datum, const char *field_name, + avro_datum_t * field) +{ + union { + avro_datum_t field; + st_data_t data; + } val; + if (is_avro_datum(datum) && is_avro_record(datum) && field_name) { + if (st_lookup + (avro_datum_to_record(datum)->fields_byname, + (st_data_t) field_name, &(val.data))) { + *field = val.field; + return 0; + } + } + avro_set_error("No field named %s", field_name); + return EINVAL; +} + +int +avro_record_set(avro_datum_t datum, const char *field_name, + const avro_datum_t field_value) +{ + check_param(EINVAL, is_avro_datum(datum), "datum"); + check_param(EINVAL, is_avro_record(datum), "record datum"); + check_param(EINVAL, field_name, "field_name"); + + char *key = (char *)field_name; + avro_datum_t old_field; + + if (avro_record_get(datum, field_name, &old_field) == 0) { + /* Overriding old value */ + avro_datum_decref(old_field); + } else { + /* Inserting new value */ + struct avro_record_datum_t *record = + avro_datum_to_record(datum); + key = avro_strdup(field_name); + if (!key) { + avro_set_error("Cannot copy field name"); + return ENOMEM; + } + st_insert(record->field_order, + record->field_order->num_entries, + (st_data_t) key); + } + avro_datum_incref(field_value); + st_insert(avro_datum_to_record(datum)->fields_byname, + (st_data_t) key, (st_data_t) field_value); + return 0; +} + +avro_datum_t avro_enum(avro_schema_t schema, int i) +{ + check_param(NULL, is_avro_schema(schema), "schema"); + + struct avro_enum_datum_t *datum = + (struct avro_enum_datum_t *) avro_new(struct avro_enum_datum_t); + if (!datum) { + avro_set_error("Cannot create new enum datum"); + return NULL; + } + datum->schema = avro_schema_incref(schema); + datum->value = i; + + avro_datum_init(&datum->obj, AVRO_ENUM); + return &datum->obj; +} + +int avro_enum_get(const avro_datum_t datum) +{ + return avro_datum_to_enum(datum)->value; +} + +const char *avro_enum_get_name(const avro_datum_t datum) +{ + int value = avro_enum_get(datum); + avro_schema_t schema = avro_datum_to_enum(datum)->schema; + return avro_schema_enum_get(schema, value); +} + +int avro_enum_set(avro_datum_t datum, const int symbol_value) +{ + check_param(EINVAL, is_avro_datum(datum), "datum"); + check_param(EINVAL, is_avro_enum(datum), "enum datum"); + + avro_datum_to_enum(datum)->value = symbol_value; + return 0; +} + +int avro_enum_set_name(avro_datum_t datum, const char *symbol_name) +{ + check_param(EINVAL, is_avro_datum(datum), "datum"); + check_param(EINVAL, is_avro_enum(datum), "enum datum"); + check_param(EINVAL, symbol_name, "symbol name"); + + avro_schema_t schema = avro_datum_to_enum(datum)->schema; + int symbol_value = avro_schema_enum_get_by_name(schema, symbol_name); + if (symbol_value == -1) { + avro_set_error("No symbol named %s", symbol_name); + return EINVAL; + } + avro_datum_to_enum(datum)->value = symbol_value; + return 0; +} + +static avro_datum_t avro_fixed_private(avro_schema_t schema, + const char *bytes, const int64_t size, + avro_free_func_t fixed_free) +{ + check_param(NULL, is_avro_schema(schema), "schema"); + struct avro_fixed_schema_t *fschema = avro_schema_to_fixed(schema); + if (size != fschema->size) { + avro_free((char *) bytes, size); + avro_set_error("Fixed size (%zu) doesn't match schema (%zu)", + (size_t) size, (size_t) fschema->size); + return NULL; + } + + struct avro_fixed_datum_t *datum = + (struct avro_fixed_datum_t *) avro_new(struct avro_fixed_datum_t); + if (!datum) { + avro_free((char *) bytes, size); + avro_set_error("Cannot create new fixed datum"); + return NULL; + } + datum->schema = avro_schema_incref(schema); + datum->size = size; + datum->bytes = (char *)bytes; + datum->free = fixed_free; + + avro_datum_init(&datum->obj, AVRO_FIXED); + return &datum->obj; +} + +avro_datum_t avro_fixed(avro_schema_t schema, + const char *bytes, const int64_t size) +{ + char *bytes_copy = (char *) avro_malloc(size); + if (!bytes_copy) { + avro_set_error("Cannot copy fixed content"); + return NULL; + } + memcpy(bytes_copy, bytes, size); + return avro_fixed_private(schema, bytes_copy, size, avro_alloc_free_func); +} + +avro_datum_t avro_givefixed(avro_schema_t schema, + const char *bytes, const int64_t size, + avro_free_func_t free) +{ + return avro_fixed_private(schema, bytes, size, free); +} + +static int avro_fixed_set_private(avro_datum_t datum, + const char *bytes, const int64_t size, + avro_free_func_t fixed_free) +{ + check_param(EINVAL, is_avro_datum(datum), "datum"); + check_param(EINVAL, is_avro_fixed(datum), "fixed datum"); + + struct avro_fixed_datum_t *fixed = avro_datum_to_fixed(datum); + struct avro_fixed_schema_t *schema = avro_schema_to_fixed(fixed->schema); + if (size != schema->size) { + avro_set_error("Fixed size doesn't match schema"); + return EINVAL; + } + + if (fixed->free) { + fixed->free(fixed->bytes, fixed->size); + } + + fixed->free = fixed_free; + fixed->bytes = (char *)bytes; + fixed->size = size; + return 0; +} + +int avro_fixed_set(avro_datum_t datum, const char *bytes, const int64_t size) +{ + int rval; + char *bytes_copy = (char *) avro_malloc(size); + if (!bytes_copy) { + avro_set_error("Cannot copy fixed content"); + return ENOMEM; + } + memcpy(bytes_copy, bytes, size); + rval = avro_fixed_set_private(datum, bytes_copy, size, avro_alloc_free_func); + if (rval) { + avro_free(bytes_copy, size); + } + return rval; +} + +int avro_givefixed_set(avro_datum_t datum, const char *bytes, + const int64_t size, avro_free_func_t free) +{ + return avro_fixed_set_private(datum, bytes, size, free); +} + +int avro_fixed_get(avro_datum_t datum, char **bytes, int64_t * size) +{ + check_param(EINVAL, is_avro_datum(datum), "datum"); + check_param(EINVAL, is_avro_fixed(datum), "fixed datum"); + check_param(EINVAL, bytes, "bytes"); + check_param(EINVAL, size, "size"); + + *bytes = avro_datum_to_fixed(datum)->bytes; + *size = avro_datum_to_fixed(datum)->size; + return 0; +} + +static int +avro_init_map(struct avro_map_datum_t *datum) +{ + datum->map = st_init_strtable_with_size(DEFAULT_TABLE_SIZE); + if (!datum->map) { + avro_set_error("Cannot create new map datum"); + return ENOMEM; + } + datum->indices_by_key = st_init_strtable_with_size(DEFAULT_TABLE_SIZE); + if (!datum->indices_by_key) { + avro_set_error("Cannot create new map datum"); + st_free_table(datum->map); + return ENOMEM; + } + datum->keys_by_index = st_init_numtable_with_size(DEFAULT_TABLE_SIZE); + if (!datum->keys_by_index) { + avro_set_error("Cannot create new map datum"); + st_free_table(datum->indices_by_key); + st_free_table(datum->map); + return ENOMEM; + } + return 0; +} + +avro_datum_t avro_map(avro_schema_t schema) +{ + check_param(NULL, is_avro_schema(schema), "schema"); + + struct avro_map_datum_t *datum = + (struct avro_map_datum_t *) avro_new(struct avro_map_datum_t); + if (!datum) { + avro_set_error("Cannot create new map datum"); + return NULL; + } + + if (avro_init_map(datum) != 0) { + avro_freet(struct avro_map_datum_t, datum); + return NULL; + } + + datum->schema = avro_schema_incref(schema); + avro_datum_init(&datum->obj, AVRO_MAP); + return &datum->obj; +} + +size_t +avro_map_size(const avro_datum_t datum) +{ + const struct avro_map_datum_t *map = avro_datum_to_map(datum); + return map->map->num_entries; +} + +int +avro_map_get(const avro_datum_t datum, const char *key, avro_datum_t * value) +{ + check_param(EINVAL, is_avro_datum(datum), "datum"); + check_param(EINVAL, is_avro_map(datum), "map datum"); + check_param(EINVAL, key, "key"); + check_param(EINVAL, value, "value"); + + union { + avro_datum_t datum; + st_data_t data; + } val; + + struct avro_map_datum_t *map = avro_datum_to_map(datum); + if (st_lookup(map->map, (st_data_t) key, &(val.data))) { + *value = val.datum; + return 0; + } + + avro_set_error("No map element named %s", key); + return EINVAL; +} + +int avro_map_get_key(const avro_datum_t datum, int index, + const char **key) +{ + check_param(EINVAL, is_avro_datum(datum), "datum"); + check_param(EINVAL, is_avro_map(datum), "map datum"); + check_param(EINVAL, index >= 0, "index"); + check_param(EINVAL, key, "key"); + + union { + st_data_t data; + char *key; + } val; + + struct avro_map_datum_t *map = avro_datum_to_map(datum); + if (st_lookup(map->keys_by_index, (st_data_t) index, &val.data)) { + *key = val.key; + return 0; + } + + avro_set_error("No map element with index %d", index); + return EINVAL; +} + +int avro_map_get_index(const avro_datum_t datum, const char *key, + int *index) +{ + check_param(EINVAL, is_avro_datum(datum), "datum"); + check_param(EINVAL, is_avro_map(datum), "map datum"); + check_param(EINVAL, key, "key"); + check_param(EINVAL, index, "index"); + + st_data_t data; + + struct avro_map_datum_t *map = avro_datum_to_map(datum); + if (st_lookup(map->indices_by_key, (st_data_t) key, &data)) { + *index = (int) data; + return 0; + } + + avro_set_error("No map element with key %s", key); + return EINVAL; +} + +int +avro_map_set(avro_datum_t datum, const char *key, + const avro_datum_t value) +{ + check_param(EINVAL, is_avro_datum(datum), "datum"); + check_param(EINVAL, is_avro_map(datum), "map datum"); + check_param(EINVAL, key, "key"); + check_param(EINVAL, is_avro_datum(value), "value"); + + char *save_key = (char *)key; + avro_datum_t old_datum; + + struct avro_map_datum_t *map = avro_datum_to_map(datum); + + if (avro_map_get(datum, key, &old_datum) == 0) { + /* Overwriting an old value */ + avro_datum_decref(old_datum); + } else { + /* Inserting a new value */ + save_key = avro_strdup(key); + if (!save_key) { + avro_set_error("Cannot copy map key"); + return ENOMEM; + } + int new_index = map->map->num_entries; + st_insert(map->indices_by_key, (st_data_t) save_key, + (st_data_t) new_index); + st_insert(map->keys_by_index, (st_data_t) new_index, + (st_data_t) save_key); + } + avro_datum_incref(value); + st_insert(map->map, (st_data_t) save_key, (st_data_t) value); + return 0; +} + +static int +avro_init_array(struct avro_array_datum_t *datum) +{ + datum->els = st_init_numtable_with_size(DEFAULT_TABLE_SIZE); + if (!datum->els) { + avro_set_error("Cannot create new array datum"); + return ENOMEM; + } + return 0; +} + +avro_datum_t avro_array(avro_schema_t schema) +{ + check_param(NULL, is_avro_schema(schema), "schema"); + + struct avro_array_datum_t *datum = + (struct avro_array_datum_t *) avro_new(struct avro_array_datum_t); + if (!datum) { + avro_set_error("Cannot create new array datum"); + return NULL; + } + + if (avro_init_array(datum) != 0) { + avro_freet(struct avro_array_datum_t, datum); + return NULL; + } + + datum->schema = avro_schema_incref(schema); + avro_datum_init(&datum->obj, AVRO_ARRAY); + return &datum->obj; +} + +int +avro_array_get(const avro_datum_t array_datum, int64_t index, avro_datum_t * value) +{ + check_param(EINVAL, is_avro_datum(array_datum), "datum"); + check_param(EINVAL, is_avro_array(array_datum), "array datum"); + check_param(EINVAL, value, "value pointer"); + + union { + st_data_t data; + avro_datum_t datum; + } val; + + const struct avro_array_datum_t * array = avro_datum_to_array(array_datum); + if (st_lookup(array->els, index, &val.data)) { + *value = val.datum; + return 0; + } + + avro_set_error("No array element with index %ld", (long) index); + return EINVAL; +} + +size_t +avro_array_size(const avro_datum_t datum) +{ + const struct avro_array_datum_t *array = avro_datum_to_array(datum); + return array->els->num_entries; +} + +int +avro_array_append_datum(avro_datum_t array_datum, + const avro_datum_t datum) +{ + check_param(EINVAL, is_avro_datum(array_datum), "datum"); + check_param(EINVAL, is_avro_array(array_datum), "array datum"); + check_param(EINVAL, is_avro_datum(datum), "element datum"); + + struct avro_array_datum_t *array = avro_datum_to_array(array_datum); + st_insert(array->els, array->els->num_entries, + (st_data_t) avro_datum_incref(datum)); + return 0; +} + +static int char_datum_free_foreach(char *key, avro_datum_t datum, void *arg) +{ + AVRO_UNUSED(arg); + + avro_datum_decref(datum); + avro_str_free(key); + return ST_DELETE; +} + +static int array_free_foreach(int i, avro_datum_t datum, void *arg) +{ + AVRO_UNUSED(i); + AVRO_UNUSED(arg); + + avro_datum_decref(datum); + return ST_DELETE; +} + +avro_schema_t avro_datum_get_schema(const avro_datum_t datum) +{ + check_param(NULL, is_avro_datum(datum), "datum"); + + switch (avro_typeof(datum)) { + /* + * For the primitive types, which don't store an + * explicit reference to their schema, we decref the + * schema before returning. This maintains the + * invariant that this function doesn't add any + * additional references to the schema. The primitive + * schemas won't be freed, because there's always at + * least 1 reference for their initial static + * initializers. + */ + + case AVRO_STRING: + { + avro_schema_t result = avro_schema_string(); + avro_schema_decref(result); + return result; + } + case AVRO_BYTES: + { + avro_schema_t result = avro_schema_bytes(); + avro_schema_decref(result); + return result; + } + case AVRO_INT32: + { + avro_schema_t result = avro_schema_int(); + avro_schema_decref(result); + return result; + } + case AVRO_INT64: + { + avro_schema_t result = avro_schema_long(); + avro_schema_decref(result); + return result; + } + case AVRO_FLOAT: + { + avro_schema_t result = avro_schema_float(); + avro_schema_decref(result); + return result; + } + case AVRO_DOUBLE: + { + avro_schema_t result = avro_schema_double(); + avro_schema_decref(result); + return result; + } + case AVRO_BOOLEAN: + { + avro_schema_t result = avro_schema_boolean(); + avro_schema_decref(result); + return result; + } + case AVRO_NULL: + { + avro_schema_t result = avro_schema_null(); + avro_schema_decref(result); + return result; + } + + case AVRO_RECORD: + return avro_datum_to_record(datum)->schema; + case AVRO_ENUM: + return avro_datum_to_enum(datum)->schema; + case AVRO_FIXED: + return avro_datum_to_fixed(datum)->schema; + case AVRO_MAP: + return avro_datum_to_map(datum)->schema; + case AVRO_ARRAY: + return avro_datum_to_array(datum)->schema; + case AVRO_UNION: + return avro_datum_to_union(datum)->schema; + + default: + return NULL; + } +} + +static void avro_datum_free(avro_datum_t datum) +{ + if (is_avro_datum(datum)) { + switch (avro_typeof(datum)) { + case AVRO_STRING:{ + struct avro_string_datum_t *string; + string = avro_datum_to_string(datum); + if (string->free) { + string->free(string->s, string->size); + } + avro_freet(struct avro_string_datum_t, string); + } + break; + case AVRO_BYTES:{ + struct avro_bytes_datum_t *bytes; + bytes = avro_datum_to_bytes(datum); + if (bytes->free) { + bytes->free(bytes->bytes, bytes->size); + } + avro_freet(struct avro_bytes_datum_t, bytes); + } + break; + case AVRO_INT32:{ + avro_freet(struct avro_int32_datum_t, datum); + } + break; + case AVRO_INT64:{ + avro_freet(struct avro_int64_datum_t, datum); + } + break; + case AVRO_FLOAT:{ + avro_freet(struct avro_float_datum_t, datum); + } + break; + case AVRO_DOUBLE:{ + avro_freet(struct avro_double_datum_t, datum); + } + break; + case AVRO_BOOLEAN:{ + avro_freet(struct avro_boolean_datum_t, datum); + } + break; + case AVRO_NULL: + /* Nothing allocated */ + break; + + case AVRO_RECORD:{ + struct avro_record_datum_t *record; + record = avro_datum_to_record(datum); + avro_schema_decref(record->schema); + st_foreach(record->fields_byname, + HASH_FUNCTION_CAST char_datum_free_foreach, 0); + st_free_table(record->field_order); + st_free_table(record->fields_byname); + avro_freet(struct avro_record_datum_t, record); + } + break; + case AVRO_ENUM:{ + struct avro_enum_datum_t *enump; + enump = avro_datum_to_enum(datum); + avro_schema_decref(enump->schema); + avro_freet(struct avro_enum_datum_t, enump); + } + break; + case AVRO_FIXED:{ + struct avro_fixed_datum_t *fixed; + fixed = avro_datum_to_fixed(datum); + avro_schema_decref(fixed->schema); + if (fixed->free) { + fixed->free((void *)fixed->bytes, + fixed->size); + } + avro_freet(struct avro_fixed_datum_t, fixed); + } + break; + case AVRO_MAP:{ + struct avro_map_datum_t *map; + map = avro_datum_to_map(datum); + avro_schema_decref(map->schema); + st_foreach(map->map, HASH_FUNCTION_CAST char_datum_free_foreach, + 0); + st_free_table(map->map); + st_free_table(map->indices_by_key); + st_free_table(map->keys_by_index); + avro_freet(struct avro_map_datum_t, map); + } + break; + case AVRO_ARRAY:{ + struct avro_array_datum_t *array; + array = avro_datum_to_array(datum); + avro_schema_decref(array->schema); + st_foreach(array->els, HASH_FUNCTION_CAST array_free_foreach, 0); + st_free_table(array->els); + avro_freet(struct avro_array_datum_t, array); + } + break; + case AVRO_UNION:{ + struct avro_union_datum_t *unionp; + unionp = avro_datum_to_union(datum); + avro_schema_decref(unionp->schema); + avro_datum_decref(unionp->value); + avro_freet(struct avro_union_datum_t, unionp); + } + break; + case AVRO_LINK:{ + /* TODO */ + } + break; + } + } +} + +static int +datum_reset_foreach(int i, avro_datum_t datum, void *arg) +{ + AVRO_UNUSED(i); + int rval; + int *result = (int *) arg; + + rval = avro_datum_reset(datum); + if (rval == 0) { + return ST_CONTINUE; + } else { + *result = rval; + return ST_STOP; + } +} + +int +avro_datum_reset(avro_datum_t datum) +{ + check_param(EINVAL, is_avro_datum(datum), "datum"); + int rval; + + switch (avro_typeof(datum)) { + case AVRO_ARRAY: + { + struct avro_array_datum_t *array; + array = avro_datum_to_array(datum); + st_foreach(array->els, HASH_FUNCTION_CAST array_free_foreach, 0); + st_free_table(array->els); + + rval = avro_init_array(array); + if (rval != 0) { + avro_freet(struct avro_array_datum_t, array); + return rval; + } + return 0; + } + + case AVRO_MAP: + { + struct avro_map_datum_t *map; + map = avro_datum_to_map(datum); + st_foreach(map->map, HASH_FUNCTION_CAST char_datum_free_foreach, 0); + st_free_table(map->map); + st_free_table(map->indices_by_key); + st_free_table(map->keys_by_index); + + rval = avro_init_map(map); + if (rval != 0) { + avro_freet(struct avro_map_datum_t, map); + return rval; + } + return 0; + } + + case AVRO_RECORD: + { + struct avro_record_datum_t *record; + record = avro_datum_to_record(datum); + rval = 0; + st_foreach(record->fields_byname, + HASH_FUNCTION_CAST datum_reset_foreach, (st_data_t) &rval); + return rval; + } + + case AVRO_UNION: + { + struct avro_union_datum_t *unionp; + unionp = avro_datum_to_union(datum); + return (unionp->value == NULL)? 0: + avro_datum_reset(unionp->value); + } + + default: + return 0; + } +} + +avro_datum_t avro_datum_incref(avro_datum_t datum) +{ + if (datum) { + avro_refcount_inc(&datum->refcount); + } + return datum; +} + +void avro_datum_decref(avro_datum_t datum) +{ + if (datum && avro_refcount_dec(&datum->refcount)) { + avro_datum_free(datum); + } +} + +void avro_datum_print(avro_datum_t value, FILE * fp) +{ + AVRO_UNUSED(value); + AVRO_UNUSED(fp); +} diff --git a/fluent-bit/lib/avro/src/datum.h b/fluent-bit/lib/avro/src/datum.h new file mode 100644 index 00000000..c09542b3 --- /dev/null +++ b/fluent-bit/lib/avro/src/datum.h @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#ifndef AVRO_DATUM_H +#define AVRO_DATUM_H +#include <avro/platform.h> +#include "avro/basics.h" +#include "avro/data.h" +#include "avro/legacy.h" +#include "avro/schema.h" +#include "avro_private.h" +#include "st.h" + +struct avro_string_datum_t { + struct avro_obj_t obj; + char *s; + int64_t size; + avro_free_func_t free; +}; + +struct avro_bytes_datum_t { + struct avro_obj_t obj; + char *bytes; + int64_t size; + avro_free_func_t free; +}; + +struct avro_int32_datum_t { + struct avro_obj_t obj; + int32_t i32; +}; + +struct avro_int64_datum_t { + struct avro_obj_t obj; + int64_t i64; +}; + +struct avro_float_datum_t { + struct avro_obj_t obj; + float f; +}; + +struct avro_double_datum_t { + struct avro_obj_t obj; + double d; +}; + +struct avro_boolean_datum_t { + struct avro_obj_t obj; + int8_t i; +}; + +struct avro_fixed_datum_t { + struct avro_obj_t obj; + avro_schema_t schema; + char *bytes; + int64_t size; + avro_free_func_t free; +}; + +struct avro_map_datum_t { + struct avro_obj_t obj; + avro_schema_t schema; + st_table *map; + st_table *indices_by_key; + st_table *keys_by_index; +}; + +struct avro_record_datum_t { + struct avro_obj_t obj; + avro_schema_t schema; + st_table *field_order; + st_table *fields_byname; +}; + +struct avro_enum_datum_t { + struct avro_obj_t obj; + avro_schema_t schema; + int value; +}; + +struct avro_array_datum_t { + struct avro_obj_t obj; + avro_schema_t schema; + st_table *els; +}; + +struct avro_union_datum_t { + struct avro_obj_t obj; + avro_schema_t schema; + int64_t discriminant; + avro_datum_t value; +}; + +#define avro_datum_to_string(datum_) (container_of(datum_, struct avro_string_datum_t, obj)) +#define avro_datum_to_bytes(datum_) (container_of(datum_, struct avro_bytes_datum_t, obj)) +#define avro_datum_to_int32(datum_) (container_of(datum_, struct avro_int32_datum_t, obj)) +#define avro_datum_to_int64(datum_) (container_of(datum_, struct avro_int64_datum_t, obj)) +#define avro_datum_to_float(datum_) (container_of(datum_, struct avro_float_datum_t, obj)) +#define avro_datum_to_double(datum_) (container_of(datum_, struct avro_double_datum_t, obj)) +#define avro_datum_to_boolean(datum_) (container_of(datum_, struct avro_boolean_datum_t, obj)) +#define avro_datum_to_fixed(datum_) (container_of(datum_, struct avro_fixed_datum_t, obj)) +#define avro_datum_to_map(datum_) (container_of(datum_, struct avro_map_datum_t, obj)) +#define avro_datum_to_record(datum_) (container_of(datum_, struct avro_record_datum_t, obj)) +#define avro_datum_to_enum(datum_) (container_of(datum_, struct avro_enum_datum_t, obj)) +#define avro_datum_to_array(datum_) (container_of(datum_, struct avro_array_datum_t, obj)) +#define avro_datum_to_union(datum_) (container_of(datum_, struct avro_union_datum_t, obj)) + +#endif diff --git a/fluent-bit/lib/avro/src/datum_equal.c b/fluent-bit/lib/avro/src/datum_equal.c new file mode 100644 index 00000000..2ef750f9 --- /dev/null +++ b/fluent-bit/lib/avro/src/datum_equal.c @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include "avro_private.h" +#include <string.h> +#include "datum.h" + +static int +array_equal(struct avro_array_datum_t *a, struct avro_array_datum_t *b) +{ + if (!avro_schema_equal(a->schema, b->schema)) { + return 0; + } + + long i; + + if (a->els->num_entries != b->els->num_entries) { + return 0; + } + for (i = 0; i < a->els->num_entries; i++) { + union { + st_data_t data; + avro_datum_t datum; + } ael, bel; + st_lookup(a->els, i, &ael.data); + st_lookup(b->els, i, &bel.data); + if (!avro_datum_equal(ael.datum, bel.datum)) { + return 0; + } + } + return 1; +} + +struct st_equal_args { + int rval; + st_table *st; +}; + +static int +st_equal_foreach(char *key, avro_datum_t datum, struct st_equal_args *args) +{ + union { + avro_datum_t datum_other; + st_data_t data; + } val; + if (!st_lookup(args->st, (st_data_t) key, &(val.data))) { + args->rval = 0; + return ST_STOP; + } + if (!avro_datum_equal(datum, val.datum_other)) { + args->rval = 0; + return ST_STOP; + } + return ST_CONTINUE; +} + +static int map_equal(struct avro_map_datum_t *a, struct avro_map_datum_t *b) +{ + if (!avro_schema_equal(a->schema, b->schema)) { + return 0; + } + + struct st_equal_args args = { 1, b->map }; + if (a->map->num_entries != b->map->num_entries) { + return 0; + } + st_foreach(a->map, HASH_FUNCTION_CAST st_equal_foreach, (st_data_t) & args); + return args.rval; +} + +static int record_equal(struct avro_record_datum_t *a, + struct avro_record_datum_t *b) +{ + if (!avro_schema_equal(a->schema, b->schema)) { + return 0; + } + + struct st_equal_args args = { 1, b->fields_byname }; + if (a->fields_byname->num_entries != b->fields_byname->num_entries) { + return 0; + } + st_foreach(a->fields_byname, HASH_FUNCTION_CAST st_equal_foreach, (st_data_t) & args); + return args.rval; +} + +static int enum_equal(struct avro_enum_datum_t *a, struct avro_enum_datum_t *b) +{ + return avro_schema_equal(a->schema, b->schema) && a->value == b->value; +} + +static int fixed_equal(struct avro_fixed_datum_t *a, + struct avro_fixed_datum_t *b) +{ + if (!avro_schema_equal(a->schema, b->schema)) { + return 0; + } + + return a->size == b->size && memcmp(a->bytes, b->bytes, a->size) == 0; +} + +static int union_equal(struct avro_union_datum_t *a, + struct avro_union_datum_t *b) +{ + if (!avro_schema_equal(a->schema, b->schema)) { + return 0; + } + + return a->discriminant == b->discriminant && avro_datum_equal(a->value, b->value); +} + +int avro_datum_equal(const avro_datum_t a, const avro_datum_t b) +{ + if (!(is_avro_datum(a) && is_avro_datum(b))) { + return 0; + } + if (avro_typeof(a) != avro_typeof(b)) { + return 0; + } + switch (avro_typeof(a)) { + case AVRO_STRING: + return strcmp(avro_datum_to_string(a)->s, + avro_datum_to_string(b)->s) == 0; + case AVRO_BYTES: + return (avro_datum_to_bytes(a)->size == + avro_datum_to_bytes(b)->size) + && memcmp(avro_datum_to_bytes(a)->bytes, + avro_datum_to_bytes(b)->bytes, + avro_datum_to_bytes(a)->size) == 0; + case AVRO_INT32: + return avro_datum_to_int32(a)->i32 == + avro_datum_to_int32(b)->i32; + case AVRO_INT64: + return avro_datum_to_int64(a)->i64 == + avro_datum_to_int64(b)->i64; + case AVRO_FLOAT: + return avro_datum_to_float(a)->f == avro_datum_to_float(b)->f; + case AVRO_DOUBLE: + return avro_datum_to_double(a)->d == avro_datum_to_double(b)->d; + case AVRO_BOOLEAN: + return avro_datum_to_boolean(a)->i == + avro_datum_to_boolean(b)->i; + case AVRO_NULL: + return 1; + case AVRO_ARRAY: + return array_equal(avro_datum_to_array(a), + avro_datum_to_array(b)); + case AVRO_MAP: + return map_equal(avro_datum_to_map(a), avro_datum_to_map(b)); + + case AVRO_RECORD: + return record_equal(avro_datum_to_record(a), + avro_datum_to_record(b)); + + case AVRO_ENUM: + return enum_equal(avro_datum_to_enum(a), avro_datum_to_enum(b)); + + case AVRO_FIXED: + return fixed_equal(avro_datum_to_fixed(a), + avro_datum_to_fixed(b)); + + case AVRO_UNION: + return union_equal(avro_datum_to_union(a), + avro_datum_to_union(b)); + + case AVRO_LINK: + /* + * TODO + */ + return 0; + } + return 0; +} diff --git a/fluent-bit/lib/avro/src/datum_read.c b/fluent-bit/lib/avro/src/datum_read.c new file mode 100644 index 00000000..97bdd54f --- /dev/null +++ b/fluent-bit/lib/avro/src/datum_read.c @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <stdlib.h> +#include <errno.h> + +#include "avro/errors.h" +#include "avro/io.h" +#include "avro/legacy.h" +#include "avro/resolver.h" +#include "avro/schema.h" +#include "avro/value.h" +#include "avro_private.h" + +int +avro_schema_match(avro_schema_t wschema, avro_schema_t rschema) +{ + check_param(0, is_avro_schema(wschema), "writer schema"); + check_param(0, is_avro_schema(rschema), "reader schema"); + + avro_value_iface_t *resolver = + avro_resolved_writer_new(wschema, rschema); + if (resolver != NULL) { + avro_value_iface_decref(resolver); + return 1; + } + + return 0; +} + +int +avro_read_data(avro_reader_t reader, avro_schema_t writers_schema, + avro_schema_t readers_schema, avro_datum_t * datum) +{ + int rval; + + check_param(EINVAL, reader, "reader"); + check_param(EINVAL, is_avro_schema(writers_schema), "writer schema"); + check_param(EINVAL, datum, "datum pointer"); + + if (!readers_schema) { + readers_schema = writers_schema; + } + + avro_datum_t result = avro_datum_from_schema(readers_schema); + if (!result) { + return EINVAL; + } + + avro_value_t value; + check(rval, avro_datum_as_value(&value, result)); + + avro_value_iface_t *resolver = + avro_resolved_writer_new(writers_schema, readers_schema); + if (!resolver) { + avro_value_decref(&value); + avro_datum_decref(result); + return EINVAL; + } + + avro_value_t resolved_value; + rval = avro_resolved_writer_new_value(resolver, &resolved_value); + if (rval) { + avro_value_iface_decref(resolver); + avro_value_decref(&value); + avro_datum_decref(result); + return rval; + } + + avro_resolved_writer_set_dest(&resolved_value, &value); + rval = avro_value_read(reader, &resolved_value); + if (rval) { + avro_value_decref(&resolved_value); + avro_value_iface_decref(resolver); + avro_value_decref(&value); + avro_datum_decref(result); + return rval; + } + + avro_value_decref(&resolved_value); + avro_value_iface_decref(resolver); + avro_value_decref(&value); + *datum = result; + return 0; +} diff --git a/fluent-bit/lib/avro/src/datum_size.c b/fluent-bit/lib/avro/src/datum_size.c new file mode 100644 index 00000000..770cb655 --- /dev/null +++ b/fluent-bit/lib/avro/src/datum_size.c @@ -0,0 +1,292 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include "avro_private.h" +#include "avro/errors.h" +#include <errno.h> +#include <assert.h> +#include <string.h> +#include "schema.h" +#include "datum.h" +#include "encoding.h" + +#define size_check(rval, call) { rval = call; if(rval) return rval; } +#define size_accum(rval, size, call) { rval = call; if (rval < 0) return rval; else size += rval; } + +static int64_t size_datum(avro_writer_t writer, const avro_encoding_t * enc, + avro_schema_t writers_schema, avro_datum_t datum); + +static int64_t +size_record(avro_writer_t writer, const avro_encoding_t * enc, + struct avro_record_schema_t *schema, avro_datum_t datum) +{ + int rval; + long i; + int64_t size; + avro_datum_t field_datum; + + size = 0; + if (schema) { + for (i = 0; i < schema->fields->num_entries; i++) { + union { + st_data_t data; + struct avro_record_field_t *field; + } val; + st_lookup(schema->fields, i, &val.data); + size_check(rval, + avro_record_get(datum, val.field->name, + &field_datum)); + size_accum(rval, size, + size_datum(writer, enc, val.field->type, + field_datum)); + } + } else { + /* No schema. Just write the record datum */ + struct avro_record_datum_t *record = + avro_datum_to_record(datum); + for (i = 0; i < record->field_order->num_entries; i++) { + union { + st_data_t data; + char *name; + } val; + st_lookup(record->field_order, i, &val.data); + size_check(rval, + avro_record_get(datum, val.name, + &field_datum)); + size_accum(rval, size, + size_datum(writer, enc, NULL, field_datum)); + } + } + return size; +} + +static int64_t +size_enum(avro_writer_t writer, const avro_encoding_t * enc, + struct avro_enum_schema_t *enump, struct avro_enum_datum_t *datum) +{ + AVRO_UNUSED(enump); + + return enc->size_long(writer, datum->value); +} + +struct size_map_args { + int rval; + int64_t size; + avro_writer_t writer; + const avro_encoding_t *enc; + avro_schema_t values_schema; +}; + +static int +size_map_foreach(char *key, avro_datum_t datum, struct size_map_args *args) +{ + int rval = args->enc->size_string(args->writer, key); + if (rval < 0) { + args->rval = rval; + return ST_STOP; + } else { + args->size += rval; + } + rval = size_datum(args->writer, args->enc, args->values_schema, datum); + if (rval < 0) { + args->rval = rval; + return ST_STOP; + } else { + args->size += rval; + } + return ST_CONTINUE; +} + +static int64_t +size_map(avro_writer_t writer, const avro_encoding_t * enc, + struct avro_map_schema_t *writers_schema, + struct avro_map_datum_t *datum) +{ + int rval; + int64_t size; + struct size_map_args args = { 0, 0, writer, enc, + writers_schema ? writers_schema->values : NULL + }; + + size = 0; + if (datum->map->num_entries) { + size_accum(rval, size, + enc->size_long(writer, datum->map->num_entries)); + st_foreach(datum->map, HASH_FUNCTION_CAST size_map_foreach, (st_data_t) & args); + size += args.size; + } + if (!args.rval) { + size_accum(rval, size, enc->size_long(writer, 0)); + } + return size; +} + +static int64_t +size_array(avro_writer_t writer, const avro_encoding_t * enc, + struct avro_array_schema_t *schema, struct avro_array_datum_t *array) +{ + int rval; + long i; + int64_t size; + + size = 0; + if (array->els->num_entries) { + size_accum(rval, size, + enc->size_long(writer, array->els->num_entries)); + for (i = 0; i < array->els->num_entries; i++) { + union { + st_data_t data; + avro_datum_t datum; + } val; + st_lookup(array->els, i, &val.data); + size_accum(rval, size, + size_datum(writer, enc, + schema ? schema->items : NULL, + val.datum)); + } + } + size_accum(rval, size, enc->size_long(writer, 0)); + return size; +} + +static int64_t +size_union(avro_writer_t writer, const avro_encoding_t * enc, + struct avro_union_schema_t *schema, + struct avro_union_datum_t *unionp) +{ + int rval; + int64_t size; + avro_schema_t write_schema = NULL; + + size = 0; + size_accum(rval, size, enc->size_long(writer, unionp->discriminant)); + if (schema) { + write_schema = + avro_schema_union_branch(&schema->obj, unionp->discriminant); + if (!write_schema) { + return -EINVAL; + } + } + size_accum(rval, size, + size_datum(writer, enc, write_schema, unionp->value)); + return size; +} + +static int64_t size_datum(avro_writer_t writer, const avro_encoding_t * enc, + avro_schema_t writers_schema, avro_datum_t datum) +{ + if (is_avro_schema(writers_schema) && is_avro_link(writers_schema)) { + return size_datum(writer, enc, + (avro_schema_to_link(writers_schema))->to, + datum); + } + + switch (avro_typeof(datum)) { + case AVRO_NULL: + return enc->size_null(writer); + + case AVRO_BOOLEAN: + return enc->size_boolean(writer, + avro_datum_to_boolean(datum)->i); + + case AVRO_STRING: + return enc->size_string(writer, avro_datum_to_string(datum)->s); + + case AVRO_BYTES: + return enc->size_bytes(writer, + avro_datum_to_bytes(datum)->bytes, + avro_datum_to_bytes(datum)->size); + + case AVRO_INT32: + case AVRO_INT64:{ + int64_t val = avro_typeof(datum) == AVRO_INT32 ? + avro_datum_to_int32(datum)->i32 : + avro_datum_to_int64(datum)->i64; + if (is_avro_schema(writers_schema)) { + /* handle promotion */ + if (is_avro_float(writers_schema)) { + return enc->size_float(writer, + (float)val); + } else if (is_avro_double(writers_schema)) { + return enc->size_double(writer, + (double)val); + } + } + return enc->size_long(writer, val); + } + + case AVRO_FLOAT:{ + float val = avro_datum_to_float(datum)->f; + if (is_avro_schema(writers_schema) + && is_avro_double(writers_schema)) { + /* handle promotion */ + return enc->size_double(writer, (double)val); + } + return enc->size_float(writer, val); + } + + case AVRO_DOUBLE: + return enc->size_double(writer, avro_datum_to_double(datum)->d); + + case AVRO_RECORD: + return size_record(writer, enc, + avro_schema_to_record(writers_schema), + datum); + + case AVRO_ENUM: + return size_enum(writer, enc, + avro_schema_to_enum(writers_schema), + avro_datum_to_enum(datum)); + + case AVRO_FIXED: + return avro_datum_to_fixed(datum)->size; + + case AVRO_MAP: + return size_map(writer, enc, + avro_schema_to_map(writers_schema), + avro_datum_to_map(datum)); + + case AVRO_ARRAY: + return size_array(writer, enc, + avro_schema_to_array(writers_schema), + avro_datum_to_array(datum)); + + case AVRO_UNION: + return size_union(writer, enc, + avro_schema_to_union(writers_schema), + avro_datum_to_union(datum)); + + case AVRO_LINK: + break; + } + + return 0; +} + +int64_t avro_size_data(avro_writer_t writer, avro_schema_t writers_schema, + avro_datum_t datum) +{ + check_param(-EINVAL, writer, "writer"); + check_param(-EINVAL, is_avro_datum(datum), "datum"); + /* Only validate datum if a writer's schema is provided */ + if (is_avro_schema(writers_schema) + && !avro_schema_datum_validate(writers_schema, datum)) { + avro_set_error("Datum doesn't validate against schema"); + return -EINVAL; + } + return size_datum(writer, &avro_binary_encoding, writers_schema, datum); +} diff --git a/fluent-bit/lib/avro/src/datum_skip.c b/fluent-bit/lib/avro/src/datum_skip.c new file mode 100644 index 00000000..aa51d793 --- /dev/null +++ b/fluent-bit/lib/avro/src/datum_skip.c @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include "avro_private.h" +#include "avro/errors.h" +#include <stdlib.h> +#include <errno.h> +#include <string.h> +#include "encoding.h" +#include "schema.h" + +static int skip_array(avro_reader_t reader, const avro_encoding_t * enc, + struct avro_array_schema_t *writers_schema) +{ + int rval; + int64_t i; + int64_t block_count; + int64_t block_size; + + check_prefix(rval, enc->read_long(reader, &block_count), + "Cannot read array block count: "); + + while (block_count != 0) { + if (block_count < 0) { + block_count = block_count * -1; + check_prefix(rval, enc->read_long(reader, &block_size), + "Cannot read array block size: "); + } + + for (i = 0; i < block_count; i++) { + check_prefix(rval, avro_skip_data(reader, writers_schema->items), + "Cannot skip array element: "); + } + + check_prefix(rval, enc->read_long(reader, &block_count), + "Cannot read array block count: "); + } + return 0; +} + +static int skip_map(avro_reader_t reader, const avro_encoding_t * enc, + struct avro_map_schema_t *writers_schema) +{ + int rval; + int64_t i, block_count; + + check_prefix(rval, enc->read_long(reader, &block_count), + "Cannot read map block count: "); + while (block_count != 0) { + int64_t block_size; + if (block_count < 0) { + block_count = block_count * -1; + check_prefix(rval, enc->read_long(reader, &block_size), + "Cannot read map block size: "); + } + for (i = 0; i < block_count; i++) { + check_prefix(rval, enc->skip_string(reader), + "Cannot skip map key: "); + check_prefix(rval, + avro_skip_data(reader, + avro_schema_to_map(writers_schema)-> + values), + "Cannot skip map value: "); + } + check_prefix(rval, enc->read_long(reader, &block_count), + "Cannot read map block count: "); + } + return 0; +} + +static int skip_union(avro_reader_t reader, const avro_encoding_t * enc, + struct avro_union_schema_t *writers_schema) +{ + int rval; + int64_t index; + avro_schema_t branch_schema; + + check_prefix(rval, enc->read_long(reader, &index), + "Cannot read union discriminant: "); + branch_schema = avro_schema_union_branch(&writers_schema->obj, index); + if (!branch_schema) { + return EILSEQ; + } + return avro_skip_data(reader, branch_schema); +} + +static int skip_record(avro_reader_t reader, const avro_encoding_t * enc, + struct avro_record_schema_t *writers_schema) +{ + int rval; + long i; + + AVRO_UNUSED(enc); + + for (i = 0; i < writers_schema->fields->num_entries; i++) { + avro_schema_t field_schema; + + field_schema = avro_schema_record_field_get_by_index + (&writers_schema->obj, i); + check_prefix(rval, avro_skip_data(reader, field_schema), + "Cannot skip record field: "); + } + return 0; +} + +int avro_skip_data(avro_reader_t reader, avro_schema_t writers_schema) +{ + check_param(EINVAL, reader, "reader"); + check_param(EINVAL, is_avro_schema(writers_schema), "writer schema"); + + int rval = EINVAL; + const avro_encoding_t *enc = &avro_binary_encoding; + + switch (avro_typeof(writers_schema)) { + case AVRO_NULL: + rval = enc->skip_null(reader); + break; + + case AVRO_BOOLEAN: + rval = enc->skip_boolean(reader); + break; + + case AVRO_STRING: + rval = enc->skip_string(reader); + break; + + case AVRO_INT32: + rval = enc->skip_int(reader); + break; + + case AVRO_INT64: + rval = enc->skip_long(reader); + break; + + case AVRO_FLOAT: + rval = enc->skip_float(reader); + break; + + case AVRO_DOUBLE: + rval = enc->skip_double(reader); + break; + + case AVRO_BYTES: + rval = enc->skip_bytes(reader); + break; + + case AVRO_FIXED: + rval = + avro_skip(reader, + avro_schema_to_fixed(writers_schema)->size); + break; + + case AVRO_ENUM: + rval = enc->skip_long(reader); + break; + + case AVRO_ARRAY: + rval = + skip_array(reader, enc, + avro_schema_to_array(writers_schema)); + break; + + case AVRO_MAP: + rval = + skip_map(reader, enc, avro_schema_to_map(writers_schema)); + break; + + case AVRO_UNION: + rval = + skip_union(reader, enc, + avro_schema_to_union(writers_schema)); + break; + + case AVRO_RECORD: + rval = + skip_record(reader, enc, + avro_schema_to_record(writers_schema)); + break; + + case AVRO_LINK: + rval = + avro_skip_data(reader, + (avro_schema_to_link(writers_schema))->to); + break; + } + + return rval; +} diff --git a/fluent-bit/lib/avro/src/datum_validate.c b/fluent-bit/lib/avro/src/datum_validate.c new file mode 100644 index 00000000..d15ebddd --- /dev/null +++ b/fluent-bit/lib/avro/src/datum_validate.c @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include "avro_private.h" +#include "avro/errors.h" +#include <limits.h> +#include <errno.h> +#include <string.h> +#include "schema.h" +#include "datum.h" +#include "st.h" + +struct validate_st { + avro_schema_t expected_schema; + int rval; +}; + +static int +schema_map_validate_foreach(char *key, avro_datum_t datum, + struct validate_st *vst) +{ + AVRO_UNUSED(key); + + if (!avro_schema_datum_validate(vst->expected_schema, datum)) { + vst->rval = 0; + return ST_STOP; + } + return ST_CONTINUE; +} + +int +avro_schema_datum_validate(avro_schema_t expected_schema, avro_datum_t datum) +{ + check_param(EINVAL, expected_schema, "expected schema"); + check_param(EINVAL, is_avro_datum(datum), "datum"); + + int rval; + long i; + + switch (avro_typeof(expected_schema)) { + case AVRO_NULL: + return is_avro_null(datum); + + case AVRO_BOOLEAN: + return is_avro_boolean(datum); + + case AVRO_STRING: + return is_avro_string(datum); + + case AVRO_BYTES: + return is_avro_bytes(datum); + + case AVRO_INT32: + return is_avro_int32(datum) + || (is_avro_int64(datum) + && (INT_MIN <= avro_datum_to_int64(datum)->i64 + && avro_datum_to_int64(datum)->i64 <= INT_MAX)); + + case AVRO_INT64: + return is_avro_int32(datum) || is_avro_int64(datum); + + case AVRO_FLOAT: + return is_avro_int32(datum) || is_avro_int64(datum) + || is_avro_float(datum); + + case AVRO_DOUBLE: + return is_avro_int32(datum) || is_avro_int64(datum) + || is_avro_float(datum) || is_avro_double(datum); + + case AVRO_FIXED: + return (is_avro_fixed(datum) + && (avro_schema_to_fixed(expected_schema)->size == + avro_datum_to_fixed(datum)->size)); + + case AVRO_ENUM: + if (is_avro_enum(datum)) { + long value = avro_datum_to_enum(datum)->value; + long max_value = + avro_schema_to_enum(expected_schema)->symbols-> + num_entries; + return 0 <= value && value <= max_value; + } + return 0; + + case AVRO_ARRAY: + if (is_avro_array(datum)) { + struct avro_array_datum_t *array = + avro_datum_to_array(datum); + + for (i = 0; i < array->els->num_entries; i++) { + union { + st_data_t data; + avro_datum_t datum; + } val; + st_lookup(array->els, i, &val.data); + if (!avro_schema_datum_validate + ((avro_schema_to_array + (expected_schema))->items, val.datum)) { + return 0; + } + } + return 1; + } + return 0; + + case AVRO_MAP: + if (is_avro_map(datum)) { + struct validate_st vst = + { avro_schema_to_map(expected_schema)->values, 1 + }; + st_foreach(avro_datum_to_map(datum)->map, + HASH_FUNCTION_CAST schema_map_validate_foreach, + (st_data_t) & vst); + return vst.rval; + } + break; + + case AVRO_UNION: + if (is_avro_union(datum)) { + struct avro_union_schema_t *union_schema = + avro_schema_to_union(expected_schema); + struct avro_union_datum_t *union_datum = + avro_datum_to_union(datum); + union { + st_data_t data; + avro_schema_t schema; + } val; + + if (!st_lookup + (union_schema->branches, union_datum->discriminant, + &val.data)) { + return 0; + } + return avro_schema_datum_validate(val.schema, + union_datum->value); + } + break; + + case AVRO_RECORD: + if (is_avro_record(datum)) { + struct avro_record_schema_t *record_schema = + avro_schema_to_record(expected_schema); + for (i = 0; i < record_schema->fields->num_entries; i++) { + avro_datum_t field_datum; + union { + st_data_t data; + struct avro_record_field_t *field; + } val; + st_lookup(record_schema->fields, i, &val.data); + + rval = + avro_record_get(datum, val.field->name, + &field_datum); + if (rval) { + /* + * TODO: check for default values + */ + return rval; + } + if (!avro_schema_datum_validate + (val.field->type, field_datum)) { + return 0; + } + } + return 1; + } + break; + + case AVRO_LINK: + { + return + avro_schema_datum_validate((avro_schema_to_link + (expected_schema))->to, + datum); + } + break; + } + return 0; +} diff --git a/fluent-bit/lib/avro/src/datum_value.c b/fluent-bit/lib/avro/src/datum_value.c new file mode 100644 index 00000000..a4fa55a0 --- /dev/null +++ b/fluent-bit/lib/avro/src/datum_value.c @@ -0,0 +1,784 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <avro/platform.h> +#include <stdlib.h> +#include <string.h> + +#include "avro/allocation.h" +#include "avro/basics.h" +#include "avro/errors.h" +#include "avro/legacy.h" +#include "avro/refcount.h" +#include "avro/schema.h" +#include "avro/value.h" +#include "avro_private.h" + +extern avro_value_iface_t AVRO_DATUM_VALUE_CLASS; + +avro_value_iface_t * +avro_datum_class(void) +{ + return &AVRO_DATUM_VALUE_CLASS; +} + +int +avro_datum_as_value(avro_value_t *value, avro_datum_t src) +{ + value->iface = &AVRO_DATUM_VALUE_CLASS; + value->self = avro_datum_incref(src); + return 0; +} + +static int +avro_datum_as_child_value(avro_value_t *value, avro_datum_t src) +{ + value->iface = &AVRO_DATUM_VALUE_CLASS; + value->self = src; + return 0; +} + +static void +avro_datum_value_incref(avro_value_t *value) +{ + avro_datum_t self = (avro_datum_t) value->self; + avro_datum_incref(self); +} + +static void +avro_datum_value_decref(avro_value_t *value) +{ + avro_datum_t self = (avro_datum_t) value->self; + avro_datum_decref(self); +} + +static int +avro_datum_value_reset(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + avro_datum_t self = (avro_datum_t) vself; + check_param(EINVAL, self, "datum instance"); + return avro_datum_reset(self); +} + +static avro_type_t +avro_datum_value_get_type(const avro_value_iface_t *iface, const void *vself) +{ + AVRO_UNUSED(iface); + const avro_datum_t self = (const avro_datum_t) vself; +#ifdef _WIN32 +#pragma message("#warning: Bug: EINVAL is not of type avro_type_t.") +#else +#warning "Bug: EINVAL is not of type avro_type_t." +#endif + /* We shouldn't use EINVAL as the return value to + * check_param(), because EINVAL (= 22) is not a valid enum + * avro_type_t. This is a structural issue -- we would need a + * different interface on all the get_type functions to fix + * this. For now, suppressing the error by casting EINVAL to + * (avro_type_t) so the code compiles under C++. + */ + check_param((avro_type_t) EINVAL, self, "datum instance"); + return avro_typeof(self); +} + +static avro_schema_t +avro_datum_value_get_schema(const avro_value_iface_t *iface, const void *vself) +{ + AVRO_UNUSED(iface); + const avro_datum_t self = (const avro_datum_t) vself; + check_param(NULL, self, "datum instance"); + return avro_datum_get_schema(self); +} + + +static int +avro_datum_value_get_boolean(const avro_value_iface_t *iface, + const void *vself, int *out) +{ + AVRO_UNUSED(iface); + const avro_datum_t self = (const avro_datum_t) vself; + check_param(EINVAL, self, "datum instance"); + + int rval; + int8_t value; + check(rval, avro_boolean_get(self, &value)); + *out = value; + return 0; +} + +static int +avro_datum_value_get_bytes(const avro_value_iface_t *iface, + const void *vself, const void **buf, size_t *size) +{ + AVRO_UNUSED(iface); + const avro_datum_t self = (const avro_datum_t) vself; + check_param(EINVAL, self, "datum instance"); + + int rval; + char *bytes; + int64_t sz; + check(rval, avro_bytes_get(self, &bytes, &sz)); + if (buf != NULL) { + *buf = (const void *) bytes; + } + if (size != NULL) { + *size = sz; + } + return 0; +} + +static int +avro_datum_value_grab_bytes(const avro_value_iface_t *iface, + const void *vself, avro_wrapped_buffer_t *dest) +{ + AVRO_UNUSED(iface); + const avro_datum_t self = (const avro_datum_t) vself; + check_param(EINVAL, self, "datum instance"); + + int rval; + char *bytes; + int64_t sz; + check(rval, avro_bytes_get(self, &bytes, &sz)); + + /* nothing clever, just make a copy */ + return avro_wrapped_buffer_new_copy(dest, bytes, sz); +} + +static int +avro_datum_value_get_double(const avro_value_iface_t *iface, + const void *vself, double *out) +{ + AVRO_UNUSED(iface); + const avro_datum_t self = (const avro_datum_t) vself; + check_param(EINVAL, self, "datum instance"); + + int rval; + double value; + check(rval, avro_double_get(self, &value)); + *out = value; + return 0; +} + +static int +avro_datum_value_get_float(const avro_value_iface_t *iface, + const void *vself, float *out) +{ + AVRO_UNUSED(iface); + const avro_datum_t self = (const avro_datum_t) vself; + check_param(EINVAL, self, "datum instance"); + + int rval; + float value; + check(rval, avro_float_get(self, &value)); + *out = value; + return 0; +} + +static int +avro_datum_value_get_int(const avro_value_iface_t *iface, + const void *vself, int32_t *out) +{ + AVRO_UNUSED(iface); + const avro_datum_t self = (const avro_datum_t) vself; + check_param(EINVAL, self, "datum instance"); + + int rval; + int32_t value; + check(rval, avro_int32_get(self, &value)); + *out = value; + return 0; +} + +static int +avro_datum_value_get_long(const avro_value_iface_t *iface, + const void *vself, int64_t *out) +{ + AVRO_UNUSED(iface); + const avro_datum_t self = (const avro_datum_t) vself; + check_param(EINVAL, self, "datum instance"); + + int rval; + int64_t value; + check(rval, avro_int64_get(self, &value)); + *out = value; + return 0; +} + +static int +avro_datum_value_get_null(const avro_value_iface_t *iface, + const void *vself) +{ + AVRO_UNUSED(iface); + const avro_datum_t self = (const avro_datum_t) vself; + check_param(EINVAL, is_avro_null(self), "datum instance"); + return 0; +} + +static int +avro_datum_value_get_string(const avro_value_iface_t *iface, + const void *vself, const char **str, size_t *size) +{ + AVRO_UNUSED(iface); + const avro_datum_t self = (const avro_datum_t) vself; + check_param(EINVAL, self, "datum instance"); + + int rval; + char *value; + check(rval, avro_string_get(self, &value)); + if (str != NULL) { + *str = (const char *) value; + } + if (size != NULL) { + *size = strlen(value)+1; + } + return 0; +} + +static int +avro_datum_value_grab_string(const avro_value_iface_t *iface, + const void *vself, avro_wrapped_buffer_t *dest) +{ + AVRO_UNUSED(iface); + const avro_datum_t self = (const avro_datum_t) vself; + check_param(EINVAL, self, "datum instance"); + + int rval; + char *str; + size_t sz; + check(rval, avro_string_get(self, &str)); + sz = strlen(str); + + /* nothing clever, just make a copy */ + /* sz doesn't contain NUL terminator */ + return avro_wrapped_buffer_new_copy(dest, str, sz+1); +} + +static int +avro_datum_value_get_enum(const avro_value_iface_t *iface, + const void *vself, int *out) +{ + AVRO_UNUSED(iface); + const avro_datum_t self = (const avro_datum_t) vself; + check_param(EINVAL, is_avro_enum(self), "datum instance"); + *out = avro_enum_get(self); + return 0; +} + +static int +avro_datum_value_get_fixed(const avro_value_iface_t *iface, + const void *vself, const void **buf, size_t *size) +{ + AVRO_UNUSED(iface); + const avro_datum_t self = (const avro_datum_t) vself; + check_param(EINVAL, self, "datum instance"); + + int rval; + char *bytes; + int64_t sz; + check(rval, avro_fixed_get(self, &bytes, &sz)); + if (buf != NULL) { + *buf = (const void *) bytes; + } + if (size != NULL) { + *size = sz; + } + return 0; +} + +static int +avro_datum_value_grab_fixed(const avro_value_iface_t *iface, + const void *vself, avro_wrapped_buffer_t *dest) +{ + AVRO_UNUSED(iface); + const avro_datum_t self = (const avro_datum_t) vself; + check_param(EINVAL, self, "datum instance"); + + int rval; + char *bytes; + int64_t sz; + check(rval, avro_fixed_get(self, &bytes, &sz)); + + /* nothing clever, just make a copy */ + return avro_wrapped_buffer_new_copy(dest, bytes, sz); +} + + +static int +avro_datum_value_set_boolean(const avro_value_iface_t *iface, + void *vself, int val) +{ + AVRO_UNUSED(iface); + avro_datum_t self = (avro_datum_t) vself; + check_param(EINVAL, self, "datum instance"); + return avro_boolean_set(self, val); +} + +static int +avro_datum_value_set_bytes(const avro_value_iface_t *iface, + void *vself, void *buf, size_t size) +{ + AVRO_UNUSED(iface); + avro_datum_t self = (avro_datum_t) vself; + check_param(EINVAL, self, "datum instance"); + return avro_bytes_set(self, (const char *) buf, size); +} + +static int +avro_datum_value_give_bytes(const avro_value_iface_t *iface, + void *vself, avro_wrapped_buffer_t *buf) +{ + /* + * We actually can't use avro_givebytes_set, since it can't + * handle the extra free_ud parameter. Ah well, this is + * deprecated, so go ahead and make a copy. + */ + + int rval = avro_datum_value_set_bytes + (iface, vself, (void *) buf->buf, buf->size); + avro_wrapped_buffer_free(buf); + return rval; +} + +static int +avro_datum_value_set_double(const avro_value_iface_t *iface, + void *vself, double val) +{ + AVRO_UNUSED(iface); + avro_datum_t self = (avro_datum_t) vself; + check_param(EINVAL, self, "datum instance"); + return avro_double_set(self, val); +} + +static int +avro_datum_value_set_float(const avro_value_iface_t *iface, + void *vself, float val) +{ + AVRO_UNUSED(iface); + avro_datum_t self = (avro_datum_t) vself; + check_param(EINVAL, self, "datum instance"); + return avro_float_set(self, val); +} + +static int +avro_datum_value_set_int(const avro_value_iface_t *iface, + void *vself, int32_t val) +{ + AVRO_UNUSED(iface); + avro_datum_t self = (avro_datum_t) vself; + check_param(EINVAL, self, "datum instance"); + return avro_int32_set(self, val); +} + +static int +avro_datum_value_set_long(const avro_value_iface_t *iface, + void *vself, int64_t val) +{ + AVRO_UNUSED(iface); + avro_datum_t self = (avro_datum_t) vself; + check_param(EINVAL, self, "datum instance"); + return avro_int64_set(self, val); +} + +static int +avro_datum_value_set_null(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + avro_datum_t self = (avro_datum_t) vself; + check_param(EINVAL, is_avro_null(self), "datum instance"); + return 0; +} + +static int +avro_datum_value_set_string(const avro_value_iface_t *iface, + void *vself, const char *str) +{ + AVRO_UNUSED(iface); + avro_datum_t self = (avro_datum_t) vself; + check_param(EINVAL, self, "datum instance"); + return avro_string_set(self, str); +} + +static int +avro_datum_value_set_string_len(const avro_value_iface_t *iface, + void *vself, const char *str, size_t size) +{ + AVRO_UNUSED(iface); + AVRO_UNUSED(size); + avro_datum_t self = (avro_datum_t) vself; + check_param(EINVAL, self, "datum instance"); + return avro_string_set(self, str); +} + +static int +avro_datum_value_give_string_len(const avro_value_iface_t *iface, + void *vself, avro_wrapped_buffer_t *buf) +{ + /* + * We actually can't use avro_givestring_set, since it can't + * handle the extra free_ud parameter. Ah well, this is + * deprecated, so go ahead and make a copy. + */ + + int rval = avro_datum_value_set_string_len + (iface, vself, (char *) buf->buf, buf->size-1); + avro_wrapped_buffer_free(buf); + return rval; +} + +static int +avro_datum_value_set_enum(const avro_value_iface_t *iface, + void *vself, int val) +{ + AVRO_UNUSED(iface); + avro_datum_t self = (avro_datum_t) vself; + check_param(EINVAL, self, "datum instance"); + return avro_enum_set(self, val); +} + +static int +avro_datum_value_set_fixed(const avro_value_iface_t *iface, + void *vself, void *buf, size_t size) +{ + AVRO_UNUSED(iface); + avro_datum_t self = (avro_datum_t) vself; + check_param(EINVAL, self, "datum instance"); + return avro_fixed_set(self, (const char *) buf, size); +} + +static int +avro_datum_value_give_fixed(const avro_value_iface_t *iface, + void *vself, avro_wrapped_buffer_t *buf) +{ + /* + * We actually can't use avro_givefixed_set, since it can't + * handle the extra free_ud parameter. Ah well, this is + * deprecated, so go ahead and make a copy. + */ + + int rval = avro_datum_value_set_fixed + (iface, vself, (void *) buf->buf, buf->size); + avro_wrapped_buffer_free(buf); + return rval; +} + + +static int +avro_datum_value_get_size(const avro_value_iface_t *iface, + const void *vself, size_t *size) +{ + AVRO_UNUSED(iface); + const avro_datum_t self = (const avro_datum_t) vself; + check_param(EINVAL, self, "datum instance"); + + if (is_avro_array(self)) { + *size = avro_array_size(self); + return 0; + } + + if (is_avro_map(self)) { + *size = avro_map_size(self); + return 0; + } + + if (is_avro_record(self)) { + avro_schema_t schema = avro_datum_get_schema(self); + *size = avro_schema_record_size(schema); + return 0; + } + + avro_set_error("Can only get size of array, map, or record, %d", avro_typeof(self)); + return EINVAL; +} + +static int +avro_datum_value_get_by_index(const avro_value_iface_t *iface, + const void *vself, size_t index, + avro_value_t *child, const char **name) +{ + AVRO_UNUSED(iface); + const avro_datum_t self = (const avro_datum_t) vself; + check_param(EINVAL, self, "datum instance"); + + int rval; + avro_datum_t child_datum; + + if (is_avro_array(self)) { + check(rval, avro_array_get(self, index, &child_datum)); + return avro_datum_as_child_value(child, child_datum); + } + + if (is_avro_map(self)) { + const char *real_key; + check(rval, avro_map_get_key(self, index, &real_key)); + if (name != NULL) { + *name = real_key; + } + check(rval, avro_map_get(self, real_key, &child_datum)); + return avro_datum_as_child_value(child, child_datum); + } + + if (is_avro_record(self)) { + avro_schema_t schema = avro_datum_get_schema(self); + const char *field_name = + avro_schema_record_field_name(schema, index); + if (field_name == NULL) { + return EINVAL; + } + if (name != NULL) { + *name = field_name; + } + check(rval, avro_record_get(self, field_name, &child_datum)); + return avro_datum_as_child_value(child, child_datum); + } + + avro_set_error("Can only get by index from array, map, or record"); + return EINVAL; +} + +static int +avro_datum_value_get_by_name(const avro_value_iface_t *iface, + const void *vself, const char *name, + avro_value_t *child, size_t *index) +{ + AVRO_UNUSED(iface); + const avro_datum_t self = (const avro_datum_t) vself; + check_param(EINVAL, self, "datum instance"); + + int rval; + avro_datum_t child_datum; + + if (is_avro_map(self)) { + if (index != NULL) { + int real_index; + check(rval, avro_map_get_index(self, name, &real_index)); + *index = real_index; + } + + check(rval, avro_map_get(self, name, &child_datum)); + return avro_datum_as_child_value(child, child_datum); + } + + if (is_avro_record(self)) { + if (index != NULL) { + avro_schema_t schema = avro_datum_get_schema(self); + *index = avro_schema_record_field_get_index(schema, name); + } + + check(rval, avro_record_get(self, name, &child_datum)); + return avro_datum_as_child_value(child, child_datum); + } + + avro_set_error("Can only get by name from map or record"); + return EINVAL; +} + +static int +avro_datum_value_get_discriminant(const avro_value_iface_t *iface, + const void *vself, int *out) +{ + AVRO_UNUSED(iface); + const avro_datum_t self = (const avro_datum_t) vself; + check_param(EINVAL, self, "datum instance"); + + if (!is_avro_union(self)) { + avro_set_error("Can only get discriminant of union"); + return EINVAL; + } + + *out = avro_union_discriminant(self); + return 0; +} + +static int +avro_datum_value_get_current_branch(const avro_value_iface_t *iface, + const void *vself, avro_value_t *branch) +{ + AVRO_UNUSED(iface); + const avro_datum_t self = (const avro_datum_t) vself; + check_param(EINVAL, self, "datum instance"); + + if (!is_avro_union(self)) { + avro_set_error("Can only get current branch of union"); + return EINVAL; + } + + avro_datum_t child_datum = avro_union_current_branch(self); + return avro_datum_as_child_value(branch, child_datum); +} + + +static int +avro_datum_value_append(const avro_value_iface_t *iface, + void *vself, avro_value_t *child_out, size_t *new_index) +{ + AVRO_UNUSED(iface); + avro_datum_t self = (avro_datum_t) vself; + check_param(EINVAL, self, "datum instance"); + + if (!is_avro_array(self)) { + avro_set_error("Can only append to array"); + return EINVAL; + } + + int rval; + + avro_schema_t array_schema = avro_datum_get_schema(self); + avro_schema_t child_schema = avro_schema_array_items(array_schema); + avro_datum_t child_datum = avro_datum_from_schema(child_schema); + if (child_datum == NULL) { + return ENOMEM; + } + + rval = avro_array_append_datum(self, child_datum); + avro_datum_decref(child_datum); + if (rval != 0) { + return rval; + } + + if (new_index != NULL) { + *new_index = avro_array_size(self) - 1; + } + return avro_datum_as_child_value(child_out, child_datum); +} + +static int +avro_datum_value_add(const avro_value_iface_t *iface, + void *vself, const char *key, + avro_value_t *child, size_t *index, int *is_new) +{ + AVRO_UNUSED(iface); + avro_datum_t self = (avro_datum_t) vself; + check_param(EINVAL, self, "datum instance"); + + if (!is_avro_map(self)) { + avro_set_error("Can only add to map"); + return EINVAL; + } + + int rval; + avro_datum_t child_datum; + + if (avro_map_get(self, key, &child_datum) == 0) { + /* key already exists */ + if (is_new != NULL) { + *is_new = 0; + } + if (index != NULL) { + int real_index; + avro_map_get_index(self, key, &real_index); + *index = real_index; + } + return avro_datum_as_child_value(child, child_datum); + } + + /* key is new */ + avro_schema_t map_schema = avro_datum_get_schema(self); + avro_schema_t child_schema = avro_schema_map_values(map_schema); + child_datum = avro_datum_from_schema(child_schema); + if (child_datum == NULL) { + return ENOMEM; + } + + rval = avro_map_set(self, key, child_datum); + avro_datum_decref(child_datum); + if (rval != 0) { + return rval; + } + + if (is_new != NULL) { + *is_new = 1; + } + if (index != NULL) { + *index = avro_map_size(self) - 1; + } + + return avro_datum_as_child_value(child, child_datum); +} + +static int +avro_datum_value_set_branch(const avro_value_iface_t *iface, + void *vself, int discriminant, + avro_value_t *branch) +{ + AVRO_UNUSED(iface); + const avro_datum_t self = (const avro_datum_t) vself; + check_param(EINVAL, self, "datum instance"); + + if (!is_avro_union(self)) { + avro_set_error("Can only set branch of union"); + return EINVAL; + } + + int rval; + avro_datum_t child_datum; + check(rval, avro_union_set_discriminant(self, discriminant, &child_datum)); + return avro_datum_as_child_value(branch, child_datum); +} + + +avro_value_iface_t AVRO_DATUM_VALUE_CLASS = +{ + /* "class" methods */ + NULL, /* incref */ + NULL, /* decref */ + /* general "instance" methods */ + avro_datum_value_incref, + avro_datum_value_decref, + avro_datum_value_reset, + avro_datum_value_get_type, + avro_datum_value_get_schema, + /* primitive getters */ + avro_datum_value_get_boolean, + avro_datum_value_get_bytes, + avro_datum_value_grab_bytes, + avro_datum_value_get_double, + avro_datum_value_get_float, + avro_datum_value_get_int, + avro_datum_value_get_long, + avro_datum_value_get_null, + avro_datum_value_get_string, + avro_datum_value_grab_string, + avro_datum_value_get_enum, + avro_datum_value_get_fixed, + avro_datum_value_grab_fixed, + /* primitive setters */ + avro_datum_value_set_boolean, + avro_datum_value_set_bytes, + avro_datum_value_give_bytes, + avro_datum_value_set_double, + avro_datum_value_set_float, + avro_datum_value_set_int, + avro_datum_value_set_long, + avro_datum_value_set_null, + avro_datum_value_set_string, + avro_datum_value_set_string_len, + avro_datum_value_give_string_len, + avro_datum_value_set_enum, + avro_datum_value_set_fixed, + avro_datum_value_give_fixed, + /* compound getters */ + avro_datum_value_get_size, + avro_datum_value_get_by_index, + avro_datum_value_get_by_name, + avro_datum_value_get_discriminant, + avro_datum_value_get_current_branch, + /* compound setters */ + avro_datum_value_append, + avro_datum_value_add, + avro_datum_value_set_branch +}; diff --git a/fluent-bit/lib/avro/src/datum_write.c b/fluent-bit/lib/avro/src/datum_write.c new file mode 100644 index 00000000..f3714ab8 --- /dev/null +++ b/fluent-bit/lib/avro/src/datum_write.c @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <assert.h> +#include <errno.h> +#include <string.h> + +#include "avro/basics.h" +#include "avro/errors.h" +#include "avro/io.h" +#include "avro/legacy.h" +#include "avro/resolver.h" +#include "avro/schema.h" +#include "avro/value.h" +#include "avro_private.h" + +int avro_write_data(avro_writer_t writer, avro_schema_t writers_schema, + avro_datum_t datum) +{ + int rval; + + check_param(EINVAL, writer, "writer"); + check_param(EINVAL, is_avro_datum(datum), "datum"); + + /* Only validate datum if a writer's schema is provided */ + if (is_avro_schema(writers_schema)) { + if (!avro_schema_datum_validate(writers_schema, datum)) { + avro_set_error("Datum doesn't validate against schema"); + return EINVAL; + } + + /* + * Some confusing terminology here. The "writers_schema" + * parameter is the schema we want to use to write the data + * into the "writer" buffer. Before doing that, we need to + * resolve the datum from its actual schema into this + * "writer" schema. For the purposes of that resolution, + * the writer schema is the datum's actual schema, and the + * reader schema is our eventual (when writing to the + * buffer) "writer" schema. + */ + + avro_schema_t datum_schema = avro_datum_get_schema(datum); + avro_value_iface_t *resolver = + avro_resolved_reader_new(datum_schema, writers_schema); + if (resolver == NULL) { + return EINVAL; + } + + avro_value_t value; + check(rval, avro_datum_as_value(&value, datum)); + + avro_value_t resolved; + rval = avro_resolved_reader_new_value(resolver, &resolved); + if (rval != 0) { + avro_value_decref(&value); + avro_value_iface_decref(resolver); + return rval; + } + + avro_resolved_reader_set_source(&resolved, &value); + rval = avro_value_write(writer, &resolved); + avro_value_decref(&resolved); + avro_value_decref(&value); + avro_value_iface_decref(resolver); + return rval; + } + + /* If we're writing using the datum's actual schema, we don't + * need a resolver. */ + + avro_value_t value; + check(rval, avro_datum_as_value(&value, datum)); + check(rval, avro_value_write(writer, &value)); + avro_value_decref(&value); + return 0; +} diff --git a/fluent-bit/lib/avro/src/dump.c b/fluent-bit/lib/avro/src/dump.c new file mode 100644 index 00000000..5b8f1c99 --- /dev/null +++ b/fluent-bit/lib/avro/src/dump.c @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <ctype.h> +#include <string.h> +#include "avro_private.h" +#include "dump.h" + +static void dump_line(FILE * out, const char *addr, const long len) +{ + int i; + fprintf(out, "|"); + for (i = 0; i < 16; i++) { + if (i < len) { + fprintf(out, " %02X", ((uint8_t *) addr)[i]); + } else { + fprintf(out, " .."); + } + if (!((i + 1) % 8)) { + fprintf(out, " |"); + } + } + fprintf(out, "\t"); + for (i = 0; i < 16; i++) { + char c = 0x7f & ((uint8_t *) addr)[i]; + if (i < len && isprint(c)) { + fprintf(out, "%c", c); + } else { + fprintf(out, "."); + } + } +} + +void dump(FILE * out, const char *addr, const long len) +{ + int i; + for (i = 0; i < len; i += 16) { + dump_line(out, addr + i, (len - i) < 16 ? (len - i) : 16); + fprintf(out, "\n"); + } + fflush(out); +} diff --git a/fluent-bit/lib/avro/src/dump.h b/fluent-bit/lib/avro/src/dump.h new file mode 100644 index 00000000..23e80667 --- /dev/null +++ b/fluent-bit/lib/avro/src/dump.h @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#ifndef DUMP_H +#define DUMP_H +#ifdef __cplusplus +extern "C" { +#define CLOSE_EXTERN } +#else +#define CLOSE_EXTERN +#endif + +#include <stdio.h> + +#pragma GCC visibility push(hidden) +void dump(FILE * out, const char *addr, const long len); +#pragma GCC visibility pop + +CLOSE_EXTERN +#endif diff --git a/fluent-bit/lib/avro/src/encoding.h b/fluent-bit/lib/avro/src/encoding.h new file mode 100644 index 00000000..6333d588 --- /dev/null +++ b/fluent-bit/lib/avro/src/encoding.h @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +#ifndef AVRO_ENCODING_H +#define AVRO_ENCODING_H +#ifdef __cplusplus +extern "C" { +#define CLOSE_EXTERN } +#else +#define CLOSE_EXTERN +#endif + +#include <avro/platform.h> +#include "avro/io.h" + +/* + * TODO: this will need more functions when JSON encoding is added + */ +struct avro_encoding_t { + const char *description; + /* + * string + */ + int (*read_string) (avro_reader_t reader, char **s, int64_t *len); + int (*skip_string) (avro_reader_t reader); + int (*write_string) (avro_writer_t writer, const char *s); + int64_t(*size_string) (avro_writer_t writer, const char *s); + /* + * bytes + */ + int (*read_bytes) (avro_reader_t reader, char **bytes, int64_t * len); + int (*skip_bytes) (avro_reader_t reader); + int (*write_bytes) (avro_writer_t writer, + const char *bytes, const int64_t len); + int64_t(*size_bytes) (avro_writer_t writer, + const char *bytes, const int64_t len); + /* + * int + */ + int (*read_int) (avro_reader_t reader, int32_t * i); + int (*skip_int) (avro_reader_t reader); + int (*write_int) (avro_writer_t writer, const int32_t i); + int64_t(*size_int) (avro_writer_t writer, const int32_t i); + /* + * long + */ + int (*read_long) (avro_reader_t reader, int64_t * l); + int (*skip_long) (avro_reader_t reader); + int (*write_long) (avro_writer_t writer, const int64_t l); + int64_t(*size_long) (avro_writer_t writer, const int64_t l); + /* + * float + */ + int (*read_float) (avro_reader_t reader, float *f); + int (*skip_float) (avro_reader_t reader); + int (*write_float) (avro_writer_t writer, const float f); + int64_t(*size_float) (avro_writer_t writer, const float f); + /* + * double + */ + int (*read_double) (avro_reader_t reader, double *d); + int (*skip_double) (avro_reader_t reader); + int (*write_double) (avro_writer_t writer, const double d); + int64_t(*size_double) (avro_writer_t writer, const double d); + /* + * boolean + */ + int (*read_boolean) (avro_reader_t reader, int8_t * b); + int (*skip_boolean) (avro_reader_t reader); + int (*write_boolean) (avro_writer_t writer, const int8_t b); + int64_t(*size_boolean) (avro_writer_t writer, const int8_t b); + /* + * null + */ + int (*read_null) (avro_reader_t reader); + int (*skip_null) (avro_reader_t reader); + int (*write_null) (avro_writer_t writer); + int64_t(*size_null) (avro_writer_t writer); +}; +typedef struct avro_encoding_t avro_encoding_t; + +#define AVRO_WRITE(writer, buf, len) \ +{ int rval = avro_write( writer, buf, len ); if(rval) return rval; } +#define AVRO_READ(reader, buf, len) \ +{ int rval = avro_read( reader, buf, len ); if(rval) return rval; } +#define AVRO_SKIP(reader, len) \ +{ int rval = avro_skip( reader, len); if (rval) return rval; } + +extern const avro_encoding_t avro_binary_encoding; /* in + * encoding_binary + */ +CLOSE_EXTERN +#endif diff --git a/fluent-bit/lib/avro/src/encoding_binary.c b/fluent-bit/lib/avro/src/encoding_binary.c new file mode 100644 index 00000000..1fc5f0c9 --- /dev/null +++ b/fluent-bit/lib/avro/src/encoding_binary.c @@ -0,0 +1,446 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include "avro_private.h" +#include "avro/allocation.h" +#include "avro/errors.h" +#include "encoding.h" +#include <stdlib.h> +#include <limits.h> +#include <errno.h> +#include <string.h> + +#define MAX_VARINT_BUF_SIZE 10 + +static int read_long(avro_reader_t reader, int64_t * l) +{ + uint64_t value = 0; + uint8_t b; + int offset = 0; + do { + if (offset == MAX_VARINT_BUF_SIZE) { + /* + * illegal byte sequence + */ + avro_set_error("Varint too long"); + return EILSEQ; + } + AVRO_READ(reader, &b, 1); + value |= (int64_t) (b & 0x7F) << (7 * offset); + ++offset; + } + while (b & 0x80); + *l = ((value >> 1) ^ -(value & 1)); + return 0; +} + +static int skip_long(avro_reader_t reader) +{ + uint8_t b; + int offset = 0; + do { + if (offset == MAX_VARINT_BUF_SIZE) { + avro_set_error("Varint too long"); + return EILSEQ; + } + AVRO_READ(reader, &b, 1); + ++offset; + } + while (b & 0x80); + return 0; +} + +static int write_long(avro_writer_t writer, int64_t l) +{ + char buf[MAX_VARINT_BUF_SIZE]; + uint8_t bytes_written = 0; + uint64_t n = (l << 1) ^ (l >> 63); + while (n & ~0x7F) { + buf[bytes_written++] = (char)((((uint8_t) n) & 0x7F) | 0x80); + n >>= 7; + } + buf[bytes_written++] = (char)n; + AVRO_WRITE(writer, buf, bytes_written); + return 0; +} + +static int64_t size_long(avro_writer_t writer, int64_t l) +{ + AVRO_UNUSED(writer); + + int64_t len = 0; + uint64_t n = (l << 1) ^ (l >> 63); + while (n & ~0x7F) { + len++; + n >>= 7; + } + len++; + return len; +} + +static int read_int(avro_reader_t reader, int32_t * i) +{ + int64_t l; + int rval; + check(rval, read_long(reader, &l)); + if (!(INT_MIN <= l && l <= INT_MAX)) { + avro_set_error("Varint out of range for int type"); + return ERANGE; + } + *i = l; + return 0; +} + +static int skip_int(avro_reader_t reader) +{ + return skip_long(reader); +} + +static int write_int(avro_writer_t writer, const int32_t i) +{ + int64_t l = i; + return write_long(writer, l); +} + +static int64_t size_int(avro_writer_t writer, const int32_t i) +{ + int64_t l = i; + return size_long(writer, l); +} + +static int read_bytes(avro_reader_t reader, char **bytes, int64_t * len) +{ + int rval; + check_prefix(rval, read_long(reader, len), + "Cannot read bytes length: "); + *bytes = (char *) avro_malloc(*len + 1); + if (!*bytes) { + avro_set_error("Cannot allocate buffer for bytes value"); + return ENOMEM; + } + AVRO_READ(reader, *bytes, *len); + (*bytes)[*len] = '\0'; + return 0; +} + +static int skip_bytes(avro_reader_t reader) +{ + int64_t len = 0; + int rval; + check_prefix(rval, read_long(reader, &len), + "Cannot read bytes length: "); + AVRO_SKIP(reader, len); + return 0; +} + +static int +write_bytes(avro_writer_t writer, const char *bytes, const int64_t len) +{ + int rval; + if (len < 0) { + avro_set_error("Invalid bytes value length"); + return EINVAL; + } + check_prefix(rval, write_long(writer, len), + "Cannot write bytes length: "); + AVRO_WRITE(writer, (char *)bytes, len); + return 0; +} + +static int64_t +size_bytes(avro_writer_t writer, const char *bytes, const int64_t len) +{ + AVRO_UNUSED(bytes); + + return size_long(writer, len) + len; +} + +static int read_string(avro_reader_t reader, char **s, int64_t *len) +{ + int64_t str_len = 0; + int rval; + check_prefix(rval, read_long(reader, &str_len), + "Cannot read string length: "); + *len = str_len + 1; + *s = (char *) avro_malloc(*len); + if (!*s) { + avro_set_error("Cannot allocate buffer for string value"); + return ENOMEM; + } + (*s)[str_len] = '\0'; + AVRO_READ(reader, *s, str_len); + return 0; +} + +static int skip_string(avro_reader_t reader) +{ + return skip_bytes(reader); +} + +static int write_string(avro_writer_t writer, const char *s) +{ + int64_t len = strlen(s); + return write_bytes(writer, s, len); +} + +static int64_t size_string(avro_writer_t writer, const char *s) +{ + int64_t len = strlen(s); + return size_bytes(writer, s, len); +} + +static int read_float(avro_reader_t reader, float *f) +{ +#if AVRO_PLATFORM_IS_BIG_ENDIAN + uint8_t buf[4]; +#endif + union { + float f; + int32_t i; + } v; +#if AVRO_PLATFORM_IS_BIG_ENDIAN + AVRO_READ(reader, buf, 4); + v.i = ((int32_t) buf[0] << 0) + | ((int32_t) buf[1] << 8) + | ((int32_t) buf[2] << 16) | ((int32_t) buf[3] << 24); +#else + AVRO_READ(reader, (void *)&v.i, 4); +#endif + *f = v.f; + return 0; +} + +static int skip_float(avro_reader_t reader) +{ + AVRO_SKIP(reader, 4); + return 0; +} + +static int write_float(avro_writer_t writer, const float f) +{ +#if AVRO_PLATFORM_IS_BIG_ENDIAN + uint8_t buf[4]; +#endif + union { + float f; + int32_t i; + } v; + + v.f = f; +#if AVRO_PLATFORM_IS_BIG_ENDIAN + buf[0] = (uint8_t) (v.i >> 0); + buf[1] = (uint8_t) (v.i >> 8); + buf[2] = (uint8_t) (v.i >> 16); + buf[3] = (uint8_t) (v.i >> 24); + AVRO_WRITE(writer, buf, 4); +#else + AVRO_WRITE(writer, (void *)&v.i, 4); +#endif + return 0; +} + +static int64_t size_float(avro_writer_t writer, const float f) +{ + AVRO_UNUSED(writer); + AVRO_UNUSED(f); + + return 4; +} + +static int read_double(avro_reader_t reader, double *d) +{ +#if AVRO_PLATFORM_IS_BIG_ENDIAN + uint8_t buf[8]; +#endif + union { + double d; + int64_t l; + } v; + +#if AVRO_PLATFORM_IS_BIG_ENDIAN + AVRO_READ(reader, buf, 8); + v.l = ((int64_t) buf[0] << 0) + | ((int64_t) buf[1] << 8) + | ((int64_t) buf[2] << 16) + | ((int64_t) buf[3] << 24) + | ((int64_t) buf[4] << 32) + | ((int64_t) buf[5] << 40) + | ((int64_t) buf[6] << 48) | ((int64_t) buf[7] << 56); +#else + AVRO_READ(reader, (void *)&v.l, 8); +#endif + *d = v.d; + return 0; +} + +static int skip_double(avro_reader_t reader) +{ + AVRO_SKIP(reader, 8); + return 0; +} + +static int write_double(avro_writer_t writer, const double d) +{ +#if AVRO_PLATFORM_IS_BIG_ENDIAN + uint8_t buf[8]; +#endif + union { + double d; + int64_t l; + } v; + + v.d = d; +#if AVRO_PLATFORM_IS_BIG_ENDIAN + buf[0] = (uint8_t) (v.l >> 0); + buf[1] = (uint8_t) (v.l >> 8); + buf[2] = (uint8_t) (v.l >> 16); + buf[3] = (uint8_t) (v.l >> 24); + buf[4] = (uint8_t) (v.l >> 32); + buf[5] = (uint8_t) (v.l >> 40); + buf[6] = (uint8_t) (v.l >> 48); + buf[7] = (uint8_t) (v.l >> 56); + AVRO_WRITE(writer, buf, 8); +#else + AVRO_WRITE(writer, (void *)&v.l, 8); +#endif + return 0; +} + +static int64_t size_double(avro_writer_t writer, const double d) +{ + AVRO_UNUSED(writer); + AVRO_UNUSED(d); + + return 8; +} + +static int read_boolean(avro_reader_t reader, int8_t * b) +{ + AVRO_READ(reader, b, 1); + return 0; +} + +static int skip_boolean(avro_reader_t reader) +{ + AVRO_SKIP(reader, 1); + return 0; +} + +static int write_boolean(avro_writer_t writer, const int8_t b) +{ + AVRO_WRITE(writer, (char *)&b, 1); + return 0; +} + +static int64_t size_boolean(avro_writer_t writer, const int8_t b) +{ + AVRO_UNUSED(writer); + AVRO_UNUSED(b); + + return 1; +} + +static int read_skip_null(avro_reader_t reader) +{ + /* + * no-op + */ + AVRO_UNUSED(reader); + + return 0; +} + +static int write_null(avro_writer_t writer) +{ + /* + * no-op + */ + AVRO_UNUSED(writer); + + return 0; +} + +static int64_t size_null(avro_writer_t writer) +{ + AVRO_UNUSED(writer); + + return 0; +} + +/* Win32 doesn't support the C99 method of initializing named elements + * in a struct declaration. So hide the named parameters for Win32, + * and initialize in the order the code was written. + */ +const avro_encoding_t avro_binary_encoding = { + /* .description = */ "BINARY FORMAT", + /* + * string + */ + /* .read_string = */ read_string, + /* .skip_string = */ skip_string, + /* .write_string = */ write_string, + /* .size_string = */ size_string, + /* + * bytes + */ + /* .read_bytes = */ read_bytes, + /* .skip_bytes = */ skip_bytes, + /* .write_bytes = */ write_bytes, + /* .size_bytes = */ size_bytes, + /* + * int + */ + /* .read_int = */ read_int, + /* .skip_int = */ skip_int, + /* .write_int = */ write_int, + /* .size_int = */ size_int, + /* + * long + */ + /* .read_long = */ read_long, + /* .skip_long = */ skip_long, + /* .write_long = */ write_long, + /* .size_long = */ size_long, + /* + * float + */ + /* .read_float = */ read_float, + /* .skip_float = */ skip_float, + /* .write_float = */ write_float, + /* .size_float = */ size_float, + /* + * double + */ + /* .read_double = */ read_double, + /* .skip_double = */ skip_double, + /* .write_double = */ write_double, + /* .size_double = */ size_double, + /* + * boolean + */ + /* .read_boolean = */ read_boolean, + /* .skip_boolean = */ skip_boolean, + /* .write_boolean = */ write_boolean, + /* .size_boolean = */ size_boolean, + /* + * null + */ + /* .read_null = */ read_skip_null, + /* .skip_null = */ read_skip_null, + /* .write_null = */ write_null, + /* .size_null = */ size_null +}; diff --git a/fluent-bit/lib/avro/src/errors.c b/fluent-bit/lib/avro/src/errors.c new file mode 100644 index 00000000..8abd8c83 --- /dev/null +++ b/fluent-bit/lib/avro/src/errors.c @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <stdarg.h> +#include <stdio.h> +#include <string.h> +#include <stdlib.h> + +#include "avro/errors.h" + +/* 4K should be enough, right? */ +#define AVRO_ERROR_SIZE 4096 + +/* + * To support the avro_prefix_error function, we keep two string buffers + * around. The AVRO_CURRENT_ERROR points at the buffer that's holding + * the current error message. avro_prefix error writes into the other + * buffer, and then swaps them. + */ + +struct avro_error_data_t { + char AVRO_ERROR1[AVRO_ERROR_SIZE]; + char AVRO_ERROR2[AVRO_ERROR_SIZE]; + + char *AVRO_CURRENT_ERROR; + char *AVRO_OTHER_ERROR; +}; + + +#if defined THREADSAFE +#if ( defined __unix__ || defined __unix ) +#include <pthread.h> +static pthread_key_t error_data_key; +static pthread_once_t error_data_key_once = PTHREAD_ONCE_INIT; + +static void make_error_data_key() +{ + pthread_key_create(&error_data_key, free); +} +#elif defined _WIN32 +#include <Windows.h> + +static __declspec( thread ) struct avro_error_data_t TLS_ERROR_DATA = { "", "", NULL, NULL }; + +#endif /* unix||_unix||_WIN32 */ +#endif /* THREADSAFE */ + +static struct avro_error_data_t * +avro_get_error_data(void) +{ +#if defined THREADSAFE +#if defined __unix__ || defined __unix + + pthread_once(&error_data_key_once, make_error_data_key); + + struct avro_error_data_t *ERROR_DATA = + (struct avro_error_data_t*) pthread_getspecific(error_data_key); + + if (!ERROR_DATA) { + ERROR_DATA = (struct avro_error_data_t*) malloc(sizeof(struct avro_error_data_t)); + pthread_setspecific(error_data_key, ERROR_DATA); + + ERROR_DATA->AVRO_ERROR1[0] = '\0'; + ERROR_DATA->AVRO_ERROR2[0] = '\0'; + ERROR_DATA->AVRO_CURRENT_ERROR = ERROR_DATA->AVRO_ERROR1; + ERROR_DATA->AVRO_OTHER_ERROR = ERROR_DATA->AVRO_ERROR2; + } + + return ERROR_DATA; + +#elif defined _WIN32 + + if ( TLS_ERROR_DATA.AVRO_CURRENT_ERROR == NULL ) + { + //first usage of the ERROR_DATA, initialize 'current' and 'other' pointers. + TLS_ERROR_DATA.AVRO_CURRENT_ERROR = TLS_ERROR_DATA.AVRO_ERROR1; + TLS_ERROR_DATA.AVRO_OTHER_ERROR = TLS_ERROR_DATA.AVRO_ERROR2; + } + return &TLS_ERROR_DATA; + + #endif /* UNIX and WIN32 threadsafe handling */ + +#else /* not thread-safe */ + static struct avro_error_data_t ERROR_DATA = { + /* .AVRO_ERROR1 = */ {'\0'}, + /* .AVRO_ERROR2 = */ {'\0'}, + /* .AVRO_CURRENT_ERROR = */ ERROR_DATA.AVRO_ERROR1, + /* .AVRO_OTHER_ERROR = */ ERROR_DATA.AVRO_ERROR2, + }; + + return &ERROR_DATA; +#endif +} + + +void +avro_set_error(const char *fmt, ...) +{ + va_list args; + va_start(args, fmt); + vsnprintf(avro_get_error_data()->AVRO_CURRENT_ERROR, AVRO_ERROR_SIZE, fmt, args); + va_end(args); + //fprintf(stderr, "--- %s\n", AVRO_CURRENT_ERROR); +} + + +void +avro_prefix_error(const char *fmt, ...) +{ + struct avro_error_data_t *ERROR_DATA = avro_get_error_data(); + + /* + * First render the prefix into OTHER_ERROR. + */ + + va_list args; + va_start(args, fmt); + int bytes_written = vsnprintf(ERROR_DATA->AVRO_OTHER_ERROR, AVRO_ERROR_SIZE, fmt, args); + va_end(args); + + /* + * Then concatenate the existing error onto the end. + */ + + if (bytes_written < AVRO_ERROR_SIZE) { + strncpy(&ERROR_DATA->AVRO_OTHER_ERROR[bytes_written], ERROR_DATA->AVRO_CURRENT_ERROR, + AVRO_ERROR_SIZE - bytes_written); + ERROR_DATA->AVRO_OTHER_ERROR[AVRO_ERROR_SIZE-1] = '\0'; + } + + /* + * Swap the two error pointers. + */ + + char *tmp; + tmp = ERROR_DATA->AVRO_OTHER_ERROR; + ERROR_DATA->AVRO_OTHER_ERROR = ERROR_DATA->AVRO_CURRENT_ERROR; + ERROR_DATA->AVRO_CURRENT_ERROR = tmp; + //fprintf(stderr, "+++ %s\n", AVRO_CURRENT_ERROR); +} + + +const char *avro_strerror(void) +{ + return avro_get_error_data()->AVRO_CURRENT_ERROR; +} diff --git a/fluent-bit/lib/avro/src/generic.c b/fluent-bit/lib/avro/src/generic.c new file mode 100644 index 00000000..e614eb3f --- /dev/null +++ b/fluent-bit/lib/avro/src/generic.c @@ -0,0 +1,3707 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <avro/platform.h> +#include <stdlib.h> +#include <string.h> + +#include "avro/allocation.h" +#include "avro/data.h" +#include "avro/errors.h" +#include "avro/generic.h" +#include "avro/refcount.h" +#include "avro/schema.h" +#include "avro/value.h" +#include "avro_generic_internal.h" +#include "avro_private.h" + + +/*----------------------------------------------------------------------- + * Forward definitions + */ + +typedef struct avro_generic_link_value_iface avro_generic_link_value_iface_t; + +typedef struct memoize_state_t { + avro_memoize_t mem; + avro_generic_link_value_iface_t *links; +} memoize_state_t; + +static avro_generic_value_iface_t * +avro_generic_class_from_schema_memoized(avro_schema_t schema, + memoize_state_t *state); + + +/*----------------------------------------------------------------------- + * Generic support functions + */ + +int +avro_generic_value_new(avro_value_iface_t *iface, avro_value_t *dest) +{ + int rval; + avro_generic_value_iface_t *giface = + container_of(iface, avro_generic_value_iface_t, parent); + size_t instance_size = avro_value_instance_size(giface); + void *self = avro_malloc(instance_size + sizeof(volatile int)); + if (self == NULL) { + avro_set_error(strerror(ENOMEM)); + dest->iface = NULL; + dest->self = NULL; + return ENOMEM; + } + + volatile int *refcount = (volatile int *) self; + self = (char *) self + sizeof(volatile int); + + *refcount = 1; + rval = avro_value_init(giface, self); + if (rval != 0) { + avro_free(self, instance_size); + dest->iface = NULL; + dest->self = NULL; + return rval; + } + + dest->iface = avro_value_iface_incref(&giface->parent); + dest->self = self; + return 0; +} + +static void +avro_generic_value_free(const avro_value_iface_t *iface, void *self) +{ + if (self != NULL) { + const avro_generic_value_iface_t *giface = + container_of(iface, avro_generic_value_iface_t, parent); + size_t instance_size = avro_value_instance_size(giface); + avro_value_done(giface, self); + self = (char *) self - sizeof(volatile int); + avro_free(self, instance_size + sizeof(volatile int)); + } +} + +static void +avro_generic_value_incref(avro_value_t *value) +{ + /* + * This only works if you pass in the top-level value. + */ + + volatile int *refcount = (volatile int *) ((char *) value->self - sizeof(volatile int)); + avro_refcount_inc(refcount); +} + +static void +avro_generic_value_decref(avro_value_t *value) +{ + /* + * This only works if you pass in the top-level value. + */ + + volatile int *refcount = (volatile int *) ((char *) value->self - sizeof(volatile int)); + if (avro_refcount_dec(refcount)) { + avro_generic_value_free(value->iface, value->self); + } +} + + +/*----------------------------------------------------------------------- + * Recursive schemas + */ + +/* + * Recursive schemas are handled specially; the value implementation for + * an AVRO_LINK schema is simply a wrapper around the value + * implementation for the link's target schema. The value methods all + * delegate to the wrapped implementation. + * + * We don't set the target_iface pointer when the link implementation is + * first created, since we might not have finished creating the + * implementation for the target schema. (We create the implementations + * for child schemas depth-first, so the target schema's implementation + * won't be done until all of its descendants — including the link + * schema — have been instantiated.) + * + * So anyway, we set the target_iface pointer to NULL at first. And + * then in a fix-up stage, once all of the non-link schemas have been + * instantiated, we go through and set the target_iface pointers for any + * link schemas we encountered. + */ + +struct avro_generic_link_value_iface { + avro_generic_value_iface_t parent; + + /** The reference count for this interface. */ + volatile int refcount; + + /** The schema for this interface. */ + avro_schema_t schema; + + /** The target's implementation. */ + avro_generic_value_iface_t *target_giface; + + /** + * A pointer to the “next” link interface that we've had to + * create. We use this as we're creating the overall top-level + * value interface to keep track of which ones we have to fix up + * afterwards. + */ + avro_generic_link_value_iface_t *next; +}; + + +static avro_value_iface_t * +avro_generic_link_incref_iface(avro_value_iface_t *viface) +{ + avro_generic_link_value_iface_t *iface = + container_of(viface, avro_generic_link_value_iface_t, parent); + avro_refcount_inc(&iface->refcount); + return viface; +} + +static void +avro_generic_link_decref_iface(avro_value_iface_t *viface) +{ + avro_generic_link_value_iface_t *iface = + container_of(viface, avro_generic_link_value_iface_t, parent.parent); + + if (avro_refcount_dec(&iface->refcount)) { + /* We don't keep a reference to the target + * implementation, since that would give us a reference + * cycle. */ + /* We do however keep a reference to the target + * schema, which we need to decrement before freeing + * the link */ + avro_schema_decref(iface->schema); + avro_freet(avro_generic_link_value_iface_t, iface); + } +} + + +static int +avro_generic_link_reset(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + avro_value_t *self = (avro_value_t *) vself; + return avro_value_reset(self); +} + +static avro_type_t +avro_generic_link_get_type(const avro_value_iface_t *viface, const void *vself) +{ + AVRO_UNUSED(viface); + const avro_value_t *self = (const avro_value_t *) vself; + return avro_value_get_type(self); +} + +static avro_schema_t +avro_generic_link_get_schema(const avro_value_iface_t *viface, const void *vself) +{ + AVRO_UNUSED(viface); + const avro_value_t *self = (const avro_value_t *) vself; + return avro_value_get_schema(self); +} + +static int +avro_generic_link_get_boolean(const avro_value_iface_t *iface, + const void *vself, int *out) +{ + AVRO_UNUSED(iface); + const avro_value_t *self = (const avro_value_t *) vself; + return avro_value_get_boolean(self, out); +} + +static int +avro_generic_link_get_bytes(const avro_value_iface_t *iface, + const void *vself, const void **buf, size_t *size) +{ + AVRO_UNUSED(iface); + const avro_value_t *self = (const avro_value_t *) vself; + return avro_value_get_bytes(self, buf, size); +} + +static int +avro_generic_link_grab_bytes(const avro_value_iface_t *iface, + const void *vself, avro_wrapped_buffer_t *dest) +{ + AVRO_UNUSED(iface); + const avro_value_t *self = (const avro_value_t *) vself; + return avro_value_grab_bytes(self, dest); +} + +static int +avro_generic_link_get_double(const avro_value_iface_t *iface, + const void *vself, double *out) +{ + AVRO_UNUSED(iface); + const avro_value_t *self = (const avro_value_t *) vself; + return avro_value_get_double(self, out); +} + +static int +avro_generic_link_get_float(const avro_value_iface_t *iface, + const void *vself, float *out) +{ + AVRO_UNUSED(iface); + const avro_value_t *self = (const avro_value_t *) vself; + return avro_value_get_float(self, out); +} + +static int +avro_generic_link_get_int(const avro_value_iface_t *iface, + const void *vself, int32_t *out) +{ + AVRO_UNUSED(iface); + const avro_value_t *self = (const avro_value_t *) vself; + return avro_value_get_int(self, out); +} + +static int +avro_generic_link_get_long(const avro_value_iface_t *iface, + const void *vself, int64_t *out) +{ + AVRO_UNUSED(iface); + const avro_value_t *self = (const avro_value_t *) vself; + return avro_value_get_long(self, out); +} + +static int +avro_generic_link_get_null(const avro_value_iface_t *iface, const void *vself) +{ + AVRO_UNUSED(iface); + const avro_value_t *self = (const avro_value_t *) vself; + return avro_value_get_null(self); +} + +static int +avro_generic_link_get_string(const avro_value_iface_t *iface, + const void *vself, const char **str, size_t *size) +{ + AVRO_UNUSED(iface); + const avro_value_t *self = (const avro_value_t *) vself; + return avro_value_get_string(self, str, size); +} + +static int +avro_generic_link_grab_string(const avro_value_iface_t *iface, + const void *vself, avro_wrapped_buffer_t *dest) +{ + AVRO_UNUSED(iface); + const avro_value_t *self = (const avro_value_t *) vself; + return avro_value_grab_string(self, dest); +} + +static int +avro_generic_link_get_enum(const avro_value_iface_t *iface, + const void *vself, int *out) +{ + AVRO_UNUSED(iface); + const avro_value_t *self = (const avro_value_t *) vself; + return avro_value_get_enum(self, out); +} + +static int +avro_generic_link_get_fixed(const avro_value_iface_t *iface, + const void *vself, const void **buf, size_t *size) +{ + AVRO_UNUSED(iface); + const avro_value_t *self = (const avro_value_t *) vself; + return avro_value_get_fixed(self, buf, size); +} + +static int +avro_generic_link_grab_fixed(const avro_value_iface_t *iface, + const void *vself, avro_wrapped_buffer_t *dest) +{ + AVRO_UNUSED(iface); + const avro_value_t *self = (const avro_value_t *) vself; + return avro_value_grab_fixed(self, dest); +} + +static int +avro_generic_link_set_boolean(const avro_value_iface_t *iface, + void *vself, int val) +{ + AVRO_UNUSED(iface); + avro_value_t *self = (avro_value_t *) vself; + return avro_value_set_boolean(self, val); +} + +static int +avro_generic_link_set_bytes(const avro_value_iface_t *iface, + void *vself, void *buf, size_t size) +{ + AVRO_UNUSED(iface); + avro_value_t *self = (avro_value_t *) vself; + return avro_value_set_bytes(self, buf, size); +} + +static int +avro_generic_link_give_bytes(const avro_value_iface_t *iface, + void *vself, avro_wrapped_buffer_t *buf) +{ + AVRO_UNUSED(iface); + avro_value_t *self = (avro_value_t *) vself; + return avro_value_give_bytes(self, buf); +} + +static int +avro_generic_link_set_double(const avro_value_iface_t *iface, + void *vself, double val) +{ + AVRO_UNUSED(iface); + avro_value_t *self = (avro_value_t *) vself; + return avro_value_set_double(self, val); +} + +static int +avro_generic_link_set_float(const avro_value_iface_t *iface, + void *vself, float val) +{ + AVRO_UNUSED(iface); + avro_value_t *self = (avro_value_t *) vself; + return avro_value_set_float(self, val); +} + +static int +avro_generic_link_set_int(const avro_value_iface_t *iface, + void *vself, int32_t val) +{ + AVRO_UNUSED(iface); + avro_value_t *self = (avro_value_t *) vself; + return avro_value_set_int(self, val); +} + +static int +avro_generic_link_set_long(const avro_value_iface_t *iface, + void *vself, int64_t val) +{ + AVRO_UNUSED(iface); + avro_value_t *self = (avro_value_t *) vself; + return avro_value_set_long(self, val); +} + +static int +avro_generic_link_set_null(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + avro_value_t *self = (avro_value_t *) vself; + return avro_value_set_null(self); +} + +static int +avro_generic_link_set_string(const avro_value_iface_t *iface, + void *vself, const char *str) +{ + AVRO_UNUSED(iface); + avro_value_t *self = (avro_value_t *) vself; + return avro_value_set_string(self, str); +} + +static int +avro_generic_link_set_string_len(const avro_value_iface_t *iface, + void *vself, const char *str, size_t size) +{ + AVRO_UNUSED(iface); + avro_value_t *self = (avro_value_t *) vself; + return avro_value_set_string_len(self, str, size); +} + +static int +avro_generic_link_give_string_len(const avro_value_iface_t *iface, + void *vself, avro_wrapped_buffer_t *buf) +{ + AVRO_UNUSED(iface); + avro_value_t *self = (avro_value_t *) vself; + return avro_value_give_string_len(self, buf); +} + +static int +avro_generic_link_set_enum(const avro_value_iface_t *iface, + void *vself, int val) +{ + AVRO_UNUSED(iface); + avro_value_t *self = (avro_value_t *) vself; + return avro_value_set_enum(self, val); +} + +static int +avro_generic_link_set_fixed(const avro_value_iface_t *iface, + void *vself, void *buf, size_t size) +{ + AVRO_UNUSED(iface); + avro_value_t *self = (avro_value_t *) vself; + return avro_value_set_fixed(self, buf, size); +} + +static int +avro_generic_link_give_fixed(const avro_value_iface_t *iface, + void *vself, avro_wrapped_buffer_t *buf) +{ + AVRO_UNUSED(iface); + avro_value_t *self = (avro_value_t *) vself; + return avro_value_give_fixed(self, buf); +} + +static int +avro_generic_link_get_size(const avro_value_iface_t *iface, + const void *vself, size_t *size) +{ + AVRO_UNUSED(iface); + const avro_value_t *self = (const avro_value_t *) vself; + return avro_value_get_size(self, size); +} + +static int +avro_generic_link_get_by_index(const avro_value_iface_t *iface, + const void *vself, size_t index, + avro_value_t *child, const char **name) +{ + AVRO_UNUSED(iface); + const avro_value_t *self = (const avro_value_t *) vself; + return avro_value_get_by_index(self, index, child, name); +} + +static int +avro_generic_link_get_by_name(const avro_value_iface_t *iface, + const void *vself, const char *name, + avro_value_t *child, size_t *index) +{ + AVRO_UNUSED(iface); + const avro_value_t *self = (const avro_value_t *) vself; + return avro_value_get_by_name(self, name, child, index); +} + +static int +avro_generic_link_get_discriminant(const avro_value_iface_t *iface, + const void *vself, int *out) +{ + AVRO_UNUSED(iface); + const avro_value_t *self = (const avro_value_t *) vself; + return avro_value_get_discriminant(self, out); +} + +static int +avro_generic_link_get_current_branch(const avro_value_iface_t *iface, + const void *vself, avro_value_t *branch) +{ + AVRO_UNUSED(iface); + const avro_value_t *self = (const avro_value_t *) vself; + return avro_value_get_current_branch(self, branch); +} + +static int +avro_generic_link_append(const avro_value_iface_t *iface, + void *vself, avro_value_t *child_out, + size_t *new_index) +{ + AVRO_UNUSED(iface); + avro_value_t *self = (avro_value_t *) vself; + return avro_value_append(self, child_out, new_index); +} + +static int +avro_generic_link_add(const avro_value_iface_t *iface, + void *vself, const char *key, + avro_value_t *child, size_t *index, int *is_new) +{ + AVRO_UNUSED(iface); + avro_value_t *self = (avro_value_t *) vself; + return avro_value_add(self, key, child, index, is_new); +} + +static int +avro_generic_link_set_branch(const avro_value_iface_t *iface, + void *vself, int discriminant, + avro_value_t *branch) +{ + AVRO_UNUSED(iface); + avro_value_t *self = (avro_value_t *) vself; + return avro_value_set_branch(self, discriminant, branch); +} + +static size_t +avro_generic_link_instance_size(const avro_value_iface_t *viface) +{ + AVRO_UNUSED(viface); + return sizeof(avro_value_t); +} + +static int +avro_generic_link_init(const avro_value_iface_t *viface, void *vself) +{ + int rval; + + avro_generic_link_value_iface_t *iface = + container_of(viface, avro_generic_link_value_iface_t, parent.parent); + + avro_value_t *self = (avro_value_t *) vself; + ssize_t target_instance_size = + avro_value_instance_size(iface->target_giface); + if (target_instance_size < 0) { + return EINVAL; + } + + self->iface = &iface->target_giface->parent; + + if (target_instance_size == 0) { + self->self = NULL; + } else { + self->self = avro_malloc(target_instance_size); + if (self->self == NULL) { + return ENOMEM; + } + } + + rval = avro_value_init(iface->target_giface, self->self); + if (rval != 0) { + avro_free(self->self, target_instance_size); + } + return rval; +} + +static void +avro_generic_link_done(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + avro_value_t *self = (avro_value_t *) vself; + avro_generic_value_iface_t *target_giface = + container_of(self->iface, avro_generic_value_iface_t, parent); + size_t target_instance_size = avro_value_instance_size(target_giface); + avro_value_done(target_giface, self->self); + avro_free(self->self, target_instance_size); + self->iface = NULL; + self->self = NULL; +} + +static avro_generic_value_iface_t AVRO_GENERIC_LINK_CLASS = +{ + { + /* "class" methods */ + avro_generic_link_incref_iface, + avro_generic_link_decref_iface, + /* general "instance" methods */ + avro_generic_value_incref, + avro_generic_value_decref, + avro_generic_link_reset, + avro_generic_link_get_type, + avro_generic_link_get_schema, + /* primitive getters */ + avro_generic_link_get_boolean, + avro_generic_link_get_bytes, + avro_generic_link_grab_bytes, + avro_generic_link_get_double, + avro_generic_link_get_float, + avro_generic_link_get_int, + avro_generic_link_get_long, + avro_generic_link_get_null, + avro_generic_link_get_string, + avro_generic_link_grab_string, + avro_generic_link_get_enum, + avro_generic_link_get_fixed, + avro_generic_link_grab_fixed, + /* primitive setters */ + avro_generic_link_set_boolean, + avro_generic_link_set_bytes, + avro_generic_link_give_bytes, + avro_generic_link_set_double, + avro_generic_link_set_float, + avro_generic_link_set_int, + avro_generic_link_set_long, + avro_generic_link_set_null, + avro_generic_link_set_string, + avro_generic_link_set_string_len, + avro_generic_link_give_string_len, + avro_generic_link_set_enum, + avro_generic_link_set_fixed, + avro_generic_link_give_fixed, + /* compound getters */ + avro_generic_link_get_size, + avro_generic_link_get_by_index, + avro_generic_link_get_by_name, + avro_generic_link_get_discriminant, + avro_generic_link_get_current_branch, + /* compound setters */ + avro_generic_link_append, + avro_generic_link_add, + avro_generic_link_set_branch + }, + avro_generic_link_instance_size, + avro_generic_link_init, + avro_generic_link_done +}; + +static avro_generic_link_value_iface_t * +avro_generic_link_class(avro_schema_t schema) +{ + if (!is_avro_link(schema)) { + avro_set_error("Expected link schema"); + return NULL; + } + + avro_generic_link_value_iface_t *iface = + (avro_generic_link_value_iface_t *) avro_new(avro_generic_link_value_iface_t); + if (iface == NULL) { + return NULL; + } + + iface->parent = AVRO_GENERIC_LINK_CLASS; + iface->refcount = 1; + iface->schema = avro_schema_incref(schema); + iface->next = NULL; + return iface; +} + + +/*----------------------------------------------------------------------- + * boolean + */ + +static int +avro_generic_boolean_reset(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + int *self = (int *) vself; + *self = 0; + return 0; +} + +static avro_type_t +avro_generic_boolean_get_type(const avro_value_iface_t *iface, const void *vself) +{ + AVRO_UNUSED(iface); + AVRO_UNUSED(vself); + return AVRO_BOOLEAN; +} + +static avro_schema_t +avro_generic_boolean_get_schema(const avro_value_iface_t *iface, const void *vself) +{ + AVRO_UNUSED(iface); + AVRO_UNUSED(vself); + return avro_schema_boolean(); +} + +static int +avro_generic_boolean_get(const avro_value_iface_t *iface, + const void *vself, int *out) +{ + AVRO_UNUSED(iface); + const int *self = (const int *) vself; + *out = *self; + return 0; +} + +static int +avro_generic_boolean_set(const avro_value_iface_t *iface, + void *vself, int val) +{ + AVRO_UNUSED(iface); + int *self = (int *) vself; + *self = val; + return 0; +} + +static size_t +avro_generic_boolean_instance_size(const avro_value_iface_t *iface) +{ + AVRO_UNUSED(iface); + return sizeof(int); +} + +static int +avro_generic_boolean_init(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + int *self = (int *) vself; + *self = 0; + return 0; +} + +static void +avro_generic_boolean_done(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + AVRO_UNUSED(vself); +} + +static avro_generic_value_iface_t AVRO_GENERIC_BOOLEAN_CLASS = +{ + { + /* "class" methods */ + NULL, /* incref_iface */ + NULL, /* decref_iface */ + /* general "instance" methods */ + avro_generic_value_incref, + avro_generic_value_decref, + avro_generic_boolean_reset, + avro_generic_boolean_get_type, + avro_generic_boolean_get_schema, + /* primitive getters */ + avro_generic_boolean_get, + NULL, /* get_bytes */ + NULL, /* grab_bytes */ + NULL, /* get_double */ + NULL, /* get_float */ + NULL, /* get_int */ + NULL, /* get_long */ + NULL, /* get_null */ + NULL, /* get_string */ + NULL, /* grab_string */ + NULL, /* get_enum */ + NULL, /* get_fixed */ + NULL, /* grab_fixed */ + /* primitive setters */ + avro_generic_boolean_set, + NULL, /* set_bytes */ + NULL, /* give_bytes */ + NULL, /* set_double */ + NULL, /* set_float */ + NULL, /* set_int */ + NULL, /* set_long */ + NULL, /* set_null */ + NULL, /* set_string */ + NULL, /* set_string_length */ + NULL, /* give_string_length */ + NULL, /* set_enum */ + NULL, /* set_fixed */ + NULL, /* give_fixed */ + /* compound getters */ + NULL, /* get_size */ + NULL, /* get_by_index */ + NULL, /* get_by_name */ + NULL, /* get_discriminant */ + NULL, /* get_current_branch */ + /* compound setters */ + NULL, /* append */ + NULL, /* add */ + NULL /* set_branch */ + }, + avro_generic_boolean_instance_size, + avro_generic_boolean_init, + avro_generic_boolean_done +}; + +avro_value_iface_t * +avro_generic_boolean_class(void) +{ + return &AVRO_GENERIC_BOOLEAN_CLASS.parent; +} + +int +avro_generic_boolean_new(avro_value_t *value, int val) +{ + int rval; + check(rval, avro_generic_value_new(&AVRO_GENERIC_BOOLEAN_CLASS.parent, value)); + return avro_generic_boolean_set(value->iface, value->self, val); +} + +/*----------------------------------------------------------------------- + * bytes + */ + +static int +avro_generic_bytes_reset(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + avro_raw_string_t *self = (avro_raw_string_t *) vself; + avro_raw_string_clear(self); + return 0; +} + +static avro_type_t +avro_generic_bytes_get_type(const avro_value_iface_t *iface, const void *vself) +{ + AVRO_UNUSED(iface); + AVRO_UNUSED(vself); + return AVRO_BYTES; +} + +static avro_schema_t +avro_generic_bytes_get_schema(const avro_value_iface_t *iface, const void *vself) +{ + AVRO_UNUSED(iface); + AVRO_UNUSED(vself); + return avro_schema_bytes(); +} + +static int +avro_generic_bytes_get(const avro_value_iface_t *iface, + const void *vself, const void **buf, size_t *size) +{ + AVRO_UNUSED(iface); + const avro_raw_string_t *self = (const avro_raw_string_t *) vself; + if (buf != NULL) { + *buf = avro_raw_string_get(self); + } + if (size != NULL) { + *size = avro_raw_string_length(self); + } + return 0; +} + +static int +avro_generic_bytes_grab(const avro_value_iface_t *iface, + const void *vself, avro_wrapped_buffer_t *dest) +{ + AVRO_UNUSED(iface); + const avro_raw_string_t *self = (const avro_raw_string_t *) vself; + return avro_raw_string_grab(self, dest); +} + +static int +avro_generic_bytes_set(const avro_value_iface_t *iface, + void *vself, void *buf, size_t size) +{ + AVRO_UNUSED(iface); + check_param(EINVAL, buf != NULL, "bytes contents"); + avro_raw_string_t *self = (avro_raw_string_t *) vself; + avro_raw_string_set_length(self, buf, size); + return 0; +} + +static int +avro_generic_bytes_give(const avro_value_iface_t *iface, + void *vself, avro_wrapped_buffer_t *buf) +{ + AVRO_UNUSED(iface); + avro_raw_string_t *self = (avro_raw_string_t *) vself; + avro_raw_string_give(self, buf); + return 0; +} + +static size_t +avro_generic_bytes_instance_size(const avro_value_iface_t *iface) +{ + AVRO_UNUSED(iface); + return sizeof(avro_raw_string_t); +} + +static int +avro_generic_bytes_init(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + avro_raw_string_t *self = (avro_raw_string_t *) vself; + avro_raw_string_init(self); + return 0; +} + +static void +avro_generic_bytes_done(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + avro_raw_string_t *self = (avro_raw_string_t *) vself; + avro_raw_string_done(self); +} + +static avro_generic_value_iface_t AVRO_GENERIC_BYTES_CLASS = +{ + { + /* "class" methods */ + NULL, /* incref_iface */ + NULL, /* decref_iface */ + /* general "instance" methods */ + avro_generic_value_incref, + avro_generic_value_decref, + avro_generic_bytes_reset, + avro_generic_bytes_get_type, + avro_generic_bytes_get_schema, + /* primitive getters */ + NULL, /* get_boolean */ + avro_generic_bytes_get, + avro_generic_bytes_grab, + NULL, /* get_double */ + NULL, /* get_float */ + NULL, /* get_int */ + NULL, /* get_long */ + NULL, /* get_null */ + NULL, /* get_string */ + NULL, /* grab_string */ + NULL, /* get_enum */ + NULL, /* get_fixed */ + NULL, /* grab_fixed */ + /* primitive setters */ + NULL, /* set_boolean */ + avro_generic_bytes_set, + avro_generic_bytes_give, + NULL, /* set_double */ + NULL, /* set_float */ + NULL, /* set_int */ + NULL, /* set_long */ + NULL, /* set_null */ + NULL, /* set_string */ + NULL, /* set_string_length */ + NULL, /* give_string_length */ + NULL, /* set_enum */ + NULL, /* set_fixed */ + NULL, /* give_fixed */ + /* compound getters */ + NULL, /* get_size */ + NULL, /* get_by_index */ + NULL, /* get_by_name */ + NULL, /* get_discriminant */ + NULL, /* get_current_branch */ + /* compound setters */ + NULL, /* append */ + NULL, /* add */ + NULL /* set_branch */ + }, + avro_generic_bytes_instance_size, + avro_generic_bytes_init, + avro_generic_bytes_done +}; + +avro_value_iface_t * +avro_generic_bytes_class(void) +{ + return &AVRO_GENERIC_BYTES_CLASS.parent; +} + +int +avro_generic_bytes_new(avro_value_t *value, void *buf, size_t size) +{ + int rval; + check(rval, avro_generic_value_new(&AVRO_GENERIC_BYTES_CLASS.parent, value)); + return avro_generic_bytes_set(value->iface, value->self, buf, size); +} + +/*----------------------------------------------------------------------- + * double + */ + +static int +avro_generic_double_reset(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + double *self = (double *) vself; + *self = 0.0; + return 0; +} + +static avro_type_t +avro_generic_double_get_type(const avro_value_iface_t *iface, const void *vself) +{ + AVRO_UNUSED(iface); + AVRO_UNUSED(vself); + return AVRO_DOUBLE; +} + +static avro_schema_t +avro_generic_double_get_schema(const avro_value_iface_t *iface, const void *vself) +{ + AVRO_UNUSED(iface); + AVRO_UNUSED(vself); + return avro_schema_double(); +} + +static int +avro_generic_double_get(const avro_value_iface_t *iface, + const void *vself, double *out) +{ + AVRO_UNUSED(iface); + const double *self = (const double *) vself; + *out = *self; + return 0; +} + +static int +avro_generic_double_set(const avro_value_iface_t *iface, + void *vself, double val) +{ + AVRO_UNUSED(iface); + double *self = (double *) vself; + *self = val; + return 0; +} + +static size_t +avro_generic_double_instance_size(const avro_value_iface_t *iface) +{ + AVRO_UNUSED(iface); + return sizeof(double); +} + +static int +avro_generic_double_init(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + double *self = (double *) vself; + *self = 0.0; + return 0; +} + +static void +avro_generic_double_done(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + AVRO_UNUSED(vself); +} + +static avro_generic_value_iface_t AVRO_GENERIC_DOUBLE_CLASS = +{ + { + /* "class" methods */ + NULL, /* incref_iface */ + NULL, /* decref_iface */ + /* general "instance" methods */ + avro_generic_value_incref, + avro_generic_value_decref, + avro_generic_double_reset, + avro_generic_double_get_type, + avro_generic_double_get_schema, + /* primitive getters */ + NULL, /* get_boolean */ + NULL, /* get_bytes */ + NULL, /* grab_bytes */ + avro_generic_double_get, + NULL, /* get_float */ + NULL, /* get_int */ + NULL, /* get_long */ + NULL, /* get_null */ + NULL, /* get_string */ + NULL, /* grab_string */ + NULL, /* get_enum */ + NULL, /* get_fixed */ + NULL, /* grab_fixed */ + /* primitive setters */ + NULL, /* set_boolean */ + NULL, /* set_bytes */ + NULL, /* give_bytes */ + avro_generic_double_set, + NULL, /* set_float */ + NULL, /* set_int */ + NULL, /* set_long */ + NULL, /* set_null */ + NULL, /* set_string */ + NULL, /* set_string_length */ + NULL, /* give_string_length */ + NULL, /* set_enum */ + NULL, /* set_fixed */ + NULL, /* give_fixed */ + /* compound getters */ + NULL, /* get_size */ + NULL, /* get_by_index */ + NULL, /* get_by_name */ + NULL, /* get_discriminant */ + NULL, /* get_current_branch */ + /* compound setters */ + NULL, /* append */ + NULL, /* add */ + NULL /* set_branch */ + }, + avro_generic_double_instance_size, + avro_generic_double_init, + avro_generic_double_done +}; + +avro_value_iface_t * +avro_generic_double_class(void) +{ + return &AVRO_GENERIC_DOUBLE_CLASS.parent; +} + +int +avro_generic_double_new(avro_value_t *value, double val) +{ + int rval; + check(rval, avro_generic_value_new(&AVRO_GENERIC_DOUBLE_CLASS.parent, value)); + return avro_generic_double_set(value->iface, value->self, val); +} + +/*----------------------------------------------------------------------- + * float + */ + +static int +avro_generic_float_reset(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + float *self = (float *) vself; + *self = 0.0f; + return 0; +} + +static avro_type_t +avro_generic_float_get_type(const avro_value_iface_t *iface, const void *vself) +{ + AVRO_UNUSED(iface); + AVRO_UNUSED(vself); + return AVRO_FLOAT; +} + +static avro_schema_t +avro_generic_float_get_schema(const avro_value_iface_t *iface, const void *vself) +{ + AVRO_UNUSED(iface); + AVRO_UNUSED(vself); + return avro_schema_float(); +} + +static int +avro_generic_float_get(const avro_value_iface_t *iface, + const void *vself, float *out) +{ + AVRO_UNUSED(iface); + const float *self = (const float *) vself; + *out = *self; + return 0; +} + +static int +avro_generic_float_set(const avro_value_iface_t *iface, + void *vself, float val) +{ + AVRO_UNUSED(iface); + float *self = (float *) vself; + *self = val; + return 0; +} + +static size_t +avro_generic_float_instance_size(const avro_value_iface_t *iface) +{ + AVRO_UNUSED(iface); + return sizeof(float); +} + +static int +avro_generic_float_init(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + float *self = (float *) vself; + *self = 0.0f; + return 0; +} + +static void +avro_generic_float_done(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + AVRO_UNUSED(vself); +} + +static avro_generic_value_iface_t AVRO_GENERIC_FLOAT_CLASS = +{ + { + /* "class" methods */ + NULL, /* incref_iface */ + NULL, /* decref_iface */ + /* general "instance" methods */ + avro_generic_value_incref, + avro_generic_value_decref, + avro_generic_float_reset, + avro_generic_float_get_type, + avro_generic_float_get_schema, + /* primitive getters */ + NULL, /* get_boolean */ + NULL, /* get_bytes */ + NULL, /* grab_bytes */ + NULL, /* get_double */ + avro_generic_float_get, + NULL, /* get_int */ + NULL, /* get_long */ + NULL, /* get_null */ + NULL, /* get_string */ + NULL, /* grab_string */ + NULL, /* get_enum */ + NULL, /* get_fixed */ + NULL, /* grab_fixed */ + /* primitive setters */ + NULL, /* set_boolean */ + NULL, /* set_bytes */ + NULL, /* give_bytes */ + NULL, /* set_double */ + avro_generic_float_set, + NULL, /* set_int */ + NULL, /* set_long */ + NULL, /* set_null */ + NULL, /* set_string */ + NULL, /* set_string_length */ + NULL, /* give_string_length */ + NULL, /* set_enum */ + NULL, /* set_fixed */ + NULL, /* give_fixed */ + /* compound getters */ + NULL, /* get_size */ + NULL, /* get_by_index */ + NULL, /* get_by_name */ + NULL, /* get_discriminant */ + NULL, /* get_current_branch */ + /* compound setters */ + NULL, /* append */ + NULL, /* add */ + NULL /* set_branch */ + }, + avro_generic_float_instance_size, + avro_generic_float_init, + avro_generic_float_done +}; + +avro_value_iface_t * +avro_generic_float_class(void) +{ + return &AVRO_GENERIC_FLOAT_CLASS.parent; +} + +int +avro_generic_float_new(avro_value_t *value, float val) +{ + int rval; + check(rval, avro_generic_value_new(&AVRO_GENERIC_FLOAT_CLASS.parent, value)); + return avro_generic_float_set(value->iface, value->self, val); +} + +/*----------------------------------------------------------------------- + * int + */ + +static int +avro_generic_int_reset(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + int32_t *self = (int32_t *) vself; + *self = 0; + return 0; +} + +static avro_type_t +avro_generic_int_get_type(const avro_value_iface_t *iface, const void *vself) +{ + AVRO_UNUSED(iface); + AVRO_UNUSED(vself); + return AVRO_INT32; +} + +static avro_schema_t +avro_generic_int_get_schema(const avro_value_iface_t *iface, const void *vself) +{ + AVRO_UNUSED(iface); + AVRO_UNUSED(vself); + return avro_schema_int(); +} + +static int +avro_generic_int_get(const avro_value_iface_t *iface, + const void *vself, int32_t *out) +{ + AVRO_UNUSED(iface); + const int32_t *self = (const int32_t *) vself; + *out = *self; + return 0; +} + +static int +avro_generic_int_set(const avro_value_iface_t *iface, + void *vself, int32_t val) +{ + AVRO_UNUSED(iface); + int32_t *self = (int32_t *) vself; + *self = val; + return 0; +} + +static size_t +avro_generic_int_instance_size(const avro_value_iface_t *iface) +{ + AVRO_UNUSED(iface); + return sizeof(int32_t); +} + +static int +avro_generic_int_init(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + int32_t *self = (int32_t *) vself; + *self = 0; + return 0; +} + +static void +avro_generic_int_done(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + AVRO_UNUSED(vself); +} + +static avro_generic_value_iface_t AVRO_GENERIC_INT_CLASS = +{ + { + /* "class" methods */ + NULL, /* incref_iface */ + NULL, /* decref_iface */ + /* general "instance" methods */ + avro_generic_value_incref, + avro_generic_value_decref, + avro_generic_int_reset, + avro_generic_int_get_type, + avro_generic_int_get_schema, + /* primitive getters */ + NULL, /* get_boolean */ + NULL, /* get_bytes */ + NULL, /* grab_bytes */ + NULL, /* get_double */ + NULL, /* get_float */ + avro_generic_int_get, + NULL, /* get_long */ + NULL, /* get_null */ + NULL, /* get_string */ + NULL, /* grab_string */ + NULL, /* get_enum */ + NULL, /* get_fixed */ + NULL, /* grab_fixed */ + /* primitive setters */ + NULL, /* set_boolean */ + NULL, /* set_bytes */ + NULL, /* give_bytes */ + NULL, /* set_double */ + NULL, /* set_float */ + avro_generic_int_set, + NULL, /* set_long */ + NULL, /* set_null */ + NULL, /* set_string */ + NULL, /* set_string_length */ + NULL, /* give_string_length */ + NULL, /* set_enum */ + NULL, /* set_fixed */ + NULL, /* give_fixed */ + /* compound getters */ + NULL, /* get_size */ + NULL, /* get_by_index */ + NULL, /* get_by_name */ + NULL, /* get_discriminant */ + NULL, /* get_current_branch */ + /* compound setters */ + NULL, /* append */ + NULL, /* add */ + NULL /* set_branch */ + }, + avro_generic_int_instance_size, + avro_generic_int_init, + avro_generic_int_done +}; + +avro_value_iface_t * +avro_generic_int_class(void) +{ + return &AVRO_GENERIC_INT_CLASS.parent; +} + +int +avro_generic_int_new(avro_value_t *value, int32_t val) +{ + int rval; + check(rval, avro_generic_value_new(&AVRO_GENERIC_INT_CLASS.parent, value)); + return avro_generic_int_set(value->iface, value->self, val); +} + +/*----------------------------------------------------------------------- + * long + */ + +static int +avro_generic_long_reset(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + int64_t *self = (int64_t *) vself; + *self = 0; + return 0; +} + +static avro_type_t +avro_generic_long_get_type(const avro_value_iface_t *iface, const void *vself) +{ + AVRO_UNUSED(iface); + AVRO_UNUSED(vself); + return AVRO_INT64; +} + +static avro_schema_t +avro_generic_long_get_schema(const avro_value_iface_t *iface, const void *vself) +{ + AVRO_UNUSED(iface); + AVRO_UNUSED(vself); + return avro_schema_long(); +} + +static int +avro_generic_long_get(const avro_value_iface_t *iface, + const void *vself, int64_t *out) +{ + AVRO_UNUSED(iface); + const int64_t *self = (const int64_t *) vself; + *out = *self; + return 0; +} + +static int +avro_generic_long_set(const avro_value_iface_t *iface, + void *vself, int64_t val) +{ + AVRO_UNUSED(iface); + int64_t *self = (int64_t *) vself; + *self = val; + return 0; +} + +static size_t +avro_generic_long_instance_size(const avro_value_iface_t *iface) +{ + AVRO_UNUSED(iface); + return sizeof(int64_t); +} + +static int +avro_generic_long_init(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + int64_t *self = (int64_t *) vself; + *self = 0; + return 0; +} + +static void +avro_generic_long_done(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + AVRO_UNUSED(vself); +} + +static avro_generic_value_iface_t AVRO_GENERIC_LONG_CLASS = +{ + { + /* "class" methods */ + NULL, /* incref_iface */ + NULL, /* decref_iface */ + /* general "instance" methods */ + avro_generic_value_incref, + avro_generic_value_decref, + avro_generic_long_reset, + avro_generic_long_get_type, + avro_generic_long_get_schema, + /* primitive getters */ + NULL, /* get_boolean */ + NULL, /* get_bytes */ + NULL, /* grab_bytes */ + NULL, /* get_double */ + NULL, /* get_float */ + NULL, /* get_int */ + avro_generic_long_get, + NULL, /* get_null */ + NULL, /* get_string */ + NULL, /* grab_string */ + NULL, /* get_enum */ + NULL, /* get_fixed */ + NULL, /* grab_fixed */ + /* primitive setters */ + NULL, /* set_boolean */ + NULL, /* set_bytes */ + NULL, /* give_bytes */ + NULL, /* set_double */ + NULL, /* set_float */ + NULL, /* set_int */ + avro_generic_long_set, + NULL, /* set_null */ + NULL, /* set_string */ + NULL, /* set_string_length */ + NULL, /* give_string_length */ + NULL, /* set_enum */ + NULL, /* set_fixed */ + NULL, /* give_fixed */ + /* compound getters */ + NULL, /* get_size */ + NULL, /* get_by_index */ + NULL, /* get_by_name */ + NULL, /* get_discriminant */ + NULL, /* get_current_branch */ + /* compound setters */ + NULL, /* append */ + NULL, /* add */ + NULL /* set_branch */ + }, + avro_generic_long_instance_size, + avro_generic_long_init, + avro_generic_long_done +}; + +avro_value_iface_t * +avro_generic_long_class(void) +{ + return &AVRO_GENERIC_LONG_CLASS.parent; +} + +int +avro_generic_long_new(avro_value_t *value, int64_t val) +{ + int rval; + check(rval, avro_generic_value_new(&AVRO_GENERIC_LONG_CLASS.parent, value)); + return avro_generic_long_set(value->iface, value->self, val); +} + +/*----------------------------------------------------------------------- + * null + */ + +static int +avro_generic_null_reset(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + int *self = (int *) vself; + *self = 0; + return 0; +} + +static avro_type_t +avro_generic_null_get_type(const avro_value_iface_t *iface, const void *vself) +{ + AVRO_UNUSED(iface); + AVRO_UNUSED(vself); + return AVRO_NULL; +} + +static avro_schema_t +avro_generic_null_get_schema(const avro_value_iface_t *iface, const void *vself) +{ + AVRO_UNUSED(iface); + AVRO_UNUSED(vself); + return avro_schema_null(); +} + +static int +avro_generic_null_get(const avro_value_iface_t *iface, const void *vself) +{ + AVRO_UNUSED(iface); + AVRO_UNUSED(vself); + return 0; +} + +static int +avro_generic_null_set(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + AVRO_UNUSED(vself); + return 0; +} + +static size_t +avro_generic_null_instance_size(const avro_value_iface_t *iface) +{ + AVRO_UNUSED(iface); + return sizeof(int); +} + +static int +avro_generic_null_init(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + int *self = (int *) vself; + *self = 0; + return 0; +} + +static void +avro_generic_null_done(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + AVRO_UNUSED(vself); +} + +static avro_generic_value_iface_t AVRO_GENERIC_NULL_CLASS = +{ + { + /* "class" methods */ + NULL, /* incref_iface */ + NULL, /* decref_iface */ + /* general "instance" methods */ + avro_generic_value_incref, + avro_generic_value_decref, + avro_generic_null_reset, + avro_generic_null_get_type, + avro_generic_null_get_schema, + /* primitive getters */ + NULL, /* get_boolean */ + NULL, /* get_bytes */ + NULL, /* grab_bytes */ + NULL, /* get_double */ + NULL, /* get_float */ + NULL, /* get_int */ + NULL, /* get_long */ + avro_generic_null_get, + NULL, /* get_string */ + NULL, /* grab_string */ + NULL, /* get_enum */ + NULL, /* get_fixed */ + NULL, /* grab_fixed */ + /* primitive setters */ + NULL, /* set_boolean */ + NULL, /* set_bytes */ + NULL, /* give_bytes */ + NULL, /* set_double */ + NULL, /* set_float */ + NULL, /* set_int */ + NULL, /* set_long */ + avro_generic_null_set, + NULL, /* set_string */ + NULL, /* set_string_length */ + NULL, /* give_string_length */ + NULL, /* set_enum */ + NULL, /* set_fixed */ + NULL, /* give_fixed */ + /* compound getters */ + NULL, /* get_size */ + NULL, /* get_by_index */ + NULL, /* get_by_name */ + NULL, /* get_discriminant */ + NULL, /* get_current_branch */ + /* compound setters */ + NULL, /* append */ + NULL, /* add */ + NULL /* set_branch */ + }, + avro_generic_null_instance_size, + avro_generic_null_init, + avro_generic_null_done +}; + +avro_value_iface_t * +avro_generic_null_class(void) +{ + return &AVRO_GENERIC_NULL_CLASS.parent; +} + +int +avro_generic_null_new(avro_value_t *value) +{ + return avro_generic_value_new(&AVRO_GENERIC_NULL_CLASS.parent, value); +} + +/*----------------------------------------------------------------------- + * string + */ + +static int +avro_generic_string_reset(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + avro_raw_string_t *self = (avro_raw_string_t *) vself; + avro_raw_string_clear(self); + return 0; +} + +static avro_type_t +avro_generic_string_get_type(const avro_value_iface_t *iface, const void *vself) +{ + AVRO_UNUSED(iface); + AVRO_UNUSED(vself); + return AVRO_STRING; +} + +static avro_schema_t +avro_generic_string_get_schema(const avro_value_iface_t *iface, const void *vself) +{ + AVRO_UNUSED(iface); + AVRO_UNUSED(vself); + return avro_schema_string(); +} + +static int +avro_generic_string_get(const avro_value_iface_t *iface, + const void *vself, const char **str, size_t *size) +{ + AVRO_UNUSED(iface); + const avro_raw_string_t *self = (const avro_raw_string_t *) vself; + const char *contents = (const char *) avro_raw_string_get(self); + + if (str != NULL) { + /* + * We can't return a NULL string, we have to return an + * *empty* string + */ + + *str = (contents == NULL)? "": contents; + } + if (size != NULL) { + /* raw_string's length includes the NUL terminator, + * unless it's empty */ + *size = (contents == NULL)? 1: avro_raw_string_length(self); + } + return 0; +} + +static int +avro_generic_string_grab(const avro_value_iface_t *iface, + const void *vself, avro_wrapped_buffer_t *dest) +{ + AVRO_UNUSED(iface); + const avro_raw_string_t *self = (const avro_raw_string_t *) vself; + const char *contents = (const char *) avro_raw_string_get(self); + + if (contents == NULL) { + return avro_wrapped_buffer_new(dest, "", 1); + } else { + return avro_raw_string_grab(self, dest); + } +} + +static int +avro_generic_string_set(const avro_value_iface_t *iface, + void *vself, const char *val) +{ + AVRO_UNUSED(iface); + check_param(EINVAL, val != NULL, "string contents"); + + /* + * This raw_string method ensures that we copy the NUL + * terminator from val, and will include the NUL terminator in + * the raw_string's length, which is what we want. + */ + avro_raw_string_t *self = (avro_raw_string_t *) vself; + avro_raw_string_set(self, val); + return 0; +} + +static int +avro_generic_string_set_length(const avro_value_iface_t *iface, + void *vself, const char *val, size_t size) +{ + AVRO_UNUSED(iface); + check_param(EINVAL, val != NULL, "string contents"); + avro_raw_string_t *self = (avro_raw_string_t *) vself; + avro_raw_string_set_length(self, val, size); + return 0; +} + +static int +avro_generic_string_give_length(const avro_value_iface_t *iface, + void *vself, avro_wrapped_buffer_t *buf) +{ + AVRO_UNUSED(iface); + avro_raw_string_t *self = (avro_raw_string_t *) vself; + avro_raw_string_give(self, buf); + return 0; +} + +static size_t +avro_generic_string_instance_size(const avro_value_iface_t *iface) +{ + AVRO_UNUSED(iface); + return sizeof(avro_raw_string_t); +} + +static int +avro_generic_string_init(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + avro_raw_string_t *self = (avro_raw_string_t *) vself; + avro_raw_string_init(self); + return 0; +} + +static void +avro_generic_string_done(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + avro_raw_string_t *self = (avro_raw_string_t *) vself; + avro_raw_string_done(self); +} + +static avro_generic_value_iface_t AVRO_GENERIC_STRING_CLASS = +{ + { + /* "class" methods */ + NULL, /* incref_iface */ + NULL, /* decref_iface */ + /* general "instance" methods */ + avro_generic_value_incref, + avro_generic_value_decref, + avro_generic_string_reset, + avro_generic_string_get_type, + avro_generic_string_get_schema, + /* primitive getters */ + NULL, /* get_boolean */ + NULL, /* get_bytes */ + NULL, /* grab_bytes */ + NULL, /* get_double */ + NULL, /* get_float */ + NULL, /* get_int */ + NULL, /* get_long */ + NULL, /* get_null */ + avro_generic_string_get, + avro_generic_string_grab, + NULL, /* get_enum */ + NULL, /* get_fixed */ + NULL, /* grab_fixed */ + /* primitive setters */ + NULL, /* set_boolean */ + NULL, /* set_bytes */ + NULL, /* give_bytes */ + NULL, /* set_double */ + NULL, /* set_float */ + NULL, /* set_int */ + NULL, /* set_long */ + NULL, /* set_null */ + avro_generic_string_set, + avro_generic_string_set_length, + avro_generic_string_give_length, + NULL, /* set_enum */ + NULL, /* set_fixed */ + NULL, /* give_fixed */ + /* compound getters */ + NULL, /* get_size */ + NULL, /* get_by_index */ + NULL, /* get_by_name */ + NULL, /* get_discriminant */ + NULL, /* get_current_branch */ + /* compound setters */ + NULL, /* append */ + NULL, /* add */ + NULL /* set_branch */ + }, + avro_generic_string_instance_size, + avro_generic_string_init, + avro_generic_string_done +}; + +avro_value_iface_t * +avro_generic_string_class(void) +{ + return &AVRO_GENERIC_STRING_CLASS.parent; +} + +int +avro_generic_string_new(avro_value_t *value, const char *str) +{ + int rval; + check(rval, avro_generic_value_new(&AVRO_GENERIC_STRING_CLASS.parent, value)); + return avro_generic_string_set(value->iface, value->self, str); +} + +int +avro_generic_string_new_length(avro_value_t *value, const char *str, size_t size) +{ + int rval; + check(rval, avro_generic_value_new(&AVRO_GENERIC_STRING_CLASS.parent, value)); + return avro_generic_string_set_length(value->iface, value->self, str, size); +} + + +/*----------------------------------------------------------------------- + * array + */ + +/* + * For generic arrays, we need to store the value implementation for the + * array's elements. + */ + +typedef struct avro_generic_array_value_iface { + avro_generic_value_iface_t parent; + volatile int refcount; + avro_schema_t schema; + avro_generic_value_iface_t *child_giface; +} avro_generic_array_value_iface_t; + +typedef struct avro_generic_array { + avro_raw_array_t array; +} avro_generic_array_t; + + +static avro_value_iface_t * +avro_generic_array_incref_iface(avro_value_iface_t *viface) +{ + avro_generic_array_value_iface_t *iface = + container_of(viface, avro_generic_array_value_iface_t, parent); + avro_refcount_inc(&iface->refcount); + return viface; +} + +static void +avro_generic_array_decref_iface(avro_value_iface_t *viface) +{ + avro_generic_array_value_iface_t *iface = + container_of(viface, avro_generic_array_value_iface_t, parent); + if (avro_refcount_dec(&iface->refcount)) { + avro_schema_decref(iface->schema); + avro_value_iface_decref(&iface->child_giface->parent); + avro_freet(avro_generic_array_value_iface_t, iface); + } +} + + +static void +avro_generic_array_free_elements(const avro_generic_value_iface_t *child_giface, + avro_generic_array_t *self) +{ + size_t i; + for (i = 0; i < avro_raw_array_size(&self->array); i++) { + void *child_self = avro_raw_array_get_raw(&self->array, i); + avro_value_done(child_giface, child_self); + } +} + +static int +avro_generic_array_reset(const avro_value_iface_t *viface, void *vself) +{ + const avro_generic_array_value_iface_t *iface = + container_of(viface, avro_generic_array_value_iface_t, parent); + avro_generic_array_t *self = (avro_generic_array_t *) vself; + avro_generic_array_free_elements(iface->child_giface, self); + avro_raw_array_clear(&self->array); + return 0; +} + +static avro_type_t +avro_generic_array_get_type(const avro_value_iface_t *viface, const void *vself) +{ + AVRO_UNUSED(viface); + AVRO_UNUSED(vself); + return AVRO_ARRAY; +} + +static avro_schema_t +avro_generic_array_get_schema(const avro_value_iface_t *viface, const void *vself) +{ + const avro_generic_array_value_iface_t *iface = + container_of(viface, avro_generic_array_value_iface_t, parent); + AVRO_UNUSED(vself); + return iface->schema; +} + +static int +avro_generic_array_get_size(const avro_value_iface_t *viface, + const void *vself, size_t *size) +{ + AVRO_UNUSED(viface); + const avro_generic_array_t *self = (const avro_generic_array_t *) vself; + if (size != NULL) { + *size = avro_raw_array_size(&self->array); + } + return 0; +} + +static int +avro_generic_array_get_by_index(const avro_value_iface_t *viface, + const void *vself, size_t index, + avro_value_t *child, const char **name) +{ + const avro_generic_array_value_iface_t *iface = + container_of(viface, avro_generic_array_value_iface_t, parent); + AVRO_UNUSED(name); + const avro_generic_array_t *self = (avro_generic_array_t *) vself; + if (index >= avro_raw_array_size(&self->array)) { + avro_set_error("Array index %" PRIsz " out of range", index); + return EINVAL; + } + child->iface = &iface->child_giface->parent; + child->self = avro_raw_array_get_raw(&self->array, index); + return 0; +} + +static int +avro_generic_array_append(const avro_value_iface_t *viface, + void *vself, avro_value_t *child, + size_t *new_index) +{ + int rval; + const avro_generic_array_value_iface_t *iface = + container_of(viface, avro_generic_array_value_iface_t, parent); + avro_generic_array_t *self = (avro_generic_array_t *) vself; + child->iface = &iface->child_giface->parent; + child->self = avro_raw_array_append(&self->array); + if (child->self == NULL) { + avro_set_error("Couldn't expand array"); + return ENOMEM; + } + check(rval, avro_value_init(iface->child_giface, child->self)); + if (new_index != NULL) { + *new_index = avro_raw_array_size(&self->array) - 1; + } + return 0; +} + +static size_t +avro_generic_array_instance_size(const avro_value_iface_t *viface) +{ + AVRO_UNUSED(viface); + return sizeof(avro_generic_array_t); +} + +static int +avro_generic_array_init(const avro_value_iface_t *viface, void *vself) +{ + const avro_generic_array_value_iface_t *iface = + container_of(viface, avro_generic_array_value_iface_t, parent); + avro_generic_array_t *self = (avro_generic_array_t *) vself; + + size_t child_size = avro_value_instance_size(iface->child_giface); + avro_raw_array_init(&self->array, child_size); + return 0; +} + +static void +avro_generic_array_done(const avro_value_iface_t *viface, void *vself) +{ + const avro_generic_array_value_iface_t *iface = + container_of(viface, avro_generic_array_value_iface_t, parent); + avro_generic_array_t *self = (avro_generic_array_t *) vself; + avro_generic_array_free_elements(iface->child_giface, self); + avro_raw_array_done(&self->array); +} + +static avro_generic_value_iface_t AVRO_GENERIC_ARRAY_CLASS = +{ +{ + /* "class" methods */ + avro_generic_array_incref_iface, + avro_generic_array_decref_iface, + /* general "instance" methods */ + avro_generic_value_incref, + avro_generic_value_decref, + avro_generic_array_reset, + avro_generic_array_get_type, + avro_generic_array_get_schema, + /* primitive getters */ + NULL, /* get_boolean */ + NULL, /* get_bytes */ + NULL, /* grab_bytes */ + NULL, /* get_double */ + NULL, /* get_float */ + NULL, /* get_int */ + NULL, /* get_long */ + NULL, /* get_null */ + NULL, /* get_string */ + NULL, /* grab_string */ + NULL, /* get_enum */ + NULL, /* get_fixed */ + NULL, /* grab_fixed */ + /* primitive setters */ + NULL, /* set_boolean */ + NULL, /* set_bytes */ + NULL, /* give_bytes */ + NULL, /* set_double */ + NULL, /* set_float */ + NULL, /* set_int */ + NULL, /* set_long */ + NULL, /* set_null */ + NULL, /* set_string */ + NULL, /* set_string_length */ + NULL, /* give_string_length */ + NULL, /* set_enum */ + NULL, /* set_fixed */ + NULL, /* give_fixed */ + /* compound getters */ + avro_generic_array_get_size, + avro_generic_array_get_by_index, + NULL, /* get_by_name */ + NULL, /* get_discriminant */ + NULL, /* get_current_branch */ + /* compound setters */ + avro_generic_array_append, + NULL, /* add */ + NULL /* set_branch */ +}, + avro_generic_array_instance_size, + avro_generic_array_init, + avro_generic_array_done +}; + +static avro_generic_value_iface_t * +avro_generic_array_class(avro_schema_t schema, memoize_state_t *state) +{ + avro_schema_t child_schema = avro_schema_array_items(schema); + avro_generic_value_iface_t *child_giface = + avro_generic_class_from_schema_memoized(child_schema, state); + if (child_giface == NULL) { + return NULL; + } + + ssize_t child_size = avro_value_instance_size(child_giface); + if (child_size < 0) { + avro_set_error("Array item class must provide instance_size"); + avro_value_iface_decref(&child_giface->parent); + return NULL; + } + + avro_generic_array_value_iface_t *iface = + (avro_generic_array_value_iface_t *) avro_new(avro_generic_array_value_iface_t); + if (iface == NULL) { + avro_value_iface_decref(&child_giface->parent); + return NULL; + } + + /* + * TODO: Maybe check that schema.items matches + * child_iface.get_schema? + */ + + iface->parent = AVRO_GENERIC_ARRAY_CLASS; + iface->refcount = 1; + iface->schema = avro_schema_incref(schema); + iface->child_giface = child_giface; + return &iface->parent; +} + + +/*----------------------------------------------------------------------- + * enum + */ + +typedef struct avro_generic_enum_value_iface { + avro_generic_value_iface_t parent; + volatile int refcount; + avro_schema_t schema; +} avro_generic_enum_value_iface_t; + + +static avro_value_iface_t * +avro_generic_enum_incref_iface(avro_value_iface_t *viface) +{ + avro_generic_enum_value_iface_t *iface = + (avro_generic_enum_value_iface_t *) viface; + avro_refcount_inc(&iface->refcount); + return viface; +} + +static void +avro_generic_enum_decref_iface(avro_value_iface_t *viface) +{ + avro_generic_enum_value_iface_t *iface = + (avro_generic_enum_value_iface_t *) viface; + if (avro_refcount_dec(&iface->refcount)) { + avro_schema_decref(iface->schema); + avro_freet(avro_generic_enum_value_iface_t, iface); + } +} + +static int +avro_generic_enum_reset(const avro_value_iface_t *viface, void *vself) +{ + AVRO_UNUSED(viface); + int *self = (int *) vself; + *self = 0; + return 0; +} + +static avro_type_t +avro_generic_enum_get_type(const avro_value_iface_t *iface, const void *vself) +{ + AVRO_UNUSED(iface); + AVRO_UNUSED(vself); + return AVRO_ENUM; +} + +static avro_schema_t +avro_generic_enum_get_schema(const avro_value_iface_t *viface, const void *vself) +{ + const avro_generic_enum_value_iface_t *iface = + container_of(viface, avro_generic_enum_value_iface_t, parent); + AVRO_UNUSED(vself); + return iface->schema; +} + +static int +avro_generic_enum_get(const avro_value_iface_t *viface, + const void *vself, int *out) +{ + AVRO_UNUSED(viface); + const int *self = (const int *) vself; + *out = *self; + return 0; +} + +static int +avro_generic_enum_set(const avro_value_iface_t *viface, + void *vself, int val) +{ + AVRO_UNUSED(viface); + int *self = (int *) vself; + *self = val; + return 0; +} + +static size_t +avro_generic_enum_instance_size(const avro_value_iface_t *viface) +{ + AVRO_UNUSED(viface); + return sizeof(int); +} + +static int +avro_generic_enum_init(const avro_value_iface_t *viface, void *vself) +{ + AVRO_UNUSED(viface); + int *self = (int *) vself; + *self = 0; + return 0; +} + +static void +avro_generic_enum_done(const avro_value_iface_t *viface, void *vself) +{ + AVRO_UNUSED(viface); + AVRO_UNUSED(vself); +} + +static avro_generic_value_iface_t AVRO_GENERIC_ENUM_CLASS = +{ + { + /* "class" methods */ + avro_generic_enum_incref_iface, + avro_generic_enum_decref_iface, + /* general "instance" methods */ + avro_generic_value_incref, + avro_generic_value_decref, + avro_generic_enum_reset, + avro_generic_enum_get_type, + avro_generic_enum_get_schema, + /* primitive getters */ + NULL, /* get_boolean */ + NULL, /* get_bytes */ + NULL, /* grab_bytes */ + NULL, /* get_double */ + NULL, /* get_float */ + NULL, /* get_int */ + NULL, /* get_long */ + NULL, /* get_null */ + NULL, /* get_string */ + NULL, /* grab_string */ + avro_generic_enum_get, + NULL, /* get_fixed */ + NULL, /* grab_fixed */ + /* primitive setters */ + NULL, /* set_boolean */ + NULL, /* set_bytes */ + NULL, /* give_bytes */ + NULL, /* set_double */ + NULL, /* set_float */ + NULL, /* set_int */ + NULL, /* set_long */ + NULL, /* set_null */ + NULL, /* set_string */ + NULL, /* set_string_length */ + NULL, /* give_string_length */ + avro_generic_enum_set, + NULL, /* set_fixed */ + NULL, /* give_fixed */ + /* compound getters */ + NULL, /* get_size */ + NULL, /* get_by_index */ + NULL, /* get_by_name */ + NULL, /* get_discriminant */ + NULL, /* get_current_branch */ + /* compound setters */ + NULL, /* append */ + NULL, /* add */ + NULL /* set_branch */ + }, + avro_generic_enum_instance_size, + avro_generic_enum_init, + avro_generic_enum_done +}; + +static avro_generic_value_iface_t * +avro_generic_enum_class(avro_schema_t schema) +{ + avro_generic_enum_value_iface_t *iface = + (avro_generic_enum_value_iface_t *) avro_new(avro_generic_enum_value_iface_t); + if (iface == NULL) { + return NULL; + } + + iface->parent = AVRO_GENERIC_ENUM_CLASS; + iface->refcount = 1; + iface->schema = avro_schema_incref(schema); + return &iface->parent; +} + + +/*----------------------------------------------------------------------- + * fixed + */ + +typedef struct avro_generic_fixed_value_iface { + avro_generic_value_iface_t parent; + volatile int refcount; + avro_schema_t schema; + size_t data_size; +} avro_generic_fixed_value_iface_t; + + +static avro_value_iface_t * +avro_generic_fixed_incref_iface(avro_value_iface_t *viface) +{ + avro_generic_fixed_value_iface_t *iface = + container_of(viface, avro_generic_fixed_value_iface_t, parent); + avro_refcount_inc(&iface->refcount); + return viface; +} + +static void +avro_generic_fixed_decref_iface(avro_value_iface_t *viface) +{ + avro_generic_fixed_value_iface_t *iface = + container_of(viface, avro_generic_fixed_value_iface_t, parent); + if (avro_refcount_dec(&iface->refcount)) { + avro_schema_decref(iface->schema); + avro_freet(avro_generic_fixed_value_iface_t, iface); + } +} + +static int +avro_generic_fixed_reset(const avro_value_iface_t *viface, void *vself) +{ + const avro_generic_fixed_value_iface_t *iface = + container_of(viface, avro_generic_fixed_value_iface_t, parent); + memset(vself, 0, iface->data_size); + return 0; +} + +static avro_type_t +avro_generic_fixed_get_type(const avro_value_iface_t *iface, const void *vself) +{ + AVRO_UNUSED(iface); + AVRO_UNUSED(vself); + return AVRO_FIXED; +} + +static avro_schema_t +avro_generic_fixed_get_schema(const avro_value_iface_t *viface, const void *vself) +{ + const avro_generic_fixed_value_iface_t *iface = + container_of(viface, avro_generic_fixed_value_iface_t, parent); + AVRO_UNUSED(vself); + return iface->schema; +} + +static int +avro_generic_fixed_get(const avro_value_iface_t *viface, + const void *vself, const void **buf, size_t *size) +{ + const avro_generic_fixed_value_iface_t *iface = + container_of(viface, avro_generic_fixed_value_iface_t, parent); + if (buf != NULL) { + *buf = vself; + } + if (size != NULL) { + *size = iface->data_size; + } + return 0; +} + +static int +avro_generic_fixed_grab(const avro_value_iface_t *viface, + const void *vself, avro_wrapped_buffer_t *dest) +{ + const avro_generic_fixed_value_iface_t *iface = + container_of(viface, avro_generic_fixed_value_iface_t, parent); + return avro_wrapped_buffer_new(dest, vself, iface->data_size); +} + +static int +avro_generic_fixed_set(const avro_value_iface_t *viface, + void *vself, void *buf, size_t size) +{ + check_param(EINVAL, buf != NULL, "fixed contents"); + const avro_generic_fixed_value_iface_t *iface = + container_of(viface, avro_generic_fixed_value_iface_t, parent); + if (size != iface->data_size) { + avro_set_error("Invalid data size in set_fixed"); + return EINVAL; + } + memcpy(vself, buf, size); + return 0; +} + +static int +avro_generic_fixed_give(const avro_value_iface_t *viface, + void *vself, avro_wrapped_buffer_t *buf) +{ + int rval = avro_generic_fixed_set + (viface, vself, (void *) buf->buf, buf->size); + avro_wrapped_buffer_free(buf); + return rval; +} + +static size_t +avro_generic_fixed_instance_size(const avro_value_iface_t *viface) +{ + const avro_generic_fixed_value_iface_t *iface = + container_of(viface, avro_generic_fixed_value_iface_t, parent); + return iface->data_size; +} + +static int +avro_generic_fixed_init(const avro_value_iface_t *viface, void *vself) +{ + const avro_generic_fixed_value_iface_t *iface = + container_of(viface, avro_generic_fixed_value_iface_t, parent); + memset(vself, 0, iface->data_size); + return 0; +} + +static void +avro_generic_fixed_done(const avro_value_iface_t *viface, void *vself) +{ + AVRO_UNUSED(viface); + AVRO_UNUSED(vself); +} + +static avro_generic_value_iface_t AVRO_GENERIC_FIXED_CLASS = +{ + { + /* "class" methods */ + avro_generic_fixed_incref_iface, + avro_generic_fixed_decref_iface, + /* general "instance" methods */ + avro_generic_value_incref, + avro_generic_value_decref, + avro_generic_fixed_reset, + avro_generic_fixed_get_type, + avro_generic_fixed_get_schema, + /* primitive getters */ + NULL, /* get_boolean */ + NULL, /* get_bytes */ + NULL, /* grab_bytes */ + NULL, /* get_double */ + NULL, /* get_float */ + NULL, /* get_int */ + NULL, /* get_long */ + NULL, /* get_null */ + NULL, /* get_string */ + NULL, /* grab_string */ + NULL, /* get_enum */ + avro_generic_fixed_get, + avro_generic_fixed_grab, + /* primitive setters */ + NULL, /* set_boolean */ + NULL, /* set_bytes */ + NULL, /* give_bytes */ + NULL, /* set_double */ + NULL, /* set_float */ + NULL, /* set_int */ + NULL, /* set_long */ + NULL, /* set_null */ + NULL, /* set_string */ + NULL, /* set_string_length */ + NULL, /* give_string_length */ + NULL, /* set_enum */ + avro_generic_fixed_set, + avro_generic_fixed_give, + /* compound getters */ + NULL, /* get_size */ + NULL, /* get_by_index */ + NULL, /* get_by_name */ + NULL, /* get_discriminant */ + NULL, /* get_current_branch */ + /* compound setters */ + NULL, /* append */ + NULL, /* add */ + NULL /* set_branch */ + }, + avro_generic_fixed_instance_size, + avro_generic_fixed_init, + avro_generic_fixed_done +}; + +static avro_generic_value_iface_t * +avro_generic_fixed_class(avro_schema_t schema) +{ + avro_generic_fixed_value_iface_t *iface = + (avro_generic_fixed_value_iface_t *) avro_new(avro_generic_fixed_value_iface_t); + if (iface == NULL) { + return NULL; + } + + iface->parent = AVRO_GENERIC_FIXED_CLASS; + iface->refcount = 1; + iface->schema = avro_schema_incref(schema); + iface->data_size = avro_schema_fixed_size(schema); + return &iface->parent; +} + + +/*----------------------------------------------------------------------- + * map + */ + +/* + * For generic maps, we need to store the value implementation for the + * map's elements. + */ + +typedef struct avro_generic_map_value_iface { + avro_generic_value_iface_t parent; + volatile int refcount; + avro_schema_t schema; + avro_generic_value_iface_t *child_giface; +} avro_generic_map_value_iface_t; + +typedef struct avro_generic_map { + avro_raw_map_t map; +} avro_generic_map_t; + + +static avro_value_iface_t * +avro_generic_map_incref_iface(avro_value_iface_t *viface) +{ + avro_generic_map_value_iface_t *iface = + container_of(viface, avro_generic_map_value_iface_t, parent); + avro_refcount_inc(&iface->refcount); + return viface; +} + +static void +avro_generic_map_decref_iface(avro_value_iface_t *viface) +{ + avro_generic_map_value_iface_t *iface = + container_of(viface, avro_generic_map_value_iface_t, parent); + if (avro_refcount_dec(&iface->refcount)) { + avro_schema_decref(iface->schema); + avro_value_iface_decref(&iface->child_giface->parent); + avro_freet(avro_generic_map_value_iface_t, iface); + } +} + + +static void +avro_generic_map_free_elements(const avro_generic_value_iface_t *child_giface, + avro_generic_map_t *self) +{ + size_t i; + for (i = 0; i < avro_raw_map_size(&self->map); i++) { + void *child_self = avro_raw_map_get_raw(&self->map, i); + avro_value_done(child_giface, child_self); + } +} + +static int +avro_generic_map_reset(const avro_value_iface_t *viface, void *vself) +{ + const avro_generic_map_value_iface_t *iface = + container_of(viface, avro_generic_map_value_iface_t, parent); + avro_generic_map_t *self = (avro_generic_map_t *) vself; + avro_generic_map_free_elements(iface->child_giface, self); + avro_raw_map_clear(&self->map); + return 0; +} + +static avro_type_t +avro_generic_map_get_type(const avro_value_iface_t *viface, const void *vself) +{ + AVRO_UNUSED(viface); + AVRO_UNUSED(vself); + return AVRO_MAP; +} + +static avro_schema_t +avro_generic_map_get_schema(const avro_value_iface_t *viface, const void *vself) +{ + const avro_generic_map_value_iface_t *iface = + container_of(viface, avro_generic_map_value_iface_t, parent); + AVRO_UNUSED(vself); + return iface->schema; +} + +static int +avro_generic_map_get_size(const avro_value_iface_t *viface, + const void *vself, size_t *size) +{ + AVRO_UNUSED(viface); + const avro_generic_map_t *self = (const avro_generic_map_t *) vself; + if (size != NULL) { + *size = avro_raw_map_size(&self->map); + } + return 0; +} + +static int +avro_generic_map_get_by_index(const avro_value_iface_t *viface, + const void *vself, size_t index, + avro_value_t *child, const char **name) +{ + const avro_generic_map_value_iface_t *iface = + container_of(viface, avro_generic_map_value_iface_t, parent); + const avro_generic_map_t *self = (const avro_generic_map_t *) vself; + if (index >= avro_raw_map_size(&self->map)) { + avro_set_error("Map index %" PRIsz " out of range", index); + return EINVAL; + } + child->iface = &iface->child_giface->parent; + child->self = avro_raw_map_get_raw(&self->map, index); + if (name != NULL) { + *name = avro_raw_map_get_key(&self->map, index); + } + return 0; +} + +static int +avro_generic_map_get_by_name(const avro_value_iface_t *viface, + const void *vself, const char *name, + avro_value_t *child, size_t *index) +{ + const avro_generic_map_value_iface_t *iface = + container_of(viface, avro_generic_map_value_iface_t, parent); + const avro_generic_map_t *self = (const avro_generic_map_t *) vself; + child->iface = &iface->child_giface->parent; + child->self = avro_raw_map_get(&self->map, name, index); + if (child->self == NULL) { + avro_set_error("No map element named %s", name); + return EINVAL; + } + return 0; +} + +static int +avro_generic_map_add(const avro_value_iface_t *viface, + void *vself, const char *key, + avro_value_t *child, size_t *index, int *is_new) +{ + const avro_generic_map_value_iface_t *iface = + container_of(viface, avro_generic_map_value_iface_t, parent); + int rval; + avro_generic_map_t *self = (avro_generic_map_t *) vself; + child->iface = &iface->child_giface->parent; + rval = avro_raw_map_get_or_create(&self->map, key, + &child->self, index); + if (rval < 0) { + return -rval; + } + if (is_new != NULL) { + *is_new = rval; + } + if (rval) { + check(rval, avro_value_init(iface->child_giface, child->self)); + } + return 0; +} + +static size_t +avro_generic_map_instance_size(const avro_value_iface_t *viface) +{ + AVRO_UNUSED(viface); + return sizeof(avro_generic_map_t); +} + +static int +avro_generic_map_init(const avro_value_iface_t *viface, void *vself) +{ + const avro_generic_map_value_iface_t *iface = + container_of(viface, avro_generic_map_value_iface_t, parent); + avro_generic_map_t *self = (avro_generic_map_t *) vself; + + size_t child_size = avro_value_instance_size(iface->child_giface); + avro_raw_map_init(&self->map, child_size); + return 0; +} + +static void +avro_generic_map_done(const avro_value_iface_t *viface, void *vself) +{ + const avro_generic_map_value_iface_t *iface = + container_of(viface, avro_generic_map_value_iface_t, parent); + avro_generic_map_t *self = (avro_generic_map_t *) vself; + avro_generic_map_free_elements(iface->child_giface, self); + avro_raw_map_done(&self->map); +} + +static avro_generic_value_iface_t AVRO_GENERIC_MAP_CLASS = +{ + { + /* "class" methods */ + avro_generic_map_incref_iface, + avro_generic_map_decref_iface, + /* general "instance" methods */ + avro_generic_value_incref, + avro_generic_value_decref, + avro_generic_map_reset, + avro_generic_map_get_type, + avro_generic_map_get_schema, + /* primitive getters */ + NULL, /* get_boolean */ + NULL, /* get_bytes */ + NULL, /* grab_bytes */ + NULL, /* get_double */ + NULL, /* get_float */ + NULL, /* get_int */ + NULL, /* get_long */ + NULL, /* get_null */ + NULL, /* get_string */ + NULL, /* grab_string */ + NULL, /* get_enum */ + NULL, /* get_fixed */ + NULL, /* grab_fixed */ + /* primitive setters */ + NULL, /* set_boolean */ + NULL, /* set_bytes */ + NULL, /* give_bytes */ + NULL, /* set_double */ + NULL, /* set_float */ + NULL, /* set_int */ + NULL, /* set_long */ + NULL, /* set_null */ + NULL, /* set_string */ + NULL, /* set_string_length */ + NULL, /* give_string_length */ + NULL, /* set_enum */ + NULL, /* set_fixed */ + NULL, /* give_fixed */ + /* compound getters */ + avro_generic_map_get_size, + avro_generic_map_get_by_index, + avro_generic_map_get_by_name, + NULL, /* get_discriminant */ + NULL, /* get_current_branch */ + /* compound setters */ + NULL, /* append */ + avro_generic_map_add, + NULL /* set_branch */ + }, + avro_generic_map_instance_size, + avro_generic_map_init, + avro_generic_map_done +}; + +static avro_generic_value_iface_t * +avro_generic_map_class(avro_schema_t schema, memoize_state_t *state) +{ + avro_schema_t child_schema = avro_schema_array_items(schema); + avro_generic_value_iface_t *child_giface = + avro_generic_class_from_schema_memoized(child_schema, state); + if (child_giface == NULL) { + return NULL; + } + + ssize_t child_size = avro_value_instance_size(child_giface); + if (child_size < 0) { + avro_set_error("Map value class must provide instance_size"); + avro_value_iface_decref(&child_giface->parent); + return NULL; + } + + avro_generic_map_value_iface_t *iface = + (avro_generic_map_value_iface_t *) avro_new(avro_generic_map_value_iface_t); + if (iface == NULL) { + avro_value_iface_decref(&child_giface->parent); + return NULL; + } + + /* + * TODO: Maybe check that schema.items matches + * child_iface.get_schema? + */ + + iface->parent = AVRO_GENERIC_MAP_CLASS; + iface->refcount = 1; + iface->schema = avro_schema_incref(schema); + iface->child_giface = child_giface; + return &iface->parent; +} + + +/*----------------------------------------------------------------------- + * record + */ + +#ifndef DEBUG_FIELD_OFFSETS +#define DEBUG_FIELD_OFFSETS 0 +#endif + +#if DEBUG_FIELD_OFFSETS +#include <stdio.h> +#endif + +/* + * For generic records, we need to store the value implementation for + * each field. We also need to store an offset for each field, since + * we're going to store the contents of each field directly in the + * record, rather than via pointers. + */ + +typedef struct avro_generic_record_value_iface { + avro_generic_value_iface_t parent; + volatile int refcount; + avro_schema_t schema; + + /** The total size of each value struct for this record. */ + size_t instance_size; + + /** The number of fields in this record. Yes, we could get this + * from schema, but this is easier. */ + size_t field_count; + + /** The offset of each field within the record struct. */ + size_t *field_offsets; + + /** The value implementation for each field. */ + avro_generic_value_iface_t **field_ifaces; +} avro_generic_record_value_iface_t; + +typedef struct avro_generic_record { + /* The rest of the struct is taken up by the inline storage + * needed for each field. */ +} avro_generic_record_t; + + +/** Return a pointer to the given field within a record struct. */ +#define avro_generic_record_field(iface, rec, index) \ + (((char *) (rec)) + (iface)->field_offsets[(index)]) + + +static avro_value_iface_t * +avro_generic_record_incref_iface(avro_value_iface_t *viface) +{ + avro_generic_record_value_iface_t *iface = + container_of(viface, avro_generic_record_value_iface_t, parent); + avro_refcount_inc(&iface->refcount); + return viface; +} + +static void +avro_generic_record_decref_iface(avro_value_iface_t *viface) +{ + avro_generic_record_value_iface_t *iface = + container_of(viface, avro_generic_record_value_iface_t, parent); + + if (avro_refcount_dec(&iface->refcount)) { + size_t i; + for (i = 0; i < iface->field_count; i++) { + avro_value_iface_decref(&iface->field_ifaces[i]->parent); + } + + avro_schema_decref(iface->schema); + avro_free(iface->field_offsets, + sizeof(size_t) * iface->field_count); + avro_free(iface->field_ifaces, + sizeof(avro_generic_value_iface_t *) * iface->field_count); + + avro_freet(avro_generic_record_value_iface_t, iface); + } +} + + +static int +avro_generic_record_reset(const avro_value_iface_t *viface, void *vself) +{ + const avro_generic_record_value_iface_t *iface = + container_of(viface, avro_generic_record_value_iface_t, parent); + int rval; + avro_generic_record_t *self = (avro_generic_record_t *) vself; + size_t i; + for (i = 0; i < iface->field_count; i++) { + avro_value_t value = { + &iface->field_ifaces[i]->parent, + avro_generic_record_field(iface, self, i) + }; + check(rval, avro_value_reset(&value)); + } + return 0; +} + +static avro_type_t +avro_generic_record_get_type(const avro_value_iface_t *viface, const void *vself) +{ + AVRO_UNUSED(viface); + AVRO_UNUSED(vself); + return AVRO_RECORD; +} + +static avro_schema_t +avro_generic_record_get_schema(const avro_value_iface_t *viface, const void *vself) +{ + const avro_generic_record_value_iface_t *iface = + container_of(viface, avro_generic_record_value_iface_t, parent); + AVRO_UNUSED(vself); + return iface->schema; +} + +static int +avro_generic_record_get_size(const avro_value_iface_t *viface, + const void *vself, size_t *size) +{ + const avro_generic_record_value_iface_t *iface = + container_of(viface, avro_generic_record_value_iface_t, parent); + AVRO_UNUSED(vself); + if (size != NULL) { + *size = iface->field_count; + } + return 0; +} + +static int +avro_generic_record_get_by_index(const avro_value_iface_t *viface, + const void *vself, size_t index, + avro_value_t *child, const char **name) +{ + const avro_generic_record_value_iface_t *iface = + container_of(viface, avro_generic_record_value_iface_t, parent); + const avro_generic_record_t *self = (const avro_generic_record_t *) vself; + if (index >= iface->field_count) { + avro_set_error("Field index %" PRIsz " out of range", index); + return EINVAL; + } + child->iface = &iface->field_ifaces[index]->parent; + child->self = avro_generic_record_field(iface, self, index); + + /* + * Grab the field name from the schema if asked for. + */ + if (name != NULL) { + avro_schema_t schema = iface->schema; + *name = avro_schema_record_field_name(schema, index); + } + + return 0; +} + +static int +avro_generic_record_get_by_name(const avro_value_iface_t *viface, + const void *vself, const char *name, + avro_value_t *child, size_t *index_out) +{ + const avro_generic_record_value_iface_t *iface = + container_of(viface, avro_generic_record_value_iface_t, parent); + const avro_generic_record_t *self = (const avro_generic_record_t *) vself; + + avro_schema_t schema = iface->schema; + int index = avro_schema_record_field_get_index(schema, name); + if (index < 0) { + avro_set_error("Unknown record field %s", name); + return EINVAL; + } + + child->iface = &iface->field_ifaces[index]->parent; + child->self = avro_generic_record_field(iface, self, index); + if (index_out != NULL) { + *index_out = index; + } + return 0; +} + +static size_t +avro_generic_record_instance_size(const avro_value_iface_t *viface) +{ + const avro_generic_record_value_iface_t *iface = + container_of(viface, avro_generic_record_value_iface_t, parent); + return iface->instance_size; +} + +static int +avro_generic_record_init(const avro_value_iface_t *viface, void *vself) +{ + int rval; + const avro_generic_record_value_iface_t *iface = + container_of(viface, avro_generic_record_value_iface_t, parent); + avro_generic_record_t *self = (avro_generic_record_t *) vself; + + /* Initialize each field */ + size_t i; + for (i = 0; i < iface->field_count; i++) { + check(rval, avro_value_init + (iface->field_ifaces[i], + avro_generic_record_field(iface, self, i))); + } + + return 0; +} + +static void +avro_generic_record_done(const avro_value_iface_t *viface, void *vself) +{ + const avro_generic_record_value_iface_t *iface = + container_of(viface, avro_generic_record_value_iface_t, parent); + avro_generic_record_t *self = (avro_generic_record_t *) vself; + size_t i; + for (i = 0; i < iface->field_count; i++) { + avro_value_done(iface->field_ifaces[i], + avro_generic_record_field(iface, self, i)); + } +} + +static avro_generic_value_iface_t AVRO_GENERIC_RECORD_CLASS = +{ + { + /* "class" methods */ + avro_generic_record_incref_iface, + avro_generic_record_decref_iface, + /* general "instance" methods */ + avro_generic_value_incref, + avro_generic_value_decref, + avro_generic_record_reset, + avro_generic_record_get_type, + avro_generic_record_get_schema, + /* primitive getters */ + NULL, /* get_boolean */ + NULL, /* get_bytes */ + NULL, /* grab_bytes */ + NULL, /* get_double */ + NULL, /* get_float */ + NULL, /* get_int */ + NULL, /* get_long */ + NULL, /* get_null */ + NULL, /* get_string */ + NULL, /* grab_string */ + NULL, /* get_enum */ + NULL, /* get_fixed */ + NULL, /* grab_fixed */ + /* primitive setters */ + NULL, /* set_boolean */ + NULL, /* set_bytes */ + NULL, /* give_bytes */ + NULL, /* set_double */ + NULL, /* set_float */ + NULL, /* set_int */ + NULL, /* set_long */ + NULL, /* set_null */ + NULL, /* set_string */ + NULL, /* set_string_length */ + NULL, /* give_string_length */ + NULL, /* set_enum */ + NULL, /* set_fixed */ + NULL, /* give_fixed */ + /* compound getters */ + avro_generic_record_get_size, + avro_generic_record_get_by_index, + avro_generic_record_get_by_name, + NULL, /* get_discriminant */ + NULL, /* get_current_branch */ + /* compound setters */ + NULL, /* append */ + NULL, /* add */ + NULL /* set_branch */ + }, + avro_generic_record_instance_size, + avro_generic_record_init, + avro_generic_record_done +}; + +static avro_generic_value_iface_t * +avro_generic_record_class(avro_schema_t schema, memoize_state_t *state) +{ + avro_generic_record_value_iface_t *iface = + (avro_generic_record_value_iface_t *) avro_new(avro_generic_record_value_iface_t); + if (iface == NULL) { + return NULL; + } + + memset(iface, 0, sizeof(avro_generic_record_value_iface_t)); + iface->parent = AVRO_GENERIC_RECORD_CLASS; + iface->refcount = 1; + iface->schema = avro_schema_incref(schema); + + iface->field_count = avro_schema_record_size(schema); + size_t field_offsets_size = + sizeof(size_t) * iface->field_count; + size_t field_ifaces_size = + sizeof(avro_generic_value_iface_t *) * iface->field_count; + + if (iface->field_count == 0) { + iface->field_offsets = NULL; + iface->field_ifaces = NULL; + } else { + iface->field_offsets = (size_t *) avro_malloc(field_offsets_size); + if (iface->field_offsets == NULL) { + goto error; + } + + iface->field_ifaces = (avro_generic_value_iface_t **) avro_malloc(field_ifaces_size); + if (iface->field_ifaces == NULL) { + goto error; + } + } + + size_t next_offset = sizeof(avro_generic_record_t); +#if DEBUG_FIELD_OFFSETS + fprintf(stderr, " Record %s\n Header: Offset 0, size %" PRIsz "\n", + avro_schema_type_name(schema), + sizeof(avro_generic_record_t)); +#endif + size_t i; + for (i = 0; i < iface->field_count; i++) { +#if DEBUG_FIELD_OFFSETS + fprintf(stderr, " Field %" PRIsz ":\n", i); +#endif + avro_schema_t field_schema = + avro_schema_record_field_get_by_index(schema, i); +#if DEBUG_FIELD_OFFSETS + fprintf(stderr, " Schema %s\n", + avro_schema_type_name(field_schema)); +#endif + + iface->field_offsets[i] = next_offset; + + iface->field_ifaces[i] = + avro_generic_class_from_schema_memoized(field_schema, state); + if (iface->field_ifaces[i] == NULL) { + goto error; + } + + ssize_t field_size = + avro_value_instance_size(iface->field_ifaces[i]); + if (field_size < 0) { + avro_set_error("Record field class must provide instance_size"); + goto error; + } + +#if DEBUG_FIELD_OFFSETS + fprintf(stderr, " Offset %" PRIsz ", size %" PRIsz "\n", + next_offset, field_size); +#endif + next_offset += field_size; + } + + iface->instance_size = next_offset; +#if DEBUG_FIELD_OFFSETS + fprintf(stderr, " TOTAL SIZE: %" PRIsz "\n", next_offset); +#endif + + return &iface->parent; + +error: + avro_schema_decref(iface->schema); + if (iface->field_offsets != NULL) { + avro_free(iface->field_offsets, field_offsets_size); + } + if (iface->field_ifaces != NULL) { + for (i = 0; i < iface->field_count; i++) { + if (iface->field_ifaces[i] != NULL) { + avro_value_iface_decref(&iface->field_ifaces[i]->parent); + } + } + avro_free(iface->field_ifaces, field_ifaces_size); + } + avro_freet(avro_generic_record_value_iface_t, iface); + return NULL; +} + + +/*----------------------------------------------------------------------- + * union + */ + +#ifndef DEBUG_BRANCHES_OFFSETS +#define DEBUG_BRANCHES_OFFSETS 0 +#endif + +#if DEBUG_BRANCHES_OFFSETS +#include <stdio.h> +#endif + +/* + * For generic unions, we need to store the value implementation for + * each branch, just like for generic records. However, for unions, we + * can only have one branch active at a time, so we can reuse the space + * in the union struct, just like is done with C unions. + */ + +typedef struct avro_generic_union_value_iface { + avro_generic_value_iface_t parent; + volatile int refcount; + avro_schema_t schema; + + /** The total size of each value struct for this union. */ + size_t instance_size; + + /** The number of branches in this union. Yes, we could get + * this from schema, but this is easier. */ + size_t branch_count; + + /** The value implementation for each branch. */ + avro_generic_value_iface_t **branch_ifaces; +} avro_generic_union_value_iface_t; + +typedef struct avro_generic_union { + /** The currently active branch of the union. -1 if no branch + * is selected. */ + int discriminant; + + /* The rest of the struct is taken up by the inline storage + * needed for the active branch. */ +} avro_generic_union_t; + + +/** Return the child interface for the active branch. */ +#define avro_generic_union_branch_giface(iface, _union) \ + ((iface)->branch_ifaces[(_union)->discriminant]) +#define avro_generic_union_branch_iface(iface, _union) \ + (&(avro_generic_union_branch_giface((iface), (_union)))->parent) + +/** Return a pointer to the active branch within a union struct. */ +#define avro_generic_union_branch(_union) \ + (((char *) (_union)) + sizeof(avro_generic_union_t)) + + +static avro_value_iface_t * +avro_generic_union_incref_iface(avro_value_iface_t *viface) +{ + avro_generic_union_value_iface_t *iface = + container_of(viface, avro_generic_union_value_iface_t, parent); + avro_refcount_inc(&iface->refcount); + return viface; +} + +static void +avro_generic_union_decref_iface(avro_value_iface_t *viface) +{ + avro_generic_union_value_iface_t *iface = + container_of(viface, avro_generic_union_value_iface_t, parent); + + if (avro_refcount_dec(&iface->refcount)) { + size_t i; + for (i = 0; i < iface->branch_count; i++) { + avro_value_iface_decref(&iface->branch_ifaces[i]->parent); + } + + avro_schema_decref(iface->schema); + avro_free(iface->branch_ifaces, + sizeof(avro_generic_value_iface_t *) * iface->branch_count); + + avro_freet(avro_generic_union_value_iface_t, iface); + } +} + + +static int +avro_generic_union_reset(const avro_value_iface_t *viface, void *vself) +{ + const avro_generic_union_value_iface_t *iface = + container_of(viface, avro_generic_union_value_iface_t, parent); + avro_generic_union_t *self = (avro_generic_union_t *) vself; + /* Keep the same branch selected, for the common case that we're + * about to reuse it. */ + if (self->discriminant >= 0) { +#if DEBUG_BRANCHES + fprintf(stderr, "Resetting branch %d\n", + self->discriminant); +#endif + avro_value_t value = { + avro_generic_union_branch_iface(iface, self), + avro_generic_union_branch(self) + }; + return avro_value_reset(&value); + } + return 0; +} + +static avro_type_t +avro_generic_union_get_type(const avro_value_iface_t *viface, const void *vself) +{ + AVRO_UNUSED(viface); + AVRO_UNUSED(vself); + return AVRO_UNION; +} + +static avro_schema_t +avro_generic_union_get_schema(const avro_value_iface_t *viface, const void *vself) +{ + const avro_generic_union_value_iface_t *iface = + container_of(viface, avro_generic_union_value_iface_t, parent); + AVRO_UNUSED(vself); + return iface->schema; +} + +static int +avro_generic_union_get_discriminant(const avro_value_iface_t *viface, + const void *vself, int *out) +{ + AVRO_UNUSED(viface); + const avro_generic_union_t *self = (const avro_generic_union_t *) vself; + *out = self->discriminant; + return 0; +} + +static int +avro_generic_union_get_current_branch(const avro_value_iface_t *viface, + const void *vself, avro_value_t *branch) +{ + const avro_generic_union_value_iface_t *iface = + container_of(viface, avro_generic_union_value_iface_t, parent); + const avro_generic_union_t *self = (const avro_generic_union_t *) vself; + if (self->discriminant < 0) { + avro_set_error("Union has no selected branch"); + return EINVAL; + } + branch->iface = avro_generic_union_branch_iface(iface, self); + branch->self = avro_generic_union_branch(self); + return 0; +} + +static int +avro_generic_union_set_branch(const avro_value_iface_t *viface, + void *vself, int discriminant, + avro_value_t *branch) +{ + const avro_generic_union_value_iface_t *iface = + container_of(viface, avro_generic_union_value_iface_t, parent); + int rval; + avro_generic_union_t *self = (avro_generic_union_t *) vself; + +#if DEBUG_BRANCHES + fprintf(stderr, "Selecting branch %d (was %d)\n", + discriminant, self->discriminant); +#endif + + /* + * If the new desired branch is different than the currently + * active one, then finalize the old branch and initialize the + * new one. + */ + if (self->discriminant != discriminant) { + if (self->discriminant >= 0) { +#if DEBUG_BRANCHES + fprintf(stderr, "Finalizing branch %d\n", + self->discriminant); +#endif + avro_value_done + (avro_generic_union_branch_giface(iface, self), + avro_generic_union_branch(self)); + } + self->discriminant = discriminant; + if (discriminant >= 0) { +#if DEBUG_BRANCHES + fprintf(stderr, "Initializing branch %d\n", + self->discriminant); +#endif + check(rval, avro_value_init + (avro_generic_union_branch_giface(iface, self), + avro_generic_union_branch(self))); + } + } + + if (branch != NULL) { + branch->iface = avro_generic_union_branch_iface(iface, self); + branch->self = avro_generic_union_branch(self); + } + + return 0; +} + +static size_t +avro_generic_union_instance_size(const avro_value_iface_t *viface) +{ + const avro_generic_union_value_iface_t *iface = + container_of(viface, avro_generic_union_value_iface_t, parent); + return iface->instance_size; +} + +static int +avro_generic_union_init(const avro_value_iface_t *viface, void *vself) +{ + AVRO_UNUSED(viface); + avro_generic_union_t *self = (avro_generic_union_t *) vself; + self->discriminant = -1; + return 0; +} + +static void +avro_generic_union_done(const avro_value_iface_t *viface, void *vself) +{ + const avro_generic_union_value_iface_t *iface = + container_of(viface, avro_generic_union_value_iface_t, parent); + avro_generic_union_t *self = (avro_generic_union_t *) vself; + if (self->discriminant >= 0) { +#if DEBUG_BRANCHES + fprintf(stderr, "Finalizing branch %d\n", + self->discriminant); +#endif + avro_value_done + (avro_generic_union_branch_giface(iface, self), + avro_generic_union_branch(self)); + self->discriminant = -1; + } +} + +static avro_generic_value_iface_t AVRO_GENERIC_UNION_CLASS = +{ + { + /* "class" methods */ + avro_generic_union_incref_iface, + avro_generic_union_decref_iface, + /* general "instance" methods */ + avro_generic_value_incref, + avro_generic_value_decref, + avro_generic_union_reset, + avro_generic_union_get_type, + avro_generic_union_get_schema, + /* primitive getters */ + NULL, /* get_boolean */ + NULL, /* get_bytes */ + NULL, /* grab_bytes */ + NULL, /* get_double */ + NULL, /* get_float */ + NULL, /* get_int */ + NULL, /* get_long */ + NULL, /* get_null */ + NULL, /* get_string */ + NULL, /* grab_string */ + NULL, /* get_enum */ + NULL, /* get_fixed */ + NULL, /* grab_fixed */ + /* primitive setters */ + NULL, /* set_boolean */ + NULL, /* set_bytes */ + NULL, /* give_bytes */ + NULL, /* set_double */ + NULL, /* set_float */ + NULL, /* set_int */ + NULL, /* set_long */ + NULL, /* set_null */ + NULL, /* set_string */ + NULL, /* set_string_length */ + NULL, /* give_string_length */ + NULL, /* set_enum */ + NULL, /* set_fixed */ + NULL, /* give_fixed */ + /* compound getters */ + NULL, /* get_size */ + NULL, /* get_by_index */ + NULL, /* get_by_name */ + avro_generic_union_get_discriminant, + avro_generic_union_get_current_branch, + /* compound setters */ + NULL, /* append */ + NULL, /* add */ + avro_generic_union_set_branch + }, + avro_generic_union_instance_size, + avro_generic_union_init, + avro_generic_union_done +}; + +static avro_generic_value_iface_t * +avro_generic_union_class(avro_schema_t schema, memoize_state_t *state) +{ + avro_generic_union_value_iface_t *iface = + (avro_generic_union_value_iface_t *) avro_new(avro_generic_union_value_iface_t); + if (iface == NULL) { + return NULL; + } + + memset(iface, 0, sizeof(avro_generic_union_value_iface_t)); + iface->parent = AVRO_GENERIC_UNION_CLASS; + iface->refcount = 1; + iface->schema = avro_schema_incref(schema); + + iface->branch_count = avro_schema_union_size(schema); + size_t branch_ifaces_size = + sizeof(avro_generic_value_iface_t *) * iface->branch_count; + + iface->branch_ifaces = (avro_generic_value_iface_t **) avro_malloc(branch_ifaces_size); + if (iface->branch_ifaces == NULL) { + goto error; + } + + size_t max_branch_size = 0; + size_t i; + for (i = 0; i < iface->branch_count; i++) { + avro_schema_t branch_schema = + avro_schema_union_branch(schema, i); + + iface->branch_ifaces[i] = + avro_generic_class_from_schema_memoized(branch_schema, state); + if (iface->branch_ifaces[i] == NULL) { + goto error; + } + + ssize_t branch_size = + avro_value_instance_size(iface->branch_ifaces[i]); + if (branch_size < 0) { + avro_set_error("Union branch class must provide instance_size"); + goto error; + } + +#if DEBUG_BRANCHES + fprintf(stderr, "Branch %" PRIsz ", size %" PRIsz "\n", + i, branch_size); +#endif + + if ((size_t)branch_size > max_branch_size) { + max_branch_size = (size_t)branch_size; + } + } + + iface->instance_size = + sizeof(avro_generic_union_t) + max_branch_size; +#if DEBUG_BRANCHES + fprintf(stderr, "MAX BRANCH SIZE: %" PRIsz "\n", max_branch_size); +#endif + + return &iface->parent; + +error: + avro_schema_decref(iface->schema); + if (iface->branch_ifaces != NULL) { + for (i = 0; i < iface->branch_count; i++) { + if (iface->branch_ifaces[i] != NULL) { + avro_value_iface_decref(&iface->branch_ifaces[i]->parent); + } + } + avro_free(iface->branch_ifaces, branch_ifaces_size); + } + avro_freet(avro_generic_union_value_iface_t, iface); + return NULL; +} + + +/*----------------------------------------------------------------------- + * Schema type dispatcher + */ + +static avro_generic_value_iface_t * +avro_generic_class_from_schema_memoized(avro_schema_t schema, + memoize_state_t *state) +{ + /* + * If we've already instantiated a value class for this schema, + * just return it. + */ + + avro_generic_value_iface_t *result = NULL; + if (avro_memoize_get(&state->mem, schema, NULL, (void **) &result)) { + avro_value_iface_incref(&result->parent); + return result; + } + + /* + * Otherwise instantiate the value class based on the schema + * type. + */ + + switch (schema->type) { + case AVRO_BOOLEAN: + result = &AVRO_GENERIC_BOOLEAN_CLASS; + break; + case AVRO_BYTES: + result = &AVRO_GENERIC_BYTES_CLASS; + break; + case AVRO_DOUBLE: + result = &AVRO_GENERIC_DOUBLE_CLASS; + break; + case AVRO_FLOAT: + result = &AVRO_GENERIC_FLOAT_CLASS; + break; + case AVRO_INT32: + result = &AVRO_GENERIC_INT_CLASS; + break; + case AVRO_INT64: + result = &AVRO_GENERIC_LONG_CLASS; + break; + case AVRO_NULL: + result = &AVRO_GENERIC_NULL_CLASS; + break; + case AVRO_STRING: + result = &AVRO_GENERIC_STRING_CLASS; + break; + + case AVRO_ARRAY: + result = avro_generic_array_class(schema, state); + break; + case AVRO_ENUM: + result = avro_generic_enum_class(schema); + break; + case AVRO_FIXED: + result = avro_generic_fixed_class(schema); + break; + case AVRO_MAP: + result = avro_generic_map_class(schema, state); + break; + case AVRO_RECORD: + result = avro_generic_record_class(schema, state); + break; + case AVRO_UNION: + result = avro_generic_union_class(schema, state); + break; + + case AVRO_LINK: + { + avro_generic_link_value_iface_t *lresult = + avro_generic_link_class(schema); + lresult->next = state->links; + state->links = lresult; + result = &lresult->parent; + break; + } + + default: + avro_set_error("Unknown schema type"); + return NULL; + } + + /* + * Add the new value implementation to the memoized state before + * we return. + */ + + avro_memoize_set(&state->mem, schema, NULL, result); + return result; +} + +avro_value_iface_t * +avro_generic_class_from_schema(avro_schema_t schema) +{ + /* + * Create a state to keep track of the value implementations + * that we create for each subschema. + */ + + memoize_state_t state; + avro_memoize_init(&state.mem); + state.links = NULL; + + /* + * Create the value implementations. + */ + + avro_generic_value_iface_t *result = + avro_generic_class_from_schema_memoized(schema, &state); + if (result == NULL) { + avro_memoize_done(&state.mem); + return NULL; + } + + /* + * Fix up any link schemas so that their value implementations + * point to their target schemas' implementations. + */ + + while (state.links != NULL) { + avro_generic_link_value_iface_t *link_iface = state.links; + avro_schema_t target_schema = + avro_schema_link_target(link_iface->schema); + + avro_generic_value_iface_t *target_iface = NULL; + if (!avro_memoize_get(&state.mem, target_schema, NULL, + (void **) &target_iface)) { + avro_set_error("Never created a value implementation for %s", + avro_schema_type_name(target_schema)); + return NULL; + } + + /* We don't keep a reference to the target + * implementation, since that would give us a reference + * cycle. */ + link_iface->target_giface = target_iface; + state.links = link_iface->next; + link_iface->next = NULL; + } + + /* + * And now we can return. + */ + + avro_memoize_done(&state.mem); + return &result->parent; +} diff --git a/fluent-bit/lib/avro/src/io.c b/fluent-bit/lib/avro/src/io.c new file mode 100644 index 00000000..c1e2f5dc --- /dev/null +++ b/fluent-bit/lib/avro/src/io.c @@ -0,0 +1,447 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include "avro/allocation.h" +#include "avro/refcount.h" +#include "avro/errors.h" +#include "avro/io.h" +#include "avro_private.h" +#include <stdio.h> +#include <stdlib.h> +#include <errno.h> +#include <string.h> +#include "dump.h" + +enum avro_io_type_t { + AVRO_FILE_IO, + AVRO_MEMORY_IO +}; +typedef enum avro_io_type_t avro_io_type_t; + +struct avro_reader_t_ { + avro_io_type_t type; + volatile int refcount; +}; + +struct avro_writer_t_ { + avro_io_type_t type; + volatile int refcount; +}; + +struct _avro_reader_file_t { + struct avro_reader_t_ reader; + FILE *fp; + int should_close; + char *cur; + char *end; + char buffer[4096]; +}; + +struct _avro_writer_file_t { + struct avro_writer_t_ writer; + FILE *fp; + int should_close; +}; + +struct _avro_reader_memory_t { + struct avro_reader_t_ reader; + const char *buf; + int64_t len; + int64_t read; +}; + +struct _avro_writer_memory_t { + struct avro_writer_t_ writer; + const char *buf; + int64_t len; + int64_t written; +}; + +#define avro_io_typeof(obj) ((obj)->type) +#define is_memory_io(obj) (obj && avro_io_typeof(obj) == AVRO_MEMORY_IO) +#define is_file_io(obj) (obj && avro_io_typeof(obj) == AVRO_FILE_IO) + +#define avro_reader_to_memory(reader_) container_of(reader_, struct _avro_reader_memory_t, reader) +#define avro_reader_to_file(reader_) container_of(reader_, struct _avro_reader_file_t, reader) +#define avro_writer_to_memory(writer_) container_of(writer_, struct _avro_writer_memory_t, writer) +#define avro_writer_to_file(writer_) container_of(writer_, struct _avro_writer_file_t, writer) + +static void reader_init(avro_reader_t reader, avro_io_type_t type) +{ + reader->type = type; + avro_refcount_set(&reader->refcount, 1); +} + +static void writer_init(avro_writer_t writer, avro_io_type_t type) +{ + writer->type = type; + avro_refcount_set(&writer->refcount, 1); +} + +avro_reader_t avro_reader_file_fp(FILE * fp, int should_close) +{ + struct _avro_reader_file_t *file_reader = + (struct _avro_reader_file_t *) avro_new(struct _avro_reader_file_t); + if (!file_reader) { + avro_set_error("Cannot allocate new file reader"); + return NULL; + } + memset(file_reader, 0, sizeof(struct _avro_reader_file_t)); + file_reader->fp = fp; + file_reader->should_close = should_close; + reader_init(&file_reader->reader, AVRO_FILE_IO); + return &file_reader->reader; +} + +avro_reader_t avro_reader_file(FILE * fp) +{ + return avro_reader_file_fp(fp, 1); +} + +avro_writer_t avro_writer_file_fp(FILE * fp, int should_close) +{ + struct _avro_writer_file_t *file_writer = + (struct _avro_writer_file_t *) avro_new(struct _avro_writer_file_t); + if (!file_writer) { + avro_set_error("Cannot allocate new file writer"); + return NULL; + } + file_writer->fp = fp; + file_writer->should_close = should_close; + writer_init(&file_writer->writer, AVRO_FILE_IO); + return &file_writer->writer; +} + +avro_writer_t avro_writer_file(FILE * fp) +{ + return avro_writer_file_fp(fp, 1); +} + +avro_reader_t avro_reader_memory(const char *buf, int64_t len) +{ + struct _avro_reader_memory_t *mem_reader = + (struct _avro_reader_memory_t *) avro_new(struct _avro_reader_memory_t); + if (!mem_reader) { + avro_set_error("Cannot allocate new memory reader"); + return NULL; + } + mem_reader->buf = buf; + mem_reader->len = len; + mem_reader->read = 0; + reader_init(&mem_reader->reader, AVRO_MEMORY_IO); + return &mem_reader->reader; +} + +void +avro_reader_memory_set_source(avro_reader_t reader, const char *buf, int64_t len) +{ + if (is_memory_io(reader)) { + struct _avro_reader_memory_t *mem_reader = avro_reader_to_memory(reader); + mem_reader->buf = buf; + mem_reader->len = len; + mem_reader->read = 0; + } +} + +avro_writer_t avro_writer_memory(const char *buf, int64_t len) +{ + struct _avro_writer_memory_t *mem_writer = + (struct _avro_writer_memory_t *) avro_new(struct _avro_writer_memory_t); + if (!mem_writer) { + avro_set_error("Cannot allocate new memory writer"); + return NULL; + } + mem_writer->buf = buf; + mem_writer->len = len; + mem_writer->written = 0; + writer_init(&mem_writer->writer, AVRO_MEMORY_IO); + return &mem_writer->writer; +} + +void +avro_writer_memory_set_dest(avro_writer_t writer, const char *buf, int64_t len) +{ + if (is_memory_io(writer)) { + struct _avro_writer_memory_t *mem_writer = avro_writer_to_memory(writer); + mem_writer->buf = buf; + mem_writer->len = len; + mem_writer->written = 0; + } +} + +static int +avro_read_memory(struct _avro_reader_memory_t *reader, void *buf, int64_t len) +{ + if (len > 0) { + if ((reader->len - reader->read) < len) { + avro_prefix_error("Cannot read %" PRIsz " bytes from memory buffer", + (size_t) len); + return ENOSPC; + } + memcpy(buf, reader->buf + reader->read, len); + reader->read += len; + } + return 0; +} + +#define bytes_available(reader) (reader->end - reader->cur) +#define buffer_reset(reader) {reader->cur = reader->end = reader->buffer;} + +static int +avro_read_file(struct _avro_reader_file_t *reader, void *buf, int64_t len) +{ + int64_t needed = len; + char *p = (char *) buf; + int rval; + + if (len == 0) { + return 0; + } + + if (needed > (int64_t) sizeof(reader->buffer)) { + if (bytes_available(reader) > 0) { + memcpy(p, reader->cur, bytes_available(reader)); + p += bytes_available(reader); + needed -= bytes_available(reader); + buffer_reset(reader); + } + rval = fread(p, 1, needed, reader->fp); + if (rval != needed) { + avro_set_error("Cannot read %" PRIsz " bytes from file", + (size_t) needed); + return EILSEQ; + } + return 0; + } else if (needed <= bytes_available(reader)) { + memcpy(p, reader->cur, needed); + reader->cur += needed; + return 0; + } else { + memcpy(p, reader->cur, bytes_available(reader)); + p += bytes_available(reader); + needed -= bytes_available(reader); + + rval = + fread(reader->buffer, 1, sizeof(reader->buffer), + reader->fp); + if (rval == 0) { + avro_set_error("Cannot read %" PRIsz " bytes from file", + (size_t) needed); + return EILSEQ; + } + reader->cur = reader->buffer; + reader->end = reader->cur + rval; + + if (bytes_available(reader) < needed) { + avro_set_error("Cannot read %" PRIsz " bytes from file", + (size_t) needed); + return EILSEQ; + } + memcpy(p, reader->cur, needed); + reader->cur += needed; + return 0; + } + avro_set_error("Cannot read %" PRIsz " bytes from file", + (size_t) needed); + return EILSEQ; +} + +int avro_read(avro_reader_t reader, void *buf, int64_t len) +{ + if (buf && len >= 0) { + if (is_memory_io(reader)) { + return avro_read_memory(avro_reader_to_memory(reader), + buf, len); + } else if (is_file_io(reader)) { + return avro_read_file(avro_reader_to_file(reader), buf, + len); + } + } + return EINVAL; +} + +static int avro_skip_memory(struct _avro_reader_memory_t *reader, int64_t len) +{ + if (len > 0) { + if ((reader->len - reader->read) < len) { + avro_set_error("Cannot skip %" PRIsz " bytes in memory buffer", + (size_t) len); + return ENOSPC; + } + reader->read += len; + } + return 0; +} + +static int avro_skip_file(struct _avro_reader_file_t *reader, int64_t len) +{ + int rval; + int64_t needed = len; + + if (len == 0) { + return 0; + } + if (needed <= bytes_available(reader)) { + reader->cur += needed; + } else { + needed -= bytes_available(reader); + buffer_reset(reader); + rval = fseek(reader->fp, needed, SEEK_CUR); + if (rval < 0) { + avro_set_error("Cannot skip %" PRIsz " bytes in file", + (size_t) len); + return rval; + } + } + return 0; +} + +int avro_skip(avro_reader_t reader, int64_t len) +{ + if (len >= 0) { + if (is_memory_io(reader)) { + return avro_skip_memory(avro_reader_to_memory(reader), + len); + } else if (is_file_io(reader)) { + return avro_skip_file(avro_reader_to_file(reader), len); + } + } + return 0; +} + +static int +avro_write_memory(struct _avro_writer_memory_t *writer, void *buf, int64_t len) +{ + if (len) { + if ((writer->len - writer->written) < len) { + avro_set_error("Cannot write %" PRIsz " bytes in memory buffer", + (size_t) len); + return ENOSPC; + } + memcpy((void *)(writer->buf + writer->written), buf, len); + writer->written += len; + } + return 0; +} + +static int +avro_write_file(struct _avro_writer_file_t *writer, void *buf, int64_t len) +{ + int rval; + if (len > 0) { + rval = fwrite(buf, len, 1, writer->fp); + if (rval == 0) { + return EIO; + } + } + return 0; +} + +int avro_write(avro_writer_t writer, void *buf, int64_t len) +{ + if (buf && len >= 0) { + if (is_memory_io(writer)) { + return avro_write_memory(avro_writer_to_memory(writer), + buf, len); + } else if (is_file_io(writer)) { + return avro_write_file(avro_writer_to_file(writer), buf, + len); + } + } + return EINVAL; +} + +void +avro_reader_reset(avro_reader_t reader) +{ + if (is_memory_io(reader)) { + avro_reader_to_memory(reader)->read = 0; + } +} + +void avro_writer_reset(avro_writer_t writer) +{ + if (is_memory_io(writer)) { + avro_writer_to_memory(writer)->written = 0; + } +} + +int64_t avro_writer_tell(avro_writer_t writer) +{ + if (is_memory_io(writer)) { + return avro_writer_to_memory(writer)->written; + } + return EINVAL; +} + +void avro_writer_flush(avro_writer_t writer) +{ + if (is_file_io(writer)) { + fflush(avro_writer_to_file(writer)->fp); + } +} + +void avro_writer_dump(avro_writer_t writer, FILE * fp) +{ + if (is_memory_io(writer)) { + dump(fp, (char *)avro_writer_to_memory(writer)->buf, + avro_writer_to_memory(writer)->written); + } +} + +void avro_reader_dump(avro_reader_t reader, FILE * fp) +{ + if (is_memory_io(reader)) { + dump(fp, (char *)avro_reader_to_memory(reader)->buf, + avro_reader_to_memory(reader)->read); + } +} + +void avro_reader_free(avro_reader_t reader) +{ + if (is_memory_io(reader)) { + avro_freet(struct _avro_reader_memory_t, reader); + } else if (is_file_io(reader)) { + if (avro_reader_to_file(reader)->should_close) { + fclose(avro_reader_to_file(reader)->fp); + } + avro_freet(struct _avro_reader_file_t, reader); + } +} + +void avro_writer_free(avro_writer_t writer) +{ + if (is_memory_io(writer)) { + avro_freet(struct _avro_writer_memory_t, writer); + } else if (is_file_io(writer)) { + if (avro_writer_to_file(writer)->should_close) { + fclose(avro_writer_to_file(writer)->fp); + } + avro_freet(struct _avro_writer_file_t, writer); + } +} + +int avro_reader_is_eof(avro_reader_t reader) +{ + if (is_file_io(reader)) { + struct _avro_reader_file_t *file = avro_reader_to_file(reader); + if (feof(file->fp)) { + return file->cur == file->end; + } + } + return 0; +} diff --git a/fluent-bit/lib/avro/src/map.c b/fluent-bit/lib/avro/src/map.c new file mode 100644 index 00000000..c85ffbd8 --- /dev/null +++ b/fluent-bit/lib/avro/src/map.c @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <errno.h> +#include <string.h> + +#include "avro/data.h" +#include "avro/allocation.h" +#include "avro/errors.h" +#include "st.h" + + +#define raw_entry_size(element_size) \ + (sizeof(avro_raw_map_entry_t) + element_size) + +void avro_raw_map_init(avro_raw_map_t *map, size_t element_size) +{ + memset(map, 0, sizeof(avro_raw_map_t)); + avro_raw_array_init(&map->elements, raw_entry_size(element_size)); + map->indices_by_key = st_init_strtable(); +} + + +static void +avro_raw_map_free_keys(avro_raw_map_t *map) +{ + unsigned int i; + for (i = 0; i < avro_raw_map_size(map); i++) { + void *ventry = + ((char *) map->elements.data + map->elements.element_size * i); + avro_raw_map_entry_t *entry = (avro_raw_map_entry_t *) ventry; + avro_str_free((char *) entry->key); + } +} + + +void avro_raw_map_done(avro_raw_map_t *map) +{ + avro_raw_map_free_keys(map); + avro_raw_array_done(&map->elements); + st_free_table((st_table *) map->indices_by_key); + memset(map, 0, sizeof(avro_raw_map_t)); +} + + +void avro_raw_map_clear(avro_raw_map_t *map) +{ + avro_raw_map_free_keys(map); + avro_raw_array_clear(&map->elements); + st_free_table((st_table *) map->indices_by_key); + map->indices_by_key = st_init_strtable(); +} + + +int +avro_raw_map_ensure_size(avro_raw_map_t *map, size_t desired_count) +{ + return avro_raw_array_ensure_size(&map->elements, desired_count); +} + + +void *avro_raw_map_get(const avro_raw_map_t *map, const char *key, + size_t *index) +{ + st_data_t data; + if (st_lookup((st_table *) map->indices_by_key, (st_data_t) key, &data)) { + unsigned int i = (unsigned int) data; + if (index) { + *index = i; + } + void *raw_entry = + ((char *) map->elements.data + map->elements.element_size * i); + return (char *) raw_entry + sizeof(avro_raw_map_entry_t); + } else { + return NULL; + } +} + + +int avro_raw_map_get_or_create(avro_raw_map_t *map, const char *key, + void **element, size_t *index) +{ + st_data_t data; + void *el; + unsigned int i; + int is_new; + + if (st_lookup((st_table *) map->indices_by_key, (st_data_t) key, &data)) { + i = (unsigned int) data; + void *raw_entry = + ((char *) map->elements.data + map->elements.element_size * i); + el = (char *) raw_entry + sizeof(avro_raw_map_entry_t); + is_new = 0; + } else { + i = map->elements.element_count; + avro_raw_map_entry_t *raw_entry = + (avro_raw_map_entry_t *) avro_raw_array_append(&map->elements); + raw_entry->key = avro_strdup(key); + st_insert((st_table *) map->indices_by_key, + (st_data_t) raw_entry->key, (st_data_t) i); + if (!raw_entry) { + avro_str_free((char*)raw_entry->key); + return -ENOMEM; + } + el = ((char *) raw_entry) + sizeof(avro_raw_map_entry_t); + is_new = 1; + } + + if (element) { + *element = el; + } + if (index) { + *index = i; + } + return is_new; +} diff --git a/fluent-bit/lib/avro/src/memoize.c b/fluent-bit/lib/avro/src/memoize.c new file mode 100644 index 00000000..933fecbd --- /dev/null +++ b/fluent-bit/lib/avro/src/memoize.c @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <errno.h> +#include <string.h> + +#include "avro/data.h" +#include "avro/allocation.h" +#include "avro/errors.h" +#include "avro_private.h" +#include "st.h" + + +typedef struct avro_memoize_key { + void *key1; + void *key2; +} avro_memoize_key_t; + + +static int +avro_memoize_key_cmp(avro_memoize_key_t *a, avro_memoize_key_t *b) +{ + /* + * This isn't a proper cmp operation, since it always returns 1 + * if the keys are different. But that's okay for the hash + * table implementation we're using. + */ + + return (a->key1 != b->key1) || (a->key2 != b->key2); +} + + +static int +avro_memoize_key_hash(avro_memoize_key_t *a) +{ + return ((uintptr_t) a->key1) ^ ((uintptr_t) a->key2); +} + + +static struct st_hash_type avro_memoize_hash_type = { + HASH_FUNCTION_CAST avro_memoize_key_cmp, + HASH_FUNCTION_CAST avro_memoize_key_hash +}; + + +void +avro_memoize_init(avro_memoize_t *mem) +{ + memset(mem, 0, sizeof(avro_memoize_t)); + mem->cache = st_init_table(&avro_memoize_hash_type); +} + + +static int +avro_memoize_free_key(avro_memoize_key_t *key, void *result, void *dummy) +{ + AVRO_UNUSED(result); + AVRO_UNUSED(dummy); + avro_freet(avro_memoize_key_t, key); + return ST_CONTINUE; +} + + +void +avro_memoize_done(avro_memoize_t *mem) +{ + st_foreach((st_table *) mem->cache, HASH_FUNCTION_CAST avro_memoize_free_key, 0); + st_free_table((st_table *) mem->cache); + memset(mem, 0, sizeof(avro_memoize_t)); +} + + +int +avro_memoize_get(avro_memoize_t *mem, + void *key1, void *key2, + void **result) +{ + avro_memoize_key_t key; + key.key1 = key1; + key.key2 = key2; + + union { + st_data_t data; + void *value; + } val; + + if (st_lookup((st_table *) mem->cache, (st_data_t) &key, &val.data)) { + if (result) { + *result = val.value; + } + return 1; + } else { + return 0; + } +} + + +void +avro_memoize_set(avro_memoize_t *mem, + void *key1, void *key2, + void *result) +{ + /* + * First see if there's already a cached value for this key. If + * so, we don't want to allocate a new avro_memoize_key_t + * instance. + */ + + avro_memoize_key_t key; + key.key1 = key1; + key.key2 = key2; + + union { + st_data_t data; + void *value; + } val; + + if (st_lookup((st_table *) mem->cache, (st_data_t) &key, &val.data)) { + st_insert((st_table *) mem->cache, (st_data_t) &key, (st_data_t) result); + return; + } + + /* + * If it's a new key pair, then we do need to allocate. + */ + + avro_memoize_key_t *real_key = (avro_memoize_key_t *) avro_new(avro_memoize_key_t); + real_key->key1 = key1; + real_key->key2 = key2; + + st_insert((st_table *) mem->cache, (st_data_t) real_key, (st_data_t) result); +} + + +void +avro_memoize_delete(avro_memoize_t *mem, void *key1, void *key2) +{ + avro_memoize_key_t key; + key.key1 = key1; + key.key2 = key2; + + union { + st_data_t data; + avro_memoize_key_t *key; + } real_key; + + real_key.key = &key; + if (st_delete((st_table *) mem->cache, &real_key.data, NULL)) { + avro_freet(avro_memoize_key_t, real_key.key); + } +} diff --git a/fluent-bit/lib/avro/src/resolved-reader.c b/fluent-bit/lib/avro/src/resolved-reader.c new file mode 100644 index 00000000..30d7b8c9 --- /dev/null +++ b/fluent-bit/lib/avro/src/resolved-reader.c @@ -0,0 +1,3377 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <avro/platform.h> +#include <stdlib.h> +#include <string.h> + +#include "avro_private.h" +#include "avro/allocation.h" +#include "avro/basics.h" +#include "avro/data.h" +#include "avro/errors.h" +#include "avro/refcount.h" +#include "avro/resolver.h" +#include "avro/schema.h" +#include "avro/value.h" +#include "st.h" + +#ifndef AVRO_RESOLVER_DEBUG +#define AVRO_RESOLVER_DEBUG 0 +#endif + +#if AVRO_RESOLVER_DEBUG +#include <stdio.h> +#define DEBUG(...) \ + do { \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, "\n"); \ + } while (0) +#else +#define DEBUG(...) /* don't print messages */ +#endif + + +typedef struct avro_resolved_reader avro_resolved_reader_t; + +struct avro_resolved_reader { + avro_value_iface_t parent; + + /** The reference count for this interface. */ + volatile int refcount; + + /** The writer schema. */ + avro_schema_t wschema; + + /** The reader schema. */ + avro_schema_t rschema; + + /* The size of the value instances for this resolver. */ + size_t instance_size; + + /* A function to calculate the instance size once the overall + * top-level resolver (and all of its children) have been + * constructed. */ + void + (*calculate_size)(avro_resolved_reader_t *iface); + + /* A free function for this resolver */ + void + (*free_iface)(avro_resolved_reader_t *iface, st_table *freeing); + + /* An initialization function for instances of this resolver. */ + int + (*init)(const avro_resolved_reader_t *iface, void *self); + + /* A finalization function for instances of this resolver. */ + void + (*done)(const avro_resolved_reader_t *iface, void *self); + + /* Clear out any existing wrappers, if any */ + int + (*reset_wrappers)(const avro_resolved_reader_t *iface, void *self); +}; + +#define avro_resolved_reader_calculate_size(iface) \ + do { \ + if ((iface)->calculate_size != NULL) { \ + (iface)->calculate_size((iface)); \ + } \ + } while (0) +#define avro_resolved_reader_init(iface, self) \ + ((iface)->init == NULL? 0: (iface)->init((iface), (self))) +#define avro_resolved_reader_done(iface, self) \ + ((iface)->done == NULL? (void) 0: (iface)->done((iface), (self))) +#define avro_resolved_reader_reset_wrappers(iface, self) \ + ((iface)->reset_wrappers == NULL? 0: \ + (iface)->reset_wrappers((iface), (self))) + + +/* + * We assume that each instance type in this value contains an an + * avro_value_t as its first element, which is the current wrapped + * value. + */ + +void +avro_resolved_reader_set_source(avro_value_t *resolved, + avro_value_t *dest) +{ + avro_value_t *self = (avro_value_t *) resolved->self; + if (self->self != NULL) { + avro_value_decref(self); + } + avro_value_copy_ref(self, dest); +} + +void +avro_resolved_reader_clear_source(avro_value_t *resolved) +{ + avro_value_t *self = (avro_value_t *) resolved->self; + if (self->self != NULL) { + avro_value_decref(self); + } + self->iface = NULL; + self->self = NULL; +} + +int +avro_resolved_reader_new_value(avro_value_iface_t *viface, + avro_value_t *value) +{ + int rval; + avro_resolved_reader_t *iface = + container_of(viface, avro_resolved_reader_t, parent); + void *self = avro_malloc(iface->instance_size + sizeof(volatile int)); + if (self == NULL) { + value->iface = NULL; + value->self = NULL; + return ENOMEM; + } + + memset(self, 0, iface->instance_size + sizeof(volatile int)); + volatile int *refcount = (volatile int *) self; + self = (char *) self + sizeof(volatile int); + + rval = avro_resolved_reader_init(iface, self); + if (rval != 0) { + avro_free(self, iface->instance_size + sizeof(volatile int)); + value->iface = NULL; + value->self = NULL; + return rval; + } + + *refcount = 1; + value->iface = avro_value_iface_incref(viface); + value->self = self; + return 0; +} + +static void +avro_resolved_reader_free_value(const avro_value_iface_t *viface, void *vself) +{ + avro_resolved_reader_t *iface = + container_of(viface, avro_resolved_reader_t, parent); + avro_value_t *self = (avro_value_t *) vself; + + avro_resolved_reader_done(iface, vself); + if (self->self != NULL) { + avro_value_decref(self); + } + + vself = (char *) vself - sizeof(volatile int); + avro_free(vself, iface->instance_size + sizeof(volatile int)); +} + +static void +avro_resolved_reader_incref(avro_value_t *value) +{ + /* + * This only works if you pass in the top-level value. + */ + + volatile int *refcount = (volatile int *) ((char *) value->self - sizeof(volatile int)); + avro_refcount_inc(refcount); +} + +static void +avro_resolved_reader_decref(avro_value_t *value) +{ + /* + * This only works if you pass in the top-level value. + */ + + volatile int *refcount = (volatile int *) ((char *) value->self - sizeof(volatile int)); + if (avro_refcount_dec(refcount)) { + avro_resolved_reader_free_value(value->iface, value->self); + } +} + + +static avro_value_iface_t * +avro_resolved_reader_incref_iface(avro_value_iface_t *viface) +{ + avro_resolved_reader_t *iface = + container_of(viface, avro_resolved_reader_t, parent); + avro_refcount_inc(&iface->refcount); + return viface; +} + +static void +free_resolver(avro_resolved_reader_t *iface, st_table *freeing) +{ + /* First check if we've already started freeing this resolver. */ + if (st_lookup(freeing, (st_data_t) iface, NULL)) { + DEBUG("Already freed %p", iface); + return; + } + + /* Otherwise add this resolver to the freeing set, then free it. */ + st_insert(freeing, (st_data_t) iface, (st_data_t) NULL); + DEBUG("Freeing resolver %p (%s->%s)", iface, + avro_schema_type_name(iface->wschema), + avro_schema_type_name(iface->rschema)); + + iface->free_iface(iface, freeing); +} + +static void +avro_resolved_reader_calculate_size_(avro_resolved_reader_t *iface) +{ + /* Only calculate the size for any resolver once */ + iface->calculate_size = NULL; + + DEBUG("Calculating size for %s->%s", + avro_schema_type_name((iface)->wschema), + avro_schema_type_name((iface)->rschema)); + iface->instance_size = sizeof(avro_value_t); +} + +static void +avro_resolved_reader_free_iface(avro_resolved_reader_t *iface, st_table *freeing) +{ + AVRO_UNUSED(freeing); + avro_schema_decref(iface->wschema); + avro_schema_decref(iface->rschema); + avro_freet(avro_resolved_reader_t, iface); +} + +static void +avro_resolved_reader_decref_iface(avro_value_iface_t *viface) +{ + avro_resolved_reader_t *iface = + container_of(viface, avro_resolved_reader_t, parent); + DEBUG("Decref resolver %p (before=%d)", iface, iface->refcount); + if (avro_refcount_dec(&iface->refcount)) { + st_table *freeing = st_init_numtable(); + free_resolver(iface, freeing); + st_free_table(freeing); + } +} + +static int +avro_resolved_reader_reset(const avro_value_iface_t *viface, void *vself) +{ + /* + * To reset a wrapped value, we first clear out any wrappers, + * and then have the wrapped value reset itself. + */ + + int rval; + avro_resolved_reader_t *iface = + container_of(viface, avro_resolved_reader_t, parent); + avro_value_t *self = (avro_value_t *) vself; + check(rval, avro_resolved_reader_reset_wrappers(iface, vself)); + return avro_value_reset(self); +} + +static avro_type_t +avro_resolved_reader_get_type(const avro_value_iface_t *viface, const void *vself) +{ + AVRO_UNUSED(vself); + const avro_resolved_reader_t *iface = + container_of(viface, avro_resolved_reader_t, parent); + return avro_typeof(iface->rschema); +} + +static avro_schema_t +avro_resolved_reader_get_schema(const avro_value_iface_t *viface, const void *vself) +{ + AVRO_UNUSED(vself); + avro_resolved_reader_t *iface = + container_of(viface, avro_resolved_reader_t, parent); + return iface->rschema; +} + + +static avro_resolved_reader_t * +avro_resolved_reader_create(avro_schema_t wschema, avro_schema_t rschema) +{ + avro_resolved_reader_t *self = (avro_resolved_reader_t *) avro_new(avro_resolved_reader_t); + memset(self, 0, sizeof(avro_resolved_reader_t)); + + self->parent.incref_iface = avro_resolved_reader_incref_iface; + self->parent.decref_iface = avro_resolved_reader_decref_iface; + self->parent.incref = avro_resolved_reader_incref; + self->parent.decref = avro_resolved_reader_decref; + self->parent.reset = avro_resolved_reader_reset; + self->parent.get_type = avro_resolved_reader_get_type; + self->parent.get_schema = avro_resolved_reader_get_schema; + + self->refcount = 1; + self->wschema = avro_schema_incref(wschema); + self->rschema = avro_schema_incref(rschema); + self->calculate_size = avro_resolved_reader_calculate_size_; + self->free_iface = avro_resolved_reader_free_iface; + self->reset_wrappers = NULL; + return self; +} + + +/*----------------------------------------------------------------------- + * Memoized resolvers + */ + +typedef struct avro_resolved_link_reader avro_resolved_link_reader_t; + +typedef struct memoize_state_t { + avro_memoize_t mem; + avro_resolved_link_reader_t *links; +} memoize_state_t; + +static avro_resolved_reader_t * +avro_resolved_reader_new_memoized(memoize_state_t *state, + avro_schema_t wschema, avro_schema_t rschema); + + +/*----------------------------------------------------------------------- + * Recursive schemas + */ + +/* + * Recursive schemas are handled specially; the value implementation for + * an AVRO_LINK schema is simply a wrapper around the value + * implementation for the link's target schema. The value methods all + * delegate to the wrapped implementation. + * + * Complicating the links here is that we might be linking to the writer + * schema or the reader schema. This only matters for a couple of + * methods, so instead of keeping a boolean flag in the value interface, + * we just have separate method implementations that we slot in + * appropriately. + */ + +struct avro_resolved_link_reader { + avro_resolved_reader_t parent; + + /** + * A pointer to the “next” link resolver that we've had to + * create. We use this as we're creating the overall top-level + * resolver to keep track of which ones we have to fix up + * afterwards. + */ + avro_resolved_link_reader_t *next; + + /** The target's implementation. */ + avro_resolved_reader_t *target_resolver; +}; + +typedef struct avro_resolved_link_value { + avro_value_t wrapped; + avro_value_t target; +} avro_resolved_link_value_t; + +static void +avro_resolved_wlink_reader_calculate_size(avro_resolved_reader_t *iface) +{ + /* Only calculate the size for any resolver once */ + iface->calculate_size = NULL; + + DEBUG("Calculating size for [%s]->%s", + avro_schema_type_name((iface)->wschema), + avro_schema_type_name((iface)->rschema)); + iface->instance_size = sizeof(avro_resolved_link_value_t); +} + +static void +avro_resolved_rlink_reader_calculate_size(avro_resolved_reader_t *iface) +{ + /* Only calculate the size for any resolver once */ + iface->calculate_size = NULL; + + DEBUG("Calculating size for %s->[%s]", + avro_schema_type_name((iface)->wschema), + avro_schema_type_name((iface)->rschema)); + iface->instance_size = sizeof(avro_resolved_link_value_t); +} + +static void +avro_resolved_link_reader_free_iface(avro_resolved_reader_t *iface, st_table *freeing) +{ + avro_resolved_link_reader_t *liface = + container_of(iface, avro_resolved_link_reader_t, parent); + if (liface->target_resolver != NULL) { + free_resolver(liface->target_resolver, freeing); + } + avro_schema_decref(iface->wschema); + avro_schema_decref(iface->rschema); + avro_freet(avro_resolved_link_reader_t, iface); +} + +static int +avro_resolved_link_reader_init(const avro_resolved_reader_t *iface, void *vself) +{ + int rval; + const avro_resolved_link_reader_t *liface = + container_of(iface, avro_resolved_link_reader_t, parent); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + size_t target_instance_size = liface->target_resolver->instance_size; + + self->target.iface = &liface->target_resolver->parent; + self->target.self = avro_malloc(target_instance_size); + if (self->target.self == NULL) { + return ENOMEM; + } + DEBUG("Allocated <%p:%" PRIsz "> for link", self->target.self, target_instance_size); + + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + + rval = avro_resolved_reader_init(liface->target_resolver, self->target.self); + if (rval != 0) { + avro_free(self->target.self, target_instance_size); + } + return rval; +} + +static void +avro_resolved_link_reader_done(const avro_resolved_reader_t *iface, void *vself) +{ + const avro_resolved_link_reader_t *liface = + container_of(iface, avro_resolved_link_reader_t, parent); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + size_t target_instance_size = liface->target_resolver->instance_size; + DEBUG("Freeing <%p:%" PRIsz "> for link", self->target.self, target_instance_size); + avro_resolved_reader_done(liface->target_resolver, self->target.self); + avro_free(self->target.self, target_instance_size); + self->target.iface = NULL; + self->target.self = NULL; +} + +static int +avro_resolved_link_reader_reset(const avro_resolved_reader_t *iface, void *vself) +{ + const avro_resolved_link_reader_t *liface = + container_of(iface, avro_resolved_link_reader_t, parent); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + return avro_resolved_reader_reset_wrappers + (liface->target_resolver, self->target.self); +} + +static avro_type_t +avro_resolved_link_reader_get_type(const avro_value_iface_t *iface, const void *vself) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_type(&self->target); +} + +static avro_schema_t +avro_resolved_link_reader_get_schema(const avro_value_iface_t *iface, const void *vself) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_schema(&self->target); +} + +static int +avro_resolved_link_reader_get_boolean(const avro_value_iface_t *iface, + const void *vself, int *out) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_boolean(&self->target, out); +} + +static int +avro_resolved_link_reader_get_bytes(const avro_value_iface_t *iface, + const void *vself, const void **buf, size_t *size) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_bytes(&self->target, buf, size); +} + +static int +avro_resolved_link_reader_grab_bytes(const avro_value_iface_t *iface, + const void *vself, avro_wrapped_buffer_t *dest) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_grab_bytes(&self->target, dest); +} + +static int +avro_resolved_link_reader_get_double(const avro_value_iface_t *iface, + const void *vself, double *out) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_double(&self->target, out); +} + +static int +avro_resolved_link_reader_get_float(const avro_value_iface_t *iface, + const void *vself, float *out) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_float(&self->target, out); +} + +static int +avro_resolved_link_reader_get_int(const avro_value_iface_t *iface, + const void *vself, int32_t *out) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_int(&self->target, out); +} + +static int +avro_resolved_link_reader_get_long(const avro_value_iface_t *iface, + const void *vself, int64_t *out) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_long(&self->target, out); +} + +static int +avro_resolved_link_reader_get_null(const avro_value_iface_t *iface, const void *vself) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_null(&self->target); +} + +static int +avro_resolved_link_reader_get_string(const avro_value_iface_t *iface, + const void *vself, const char **str, size_t *size) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_string(&self->target, str, size); +} + +static int +avro_resolved_link_reader_grab_string(const avro_value_iface_t *iface, + const void *vself, avro_wrapped_buffer_t *dest) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_grab_string(&self->target, dest); +} + +static int +avro_resolved_link_reader_get_enum(const avro_value_iface_t *iface, + const void *vself, int *out) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_enum(&self->target, out); +} + +static int +avro_resolved_link_reader_get_fixed(const avro_value_iface_t *iface, + const void *vself, const void **buf, size_t *size) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_fixed(&self->target, buf, size); +} + +static int +avro_resolved_link_reader_grab_fixed(const avro_value_iface_t *iface, + const void *vself, avro_wrapped_buffer_t *dest) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_grab_fixed(&self->target, dest); +} + +static int +avro_resolved_link_reader_set_boolean(const avro_value_iface_t *iface, + void *vself, int val) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_set_boolean(&self->target, val); +} + +static int +avro_resolved_link_reader_set_bytes(const avro_value_iface_t *iface, + void *vself, void *buf, size_t size) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_set_bytes(&self->target, buf, size); +} + +static int +avro_resolved_link_reader_give_bytes(const avro_value_iface_t *iface, + void *vself, avro_wrapped_buffer_t *buf) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_give_bytes(&self->target, buf); +} + +static int +avro_resolved_link_reader_set_double(const avro_value_iface_t *iface, + void *vself, double val) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_set_double(&self->target, val); +} + +static int +avro_resolved_link_reader_set_float(const avro_value_iface_t *iface, + void *vself, float val) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_set_float(&self->target, val); +} + +static int +avro_resolved_link_reader_set_int(const avro_value_iface_t *iface, + void *vself, int32_t val) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_set_int(&self->target, val); +} + +static int +avro_resolved_link_reader_set_long(const avro_value_iface_t *iface, + void *vself, int64_t val) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_set_long(&self->target, val); +} + +static int +avro_resolved_link_reader_set_null(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_set_null(&self->target); +} + +static int +avro_resolved_link_reader_set_string(const avro_value_iface_t *iface, + void *vself, const char *str) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_set_string(&self->target, str); +} + +static int +avro_resolved_link_reader_set_string_len(const avro_value_iface_t *iface, + void *vself, const char *str, size_t size) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_set_string_len(&self->target, str, size); +} + +static int +avro_resolved_link_reader_give_string_len(const avro_value_iface_t *iface, + void *vself, avro_wrapped_buffer_t *buf) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_give_string_len(&self->target, buf); +} + +static int +avro_resolved_link_reader_set_enum(const avro_value_iface_t *iface, + void *vself, int val) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_set_enum(&self->target, val); +} + +static int +avro_resolved_link_reader_set_fixed(const avro_value_iface_t *iface, + void *vself, void *buf, size_t size) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_set_fixed(&self->target, buf, size); +} + +static int +avro_resolved_link_reader_give_fixed(const avro_value_iface_t *iface, + void *vself, avro_wrapped_buffer_t *buf) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_give_fixed(&self->target, buf); +} + +static int +avro_resolved_link_reader_get_size(const avro_value_iface_t *iface, + const void *vself, size_t *size) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_size(&self->target, size); +} + +static int +avro_resolved_link_reader_get_by_index(const avro_value_iface_t *iface, + const void *vself, size_t index, + avro_value_t *child, const char **name) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_by_index(&self->target, index, child, name); +} + +static int +avro_resolved_link_reader_get_by_name(const avro_value_iface_t *iface, + const void *vself, const char *name, + avro_value_t *child, size_t *index) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_by_name(&self->target, name, child, index); +} + +static int +avro_resolved_link_reader_get_discriminant(const avro_value_iface_t *iface, + const void *vself, int *out) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_discriminant(&self->target, out); +} + +static int +avro_resolved_link_reader_get_current_branch(const avro_value_iface_t *iface, + const void *vself, avro_value_t *branch) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_current_branch(&self->target, branch); +} + +static int +avro_resolved_link_reader_append(const avro_value_iface_t *iface, + void *vself, avro_value_t *child_out, + size_t *new_index) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_append(&self->target, child_out, new_index); +} + +static int +avro_resolved_link_reader_add(const avro_value_iface_t *iface, + void *vself, const char *key, + avro_value_t *child, size_t *index, int *is_new) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_add(&self->target, key, child, index, is_new); +} + +static int +avro_resolved_link_reader_set_branch(const avro_value_iface_t *iface, + void *vself, int discriminant, + avro_value_t *branch) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_set_branch(&self->target, discriminant, branch); +} + +static avro_resolved_link_reader_t * +avro_resolved_link_reader_create(avro_schema_t wschema, avro_schema_t rschema) +{ + avro_resolved_reader_t *self = (avro_resolved_reader_t *) avro_new(avro_resolved_link_reader_t); + memset(self, 0, sizeof(avro_resolved_link_reader_t)); + + self->parent.incref_iface = avro_resolved_reader_incref_iface; + self->parent.decref_iface = avro_resolved_reader_decref_iface; + self->parent.incref = avro_resolved_reader_incref; + self->parent.decref = avro_resolved_reader_decref; + self->parent.reset = avro_resolved_reader_reset; + self->parent.get_type = avro_resolved_link_reader_get_type; + self->parent.get_schema = avro_resolved_link_reader_get_schema; + self->parent.get_size = avro_resolved_link_reader_get_size; + self->parent.get_by_index = avro_resolved_link_reader_get_by_index; + self->parent.get_by_name = avro_resolved_link_reader_get_by_name; + + self->refcount = 1; + self->wschema = avro_schema_incref(wschema); + self->rschema = avro_schema_incref(rschema); + self->free_iface = avro_resolved_link_reader_free_iface; + self->init = avro_resolved_link_reader_init; + self->done = avro_resolved_link_reader_done; + self->reset_wrappers = avro_resolved_link_reader_reset; + + self->parent.get_boolean = avro_resolved_link_reader_get_boolean; + self->parent.get_bytes = avro_resolved_link_reader_get_bytes; + self->parent.grab_bytes = avro_resolved_link_reader_grab_bytes; + self->parent.get_double = avro_resolved_link_reader_get_double; + self->parent.get_float = avro_resolved_link_reader_get_float; + self->parent.get_int = avro_resolved_link_reader_get_int; + self->parent.get_long = avro_resolved_link_reader_get_long; + self->parent.get_null = avro_resolved_link_reader_get_null; + self->parent.get_string = avro_resolved_link_reader_get_string; + self->parent.grab_string = avro_resolved_link_reader_grab_string; + self->parent.get_enum = avro_resolved_link_reader_get_enum; + self->parent.get_fixed = avro_resolved_link_reader_get_fixed; + self->parent.grab_fixed = avro_resolved_link_reader_grab_fixed; + + self->parent.set_boolean = avro_resolved_link_reader_set_boolean; + self->parent.set_bytes = avro_resolved_link_reader_set_bytes; + self->parent.give_bytes = avro_resolved_link_reader_give_bytes; + self->parent.set_double = avro_resolved_link_reader_set_double; + self->parent.set_float = avro_resolved_link_reader_set_float; + self->parent.set_int = avro_resolved_link_reader_set_int; + self->parent.set_long = avro_resolved_link_reader_set_long; + self->parent.set_null = avro_resolved_link_reader_set_null; + self->parent.set_string = avro_resolved_link_reader_set_string; + self->parent.set_string_len = avro_resolved_link_reader_set_string_len; + self->parent.give_string_len = avro_resolved_link_reader_give_string_len; + self->parent.set_enum = avro_resolved_link_reader_set_enum; + self->parent.set_fixed = avro_resolved_link_reader_set_fixed; + self->parent.give_fixed = avro_resolved_link_reader_give_fixed; + + self->parent.get_size = avro_resolved_link_reader_get_size; + self->parent.get_by_index = avro_resolved_link_reader_get_by_index; + self->parent.get_by_name = avro_resolved_link_reader_get_by_name; + self->parent.get_discriminant = avro_resolved_link_reader_get_discriminant; + self->parent.get_current_branch = avro_resolved_link_reader_get_current_branch; + + self->parent.append = avro_resolved_link_reader_append; + self->parent.add = avro_resolved_link_reader_add; + self->parent.set_branch = avro_resolved_link_reader_set_branch; + + return container_of(self, avro_resolved_link_reader_t, parent); +} + +static avro_resolved_reader_t * +try_wlink(memoize_state_t *state, + avro_schema_t wschema, avro_schema_t rschema) +{ + AVRO_UNUSED(rschema); + + /* + * For link schemas, we create a special value implementation + * that allocates space for its wrapped value at runtime. This + * lets us handle recursive types without having to instantiate + * in infinite-size value. + */ + + avro_schema_t wtarget = avro_schema_link_target(wschema); + avro_resolved_link_reader_t *lself = + avro_resolved_link_reader_create(wtarget, rschema); + avro_memoize_set(&state->mem, wschema, rschema, lself); + + avro_resolved_reader_t *target_resolver = + avro_resolved_reader_new_memoized(state, wtarget, rschema); + if (target_resolver == NULL) { + avro_memoize_delete(&state->mem, wschema, rschema); + avro_value_iface_decref(&lself->parent.parent); + avro_prefix_error("Link target isn't compatible: "); + DEBUG("%s", avro_strerror()); + return NULL; + } + + lself->parent.calculate_size = avro_resolved_wlink_reader_calculate_size; + lself->target_resolver = target_resolver; + lself->next = state->links; + state->links = lself; + + return &lself->parent; +} + +static avro_resolved_reader_t * +try_rlink(memoize_state_t *state, + avro_schema_t wschema, avro_schema_t rschema) +{ + AVRO_UNUSED(rschema); + + /* + * For link schemas, we create a special value implementation + * that allocates space for its wrapped value at runtime. This + * lets us handle recursive types without having to instantiate + * in infinite-size value. + */ + + avro_schema_t rtarget = avro_schema_link_target(rschema); + avro_resolved_link_reader_t *lself = + avro_resolved_link_reader_create(wschema, rtarget); + avro_memoize_set(&state->mem, wschema, rschema, lself); + + avro_resolved_reader_t *target_resolver = + avro_resolved_reader_new_memoized(state, wschema, rtarget); + if (target_resolver == NULL) { + avro_memoize_delete(&state->mem, wschema, rschema); + avro_value_iface_decref(&lself->parent.parent); + avro_prefix_error("Link target isn't compatible: "); + DEBUG("%s", avro_strerror()); + return NULL; + } + + lself->parent.calculate_size = avro_resolved_rlink_reader_calculate_size; + lself->target_resolver = target_resolver; + lself->next = state->links; + state->links = lself; + + return &lself->parent; +} + + +/*----------------------------------------------------------------------- + * boolean + */ + +static int +avro_resolved_reader_get_boolean(const avro_value_iface_t *viface, + const void *vself, int *val) +{ + AVRO_UNUSED(viface); + const avro_value_t *src = (const avro_value_t *) vself; + DEBUG("Getting boolean from %p", src->self); + return avro_value_get_boolean(src, val); +} + +static avro_resolved_reader_t * +try_boolean(memoize_state_t *state, + avro_schema_t wschema, avro_schema_t rschema) +{ + if (is_avro_boolean(wschema)) { + avro_resolved_reader_t *self = + avro_resolved_reader_create(wschema, rschema); + avro_memoize_set(&state->mem, wschema, rschema, self); + self->parent.get_boolean = avro_resolved_reader_get_boolean; + return self; + } + avro_set_error("Writer %s not compatible with reader boolean", + avro_schema_type_name(wschema)); + return NULL; +} + + +/*----------------------------------------------------------------------- + * bytes + */ + +static int +avro_resolved_reader_get_bytes(const avro_value_iface_t *viface, + const void *vself, const void **buf, size_t *size) +{ + AVRO_UNUSED(viface); + const avro_value_t *src = (const avro_value_t *) vself; + DEBUG("Getting bytes from %p", src->self); + return avro_value_get_bytes(src, buf, size); +} + +static int +avro_resolved_reader_grab_bytes(const avro_value_iface_t *viface, + const void *vself, avro_wrapped_buffer_t *dest) +{ + AVRO_UNUSED(viface); + const avro_value_t *src = (const avro_value_t *) vself; + DEBUG("Grabbing bytes from %p", src->self); + return avro_value_grab_bytes(src, dest); +} + +static avro_resolved_reader_t * +try_bytes(memoize_state_t *state, + avro_schema_t wschema, avro_schema_t rschema) +{ + if (is_avro_bytes(wschema)) { + avro_resolved_reader_t *self = + avro_resolved_reader_create(wschema, rschema); + avro_memoize_set(&state->mem, wschema, rschema, self); + self->parent.get_bytes = avro_resolved_reader_get_bytes; + self->parent.grab_bytes = avro_resolved_reader_grab_bytes; + return self; + } + avro_set_error("Writer %s not compatible with reader bytes", + avro_schema_type_name(wschema)); + return NULL; +} + + +/*----------------------------------------------------------------------- + * double + */ + +static int +avro_resolved_reader_get_double(const avro_value_iface_t *viface, + const void *vself, double *val) +{ + AVRO_UNUSED(viface); + const avro_value_t *src = (const avro_value_t *) vself; + DEBUG("Getting double from %p", src->self); + return avro_value_get_double(src, val); +} + +static int +avro_resolved_reader_get_double_float(const avro_value_iface_t *viface, + const void *vself, double *val) +{ + int rval; + float real_val; + AVRO_UNUSED(viface); + const avro_value_t *src = (const avro_value_t *) vself; + DEBUG("Promoting double from float %p", src->self); + check(rval, avro_value_get_float(src, &real_val)); + *val = real_val; + return 0; +} + +static int +avro_resolved_reader_get_double_int(const avro_value_iface_t *viface, + const void *vself, double *val) +{ + int rval; + int32_t real_val; + AVRO_UNUSED(viface); + const avro_value_t *src = (const avro_value_t *) vself; + DEBUG("Promoting double from int %p", src->self); + check(rval, avro_value_get_int(src, &real_val)); + *val = real_val; + return 0; +} + +static int +avro_resolved_reader_get_double_long(const avro_value_iface_t *viface, + const void *vself, double *val) +{ + int rval; + int64_t real_val; + AVRO_UNUSED(viface); + const avro_value_t *src = (const avro_value_t *) vself; + DEBUG("Promoting double from long %p", src->self); + check(rval, avro_value_get_long(src, &real_val)); + *val = (double) real_val; + return 0; +} + +static avro_resolved_reader_t * +try_double(memoize_state_t *state, + avro_schema_t wschema, avro_schema_t rschema) +{ + if (is_avro_double(wschema)) { + avro_resolved_reader_t *self = + avro_resolved_reader_create(wschema, rschema); + avro_memoize_set(&state->mem, wschema, rschema, self); + self->parent.get_double = avro_resolved_reader_get_double; + return self; + } + + else if (is_avro_float(wschema)) { + avro_resolved_reader_t *self = + avro_resolved_reader_create(wschema, rschema); + avro_memoize_set(&state->mem, wschema, rschema, self); + self->parent.get_double = avro_resolved_reader_get_double_float; + return self; + } + + else if (is_avro_int32(wschema)) { + avro_resolved_reader_t *self = + avro_resolved_reader_create(wschema, rschema); + avro_memoize_set(&state->mem, wschema, rschema, self); + self->parent.get_double = avro_resolved_reader_get_double_int; + return self; + } + + else if (is_avro_int64(wschema)) { + avro_resolved_reader_t *self = + avro_resolved_reader_create(wschema, rschema); + avro_memoize_set(&state->mem, wschema, rschema, self); + self->parent.get_double = avro_resolved_reader_get_double_long; + return self; + } + + avro_set_error("Writer %s not compatible with reader double", + avro_schema_type_name(wschema)); + return NULL; +} + + +/*----------------------------------------------------------------------- + * float + */ + +static int +avro_resolved_reader_get_float(const avro_value_iface_t *viface, + const void *vself, float *val) +{ + AVRO_UNUSED(viface); + const avro_value_t *src = (const avro_value_t *) vself; + DEBUG("Getting float from %p", src->self); + return avro_value_get_float(src, val); +} + +static int +avro_resolved_reader_get_float_int(const avro_value_iface_t *viface, + const void *vself, float *val) +{ + int rval; + int32_t real_val; + AVRO_UNUSED(viface); + const avro_value_t *src = (const avro_value_t *) vself; + DEBUG("Promoting float from int %p", src->self); + check(rval, avro_value_get_int(src, &real_val)); + *val = (float) real_val; + return 0; +} + +static int +avro_resolved_reader_get_float_long(const avro_value_iface_t *viface, + const void *vself, float *val) +{ + int rval; + int64_t real_val; + AVRO_UNUSED(viface); + const avro_value_t *src = (const avro_value_t *) vself; + DEBUG("Promoting float from long %p", src->self); + check(rval, avro_value_get_long(src, &real_val)); + *val = (float) real_val; + return 0; +} + +static avro_resolved_reader_t * +try_float(memoize_state_t *state, + avro_schema_t wschema, avro_schema_t rschema) +{ + if (is_avro_float(wschema)) { + avro_resolved_reader_t *self = + avro_resolved_reader_create(wschema, rschema); + avro_memoize_set(&state->mem, wschema, rschema, self); + self->parent.get_float = avro_resolved_reader_get_float; + return self; + } + + else if (is_avro_int32(wschema)) { + avro_resolved_reader_t *self = + avro_resolved_reader_create(wschema, rschema); + avro_memoize_set(&state->mem, wschema, rschema, self); + self->parent.get_float = avro_resolved_reader_get_float_int; + return self; + } + + else if (is_avro_int64(wschema)) { + avro_resolved_reader_t *self = + avro_resolved_reader_create(wschema, rschema); + avro_memoize_set(&state->mem, wschema, rschema, self); + self->parent.get_float = avro_resolved_reader_get_float_long; + return self; + } + + avro_set_error("Writer %s not compatible with reader float", + avro_schema_type_name(wschema)); + return NULL; +} + + +/*----------------------------------------------------------------------- + * int + */ + +static int +avro_resolved_reader_get_int(const avro_value_iface_t *viface, + const void *vself, int32_t *val) +{ + AVRO_UNUSED(viface); + const avro_value_t *src = (const avro_value_t *) vself; + DEBUG("Getting int from %p", src->self); + return avro_value_get_int(src, val); +} + +static avro_resolved_reader_t * +try_int(memoize_state_t *state, + avro_schema_t wschema, avro_schema_t rschema) +{ + if (is_avro_int32(wschema)) { + avro_resolved_reader_t *self = + avro_resolved_reader_create(wschema, rschema); + avro_memoize_set(&state->mem, wschema, rschema, self); + self->parent.get_int = avro_resolved_reader_get_int; + return self; + } + avro_set_error("Writer %s not compatible with reader int", + avro_schema_type_name(wschema)); + return NULL; +} + + +/*----------------------------------------------------------------------- + * long + */ + +static int +avro_resolved_reader_get_long(const avro_value_iface_t *viface, + const void *vself, int64_t *val) +{ + AVRO_UNUSED(viface); + const avro_value_t *src = (const avro_value_t *) vself; + DEBUG("Getting long from %p", src->self); + return avro_value_get_long(src, val); +} + +static int +avro_resolved_reader_get_long_int(const avro_value_iface_t *viface, + const void *vself, int64_t *val) +{ + int rval; + int32_t real_val; + AVRO_UNUSED(viface); + const avro_value_t *src = (const avro_value_t *) vself; + DEBUG("Promoting long from int %p", src->self); + check(rval, avro_value_get_int(src, &real_val)); + *val = real_val; + return 0; +} + +static avro_resolved_reader_t * +try_long(memoize_state_t *state, + avro_schema_t wschema, avro_schema_t rschema) +{ + if (is_avro_int64(wschema)) { + avro_resolved_reader_t *self = + avro_resolved_reader_create(wschema, rschema); + avro_memoize_set(&state->mem, wschema, rschema, self); + self->parent.get_long = avro_resolved_reader_get_long; + return self; + } + + else if (is_avro_int32(wschema)) { + avro_resolved_reader_t *self = + avro_resolved_reader_create(wschema, rschema); + avro_memoize_set(&state->mem, wschema, rschema, self); + self->parent.get_long = avro_resolved_reader_get_long_int; + return self; + } + + avro_set_error("Writer %s not compatible with reader long", + avro_schema_type_name(wschema)); + return NULL; +} + + +/*----------------------------------------------------------------------- + * null + */ + +static int +avro_resolved_reader_get_null(const avro_value_iface_t *viface, + const void *vself) +{ + AVRO_UNUSED(viface); + const avro_value_t *src = (const avro_value_t *) vself; + DEBUG("Getting null from %p", src->self); + return avro_value_get_null(src); +} + +static avro_resolved_reader_t * +try_null(memoize_state_t *state, + avro_schema_t wschema, avro_schema_t rschema) +{ + if (is_avro_null(wschema)) { + avro_resolved_reader_t *self = + avro_resolved_reader_create(wschema, rschema); + avro_memoize_set(&state->mem, wschema, rschema, self); + self->parent.get_null = avro_resolved_reader_get_null; + return self; + } + avro_set_error("Writer %s not compatible with reader null", + avro_schema_type_name(wschema)); + return NULL; +} + + +/*----------------------------------------------------------------------- + * string + */ + +static int +avro_resolved_reader_get_string(const avro_value_iface_t *viface, + const void *vself, const char **str, size_t *size) +{ + AVRO_UNUSED(viface); + const avro_value_t *src = (const avro_value_t *) vself; + DEBUG("Getting string from %p", src->self); + return avro_value_get_string(src, str, size); +} + +static int +avro_resolved_reader_grab_string(const avro_value_iface_t *viface, + const void *vself, avro_wrapped_buffer_t *dest) +{ + AVRO_UNUSED(viface); + const avro_value_t *src = (const avro_value_t *) vself; + DEBUG("Grabbing string from %p", src->self); + return avro_value_grab_string(src, dest); +} + +static avro_resolved_reader_t * +try_string(memoize_state_t *state, + avro_schema_t wschema, avro_schema_t rschema) +{ + if (is_avro_string(wschema)) { + avro_resolved_reader_t *self = + avro_resolved_reader_create(wschema, rschema); + avro_memoize_set(&state->mem, wschema, rschema, self); + self->parent.get_string = avro_resolved_reader_get_string; + self->parent.grab_string = avro_resolved_reader_grab_string; + return self; + } + avro_set_error("Writer %s not compatible with reader string", + avro_schema_type_name(wschema)); + return NULL; +} + + +/*----------------------------------------------------------------------- + * array + */ + +typedef struct avro_resolved_array_reader { + avro_resolved_reader_t parent; + avro_resolved_reader_t *child_resolver; +} avro_resolved_array_reader_t; + +typedef struct avro_resolved_array_value { + avro_value_t wrapped; + avro_raw_array_t children; +} avro_resolved_array_value_t; + +static void +avro_resolved_array_reader_calculate_size(avro_resolved_reader_t *iface) +{ + avro_resolved_array_reader_t *aiface = + container_of(iface, avro_resolved_array_reader_t, parent); + + /* Only calculate the size for any resolver once */ + iface->calculate_size = NULL; + + DEBUG("Calculating size for %s->%s", + avro_schema_type_name((iface)->wschema), + avro_schema_type_name((iface)->rschema)); + iface->instance_size = sizeof(avro_resolved_array_value_t); + + avro_resolved_reader_calculate_size(aiface->child_resolver); +} + +static void +avro_resolved_array_reader_free_iface(avro_resolved_reader_t *iface, st_table *freeing) +{ + avro_resolved_array_reader_t *aiface = + container_of(iface, avro_resolved_array_reader_t, parent); + free_resolver(aiface->child_resolver, freeing); + avro_schema_decref(iface->wschema); + avro_schema_decref(iface->rschema); + avro_freet(avro_resolved_array_reader_t, iface); +} + +static int +avro_resolved_array_reader_init(const avro_resolved_reader_t *iface, void *vself) +{ + const avro_resolved_array_reader_t *aiface = + container_of(iface, avro_resolved_array_reader_t, parent); + avro_resolved_array_value_t *self = (avro_resolved_array_value_t *) vself; + size_t child_instance_size = aiface->child_resolver->instance_size; + DEBUG("Initializing child array (child_size=%" PRIsz ")", child_instance_size); + avro_raw_array_init(&self->children, child_instance_size); + return 0; +} + +static void +avro_resolved_array_reader_free_elements(const avro_resolved_reader_t *child_iface, + avro_resolved_array_value_t *self) +{ + size_t i; + for (i = 0; i < avro_raw_array_size(&self->children); i++) { + void *child_self = avro_raw_array_get_raw(&self->children, i); + avro_resolved_reader_done(child_iface, child_self); + } +} + +static void +avro_resolved_array_reader_done(const avro_resolved_reader_t *iface, void *vself) +{ + const avro_resolved_array_reader_t *aiface = + container_of(iface, avro_resolved_array_reader_t, parent); + avro_resolved_array_value_t *self = (avro_resolved_array_value_t *) vself; + avro_resolved_array_reader_free_elements(aiface->child_resolver, self); + avro_raw_array_done(&self->children); +} + +static int +avro_resolved_array_reader_reset(const avro_resolved_reader_t *iface, void *vself) +{ + const avro_resolved_array_reader_t *aiface = + container_of(iface, avro_resolved_array_reader_t, parent); + avro_resolved_array_value_t *self = (avro_resolved_array_value_t *) vself; + + /* Clear out our cache of wrapped children */ + avro_resolved_array_reader_free_elements(aiface->child_resolver, self); + avro_raw_array_clear(&self->children); + return 0; +} + +static int +avro_resolved_array_reader_get_size(const avro_value_iface_t *viface, + const void *vself, size_t *size) +{ + AVRO_UNUSED(viface); + const avro_resolved_array_value_t *self = (const avro_resolved_array_value_t *) vself; + return avro_value_get_size(&self->wrapped, size); +} + +static int +avro_resolved_array_reader_get_by_index(const avro_value_iface_t *viface, + const void *vself, size_t index, + avro_value_t *child, const char **name) +{ + int rval; + size_t old_size; + size_t new_size; + const avro_resolved_reader_t *iface = + container_of(viface, avro_resolved_reader_t, parent); + const avro_resolved_array_reader_t *aiface = + container_of(iface, avro_resolved_array_reader_t, parent); + avro_resolved_array_value_t *self = (avro_resolved_array_value_t *) vself; + + /* + * Ensure that our child wrapper array is big enough to hold + * this many elements. + */ + new_size = index + 1; + check(rval, avro_raw_array_ensure_size0(&self->children, new_size)); + old_size = avro_raw_array_size(&self->children); + if (old_size <= index) { + size_t i; + for (i = old_size; i < new_size; i++) { + check(rval, avro_resolved_reader_init + (aiface->child_resolver, + avro_raw_array_get_raw(&self->children, i))); + } + avro_raw_array_size(&self->children) = index+1; + } + + child->iface = &aiface->child_resolver->parent; + child->self = avro_raw_array_get_raw(&self->children, index); + + DEBUG("Getting element %" PRIsz " from array %p", index, self->wrapped.self); + return avro_value_get_by_index(&self->wrapped, index, (avro_value_t *) child->self, name); +} + +static avro_resolved_array_reader_t * +avro_resolved_array_reader_create(avro_schema_t wschema, avro_schema_t rschema) +{ + avro_resolved_reader_t *self = (avro_resolved_reader_t *) avro_new(avro_resolved_array_reader_t); + memset(self, 0, sizeof(avro_resolved_array_reader_t)); + + self->parent.incref_iface = avro_resolved_reader_incref_iface; + self->parent.decref_iface = avro_resolved_reader_decref_iface; + self->parent.incref = avro_resolved_reader_incref; + self->parent.decref = avro_resolved_reader_decref; + self->parent.reset = avro_resolved_reader_reset; + self->parent.get_type = avro_resolved_reader_get_type; + self->parent.get_schema = avro_resolved_reader_get_schema; + self->parent.get_size = avro_resolved_array_reader_get_size; + self->parent.get_by_index = avro_resolved_array_reader_get_by_index; + + self->refcount = 1; + self->wschema = avro_schema_incref(wschema); + self->rschema = avro_schema_incref(rschema); + self->calculate_size = avro_resolved_array_reader_calculate_size; + self->free_iface = avro_resolved_array_reader_free_iface; + self->init = avro_resolved_array_reader_init; + self->done = avro_resolved_array_reader_done; + self->reset_wrappers = avro_resolved_array_reader_reset; + return container_of(self, avro_resolved_array_reader_t, parent); +} + +static avro_resolved_reader_t * +try_array(memoize_state_t *state, + avro_schema_t wschema, avro_schema_t rschema) +{ + /* + * First verify that the writer is an array. + */ + + if (!is_avro_array(wschema)) { + return 0; + } + + /* + * Array schemas have to have compatible element schemas to be + * compatible themselves. Try to create an resolver to check + * the compatibility. + */ + + avro_resolved_array_reader_t *aself = + avro_resolved_array_reader_create(wschema, rschema); + avro_memoize_set(&state->mem, wschema, rschema, aself); + + avro_schema_t witems = avro_schema_array_items(wschema); + avro_schema_t ritems = avro_schema_array_items(rschema); + + avro_resolved_reader_t *item_resolver = + avro_resolved_reader_new_memoized(state, witems, ritems); + if (item_resolver == NULL) { + avro_memoize_delete(&state->mem, wschema, rschema); + avro_value_iface_decref(&aself->parent.parent); + avro_prefix_error("Array values aren't compatible: "); + return NULL; + } + + /* + * The two schemas are compatible. Store the item schema's + * resolver into the child_resolver field. + */ + + aself->child_resolver = item_resolver; + return &aself->parent; +} + + +/*----------------------------------------------------------------------- + * enum + */ + +static int +avro_resolved_reader_get_enum(const avro_value_iface_t *viface, + const void *vself, int *val) +{ + AVRO_UNUSED(viface); + const avro_value_t *src = (const avro_value_t *) vself; + DEBUG("Getting enum from %p", src->self); + return avro_value_get_enum(src, val); +} + +static avro_resolved_reader_t * +try_enum(memoize_state_t *state, + avro_schema_t wschema, avro_schema_t rschema) +{ + /* + * Enum schemas have to have the same name — but not the same + * list of symbols — to be compatible. + */ + + if (is_avro_enum(wschema)) { + const char *wname = avro_schema_name(wschema); + const char *rname = avro_schema_name(rschema); + + if (strcmp(wname, rname) == 0) { + avro_resolved_reader_t *self = + avro_resolved_reader_create(wschema, rschema); + avro_memoize_set(&state->mem, wschema, rschema, self); + self->parent.get_enum = avro_resolved_reader_get_enum; + return self; + } + } + avro_set_error("Writer %s not compatible with reader %s", + avro_schema_type_name(wschema), + avro_schema_type_name(rschema)); + return NULL; +} + + +/*----------------------------------------------------------------------- + * fixed + */ + +static int +avro_resolved_reader_get_fixed(const avro_value_iface_t *viface, + const void *vself, const void **buf, size_t *size) +{ + AVRO_UNUSED(viface); + const avro_value_t *src = (const avro_value_t *) vself; + DEBUG("Getting fixed from %p", vself); + return avro_value_get_fixed(src, buf, size); +} + +static int +avro_resolved_reader_grab_fixed(const avro_value_iface_t *viface, + const void *vself, avro_wrapped_buffer_t *dest) +{ + AVRO_UNUSED(viface); + const avro_value_t *src = (const avro_value_t *) vself; + DEBUG("Grabbing fixed from %p", vself); + return avro_value_grab_fixed(src, dest); +} + +static avro_resolved_reader_t * +try_fixed(memoize_state_t *state, + avro_schema_t wschema, avro_schema_t rschema) +{ + /* + * Fixed schemas need the same name and size to be compatible. + */ + + if (avro_schema_equal(wschema, rschema)) { + avro_resolved_reader_t *self = + avro_resolved_reader_create(wschema, rschema); + avro_memoize_set(&state->mem, wschema, rschema, self); + self->parent.get_fixed = avro_resolved_reader_get_fixed; + self->parent.grab_fixed = avro_resolved_reader_grab_fixed; + return self; + } + avro_set_error("Writer %s not compatible with reader %s", + avro_schema_type_name(wschema), + avro_schema_type_name(rschema)); + return NULL; +} + + +/*----------------------------------------------------------------------- + * map + */ + +typedef struct avro_resolved_map_reader { + avro_resolved_reader_t parent; + avro_resolved_reader_t *child_resolver; +} avro_resolved_map_reader_t; + +typedef struct avro_resolved_map_value { + avro_value_t wrapped; + avro_raw_array_t children; +} avro_resolved_map_value_t; + +static void +avro_resolved_map_reader_calculate_size(avro_resolved_reader_t *iface) +{ + avro_resolved_map_reader_t *miface = + container_of(iface, avro_resolved_map_reader_t, parent); + + /* Only calculate the size for any resolver once */ + iface->calculate_size = NULL; + + DEBUG("Calculating size for %s->%s", + avro_schema_type_name((iface)->wschema), + avro_schema_type_name((iface)->rschema)); + iface->instance_size = sizeof(avro_resolved_map_value_t); + + avro_resolved_reader_calculate_size(miface->child_resolver); +} + +static void +avro_resolved_map_reader_free_iface(avro_resolved_reader_t *iface, st_table *freeing) +{ + avro_resolved_map_reader_t *miface = + container_of(iface, avro_resolved_map_reader_t, parent); + free_resolver(miface->child_resolver, freeing); + avro_schema_decref(iface->wschema); + avro_schema_decref(iface->rschema); + avro_freet(avro_resolved_map_reader_t, iface); +} + +static int +avro_resolved_map_reader_init(const avro_resolved_reader_t *iface, void *vself) +{ + const avro_resolved_map_reader_t *miface = + container_of(iface, avro_resolved_map_reader_t, parent); + avro_resolved_map_value_t *self = (avro_resolved_map_value_t *) vself; + size_t child_instance_size = miface->child_resolver->instance_size; + DEBUG("Initializing child array for map (child_size=%" PRIsz ")", child_instance_size); + avro_raw_array_init(&self->children, child_instance_size); + return 0; +} + +static void +avro_resolved_map_reader_free_elements(const avro_resolved_reader_t *child_iface, + avro_resolved_map_value_t *self) +{ + size_t i; + for (i = 0; i < avro_raw_array_size(&self->children); i++) { + void *child_self = avro_raw_array_get_raw(&self->children, i); + avro_resolved_reader_done(child_iface, child_self); + } +} + +static void +avro_resolved_map_reader_done(const avro_resolved_reader_t *iface, void *vself) +{ + const avro_resolved_map_reader_t *miface = + container_of(iface, avro_resolved_map_reader_t, parent); + avro_resolved_map_value_t *self = (avro_resolved_map_value_t *) vself; + avro_resolved_map_reader_free_elements(miface->child_resolver, self); + avro_raw_array_done(&self->children); +} + +static int +avro_resolved_map_reader_reset(const avro_resolved_reader_t *iface, void *vself) +{ + const avro_resolved_map_reader_t *miface = + container_of(iface, avro_resolved_map_reader_t, parent); + avro_resolved_map_value_t *self = (avro_resolved_map_value_t *) vself; + + /* Clear out our cache of wrapped children */ + avro_resolved_map_reader_free_elements(miface->child_resolver, self); + return 0; +} + +static int +avro_resolved_map_reader_get_size(const avro_value_iface_t *viface, + const void *vself, size_t *size) +{ + AVRO_UNUSED(viface); + const avro_value_t *src = (const avro_value_t *) vself; + return avro_value_get_size(src, size); +} + +static int +avro_resolved_map_reader_get_by_index(const avro_value_iface_t *viface, + const void *vself, size_t index, + avro_value_t *child, const char **name) +{ + int rval; + const avro_resolved_reader_t *iface = + container_of(viface, avro_resolved_reader_t, parent); + const avro_resolved_map_reader_t *miface = + container_of(iface, avro_resolved_map_reader_t, parent); + avro_resolved_map_value_t *self = (avro_resolved_map_value_t *) vself; + + /* + * Ensure that our child wrapper array is big enough to hold + * this many elements. + */ + check(rval, avro_raw_array_ensure_size0(&self->children, index+1)); + if (avro_raw_array_size(&self->children) <= index) { + avro_raw_array_size(&self->children) = index+1; + } + + child->iface = &miface->child_resolver->parent; + child->self = avro_raw_array_get_raw(&self->children, index); + + DEBUG("Getting element %" PRIsz " from map %p", index, self->wrapped.self); + return avro_value_get_by_index(&self->wrapped, index, (avro_value_t *) child->self, name); +} + +static int +avro_resolved_map_reader_get_by_name(const avro_value_iface_t *viface, + const void *vself, const char *name, + avro_value_t *child, size_t *index) +{ + int rval; + const avro_resolved_reader_t *iface = + container_of(viface, avro_resolved_reader_t, parent); + const avro_resolved_map_reader_t *miface = + container_of(iface, avro_resolved_map_reader_t, parent); + avro_resolved_map_value_t *self = (avro_resolved_map_value_t *) vself; + + /* + * This is a bit convoluted. We need to stash the wrapped child + * value somewhere in our children array. But we don't know + * where to put it until the wrapped map tells us what its index + * is. + */ + + avro_value_t real_child; + size_t real_index; + + DEBUG("Getting element %s from map %p", name, self->wrapped.self); + check(rval, avro_value_get_by_name + (&self->wrapped, name, &real_child, &real_index)); + + /* + * Ensure that our child wrapper array is big enough to hold + * this many elements. + */ + check(rval, avro_raw_array_ensure_size0(&self->children, real_index+1)); + if (avro_raw_array_size(&self->children) <= real_index) { + avro_raw_array_size(&self->children) = real_index+1; + } + + child->iface = &miface->child_resolver->parent; + child->self = avro_raw_array_get_raw(&self->children, real_index); + avro_value_t *child_vself = (avro_value_t *) child->self; + *child_vself = real_child; + + if (index != NULL) { + *index = real_index; + } + return 0; +} + +static avro_resolved_map_reader_t * +avro_resolved_map_reader_create(avro_schema_t wschema, avro_schema_t rschema) +{ + avro_resolved_reader_t *self = (avro_resolved_reader_t *) avro_new(avro_resolved_map_reader_t); + memset(self, 0, sizeof(avro_resolved_map_reader_t)); + + self->parent.incref_iface = avro_resolved_reader_incref_iface; + self->parent.decref_iface = avro_resolved_reader_decref_iface; + self->parent.incref = avro_resolved_reader_incref; + self->parent.decref = avro_resolved_reader_decref; + self->parent.reset = avro_resolved_reader_reset; + self->parent.get_type = avro_resolved_reader_get_type; + self->parent.get_schema = avro_resolved_reader_get_schema; + self->parent.get_size = avro_resolved_map_reader_get_size; + self->parent.get_by_index = avro_resolved_map_reader_get_by_index; + self->parent.get_by_name = avro_resolved_map_reader_get_by_name; + + self->refcount = 1; + self->wschema = avro_schema_incref(wschema); + self->rschema = avro_schema_incref(rschema); + self->calculate_size = avro_resolved_map_reader_calculate_size; + self->free_iface = avro_resolved_map_reader_free_iface; + self->init = avro_resolved_map_reader_init; + self->done = avro_resolved_map_reader_done; + self->reset_wrappers = avro_resolved_map_reader_reset; + return container_of(self, avro_resolved_map_reader_t, parent); +} + +static avro_resolved_reader_t * +try_map(memoize_state_t *state, + avro_schema_t wschema, avro_schema_t rschema) +{ + /* + * First verify that the reader is an map. + */ + + if (!is_avro_map(wschema)) { + return 0; + } + + /* + * Map schemas have to have compatible element schemas to be + * compatible themselves. Try to create an resolver to check + * the compatibility. + */ + + avro_resolved_map_reader_t *mself = + avro_resolved_map_reader_create(wschema, rschema); + avro_memoize_set(&state->mem, wschema, rschema, mself); + + avro_schema_t witems = avro_schema_map_values(wschema); + avro_schema_t ritems = avro_schema_map_values(rschema); + + avro_resolved_reader_t *item_resolver = + avro_resolved_reader_new_memoized(state, witems, ritems); + if (item_resolver == NULL) { + avro_memoize_delete(&state->mem, wschema, rschema); + avro_value_iface_decref(&mself->parent.parent); + avro_prefix_error("Map values aren't compatible: "); + return NULL; + } + + /* + * The two schemas are compatible. Store the item schema's + * resolver into the child_resolver field. + */ + + mself->child_resolver = item_resolver; + return &mself->parent; +} + + +/*----------------------------------------------------------------------- + * record + */ + +typedef struct avro_resolved_record_reader { + avro_resolved_reader_t parent; + size_t field_count; + size_t *field_offsets; + avro_resolved_reader_t **field_resolvers; + size_t *index_mapping; +} avro_resolved_record_reader_t; + +typedef struct avro_resolved_record_value { + avro_value_t wrapped; + /* The rest of the struct is taken up by the inline storage + * needed for each field. */ +} avro_resolved_record_value_t; + +/** Return a pointer to the given field within a record struct. */ +#define avro_resolved_record_field(iface, rec, index) \ + (((char *) (rec)) + (iface)->field_offsets[(index)]) + + +static void +avro_resolved_record_reader_calculate_size(avro_resolved_reader_t *iface) +{ + avro_resolved_record_reader_t *riface = + container_of(iface, avro_resolved_record_reader_t, parent); + + /* Only calculate the size for any resolver once */ + iface->calculate_size = NULL; + + DEBUG("Calculating size for %s->%s", + avro_schema_type_name((iface)->wschema), + avro_schema_type_name((iface)->rschema)); + + /* + * Once we've figured out which reader fields we actually need, + * calculate an offset for each one. + */ + + size_t ri; + size_t next_offset = sizeof(avro_resolved_record_value_t); + for (ri = 0; ri < riface->field_count; ri++) { + riface->field_offsets[ri] = next_offset; + if (riface->field_resolvers[ri] != NULL) { + avro_resolved_reader_calculate_size + (riface->field_resolvers[ri]); + size_t field_size = + riface->field_resolvers[ri]->instance_size; + DEBUG("Field %" PRIsz " has size %" PRIsz, ri, field_size); + next_offset += field_size; + } else { + DEBUG("Field %" PRIsz " is being skipped", ri); + } + } + + DEBUG("Record has size %" PRIsz, next_offset); + iface->instance_size = next_offset; +} + + +static void +avro_resolved_record_reader_free_iface(avro_resolved_reader_t *iface, st_table *freeing) +{ + avro_resolved_record_reader_t *riface = + container_of(iface, avro_resolved_record_reader_t, parent); + + if (riface->field_offsets != NULL) { + avro_free(riface->field_offsets, + riface->field_count * sizeof(size_t)); + } + + if (riface->field_resolvers != NULL) { + size_t i; + for (i = 0; i < riface->field_count; i++) { + if (riface->field_resolvers[i] != NULL) { + DEBUG("Freeing field %" PRIsz " %p", i, + riface->field_resolvers[i]); + free_resolver(riface->field_resolvers[i], freeing); + } + } + avro_free(riface->field_resolvers, + riface->field_count * sizeof(avro_resolved_reader_t *)); + } + + if (riface->index_mapping != NULL) { + avro_free(riface->index_mapping, + riface->field_count * sizeof(size_t)); + } + + avro_schema_decref(iface->wschema); + avro_schema_decref(iface->rschema); + avro_freet(avro_resolved_record_reader_t, iface); +} + +static int +avro_resolved_record_reader_init(const avro_resolved_reader_t *iface, void *vself) +{ + int rval; + const avro_resolved_record_reader_t *riface = + container_of(iface, avro_resolved_record_reader_t, parent); + avro_resolved_record_value_t *self = (avro_resolved_record_value_t *) vself; + + /* Initialize each field */ + size_t i; + for (i = 0; i < riface->field_count; i++) { + if (riface->field_resolvers[i] != NULL) { + check(rval, avro_resolved_reader_init + (riface->field_resolvers[i], + avro_resolved_record_field(riface, self, i))); + } + } + + return 0; +} + +static void +avro_resolved_record_reader_done(const avro_resolved_reader_t *iface, void *vself) +{ + const avro_resolved_record_reader_t *riface = + container_of(iface, avro_resolved_record_reader_t, parent); + avro_resolved_record_value_t *self = (avro_resolved_record_value_t *) vself; + + /* Finalize each field */ + size_t i; + for (i = 0; i < riface->field_count; i++) { + if (riface->field_resolvers[i] != NULL) { + avro_resolved_reader_done + (riface->field_resolvers[i], + avro_resolved_record_field(riface, self, i)); + } + } +} + +static int +avro_resolved_record_reader_reset(const avro_resolved_reader_t *iface, void *vself) +{ + int rval; + const avro_resolved_record_reader_t *riface = + container_of(iface, avro_resolved_record_reader_t, parent); + avro_resolved_record_value_t *self = (avro_resolved_record_value_t *) vself; + + /* Reset each field */ + size_t i; + for (i = 0; i < riface->field_count; i++) { + if (riface->field_resolvers[i] != NULL) { + check(rval, avro_resolved_reader_reset_wrappers + (riface->field_resolvers[i], + avro_resolved_record_field(riface, self, i))); + } + } + + return 0; +} + +static int +avro_resolved_record_reader_get_size(const avro_value_iface_t *viface, + const void *vself, size_t *size) +{ + AVRO_UNUSED(vself); + const avro_resolved_reader_t *iface = + container_of(viface, avro_resolved_reader_t, parent); + const avro_resolved_record_reader_t *riface = + container_of(iface, avro_resolved_record_reader_t, parent); + *size = riface->field_count; + return 0; +} + +static int +avro_resolved_record_reader_get_by_index(const avro_value_iface_t *viface, + const void *vself, size_t index, + avro_value_t *child, const char **name) +{ + const avro_resolved_reader_t *iface = + container_of(viface, avro_resolved_reader_t, parent); + const avro_resolved_record_reader_t *riface = + container_of(iface, avro_resolved_record_reader_t, parent); + const avro_resolved_record_value_t *self = (avro_resolved_record_value_t *) vself; + + DEBUG("Getting reader field %" PRIsz " from record %p", index, self->wrapped.self); + if (riface->field_resolvers[index] == NULL) { + /* + * TODO: Return the default value if the writer record + * doesn't contain this field. + */ + DEBUG("Writer doesn't have field"); + avro_set_error("NIY: Default values"); + return EINVAL; + } + + size_t writer_index = riface->index_mapping[index]; + DEBUG(" Writer field is %" PRIsz, writer_index); + child->iface = &riface->field_resolvers[index]->parent; + child->self = avro_resolved_record_field(riface, self, index); + return avro_value_get_by_index(&self->wrapped, writer_index, (avro_value_t *) child->self, name); +} + +static int +avro_resolved_record_reader_get_by_name(const avro_value_iface_t *viface, + const void *vself, const char *name, + avro_value_t *child, size_t *index) +{ + const avro_resolved_reader_t *iface = + container_of(viface, avro_resolved_reader_t, parent); + + int ri = avro_schema_record_field_get_index(iface->rschema, name); + if (ri == -1) { + avro_set_error("Record doesn't have field named %s", name); + return EINVAL; + } + + DEBUG("Reader field %s is at index %d", name, ri); + if (index != NULL) { + *index = ri; + } + return avro_resolved_record_reader_get_by_index(viface, vself, ri, child, NULL); +} + +static avro_resolved_record_reader_t * +avro_resolved_record_reader_create(avro_schema_t wschema, avro_schema_t rschema) +{ + avro_resolved_reader_t *self = (avro_resolved_reader_t *) avro_new(avro_resolved_record_reader_t); + memset(self, 0, sizeof(avro_resolved_record_reader_t)); + + self->parent.incref_iface = avro_resolved_reader_incref_iface; + self->parent.decref_iface = avro_resolved_reader_decref_iface; + self->parent.incref = avro_resolved_reader_incref; + self->parent.decref = avro_resolved_reader_decref; + self->parent.reset = avro_resolved_reader_reset; + self->parent.get_type = avro_resolved_reader_get_type; + self->parent.get_schema = avro_resolved_reader_get_schema; + self->parent.get_size = avro_resolved_record_reader_get_size; + self->parent.get_by_index = avro_resolved_record_reader_get_by_index; + self->parent.get_by_name = avro_resolved_record_reader_get_by_name; + + self->refcount = 1; + self->wschema = avro_schema_incref(wschema); + self->rschema = avro_schema_incref(rschema); + self->calculate_size = avro_resolved_record_reader_calculate_size; + self->free_iface = avro_resolved_record_reader_free_iface; + self->init = avro_resolved_record_reader_init; + self->done = avro_resolved_record_reader_done; + self->reset_wrappers = avro_resolved_record_reader_reset; + return container_of(self, avro_resolved_record_reader_t, parent); +} + +static avro_resolved_reader_t * +try_record(memoize_state_t *state, + avro_schema_t wschema, avro_schema_t rschema) +{ + /* + * First verify that the writer is also a record, and has the + * same name as the reader. + */ + + if (!is_avro_record(wschema)) { + return 0; + } + + const char *wname = avro_schema_name(wschema); + const char *rname = avro_schema_name(rschema); + + if (strcmp(wname, rname) != 0) { + return 0; + } + + /* + * Categorize the fields in the record schemas. Fields that are + * only in the writer are ignored. Fields that are only in the + * reader raise a schema mismatch error, unless the field has a + * default value. Fields that are in both are resolved + * recursively. + * + * The field_resolvers array will contain an avro_value_iface_t + * for each field in the reader schema. To build this array, we + * loop through the fields of the reader schema. If that field + * is also in the writer schema, we resolve them recursively, + * and store the resolver into the array. If the field isn't in + * the writer schema, we raise an error. (TODO: Eventually, + * we'll handle default values here.) After this loop finishes, + * any NULLs in the field_resolvers array will represent fields + * in the writer but not the reader; these fields should be + * skipped, and won't be accessible in the resolved reader. + */ + + avro_resolved_record_reader_t *rself = + avro_resolved_record_reader_create(wschema, rschema); + avro_memoize_set(&state->mem, wschema, rschema, rself); + + size_t rfields = avro_schema_record_size(rschema); + + DEBUG("Checking reader record schema %s", wname); + + avro_resolved_reader_t **field_resolvers = + (avro_resolved_reader_t **) avro_calloc(rfields, sizeof(avro_resolved_reader_t *)); + size_t *field_offsets = (size_t *) avro_calloc(rfields, sizeof(size_t)); + size_t *index_mapping = (size_t *) avro_calloc(rfields, sizeof(size_t)); + + size_t ri; + for (ri = 0; ri < rfields; ri++) { + avro_schema_t rfield = + avro_schema_record_field_get_by_index(rschema, ri); + const char *field_name = + avro_schema_record_field_name(rschema, ri); + + DEBUG("Resolving reader record field %" PRIsz " (%s)", ri, field_name); + + /* + * See if this field is also in the writer schema. + */ + + int wi = avro_schema_record_field_get_index(wschema, field_name); + + if (wi == -1) { + /* + * This field isn't in the writer schema — + * that's an error! TODO: Handle default + * values! + */ + + DEBUG("Field %s isn't in writer", field_name); + avro_set_error("Reader field %s doesn't appear in writer", + field_name); + goto error; + } + + /* + * Try to recursively resolve the schemas for this + * field. If they're not compatible, that's an error. + */ + + avro_schema_t wfield = + avro_schema_record_field_get_by_index(wschema, wi); + avro_resolved_reader_t *field_resolver = + avro_resolved_reader_new_memoized(state, wfield, rfield); + + if (field_resolver == NULL) { + avro_prefix_error("Field %s isn't compatible: ", field_name); + goto error; + } + + /* + * Save the details for this field. + */ + + DEBUG("Found match for field %s (%" PRIsz " in reader, %d in writer)", + field_name, ri, wi); + field_resolvers[ri] = field_resolver; + index_mapping[ri] = wi; + } + + /* + * We might not have found matches for all of the writer fields, + * but that's okay — any extras will be ignored. + */ + + rself->field_count = rfields; + rself->field_offsets = field_offsets; + rself->field_resolvers = field_resolvers; + rself->index_mapping = index_mapping; + return &rself->parent; + +error: + /* + * Clean up any resolver we might have already created. + */ + + avro_memoize_delete(&state->mem, wschema, rschema); + avro_value_iface_decref(&rself->parent.parent); + + { + unsigned int i; + for (i = 0; i < rfields; i++) { + if (field_resolvers[i]) { + avro_value_iface_decref(&field_resolvers[i]->parent); + } + } + } + + avro_free(field_resolvers, rfields * sizeof(avro_resolved_reader_t *)); + avro_free(field_offsets, rfields * sizeof(size_t)); + avro_free(index_mapping, rfields * sizeof(size_t)); + return NULL; +} + + +/*----------------------------------------------------------------------- + * writer union + */ + +/* + * For writer unions, we maintain a list of resolvers for each branch of + * the union. When we encounter a writer value, we see which branch it + * is, and choose a reader resolver based on that. + */ + +typedef struct avro_resolved_wunion_reader { + avro_resolved_reader_t parent; + + /* The number of branches in the writer union */ + size_t branch_count; + + /* A child resolver for each branch of the writer union. If any + * of these are NULL, then we don't have anything on the reader + * side that's compatible with that writer branch. */ + avro_resolved_reader_t **branch_resolvers; + +} avro_resolved_wunion_reader_t; + +typedef struct avro_resolved_wunion_value { + avro_value_t wrapped; + + /** The currently active branch of the union. -1 if no branch + * is selected. */ + int discriminant; + + /* The rest of the struct is taken up by the inline storage + * needed for the active branch. */ +} avro_resolved_wunion_value_t; + +/** Return a pointer to the active branch within a union struct. */ +#define avro_resolved_wunion_branch(_wunion) \ + (((char *) (_wunion)) + sizeof(avro_resolved_wunion_value_t)) + + +static void +avro_resolved_wunion_reader_calculate_size(avro_resolved_reader_t *iface) +{ + avro_resolved_wunion_reader_t *uiface = + container_of(iface, avro_resolved_wunion_reader_t, parent); + + /* Only calculate the size for any resolver once */ + iface->calculate_size = NULL; + + DEBUG("Calculating size for %s->%s", + avro_schema_type_name((iface)->wschema), + avro_schema_type_name((iface)->rschema)); + + size_t i; + size_t max_branch_size = 0; + for (i = 0; i < uiface->branch_count; i++) { + if (uiface->branch_resolvers[i] == NULL) { + DEBUG("No match for writer union branch %" PRIsz, i); + } else { + avro_resolved_reader_calculate_size + (uiface->branch_resolvers[i]); + size_t branch_size = + uiface->branch_resolvers[i]->instance_size; + DEBUG("Writer branch %" PRIsz " has size %" PRIsz, i, branch_size); + if (branch_size > max_branch_size) { + max_branch_size = branch_size; + } + } + } + + DEBUG("Maximum branch size is %" PRIsz, max_branch_size); + iface->instance_size = + sizeof(avro_resolved_wunion_value_t) + max_branch_size; + DEBUG("Total union size is %" PRIsz, iface->instance_size); +} + + +static void +avro_resolved_wunion_reader_free_iface(avro_resolved_reader_t *iface, st_table *freeing) +{ + avro_resolved_wunion_reader_t *uiface = + container_of(iface, avro_resolved_wunion_reader_t, parent); + + if (uiface->branch_resolvers != NULL) { + size_t i; + for (i = 0; i < uiface->branch_count; i++) { + if (uiface->branch_resolvers[i] != NULL) { + free_resolver(uiface->branch_resolvers[i], freeing); + } + } + avro_free(uiface->branch_resolvers, + uiface->branch_count * sizeof(avro_resolved_reader_t *)); + } + + avro_schema_decref(iface->wschema); + avro_schema_decref(iface->rschema); + avro_freet(avro_resolved_wunion_reader_t, iface); +} + +static int +avro_resolved_wunion_reader_init(const avro_resolved_reader_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + avro_resolved_wunion_value_t *self = (avro_resolved_wunion_value_t *) vself; + self->discriminant = -1; + return 0; +} + +static void +avro_resolved_wunion_reader_done(const avro_resolved_reader_t *iface, void *vself) +{ + const avro_resolved_wunion_reader_t *uiface = + container_of(iface, avro_resolved_wunion_reader_t, parent); + avro_resolved_wunion_value_t *self = (avro_resolved_wunion_value_t *) vself; + if (self->discriminant >= 0) { + avro_resolved_reader_done + (uiface->branch_resolvers[self->discriminant], + avro_resolved_wunion_branch(self)); + self->discriminant = -1; + } +} + +static int +avro_resolved_wunion_reader_reset(const avro_resolved_reader_t *iface, void *vself) +{ + const avro_resolved_wunion_reader_t *uiface = + container_of(iface, avro_resolved_wunion_reader_t, parent); + avro_resolved_wunion_value_t *self = (avro_resolved_wunion_value_t *) vself; + + /* Keep the same branch selected, for the common case that we're + * about to reuse it. */ + if (self->discriminant >= 0) { + return avro_resolved_reader_reset_wrappers + (uiface->branch_resolvers[self->discriminant], + avro_resolved_wunion_branch(self)); + } + + return 0; +} + +static int +avro_resolved_wunion_get_real_src(const avro_value_iface_t *viface, + const void *vself, avro_value_t *real_src) +{ + int rval; + const avro_resolved_reader_t *iface = + container_of(viface, avro_resolved_reader_t, parent); + const avro_resolved_wunion_reader_t *uiface = + container_of(iface, avro_resolved_wunion_reader_t, parent); + avro_resolved_wunion_value_t *self = (avro_resolved_wunion_value_t *) vself; + int writer_disc; + check(rval, avro_value_get_discriminant(&self->wrapped, &writer_disc)); + DEBUG("Writer is branch %d", writer_disc); + + if (uiface->branch_resolvers[writer_disc] == NULL) { + avro_set_error("Reader isn't compatible with writer branch %d", + writer_disc); + return EINVAL; + } + + if (self->discriminant == writer_disc) { + DEBUG("Writer branch %d already selected", writer_disc); + } else { + if (self->discriminant >= 0) { + DEBUG("Finalizing old writer branch %d", self->discriminant); + avro_resolved_reader_done + (uiface->branch_resolvers[self->discriminant], + avro_resolved_wunion_branch(self)); + } + DEBUG("Initializing writer branch %d", writer_disc); + check(rval, avro_resolved_reader_init + (uiface->branch_resolvers[writer_disc], + avro_resolved_wunion_branch(self))); + self->discriminant = writer_disc; + } + + real_src->iface = &uiface->branch_resolvers[writer_disc]->parent; + real_src->self = avro_resolved_wunion_branch(self); + return avro_value_get_current_branch(&self->wrapped, (avro_value_t *) real_src->self); +} + +static int +avro_resolved_wunion_reader_get_boolean(const avro_value_iface_t *viface, + const void *vself, int *out) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_get_boolean(&src, out); +} + +static int +avro_resolved_wunion_reader_get_bytes(const avro_value_iface_t *viface, + const void *vself, const void **buf, size_t *size) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_get_bytes(&src, buf, size); +} + +static int +avro_resolved_wunion_reader_grab_bytes(const avro_value_iface_t *viface, + const void *vself, avro_wrapped_buffer_t *dest) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_grab_bytes(&src, dest); +} + +static int +avro_resolved_wunion_reader_get_double(const avro_value_iface_t *viface, + const void *vself, double *out) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_get_double(&src, out); +} + +static int +avro_resolved_wunion_reader_get_float(const avro_value_iface_t *viface, + const void *vself, float *out) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_get_float(&src, out); +} + +static int +avro_resolved_wunion_reader_get_int(const avro_value_iface_t *viface, + const void *vself, int32_t *out) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_get_int(&src, out); +} + +static int +avro_resolved_wunion_reader_get_long(const avro_value_iface_t *viface, + const void *vself, int64_t *out) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_get_long(&src, out); +} + +static int +avro_resolved_wunion_reader_get_null(const avro_value_iface_t *viface, + const void *vself) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_get_null(&src); +} + +static int +avro_resolved_wunion_reader_get_string(const avro_value_iface_t *viface, + const void *vself, const char **str, size_t *size) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_get_string(&src, str, size); +} + +static int +avro_resolved_wunion_reader_grab_string(const avro_value_iface_t *viface, + const void *vself, avro_wrapped_buffer_t *dest) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_grab_string(&src, dest); +} + +static int +avro_resolved_wunion_reader_get_enum(const avro_value_iface_t *viface, + const void *vself, int *out) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_get_enum(&src, out); +} + +static int +avro_resolved_wunion_reader_get_fixed(const avro_value_iface_t *viface, + const void *vself, const void **buf, size_t *size) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_get_fixed(&src, buf, size); +} + +static int +avro_resolved_wunion_reader_grab_fixed(const avro_value_iface_t *viface, + const void *vself, avro_wrapped_buffer_t *dest) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_grab_fixed(&src, dest); +} + +static int +avro_resolved_wunion_reader_set_boolean(const avro_value_iface_t *viface, + void *vself, int val) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_set_boolean(&src, val); +} + +static int +avro_resolved_wunion_reader_set_bytes(const avro_value_iface_t *viface, + void *vself, void *buf, size_t size) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_set_bytes(&src, buf, size); +} + +static int +avro_resolved_wunion_reader_give_bytes(const avro_value_iface_t *viface, + void *vself, avro_wrapped_buffer_t *buf) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_give_bytes(&src, buf); +} + +static int +avro_resolved_wunion_reader_set_double(const avro_value_iface_t *viface, + void *vself, double val) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_set_double(&src, val); +} + +static int +avro_resolved_wunion_reader_set_float(const avro_value_iface_t *viface, + void *vself, float val) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_set_float(&src, val); +} + +static int +avro_resolved_wunion_reader_set_int(const avro_value_iface_t *viface, + void *vself, int32_t val) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_set_int(&src, val); +} + +static int +avro_resolved_wunion_reader_set_long(const avro_value_iface_t *viface, + void *vself, int64_t val) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_set_long(&src, val); +} + +static int +avro_resolved_wunion_reader_set_null(const avro_value_iface_t *viface, + void *vself) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_set_null(&src); +} + +static int +avro_resolved_wunion_reader_set_string(const avro_value_iface_t *viface, + void *vself, const char *str) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_set_string(&src, str); +} + +static int +avro_resolved_wunion_reader_set_string_len(const avro_value_iface_t *viface, + void *vself, const char *str, size_t size) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_set_string_len(&src, str, size); +} + +static int +avro_resolved_wunion_reader_give_string_len(const avro_value_iface_t *viface, + void *vself, avro_wrapped_buffer_t *buf) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_give_string_len(&src, buf); +} + +static int +avro_resolved_wunion_reader_set_enum(const avro_value_iface_t *viface, + void *vself, int val) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_set_enum(&src, val); +} + +static int +avro_resolved_wunion_reader_set_fixed(const avro_value_iface_t *viface, + void *vself, void *buf, size_t size) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_set_fixed(&src, buf, size); +} + +static int +avro_resolved_wunion_reader_give_fixed(const avro_value_iface_t *viface, + void *vself, avro_wrapped_buffer_t *dest) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_give_fixed(&src, dest); +} + +static int +avro_resolved_wunion_reader_get_size(const avro_value_iface_t *viface, + const void *vself, size_t *size) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_get_size(&src, size); +} + +static int +avro_resolved_wunion_reader_get_by_index(const avro_value_iface_t *viface, + const void *vself, size_t index, + avro_value_t *child, const char **name) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_get_by_index(&src, index, child, name); +} + +static int +avro_resolved_wunion_reader_get_by_name(const avro_value_iface_t *viface, + const void *vself, const char *name, + avro_value_t *child, size_t *index) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_get_by_name(&src, name, child, index); +} + +static int +avro_resolved_wunion_reader_get_discriminant(const avro_value_iface_t *viface, + const void *vself, int *out) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_get_discriminant(&src, out); +} + +static int +avro_resolved_wunion_reader_get_current_branch(const avro_value_iface_t *viface, + const void *vself, avro_value_t *branch) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_get_current_branch(&src, branch); +} + +static int +avro_resolved_wunion_reader_append(const avro_value_iface_t *viface, + void *vself, avro_value_t *child_out, + size_t *new_index) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_append(&src, child_out, new_index); +} + +static int +avro_resolved_wunion_reader_add(const avro_value_iface_t *viface, + void *vself, const char *key, + avro_value_t *child, size_t *index, int *is_new) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_add(&src, key, child, index, is_new); +} + +static int +avro_resolved_wunion_reader_set_branch(const avro_value_iface_t *viface, + void *vself, int discriminant, + avro_value_t *branch) +{ + int rval; + avro_value_t src; + check(rval, avro_resolved_wunion_get_real_src(viface, vself, &src)); + return avro_value_set_branch(&src, discriminant, branch); +} + +static avro_resolved_wunion_reader_t * +avro_resolved_wunion_reader_create(avro_schema_t wschema, avro_schema_t rschema) +{ + avro_resolved_reader_t *self = (avro_resolved_reader_t *) avro_new(avro_resolved_wunion_reader_t); + memset(self, 0, sizeof(avro_resolved_wunion_reader_t)); + + self->parent.incref_iface = avro_resolved_reader_incref_iface; + self->parent.decref_iface = avro_resolved_reader_decref_iface; + self->parent.incref = avro_resolved_reader_incref; + self->parent.decref = avro_resolved_reader_decref; + self->parent.reset = avro_resolved_reader_reset; + self->parent.get_type = avro_resolved_reader_get_type; + self->parent.get_schema = avro_resolved_reader_get_schema; + + self->parent.get_boolean = avro_resolved_wunion_reader_get_boolean; + self->parent.grab_bytes = avro_resolved_wunion_reader_grab_bytes; + self->parent.get_bytes = avro_resolved_wunion_reader_get_bytes; + self->parent.get_double = avro_resolved_wunion_reader_get_double; + self->parent.get_float = avro_resolved_wunion_reader_get_float; + self->parent.get_int = avro_resolved_wunion_reader_get_int; + self->parent.get_long = avro_resolved_wunion_reader_get_long; + self->parent.get_null = avro_resolved_wunion_reader_get_null; + self->parent.get_string = avro_resolved_wunion_reader_get_string; + self->parent.grab_string = avro_resolved_wunion_reader_grab_string; + self->parent.get_enum = avro_resolved_wunion_reader_get_enum; + self->parent.get_fixed = avro_resolved_wunion_reader_get_fixed; + self->parent.grab_fixed = avro_resolved_wunion_reader_grab_fixed; + + self->parent.set_boolean = avro_resolved_wunion_reader_set_boolean; + self->parent.set_bytes = avro_resolved_wunion_reader_set_bytes; + self->parent.give_bytes = avro_resolved_wunion_reader_give_bytes; + self->parent.set_double = avro_resolved_wunion_reader_set_double; + self->parent.set_float = avro_resolved_wunion_reader_set_float; + self->parent.set_int = avro_resolved_wunion_reader_set_int; + self->parent.set_long = avro_resolved_wunion_reader_set_long; + self->parent.set_null = avro_resolved_wunion_reader_set_null; + self->parent.set_string = avro_resolved_wunion_reader_set_string; + self->parent.set_string_len = avro_resolved_wunion_reader_set_string_len; + self->parent.give_string_len = avro_resolved_wunion_reader_give_string_len; + self->parent.set_enum = avro_resolved_wunion_reader_set_enum; + self->parent.set_fixed = avro_resolved_wunion_reader_set_fixed; + self->parent.give_fixed = avro_resolved_wunion_reader_give_fixed; + + self->parent.get_size = avro_resolved_wunion_reader_get_size; + self->parent.get_by_index = avro_resolved_wunion_reader_get_by_index; + self->parent.get_by_name = avro_resolved_wunion_reader_get_by_name; + self->parent.get_discriminant = avro_resolved_wunion_reader_get_discriminant; + self->parent.get_current_branch = avro_resolved_wunion_reader_get_current_branch; + + self->parent.append = avro_resolved_wunion_reader_append; + self->parent.add = avro_resolved_wunion_reader_add; + self->parent.set_branch = avro_resolved_wunion_reader_set_branch; + + self->refcount = 1; + self->wschema = avro_schema_incref(wschema); + self->rschema = avro_schema_incref(rschema); + self->calculate_size = avro_resolved_wunion_reader_calculate_size; + self->free_iface = avro_resolved_wunion_reader_free_iface; + self->init = avro_resolved_wunion_reader_init; + self->done = avro_resolved_wunion_reader_done; + self->reset_wrappers = avro_resolved_wunion_reader_reset; + return container_of(self, avro_resolved_wunion_reader_t, parent); +} + +static avro_resolved_reader_t * +try_writer_union(memoize_state_t *state, + avro_schema_t wschema, avro_schema_t rschema) +{ + /* + * For a writer union, we check each branch of the union in turn + * against the reader schema. For each one that is compatible, + * we save the child resolver that can be used to process a + * writer value of that branch. + */ + + size_t branch_count = avro_schema_union_size(wschema); + DEBUG("Checking %" PRIsz "-branch writer union schema", branch_count); + + avro_resolved_wunion_reader_t *uself = + avro_resolved_wunion_reader_create(wschema, rschema); + avro_memoize_set(&state->mem, wschema, rschema, uself); + + avro_resolved_reader_t **branch_resolvers = + (avro_resolved_reader_t **) avro_calloc(branch_count, sizeof(avro_resolved_reader_t *)); + int some_branch_compatible = 0; + + size_t i; + for (i = 0; i < branch_count; i++) { + avro_schema_t branch_schema = + avro_schema_union_branch(wschema, i); + + DEBUG("Resolving writer union branch %" PRIsz " (%s)", i, + avro_schema_type_name(branch_schema)); + + /* + * Try to recursively resolve this branch of the writer + * union against the reader schema. Don't raise + * an error if this fails — we just need one of + * the writer branches to be compatible. + */ + + branch_resolvers[i] = + avro_resolved_reader_new_memoized(state, branch_schema, rschema); + if (branch_resolvers[i] == NULL) { + DEBUG("No match for writer union branch %" PRIsz, i); + } else { + DEBUG("Found match for writer union branch %" PRIsz, i); + some_branch_compatible = 1; + } + } + + /* + * If we didn't find a match, that's an error. + */ + + if (!some_branch_compatible) { + DEBUG("No writer union branches match"); + avro_set_error("No branches in the writer are compatible " + "with reader schema %s", + avro_schema_type_name(rschema)); + goto error; + } + + uself->branch_count = branch_count; + uself->branch_resolvers = branch_resolvers; + return &uself->parent; + +error: + /* + * Clean up any resolver we might have already created. + */ + + avro_memoize_delete(&state->mem, wschema, rschema); + avro_value_iface_decref(&uself->parent.parent); + + { + unsigned int i; + for (i = 0; i < branch_count; i++) { + if (branch_resolvers[i]) { + avro_value_iface_decref(&branch_resolvers[i]->parent); + } + } + } + + avro_free(branch_resolvers, branch_count * sizeof(avro_resolved_reader_t *)); + return NULL; +} + + +/*----------------------------------------------------------------------- + * reader union + */ + +/* + * For reader unions, we only resolve them against writers which aren't + * unions. (We'll have already broken any writer union apart into its + * separate branches.) We just have to record which branch of the + * reader union the writer schema is compatible with. + */ + +typedef struct avro_resolved_runion_reader { + avro_resolved_reader_t parent; + + /* The reader union branch that's compatible with the writer + * schema. */ + size_t active_branch; + + /* A child resolver for the reader branch. */ + avro_resolved_reader_t *branch_resolver; +} avro_resolved_runion_reader_t; + + +static void +avro_resolved_runion_reader_calculate_size(avro_resolved_reader_t *iface) +{ + avro_resolved_runion_reader_t *uiface = + container_of(iface, avro_resolved_runion_reader_t, parent); + + /* Only calculate the size for any resolver once */ + iface->calculate_size = NULL; + + DEBUG("Calculating size for %s->%s", + avro_schema_type_name((iface)->wschema), + avro_schema_type_name((iface)->rschema)); + + avro_resolved_reader_calculate_size(uiface->branch_resolver); + iface->instance_size = uiface->branch_resolver->instance_size; +} + + +static void +avro_resolved_runion_reader_free_iface(avro_resolved_reader_t *iface, st_table *freeing) +{ + avro_resolved_runion_reader_t *uiface = + container_of(iface, avro_resolved_runion_reader_t, parent); + + if (uiface->branch_resolver != NULL) { + free_resolver(uiface->branch_resolver, freeing); + } + + avro_schema_decref(iface->wschema); + avro_schema_decref(iface->rschema); + avro_freet(avro_resolved_runion_reader_t, iface); +} + +static int +avro_resolved_runion_reader_init(const avro_resolved_reader_t *iface, void *vself) +{ + avro_resolved_runion_reader_t *uiface = + container_of(iface, avro_resolved_runion_reader_t, parent); + return avro_resolved_reader_init(uiface->branch_resolver, vself); +} + +static void +avro_resolved_runion_reader_done(const avro_resolved_reader_t *iface, void *vself) +{ + avro_resolved_runion_reader_t *uiface = + container_of(iface, avro_resolved_runion_reader_t, parent); + avro_resolved_reader_done(uiface->branch_resolver, vself); +} + +static int +avro_resolved_runion_reader_reset(const avro_resolved_reader_t *iface, void *vself) +{ + avro_resolved_runion_reader_t *uiface = + container_of(iface, avro_resolved_runion_reader_t, parent); + return avro_resolved_reader_reset_wrappers(uiface->branch_resolver, vself); +} + +static int +avro_resolved_runion_reader_get_discriminant(const avro_value_iface_t *viface, + const void *vself, int *out) +{ + AVRO_UNUSED(vself); + const avro_resolved_reader_t *iface = + container_of(viface, avro_resolved_reader_t, parent); + const avro_resolved_runion_reader_t *uiface = + container_of(iface, avro_resolved_runion_reader_t, parent); + DEBUG("Reader union is branch %" PRIsz, uiface->active_branch); + *out = uiface->active_branch; + return 0; +} + +static int +avro_resolved_runion_reader_get_current_branch(const avro_value_iface_t *viface, + const void *vself, avro_value_t *branch) +{ + const avro_resolved_reader_t *iface = + container_of(viface, avro_resolved_reader_t, parent); + const avro_resolved_runion_reader_t *uiface = + container_of(iface, avro_resolved_runion_reader_t, parent); + DEBUG("Getting reader branch %" PRIsz " for union %p", uiface->active_branch, vself); + branch->iface = &uiface->branch_resolver->parent; + branch->self = (void *) vself; + return 0; +} + +static avro_resolved_runion_reader_t * +avro_resolved_runion_reader_create(avro_schema_t wschema, avro_schema_t rschema) +{ + avro_resolved_reader_t *self = (avro_resolved_reader_t *) avro_new(avro_resolved_runion_reader_t); + memset(self, 0, sizeof(avro_resolved_runion_reader_t)); + + self->parent.incref_iface = avro_resolved_reader_incref_iface; + self->parent.decref_iface = avro_resolved_reader_decref_iface; + self->parent.incref = avro_resolved_reader_incref; + self->parent.decref = avro_resolved_reader_decref; + self->parent.reset = avro_resolved_reader_reset; + self->parent.get_type = avro_resolved_reader_get_type; + self->parent.get_schema = avro_resolved_reader_get_schema; + self->parent.get_discriminant = avro_resolved_runion_reader_get_discriminant; + self->parent.get_current_branch = avro_resolved_runion_reader_get_current_branch; + + self->refcount = 1; + self->wschema = avro_schema_incref(wschema); + self->rschema = avro_schema_incref(rschema); + self->calculate_size = avro_resolved_runion_reader_calculate_size; + self->free_iface = avro_resolved_runion_reader_free_iface; + self->init = avro_resolved_runion_reader_init; + self->done = avro_resolved_runion_reader_done; + self->reset_wrappers = avro_resolved_runion_reader_reset; + return container_of(self, avro_resolved_runion_reader_t, parent); +} + +static avro_resolved_reader_t * +try_reader_union(memoize_state_t *state, + avro_schema_t wschema, avro_schema_t rschema) +{ + /* + * For a reader union, we have to identify which branch + * corresponds to the writer schema. (The writer won't be a + * union, since we'll have already broken it into its branches.) + */ + + size_t branch_count = avro_schema_union_size(rschema); + DEBUG("Checking %" PRIsz "-branch reader union schema", branch_count); + + avro_resolved_runion_reader_t *uself = + avro_resolved_runion_reader_create(wschema, rschema); + avro_memoize_set(&state->mem, wschema, rschema, uself); + + size_t i; + for (i = 0; i < branch_count; i++) { + avro_schema_t branch_schema = + avro_schema_union_branch(rschema, i); + + DEBUG("Resolving reader union branch %" PRIsz " (%s)", i, + avro_schema_type_name(branch_schema)); + + /* + * Try to recursively resolve this branch of the reader + * union against the writer schema. Don't raise + * an error if this fails — we just need one of + * the reader branches to be compatible. + */ + + uself->branch_resolver = + avro_resolved_reader_new_memoized(state, wschema, branch_schema); + if (uself->branch_resolver == NULL) { + DEBUG("No match for reader union branch %" PRIsz, i); + } else { + DEBUG("Found match for reader union branch %" PRIsz, i); + uself->active_branch = i; + return &uself->parent; + } + } + + /* + * If we didn't find a match, that's an error. + */ + + DEBUG("No reader union branches match"); + avro_set_error("No branches in the reader are compatible " + "with writer schema %s", + avro_schema_type_name(wschema)); + goto error; + +error: + /* + * Clean up any resolver we might have already created. + */ + + avro_memoize_delete(&state->mem, wschema, rschema); + avro_value_iface_decref(&uself->parent.parent); + return NULL; +} + + +/*----------------------------------------------------------------------- + * Schema type dispatcher + */ + +static avro_resolved_reader_t * +avro_resolved_reader_new_memoized(memoize_state_t *state, + avro_schema_t wschema, avro_schema_t rschema) +{ + check_param(NULL, is_avro_schema(wschema), "writer schema"); + check_param(NULL, is_avro_schema(rschema), "reader schema"); + + /* + * First see if we've already matched these two schemas. If so, + * just return that resolver. + */ + + avro_resolved_reader_t *saved = NULL; + if (avro_memoize_get(&state->mem, wschema, rschema, (void **) &saved)) { + DEBUG("Already resolved %s%s%s->%s%s%s", + is_avro_link(wschema)? "[": "", + avro_schema_type_name(wschema), + is_avro_link(wschema)? "]": "", + is_avro_link(rschema)? "[": "", + avro_schema_type_name(rschema), + is_avro_link(rschema)? "]": ""); + return saved; + } else { + DEBUG("Resolving %s%s%s->%s%s%s", + is_avro_link(wschema)? "[": "", + avro_schema_type_name(wschema), + is_avro_link(wschema)? "]": "", + is_avro_link(rschema)? "[": "", + avro_schema_type_name(rschema), + is_avro_link(rschema)? "]": ""); + } + + /* + * Otherwise we have some work to do. First check if the writer + * schema is a union. If so, break it apart. + */ + + if (is_avro_union(wschema)) { + return try_writer_union(state, wschema, rschema); + } + + else if (is_avro_link(wschema)) { + return try_wlink(state, wschema, rschema); + } + + /* + * If the writer isn't a union, than choose a resolver based on + * the reader schema. + */ + + switch (avro_typeof(rschema)) + { + case AVRO_BOOLEAN: + return try_boolean(state, wschema, rschema); + + case AVRO_BYTES: + return try_bytes(state, wschema, rschema); + + case AVRO_DOUBLE: + return try_double(state, wschema, rschema); + + case AVRO_FLOAT: + return try_float(state, wschema, rschema); + + case AVRO_INT32: + return try_int(state, wschema, rschema); + + case AVRO_INT64: + return try_long(state, wschema, rschema); + + case AVRO_NULL: + return try_null(state, wschema, rschema); + + case AVRO_STRING: + return try_string(state, wschema, rschema); + + case AVRO_ARRAY: + return try_array(state, wschema, rschema); + + case AVRO_ENUM: + return try_enum(state, wschema, rschema); + + case AVRO_FIXED: + return try_fixed(state, wschema, rschema); + + case AVRO_MAP: + return try_map(state, wschema, rschema); + + case AVRO_RECORD: + return try_record(state, wschema, rschema); + + case AVRO_UNION: + return try_reader_union(state, wschema, rschema); + + case AVRO_LINK: + return try_rlink(state, wschema, rschema); + + default: + avro_set_error("Unknown reader schema type"); + return NULL; + } + + return NULL; +} + + +avro_value_iface_t * +avro_resolved_reader_new(avro_schema_t wschema, avro_schema_t rschema) +{ + /* + * Create a state to keep track of the value implementations + * that we create for each subschema. + */ + + memoize_state_t state; + avro_memoize_init(&state.mem); + state.links = NULL; + + /* + * Create the value implementations. + */ + + avro_resolved_reader_t *result = + avro_resolved_reader_new_memoized(&state, wschema, rschema); + if (result == NULL) { + avro_memoize_done(&state.mem); + return NULL; + } + + /* + * Fix up any link schemas so that their value implementations + * point to their target schemas' implementations. + */ + + avro_resolved_reader_calculate_size(result); + while (state.links != NULL) { + avro_resolved_link_reader_t *liface = state.links; + avro_resolved_reader_calculate_size(liface->target_resolver); + state.links = liface->next; + liface->next = NULL; + } + + /* + * And now we can return. + */ + + avro_memoize_done(&state.mem); + return &result->parent; +} diff --git a/fluent-bit/lib/avro/src/resolved-writer.c b/fluent-bit/lib/avro/src/resolved-writer.c new file mode 100644 index 00000000..0eafba00 --- /dev/null +++ b/fluent-bit/lib/avro/src/resolved-writer.c @@ -0,0 +1,2911 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <avro/platform.h> +#include <stdlib.h> +#include <string.h> + +#include "avro_private.h" +#include "avro/allocation.h" +#include "avro/basics.h" +#include "avro/data.h" +#include "avro/errors.h" +#include "avro/refcount.h" +#include "avro/resolver.h" +#include "avro/schema.h" +#include "avro/value.h" +#include "st.h" + +#ifndef AVRO_RESOLVER_DEBUG +#define AVRO_RESOLVER_DEBUG 0 +#endif + +#if AVRO_RESOLVER_DEBUG +#include <stdio.h> +#define DEBUG(...) \ + do { \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, "\n"); \ + } while (0) +#else +#define DEBUG(...) /* don't print messages */ +#endif + + +typedef struct avro_resolved_writer avro_resolved_writer_t; + +struct avro_resolved_writer { + avro_value_iface_t parent; + + /** The reference count for this interface. */ + volatile int refcount; + + /** The writer schema. */ + avro_schema_t wschema; + + /** The reader schema. */ + avro_schema_t rschema; + + /* If the reader schema is a union, but the writer schema is + * not, this field indicates which branch of the reader union + * should be selected. */ + int reader_union_branch; + + /* The size of the value instances for this resolver. */ + size_t instance_size; + + /* A function to calculate the instance size once the overall + * top-level resolver (and all of its children) have been + * constructed. */ + void + (*calculate_size)(avro_resolved_writer_t *iface); + + /* A free function for this resolver interface */ + void + (*free_iface)(avro_resolved_writer_t *iface, st_table *freeing); + + /* An initialization function for instances of this resolver. */ + int + (*init)(const avro_resolved_writer_t *iface, void *self); + + /* A finalization function for instances of this resolver. */ + void + (*done)(const avro_resolved_writer_t *iface, void *self); + + /* Clear out any existing wrappers, if any */ + int + (*reset_wrappers)(const avro_resolved_writer_t *iface, void *self); +}; + +#define avro_resolved_writer_calculate_size(iface) \ + do { \ + if ((iface)->calculate_size != NULL) { \ + (iface)->calculate_size((iface)); \ + } \ + } while (0) +#define avro_resolved_writer_init(iface, self) \ + ((iface)->init == NULL? 0: (iface)->init((iface), (self))) +#define avro_resolved_writer_done(iface, self) \ + ((iface)->done == NULL? (void) 0: (iface)->done((iface), (self))) +#define avro_resolved_writer_reset_wrappers(iface, self) \ + ((iface)->reset_wrappers == NULL? 0: \ + (iface)->reset_wrappers((iface), (self))) + + +/* + * We assume that each instance type in this value contains an an + * avro_value_t as its first element, which is the current wrapped + * value. + */ + +void +avro_resolved_writer_set_dest(avro_value_t *resolved, + avro_value_t *dest) +{ + avro_value_t *self = (avro_value_t *) resolved->self; + if (self->self != NULL) { + avro_value_decref(self); + } + avro_value_copy_ref(self, dest); +} + +void +avro_resolved_writer_clear_dest(avro_value_t *resolved) +{ + avro_value_t *self = (avro_value_t *) resolved->self; + if (self->self != NULL) { + avro_value_decref(self); + } + self->iface = NULL; + self->self = NULL; +} + +int +avro_resolved_writer_new_value(avro_value_iface_t *viface, + avro_value_t *value) +{ + int rval; + avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + void *self = avro_malloc(iface->instance_size + sizeof(volatile int)); + if (self == NULL) { + value->iface = NULL; + value->self = NULL; + return ENOMEM; + } + + memset(self, 0, iface->instance_size + sizeof(volatile int)); + volatile int *refcount = (volatile int *) self; + self = (char *) self + sizeof(volatile int); + + rval = avro_resolved_writer_init(iface, self); + if (rval != 0) { + avro_free(self, iface->instance_size + sizeof(volatile int)); + value->iface = NULL; + value->self = NULL; + return rval; + } + + *refcount = 1; + value->iface = avro_value_iface_incref(viface); + value->self = self; + return 0; +} + +static void +avro_resolved_writer_free_value(const avro_value_iface_t *viface, void *vself) +{ + avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + avro_value_t *self = (avro_value_t *) vself; + + avro_resolved_writer_done(iface, vself); + if (self->self != NULL) { + avro_value_decref(self); + } + + vself = (char *) vself - sizeof(volatile int); + avro_free(vself, iface->instance_size + sizeof(volatile int)); +} + +static void +avro_resolved_writer_incref(avro_value_t *value) +{ + /* + * This only works if you pass in the top-level value. + */ + + volatile int *refcount = (volatile int *) ((char *) value->self - sizeof(volatile int)); + avro_refcount_inc(refcount); +} + +static void +avro_resolved_writer_decref(avro_value_t *value) +{ + /* + * This only works if you pass in the top-level value. + */ + + volatile int *refcount = (volatile int *) ((char *) value->self - sizeof(volatile int)); + if (avro_refcount_dec(refcount)) { + avro_resolved_writer_free_value(value->iface, value->self); + } +} + + +static avro_value_iface_t * +avro_resolved_writer_incref_iface(avro_value_iface_t *viface) +{ + avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + avro_refcount_inc(&iface->refcount); + return viface; +} + +static void +free_resolver(avro_resolved_writer_t *iface, st_table *freeing) +{ + /* First check if we've already started freeing this resolver. */ + if (st_lookup(freeing, (st_data_t) iface, NULL)) { + DEBUG("Already freed %p", iface); + return; + } + + /* Otherwise add this resolver to the freeing set, then free it. */ + st_insert(freeing, (st_data_t) iface, (st_data_t) NULL); + DEBUG("Freeing resolver %p (%s->%s)", iface, + avro_schema_type_name(iface->wschema), + avro_schema_type_name(iface->rschema)); + + iface->free_iface(iface, freeing); +} + +static void +avro_resolved_writer_calculate_size_(avro_resolved_writer_t *iface) +{ + /* Only calculate the size for any resolver once */ + iface->calculate_size = NULL; + + DEBUG("Calculating size for %s->%s", + avro_schema_type_name((iface)->wschema), + avro_schema_type_name((iface)->rschema)); + iface->instance_size = sizeof(avro_value_t); +} + +static void +avro_resolved_writer_free_iface(avro_resolved_writer_t *iface, st_table *freeing) +{ + AVRO_UNUSED(freeing); + avro_schema_decref(iface->wschema); + avro_schema_decref(iface->rschema); + avro_freet(avro_resolved_writer_t, iface); +} + +static void +avro_resolved_writer_decref_iface(avro_value_iface_t *viface) +{ + avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + DEBUG("Decref resolver %p (before=%d)", iface, iface->refcount); + if (avro_refcount_dec(&iface->refcount)) { + avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + + st_table *freeing = st_init_numtable(); + free_resolver(iface, freeing); + st_free_table(freeing); + } +} + + +static int +avro_resolved_writer_reset(const avro_value_iface_t *viface, void *vself) +{ + /* + * To reset a wrapped value, we first clear out any wrappers, + * and then have the wrapped value reset itself. + */ + + int rval; + avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + avro_value_t *self = (avro_value_t *) vself; + check(rval, avro_resolved_writer_reset_wrappers(iface, vself)); + return avro_value_reset(self); +} + +static avro_type_t +avro_resolved_writer_get_type(const avro_value_iface_t *viface, const void *vself) +{ + AVRO_UNUSED(vself); + const avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + return avro_typeof(iface->wschema); +} + +static avro_schema_t +avro_resolved_writer_get_schema(const avro_value_iface_t *viface, const void *vself) +{ + AVRO_UNUSED(vself); + avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + return iface->wschema; +} + + +static avro_resolved_writer_t * +avro_resolved_writer_create(avro_schema_t wschema, avro_schema_t rschema) +{ + avro_resolved_writer_t *self = (avro_resolved_writer_t *) avro_new(avro_resolved_writer_t); + memset(self, 0, sizeof(avro_resolved_writer_t)); + + self->parent.incref_iface = avro_resolved_writer_incref_iface; + self->parent.decref_iface = avro_resolved_writer_decref_iface; + self->parent.incref = avro_resolved_writer_incref; + self->parent.decref = avro_resolved_writer_decref; + self->parent.reset = avro_resolved_writer_reset; + self->parent.get_type = avro_resolved_writer_get_type; + self->parent.get_schema = avro_resolved_writer_get_schema; + + self->refcount = 1; + self->wschema = avro_schema_incref(wschema); + self->rschema = avro_schema_incref(rschema); + self->reader_union_branch = -1; + self->calculate_size = avro_resolved_writer_calculate_size_; + self->free_iface = avro_resolved_writer_free_iface; + self->reset_wrappers = NULL; + return self; +} + +static inline int +avro_resolved_writer_get_real_dest(const avro_resolved_writer_t *iface, + const avro_value_t *dest, avro_value_t *real_dest) +{ + if (iface->reader_union_branch < 0) { + /* + * The reader schema isn't a union, so use the dest + * field as-is. + */ + + *real_dest = *dest; + return 0; + } + + DEBUG("Retrieving union branch %d for %s value", + iface->reader_union_branch, + avro_schema_type_name(iface->wschema)); + + return avro_value_set_branch(dest, iface->reader_union_branch, real_dest); +} + + +#define skip_links(schema) \ + while (is_avro_link(schema)) { \ + schema = avro_schema_link_target(schema); \ + } + + +/*----------------------------------------------------------------------- + * Memoized resolvers + */ + +typedef struct avro_resolved_link_writer avro_resolved_link_writer_t; + +typedef struct memoize_state_t { + avro_memoize_t mem; + avro_resolved_link_writer_t *links; +} memoize_state_t; + +static avro_resolved_writer_t * +avro_resolved_writer_new_memoized(memoize_state_t *state, + avro_schema_t wschema, avro_schema_t rschema); + + +/*----------------------------------------------------------------------- + * Reader unions + */ + +/* + * For each Avro type, we have to check whether the reader schema on its + * own is compatible, and also whether the reader is a union that + * contains a compatible type. The macros in this section help us + * perform both of these checks with less code. + */ + + +/** + * A helper macro that handles the case where neither writer nor reader + * are unions. Uses @ref check_func to see if the two schemas are + * compatible. + */ + +#define check_non_union(saved, wschema, rschema, check_func) \ +do { \ + avro_resolved_writer_t *self = NULL; \ + int rc = check_func(saved, &self, wschema, rschema, \ + rschema); \ + if (self) { \ + DEBUG("Non-union schemas %s (writer) " \ + "and %s (reader) match", \ + avro_schema_type_name(wschema), \ + avro_schema_type_name(rschema)); \ + \ + self->reader_union_branch = -1; \ + return self; \ + } \ + \ + if (rc) { \ + return NULL; \ + } \ +} while (0) + + +/** + * Helper macro that handles the case where the reader is a union, and + * the writer is not. Checks each branch of the reader union schema, + * looking for the first branch that is compatible with the writer + * schema. The @ref check_func argument should be a function that can + * check the compatiblity of each branch schema. + */ + +#define check_reader_union(saved, wschema, rschema, check_func) \ +do { \ + if (!is_avro_union(rschema)) { \ + break; \ + } \ + \ + DEBUG("Checking reader union schema"); \ + size_t num_branches = avro_schema_union_size(rschema); \ + unsigned int i; \ + \ + for (i = 0; i < num_branches; i++) { \ + avro_schema_t branch_schema = \ + avro_schema_union_branch(rschema, i); \ + skip_links(branch_schema); \ + \ + DEBUG("Trying branch %u %s%s%s->%s", i, \ + is_avro_link(wschema)? "[": "", \ + avro_schema_type_name(wschema), \ + is_avro_link(wschema)? "]": "", \ + avro_schema_type_name(branch_schema)); \ + \ + avro_resolved_writer_t *self = NULL; \ + int rc = check_func(saved, &self, \ + wschema, branch_schema, rschema); \ + if (self) { \ + DEBUG("Reader union branch %d (%s) " \ + "and writer %s match", \ + i, avro_schema_type_name(branch_schema), \ + avro_schema_type_name(wschema)); \ + self->reader_union_branch = i; \ + return self; \ + } else { \ + DEBUG("Reader union branch %d (%s) " \ + "doesn't match", \ + i, avro_schema_type_name(branch_schema)); \ + } \ + \ + if (rc) { \ + return NULL; \ + } \ + } \ + \ + DEBUG("No reader union branches match"); \ +} while (0) + +/** + * A helper macro that wraps together check_non_union and + * check_reader_union for a simple (non-union) writer schema type. + */ + +#define check_simple_writer(saved, wschema, rschema, type_name) \ +do { \ + check_non_union(saved, wschema, rschema, try_##type_name); \ + check_reader_union(saved, wschema, rschema, try_##type_name); \ + DEBUG("Writer %s doesn't match reader %s", \ + avro_schema_type_name(wschema), \ + avro_schema_type_name(rschema)); \ + avro_set_error("Cannot store " #type_name " into %s", \ + avro_schema_type_name(rschema)); \ + return NULL; \ +} while (0) + + +/*----------------------------------------------------------------------- + * Recursive schemas + */ + +/* + * Recursive schemas are handled specially; the value implementation for + * an AVRO_LINK schema is simply a wrapper around the value + * implementation for the link's target schema. The value methods all + * delegate to the wrapped implementation. + */ + +struct avro_resolved_link_writer { + avro_resolved_writer_t parent; + + /** + * A pointer to the “next” link resolver that we've had to + * create. We use this as we're creating the overall top-level + * resolver to keep track of which ones we have to fix up + * afterwards. + */ + avro_resolved_link_writer_t *next; + + /** The target's implementation. */ + avro_resolved_writer_t *target_resolver; +}; + +typedef struct avro_resolved_link_value { + avro_value_t wrapped; + avro_value_t target; +} avro_resolved_link_value_t; + +static void +avro_resolved_link_writer_calculate_size(avro_resolved_writer_t *iface) +{ + /* Only calculate the size for any resolver once */ + iface->calculate_size = NULL; + + DEBUG("Calculating size for [%s]->%s", + avro_schema_type_name((iface)->wschema), + avro_schema_type_name((iface)->rschema)); + iface->instance_size = sizeof(avro_resolved_link_value_t); +} + +static void +avro_resolved_link_writer_free_iface(avro_resolved_writer_t *iface, st_table *freeing) +{ + avro_resolved_link_writer_t *liface = + container_of(iface, avro_resolved_link_writer_t, parent); + if (liface->target_resolver != NULL) { + free_resolver(liface->target_resolver, freeing); + } + avro_schema_decref(iface->wschema); + avro_schema_decref(iface->rschema); + avro_freet(avro_resolved_link_writer_t, iface); +} + +static int +avro_resolved_link_writer_init(const avro_resolved_writer_t *iface, void *vself) +{ + int rval; + const avro_resolved_link_writer_t *liface = + container_of(iface, avro_resolved_link_writer_t, parent); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + size_t target_instance_size = liface->target_resolver->instance_size; + + self->target.iface = &liface->target_resolver->parent; + self->target.self = avro_malloc(target_instance_size); + if (self->target.self == NULL) { + return ENOMEM; + } + DEBUG("Allocated <%p:%" PRIsz "> for link", self->target.self, target_instance_size); + + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + + rval = avro_resolved_writer_init(liface->target_resolver, self->target.self); + if (rval != 0) { + avro_free(self->target.self, target_instance_size); + } + return rval; +} + +static void +avro_resolved_link_writer_done(const avro_resolved_writer_t *iface, void *vself) +{ + const avro_resolved_link_writer_t *liface = + container_of(iface, avro_resolved_link_writer_t, parent); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + size_t target_instance_size = liface->target_resolver->instance_size; + DEBUG("Freeing <%p:%" PRIsz "> for link", self->target.self, target_instance_size); + avro_resolved_writer_done(liface->target_resolver, self->target.self); + avro_free(self->target.self, target_instance_size); + self->target.iface = NULL; + self->target.self = NULL; +} + +static int +avro_resolved_link_writer_reset(const avro_resolved_writer_t *iface, void *vself) +{ + const avro_resolved_link_writer_t *liface = + container_of(iface, avro_resolved_link_writer_t, parent); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + return avro_resolved_writer_reset_wrappers + (liface->target_resolver, self->target.self); +} + +static avro_type_t +avro_resolved_link_writer_get_type(const avro_value_iface_t *iface, const void *vself) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_type(&self->target); +} + +static avro_schema_t +avro_resolved_link_writer_get_schema(const avro_value_iface_t *iface, const void *vself) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_schema(&self->target); +} + +static int +avro_resolved_link_writer_get_boolean(const avro_value_iface_t *iface, + const void *vself, int *out) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_boolean(&self->target, out); +} + +static int +avro_resolved_link_writer_get_bytes(const avro_value_iface_t *iface, + const void *vself, const void **buf, size_t *size) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_bytes(&self->target, buf, size); +} + +static int +avro_resolved_link_writer_grab_bytes(const avro_value_iface_t *iface, + const void *vself, avro_wrapped_buffer_t *dest) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_grab_bytes(&self->target, dest); +} + +static int +avro_resolved_link_writer_get_double(const avro_value_iface_t *iface, + const void *vself, double *out) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_double(&self->target, out); +} + +static int +avro_resolved_link_writer_get_float(const avro_value_iface_t *iface, + const void *vself, float *out) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_float(&self->target, out); +} + +static int +avro_resolved_link_writer_get_int(const avro_value_iface_t *iface, + const void *vself, int32_t *out) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_int(&self->target, out); +} + +static int +avro_resolved_link_writer_get_long(const avro_value_iface_t *iface, + const void *vself, int64_t *out) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_long(&self->target, out); +} + +static int +avro_resolved_link_writer_get_null(const avro_value_iface_t *iface, const void *vself) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_null(&self->target); +} + +static int +avro_resolved_link_writer_get_string(const avro_value_iface_t *iface, + const void *vself, const char **str, size_t *size) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_string(&self->target, str, size); +} + +static int +avro_resolved_link_writer_grab_string(const avro_value_iface_t *iface, + const void *vself, avro_wrapped_buffer_t *dest) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_grab_string(&self->target, dest); +} + +static int +avro_resolved_link_writer_get_enum(const avro_value_iface_t *iface, + const void *vself, int *out) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_enum(&self->target, out); +} + +static int +avro_resolved_link_writer_get_fixed(const avro_value_iface_t *iface, + const void *vself, const void **buf, size_t *size) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_fixed(&self->target, buf, size); +} + +static int +avro_resolved_link_writer_grab_fixed(const avro_value_iface_t *iface, + const void *vself, avro_wrapped_buffer_t *dest) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_grab_fixed(&self->target, dest); +} + +static int +avro_resolved_link_writer_set_boolean(const avro_value_iface_t *iface, + void *vself, int val) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_set_boolean(&self->target, val); +} + +static int +avro_resolved_link_writer_set_bytes(const avro_value_iface_t *iface, + void *vself, void *buf, size_t size) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_set_bytes(&self->target, buf, size); +} + +static int +avro_resolved_link_writer_give_bytes(const avro_value_iface_t *iface, + void *vself, avro_wrapped_buffer_t *buf) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_give_bytes(&self->target, buf); +} + +static int +avro_resolved_link_writer_set_double(const avro_value_iface_t *iface, + void *vself, double val) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_set_double(&self->target, val); +} + +static int +avro_resolved_link_writer_set_float(const avro_value_iface_t *iface, + void *vself, float val) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_set_float(&self->target, val); +} + +static int +avro_resolved_link_writer_set_int(const avro_value_iface_t *iface, + void *vself, int32_t val) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_set_int(&self->target, val); +} + +static int +avro_resolved_link_writer_set_long(const avro_value_iface_t *iface, + void *vself, int64_t val) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_set_long(&self->target, val); +} + +static int +avro_resolved_link_writer_set_null(const avro_value_iface_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_set_null(&self->target); +} + +static int +avro_resolved_link_writer_set_string(const avro_value_iface_t *iface, + void *vself, const char *str) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_set_string(&self->target, str); +} + +static int +avro_resolved_link_writer_set_string_len(const avro_value_iface_t *iface, + void *vself, const char *str, size_t size) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_set_string_len(&self->target, str, size); +} + +static int +avro_resolved_link_writer_give_string_len(const avro_value_iface_t *iface, + void *vself, avro_wrapped_buffer_t *buf) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_give_string_len(&self->target, buf); +} + +static int +avro_resolved_link_writer_set_enum(const avro_value_iface_t *iface, + void *vself, int val) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_set_enum(&self->target, val); +} + +static int +avro_resolved_link_writer_set_fixed(const avro_value_iface_t *iface, + void *vself, void *buf, size_t size) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_set_fixed(&self->target, buf, size); +} + +static int +avro_resolved_link_writer_give_fixed(const avro_value_iface_t *iface, + void *vself, avro_wrapped_buffer_t *buf) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_give_fixed(&self->target, buf); +} + +static int +avro_resolved_link_writer_get_size(const avro_value_iface_t *iface, + const void *vself, size_t *size) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_size(&self->target, size); +} + +static int +avro_resolved_link_writer_get_by_index(const avro_value_iface_t *iface, + const void *vself, size_t index, + avro_value_t *child, const char **name) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_by_index(&self->target, index, child, name); +} + +static int +avro_resolved_link_writer_get_by_name(const avro_value_iface_t *iface, + const void *vself, const char *name, + avro_value_t *child, size_t *index) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_by_name(&self->target, name, child, index); +} + +static int +avro_resolved_link_writer_get_discriminant(const avro_value_iface_t *iface, + const void *vself, int *out) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_discriminant(&self->target, out); +} + +static int +avro_resolved_link_writer_get_current_branch(const avro_value_iface_t *iface, + const void *vself, avro_value_t *branch) +{ + AVRO_UNUSED(iface); + const avro_resolved_link_value_t *self = (const avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_get_current_branch(&self->target, branch); +} + +static int +avro_resolved_link_writer_append(const avro_value_iface_t *iface, + void *vself, avro_value_t *child_out, + size_t *new_index) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_append(&self->target, child_out, new_index); +} + +static int +avro_resolved_link_writer_add(const avro_value_iface_t *iface, + void *vself, const char *key, + avro_value_t *child, size_t *index, int *is_new) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_add(&self->target, key, child, index, is_new); +} + +static int +avro_resolved_link_writer_set_branch(const avro_value_iface_t *iface, + void *vself, int discriminant, + avro_value_t *branch) +{ + AVRO_UNUSED(iface); + avro_resolved_link_value_t *self = (avro_resolved_link_value_t *) vself; + avro_value_t *target_vself = (avro_value_t *) self->target.self; + *target_vself = self->wrapped; + return avro_value_set_branch(&self->target, discriminant, branch); +} + +static avro_resolved_link_writer_t * +avro_resolved_link_writer_create(avro_schema_t wschema, avro_schema_t rschema) +{ + avro_resolved_writer_t *self = (avro_resolved_writer_t *) avro_new(avro_resolved_link_writer_t); + memset(self, 0, sizeof(avro_resolved_link_writer_t)); + + self->parent.incref_iface = avro_resolved_writer_incref_iface; + self->parent.decref_iface = avro_resolved_writer_decref_iface; + self->parent.incref = avro_resolved_writer_incref; + self->parent.decref = avro_resolved_writer_decref; + self->parent.reset = avro_resolved_writer_reset; + self->parent.get_type = avro_resolved_link_writer_get_type; + self->parent.get_schema = avro_resolved_link_writer_get_schema; + self->parent.get_size = avro_resolved_link_writer_get_size; + self->parent.get_by_index = avro_resolved_link_writer_get_by_index; + self->parent.get_by_name = avro_resolved_link_writer_get_by_name; + + self->refcount = 1; + self->wschema = avro_schema_incref(wschema); + self->rschema = avro_schema_incref(rschema); + self->reader_union_branch = -1; + self->calculate_size = avro_resolved_link_writer_calculate_size; + self->free_iface = avro_resolved_link_writer_free_iface; + self->init = avro_resolved_link_writer_init; + self->done = avro_resolved_link_writer_done; + self->reset_wrappers = avro_resolved_link_writer_reset; + + self->parent.get_boolean = avro_resolved_link_writer_get_boolean; + self->parent.get_bytes = avro_resolved_link_writer_get_bytes; + self->parent.grab_bytes = avro_resolved_link_writer_grab_bytes; + self->parent.get_double = avro_resolved_link_writer_get_double; + self->parent.get_float = avro_resolved_link_writer_get_float; + self->parent.get_int = avro_resolved_link_writer_get_int; + self->parent.get_long = avro_resolved_link_writer_get_long; + self->parent.get_null = avro_resolved_link_writer_get_null; + self->parent.get_string = avro_resolved_link_writer_get_string; + self->parent.grab_string = avro_resolved_link_writer_grab_string; + self->parent.get_enum = avro_resolved_link_writer_get_enum; + self->parent.get_fixed = avro_resolved_link_writer_get_fixed; + self->parent.grab_fixed = avro_resolved_link_writer_grab_fixed; + + self->parent.set_boolean = avro_resolved_link_writer_set_boolean; + self->parent.set_bytes = avro_resolved_link_writer_set_bytes; + self->parent.give_bytes = avro_resolved_link_writer_give_bytes; + self->parent.set_double = avro_resolved_link_writer_set_double; + self->parent.set_float = avro_resolved_link_writer_set_float; + self->parent.set_int = avro_resolved_link_writer_set_int; + self->parent.set_long = avro_resolved_link_writer_set_long; + self->parent.set_null = avro_resolved_link_writer_set_null; + self->parent.set_string = avro_resolved_link_writer_set_string; + self->parent.set_string_len = avro_resolved_link_writer_set_string_len; + self->parent.give_string_len = avro_resolved_link_writer_give_string_len; + self->parent.set_enum = avro_resolved_link_writer_set_enum; + self->parent.set_fixed = avro_resolved_link_writer_set_fixed; + self->parent.give_fixed = avro_resolved_link_writer_give_fixed; + + self->parent.get_size = avro_resolved_link_writer_get_size; + self->parent.get_by_index = avro_resolved_link_writer_get_by_index; + self->parent.get_by_name = avro_resolved_link_writer_get_by_name; + self->parent.get_discriminant = avro_resolved_link_writer_get_discriminant; + self->parent.get_current_branch = avro_resolved_link_writer_get_current_branch; + + self->parent.append = avro_resolved_link_writer_append; + self->parent.add = avro_resolved_link_writer_add; + self->parent.set_branch = avro_resolved_link_writer_set_branch; + + return container_of(self, avro_resolved_link_writer_t, parent); +} + +static int +try_link(memoize_state_t *state, avro_resolved_writer_t **self, + avro_schema_t wschema, avro_schema_t rschema, + avro_schema_t root_rschema) +{ + AVRO_UNUSED(rschema); + + /* + * For link schemas, we create a special value implementation + * that allocates space for its wrapped value at runtime. This + * lets us handle recursive types without having to instantiate + * in infinite-size value. + */ + + avro_schema_t wtarget = avro_schema_link_target(wschema); + avro_resolved_link_writer_t *lself = + avro_resolved_link_writer_create(wtarget, root_rschema); + avro_memoize_set(&state->mem, wschema, root_rschema, lself); + + avro_resolved_writer_t *target_resolver = + avro_resolved_writer_new_memoized(state, wtarget, rschema); + if (target_resolver == NULL) { + avro_memoize_delete(&state->mem, wschema, root_rschema); + avro_value_iface_decref(&lself->parent.parent); + avro_prefix_error("Link target isn't compatible: "); + DEBUG("%s", avro_strerror()); + return EINVAL; + } + + lself->target_resolver = target_resolver; + lself->next = state->links; + state->links = lself; + + *self = &lself->parent; + return 0; +} + + +/*----------------------------------------------------------------------- + * boolean + */ + +static int +avro_resolved_writer_set_boolean(const avro_value_iface_t *viface, + void *vself, int val) +{ + int rval; + const avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + avro_value_t *self = (avro_value_t *) vself; + avro_value_t dest; + check(rval, avro_resolved_writer_get_real_dest(iface, self, &dest)); + DEBUG("Storing %s into %p", val? "TRUE": "FALSE", dest.self); + return avro_value_set_boolean(&dest, val); +} + +static int +try_boolean(memoize_state_t *state, avro_resolved_writer_t **self, + avro_schema_t wschema, avro_schema_t rschema, + avro_schema_t root_rschema) +{ + if (is_avro_boolean(rschema)) { + *self = avro_resolved_writer_create(wschema, root_rschema); + avro_memoize_set(&state->mem, wschema, root_rschema, *self); + (*self)->parent.set_boolean = avro_resolved_writer_set_boolean; + } + return 0; +} + + +/*----------------------------------------------------------------------- + * bytes + */ + +static int +avro_resolved_writer_set_bytes(const avro_value_iface_t *viface, + void *vself, void *buf, size_t size) +{ + int rval; + const avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + avro_value_t *self = (avro_value_t *) vself; + avro_value_t dest; + check(rval, avro_resolved_writer_get_real_dest(iface, self, &dest)); + DEBUG("Storing <%p:%" PRIsz "> into %p", buf, size, dest.self); + return avro_value_set_bytes(&dest, buf, size); +} + +static int +avro_resolved_writer_give_bytes(const avro_value_iface_t *viface, + void *vself, avro_wrapped_buffer_t *buf) +{ + int rval; + const avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + avro_value_t *self = (avro_value_t *) vself; + avro_value_t dest; + check(rval, avro_resolved_writer_get_real_dest(iface, self, &dest)); + DEBUG("Storing [%p] into %p", buf, dest.self); + return avro_value_give_bytes(&dest, buf); +} + +static int +try_bytes(memoize_state_t *state, avro_resolved_writer_t **self, + avro_schema_t wschema, avro_schema_t rschema, + avro_schema_t root_rschema) +{ + if (is_avro_bytes(rschema)) { + *self = avro_resolved_writer_create(wschema, root_rschema); + avro_memoize_set(&state->mem, wschema, root_rschema, *self); + (*self)->parent.set_bytes = avro_resolved_writer_set_bytes; + (*self)->parent.give_bytes = avro_resolved_writer_give_bytes; + } + return 0; +} + + +/*----------------------------------------------------------------------- + * double + */ + +static int +avro_resolved_writer_set_double(const avro_value_iface_t *viface, + void *vself, double val) +{ + int rval; + const avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + avro_value_t *self = (avro_value_t *) vself; + avro_value_t dest; + check(rval, avro_resolved_writer_get_real_dest(iface, self, &dest)); + DEBUG("Storing %le into %p", val, dest.self); + return avro_value_set_double(&dest, val); +} + +static int +try_double(memoize_state_t *state, avro_resolved_writer_t **self, + avro_schema_t wschema, avro_schema_t rschema, + avro_schema_t root_rschema) +{ + if (is_avro_double(rschema)) { + *self = avro_resolved_writer_create(wschema, root_rschema); + avro_memoize_set(&state->mem, wschema, root_rschema, *self); + (*self)->parent.set_double = avro_resolved_writer_set_double; + } + return 0; +} + + +/*----------------------------------------------------------------------- + * float + */ + +static int +avro_resolved_writer_set_float(const avro_value_iface_t *viface, + void *vself, float val) +{ + int rval; + const avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + avro_value_t *self = (avro_value_t *) vself; + avro_value_t dest; + check(rval, avro_resolved_writer_get_real_dest(iface, self, &dest)); + DEBUG("Storing %e into %p", val, dest.self); + return avro_value_set_float(&dest, val); +} + +static int +avro_resolved_writer_set_float_double(const avro_value_iface_t *viface, + void *vself, float val) +{ + int rval; + const avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + avro_value_t *self = (avro_value_t *) vself; + avro_value_t dest; + check(rval, avro_resolved_writer_get_real_dest(iface, self, &dest)); + DEBUG("Promoting float %e into double %p", val, dest.self); + return avro_value_set_double(&dest, val); +} + +static int +try_float(memoize_state_t *state, avro_resolved_writer_t **self, + avro_schema_t wschema, avro_schema_t rschema, + avro_schema_t root_rschema) +{ + if (is_avro_float(rschema)) { + *self = avro_resolved_writer_create(wschema, root_rschema); + avro_memoize_set(&state->mem, wschema, root_rschema, *self); + (*self)->parent.set_float = avro_resolved_writer_set_float; + } + + else if (is_avro_double(rschema)) { + *self = avro_resolved_writer_create(wschema, root_rschema); + avro_memoize_set(&state->mem, wschema, root_rschema, *self); + (*self)->parent.set_float = avro_resolved_writer_set_float_double; + } + + return 0; +} + + +/*----------------------------------------------------------------------- + * int + */ + +static int +avro_resolved_writer_set_int(const avro_value_iface_t *viface, + void *vself, int32_t val) +{ + int rval; + const avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + avro_value_t *self = (avro_value_t *) vself; + avro_value_t dest; + check(rval, avro_resolved_writer_get_real_dest(iface, self, &dest)); + DEBUG("Storing %" PRId32 " into %p", val, dest.self); + return avro_value_set_int(&dest, val); +} + +static int +avro_resolved_writer_set_int_double(const avro_value_iface_t *viface, + void *vself, int32_t val) +{ + int rval; + const avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + avro_value_t *self = (avro_value_t *) vself; + avro_value_t dest; + check(rval, avro_resolved_writer_get_real_dest(iface, self, &dest)); + DEBUG("Promoting int %" PRId32 " into double %p", val, dest.self); + return avro_value_set_double(&dest, val); +} + +static int +avro_resolved_writer_set_int_float(const avro_value_iface_t *viface, + void *vself, int32_t val) +{ + int rval; + const avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + avro_value_t *self = (avro_value_t *) vself; + avro_value_t dest; + check(rval, avro_resolved_writer_get_real_dest(iface, self, &dest)); + DEBUG("Promoting int %" PRId32 " into float %p", val, dest.self); + return avro_value_set_float(&dest, (float) val); +} + +static int +avro_resolved_writer_set_int_long(const avro_value_iface_t *viface, + void *vself, int32_t val) +{ + int rval; + const avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + avro_value_t *self = (avro_value_t *) vself; + avro_value_t dest; + check(rval, avro_resolved_writer_get_real_dest(iface, self, &dest)); + DEBUG("Promoting int %" PRId32 " into long %p", val, dest.self); + return avro_value_set_long(&dest, val); +} + +static int +try_int(memoize_state_t *state, avro_resolved_writer_t **self, + avro_schema_t wschema, avro_schema_t rschema, + avro_schema_t root_rschema) +{ + if (is_avro_int32(rschema)) { + *self = avro_resolved_writer_create(wschema, root_rschema); + avro_memoize_set(&state->mem, wschema, root_rschema, *self); + (*self)->parent.set_int = avro_resolved_writer_set_int; + } + + else if (is_avro_int64(rschema)) { + *self = avro_resolved_writer_create(wschema, root_rschema); + avro_memoize_set(&state->mem, wschema, root_rschema, *self); + (*self)->parent.set_int = avro_resolved_writer_set_int_long; + } + + else if (is_avro_double(rschema)) { + *self = avro_resolved_writer_create(wschema, root_rschema); + avro_memoize_set(&state->mem, wschema, root_rschema, *self); + (*self)->parent.set_int = avro_resolved_writer_set_int_double; + } + + else if (is_avro_float(rschema)) { + *self = avro_resolved_writer_create(wschema, root_rschema); + avro_memoize_set(&state->mem, wschema, root_rschema, *self); + (*self)->parent.set_int = avro_resolved_writer_set_int_float; + } + + return 0; +} + + +/*----------------------------------------------------------------------- + * long + */ + +static int +avro_resolved_writer_set_long(const avro_value_iface_t *viface, + void *vself, int64_t val) +{ + int rval; + const avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + avro_value_t *self = (avro_value_t *) vself; + avro_value_t dest; + check(rval, avro_resolved_writer_get_real_dest(iface, self, &dest)); + DEBUG("Storing %" PRId64 " into %p", val, dest.self); + return avro_value_set_long(&dest, val); +} + +static int +avro_resolved_writer_set_long_double(const avro_value_iface_t *viface, + void *vself, int64_t val) +{ + int rval; + const avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + avro_value_t *self = (avro_value_t *) vself; + avro_value_t dest; + check(rval, avro_resolved_writer_get_real_dest(iface, self, &dest)); + DEBUG("Promoting long %" PRId64 " into double %p", val, dest.self); + return avro_value_set_double(&dest, (double) val); +} + +static int +avro_resolved_writer_set_long_float(const avro_value_iface_t *viface, + void *vself, int64_t val) +{ + int rval; + const avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + avro_value_t *self = (avro_value_t *) vself; + avro_value_t dest; + check(rval, avro_resolved_writer_get_real_dest(iface, self, &dest)); + DEBUG("Promoting long %" PRId64 " into float %p", val, dest.self); + return avro_value_set_float(&dest, (float) val); +} + +static int +try_long(memoize_state_t *state, avro_resolved_writer_t **self, + avro_schema_t wschema, avro_schema_t rschema, + avro_schema_t root_rschema) +{ + if (is_avro_int64(rschema)) { + *self = avro_resolved_writer_create(wschema, root_rschema); + avro_memoize_set(&state->mem, wschema, root_rschema, *self); + (*self)->parent.set_long = avro_resolved_writer_set_long; + } + + else if (is_avro_double(rschema)) { + *self = avro_resolved_writer_create(wschema, root_rschema); + avro_memoize_set(&state->mem, wschema, root_rschema, *self); + (*self)->parent.set_long = avro_resolved_writer_set_long_double; + } + + else if (is_avro_float(rschema)) { + *self = avro_resolved_writer_create(wschema, root_rschema); + avro_memoize_set(&state->mem, wschema, root_rschema, *self); + (*self)->parent.set_long = avro_resolved_writer_set_long_float; + } + + return 0; +} + + +/*----------------------------------------------------------------------- + * null + */ + +static int +avro_resolved_writer_set_null(const avro_value_iface_t *viface, + void *vself) +{ + int rval; + const avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + avro_value_t *self = (avro_value_t *) vself; + avro_value_t dest; + check(rval, avro_resolved_writer_get_real_dest(iface, self, &dest)); + DEBUG("Storing NULL into %p", dest.self); + return avro_value_set_null(&dest); +} + +static int +try_null(memoize_state_t *state, avro_resolved_writer_t **self, + avro_schema_t wschema, avro_schema_t rschema, + avro_schema_t root_rschema) +{ + if (is_avro_null(rschema)) { + *self = avro_resolved_writer_create(wschema, root_rschema); + avro_memoize_set(&state->mem, wschema, root_rschema, *self); + (*self)->parent.set_null = avro_resolved_writer_set_null; + } + return 0; +} + + +/*----------------------------------------------------------------------- + * string + */ + +static int +avro_resolved_writer_set_string(const avro_value_iface_t *viface, + void *vself, const char *str) +{ + int rval; + const avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + avro_value_t *self = (avro_value_t *) vself; + avro_value_t dest; + check(rval, avro_resolved_writer_get_real_dest(iface, self, &dest)); + DEBUG("Storing \"%s\" into %p", str, dest.self); + return avro_value_set_string(&dest, str); +} + +static int +avro_resolved_writer_set_string_len(const avro_value_iface_t *viface, + void *vself, const char *str, size_t size) +{ + int rval; + const avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + avro_value_t *self = (avro_value_t *) vself; + avro_value_t dest; + check(rval, avro_resolved_writer_get_real_dest(iface, self, &dest)); + DEBUG("Storing <%p:%" PRIsz "> into %p", str, size, dest.self); + return avro_value_set_string_len(&dest, str, size); +} + +static int +avro_resolved_writer_give_string_len(const avro_value_iface_t *viface, + void *vself, avro_wrapped_buffer_t *buf) +{ + int rval; + const avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + avro_value_t *self = (avro_value_t *) vself; + avro_value_t dest; + check(rval, avro_resolved_writer_get_real_dest(iface, self, &dest)); + DEBUG("Storing [%p] into %p", buf, dest.self); + return avro_value_give_string_len(&dest, buf); +} + +static int +try_string(memoize_state_t *state, avro_resolved_writer_t **self, + avro_schema_t wschema, avro_schema_t rschema, + avro_schema_t root_rschema) +{ + if (is_avro_string(rschema)) { + *self = avro_resolved_writer_create(wschema, root_rschema); + avro_memoize_set(&state->mem, wschema, root_rschema, *self); + (*self)->parent.set_string = avro_resolved_writer_set_string; + (*self)->parent.set_string_len = avro_resolved_writer_set_string_len; + (*self)->parent.give_string_len = avro_resolved_writer_give_string_len; + } + return 0; +} + + +/*----------------------------------------------------------------------- + * array + */ + +typedef struct avro_resolved_array_writer { + avro_resolved_writer_t parent; + avro_resolved_writer_t *child_resolver; +} avro_resolved_array_writer_t; + +typedef struct avro_resolved_array_value { + avro_value_t wrapped; + avro_raw_array_t children; +} avro_resolved_array_value_t; + +static void +avro_resolved_array_writer_calculate_size(avro_resolved_writer_t *iface) +{ + avro_resolved_array_writer_t *aiface = + container_of(iface, avro_resolved_array_writer_t, parent); + + /* Only calculate the size for any resolver once */ + iface->calculate_size = NULL; + + DEBUG("Calculating size for %s->%s", + avro_schema_type_name((iface)->wschema), + avro_schema_type_name((iface)->rschema)); + iface->instance_size = sizeof(avro_resolved_array_value_t); + + avro_resolved_writer_calculate_size(aiface->child_resolver); +} + +static void +avro_resolved_array_writer_free_iface(avro_resolved_writer_t *iface, st_table *freeing) +{ + avro_resolved_array_writer_t *aiface = + container_of(iface, avro_resolved_array_writer_t, parent); + free_resolver(aiface->child_resolver, freeing); + avro_schema_decref(iface->wschema); + avro_schema_decref(iface->rschema); + avro_freet(avro_resolved_array_writer_t, iface); +} + +static int +avro_resolved_array_writer_init(const avro_resolved_writer_t *iface, void *vself) +{ + const avro_resolved_array_writer_t *aiface = + container_of(iface, avro_resolved_array_writer_t, parent); + avro_resolved_array_value_t *self = (avro_resolved_array_value_t *) vself; + size_t child_instance_size = aiface->child_resolver->instance_size; + DEBUG("Initializing child array (child_size=%" PRIsz ")", child_instance_size); + avro_raw_array_init(&self->children, child_instance_size); + return 0; +} + +static void +avro_resolved_array_writer_free_elements(const avro_resolved_writer_t *child_iface, + avro_resolved_array_value_t *self) +{ + size_t i; + for (i = 0; i < avro_raw_array_size(&self->children); i++) { + void *child_self = avro_raw_array_get_raw(&self->children, i); + avro_resolved_writer_done(child_iface, child_self); + } +} + +static void +avro_resolved_array_writer_done(const avro_resolved_writer_t *iface, void *vself) +{ + const avro_resolved_array_writer_t *aiface = + container_of(iface, avro_resolved_array_writer_t, parent); + avro_resolved_array_value_t *self = (avro_resolved_array_value_t *) vself; + avro_resolved_array_writer_free_elements(aiface->child_resolver, self); + avro_raw_array_done(&self->children); +} + +static int +avro_resolved_array_writer_reset(const avro_resolved_writer_t *iface, void *vself) +{ + const avro_resolved_array_writer_t *aiface = + container_of(iface, avro_resolved_array_writer_t, parent); + avro_resolved_array_value_t *self = (avro_resolved_array_value_t *) vself; + + /* Clear out our cache of wrapped children */ + avro_resolved_array_writer_free_elements(aiface->child_resolver, self); + avro_raw_array_clear(&self->children); + return 0; +} + +static int +avro_resolved_array_writer_get_size(const avro_value_iface_t *viface, + const void *vself, size_t *size) +{ + int rval; + const avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + const avro_resolved_array_value_t *self = (const avro_resolved_array_value_t *) vself; + avro_value_t dest; + check(rval, avro_resolved_writer_get_real_dest(iface, &self->wrapped, &dest)); + return avro_value_get_size(&dest, size); +} + +static int +avro_resolved_array_writer_append(const avro_value_iface_t *viface, + void *vself, avro_value_t *child_out, + size_t *new_index) +{ + int rval; + const avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + const avro_resolved_array_writer_t *aiface = + container_of(iface, avro_resolved_array_writer_t, parent); + avro_resolved_array_value_t *self = (avro_resolved_array_value_t *) vself; + avro_value_t dest; + check(rval, avro_resolved_writer_get_real_dest(iface, &self->wrapped, &dest)); + + child_out->iface = &aiface->child_resolver->parent; + child_out->self = avro_raw_array_append(&self->children); + if (child_out->self == NULL) { + avro_set_error("Couldn't expand array"); + return ENOMEM; + } + + DEBUG("Appending to array %p", dest.self); + check(rval, avro_value_append(&dest, (avro_value_t *) child_out->self, new_index)); + return avro_resolved_writer_init(aiface->child_resolver, child_out->self); +} + +static avro_resolved_array_writer_t * +avro_resolved_array_writer_create(avro_schema_t wschema, avro_schema_t rschema) +{ + avro_resolved_writer_t *self = (avro_resolved_writer_t *) avro_new(avro_resolved_array_writer_t); + memset(self, 0, sizeof(avro_resolved_array_writer_t)); + + self->parent.incref_iface = avro_resolved_writer_incref_iface; + self->parent.decref_iface = avro_resolved_writer_decref_iface; + self->parent.incref = avro_resolved_writer_incref; + self->parent.decref = avro_resolved_writer_decref; + self->parent.reset = avro_resolved_writer_reset; + self->parent.get_type = avro_resolved_writer_get_type; + self->parent.get_schema = avro_resolved_writer_get_schema; + self->parent.get_size = avro_resolved_array_writer_get_size; + self->parent.append = avro_resolved_array_writer_append; + + self->refcount = 1; + self->wschema = avro_schema_incref(wschema); + self->rschema = avro_schema_incref(rschema); + self->reader_union_branch = -1; + self->calculate_size = avro_resolved_array_writer_calculate_size; + self->free_iface = avro_resolved_array_writer_free_iface; + self->init = avro_resolved_array_writer_init; + self->done = avro_resolved_array_writer_done; + self->reset_wrappers = avro_resolved_array_writer_reset; + return container_of(self, avro_resolved_array_writer_t, parent); +} + +static int +try_array(memoize_state_t *state, avro_resolved_writer_t **self, + avro_schema_t wschema, avro_schema_t rschema, + avro_schema_t root_rschema) +{ + /* + * First verify that the reader is an array. + */ + + if (!is_avro_array(rschema)) { + return 0; + } + + /* + * Array schemas have to have compatible element schemas to be + * compatible themselves. Try to create an resolver to check + * the compatibility. + */ + + avro_resolved_array_writer_t *aself = + avro_resolved_array_writer_create(wschema, root_rschema); + avro_memoize_set(&state->mem, wschema, root_rschema, aself); + + avro_schema_t witems = avro_schema_array_items(wschema); + avro_schema_t ritems = avro_schema_array_items(rschema); + + avro_resolved_writer_t *item_resolver = + avro_resolved_writer_new_memoized(state, witems, ritems); + if (item_resolver == NULL) { + avro_memoize_delete(&state->mem, wschema, root_rschema); + avro_value_iface_decref(&aself->parent.parent); + avro_prefix_error("Array values aren't compatible: "); + return EINVAL; + } + + /* + * The two schemas are compatible. Store the item schema's + * resolver into the child_resolver field. + */ + + aself->child_resolver = item_resolver; + *self = &aself->parent; + return 0; +} + + +/*----------------------------------------------------------------------- + * enum + */ + +static int +avro_resolved_writer_set_enum(const avro_value_iface_t *viface, + void *vself, int val) +{ + int rval; + const avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + avro_value_t *self = (avro_value_t *) vself; + avro_value_t dest; + check(rval, avro_resolved_writer_get_real_dest(iface, self, &dest)); + DEBUG("Storing %d into %p", val, dest.self); + return avro_value_set_enum(&dest, val); +} + +static int +try_enum(memoize_state_t *state, avro_resolved_writer_t **self, + avro_schema_t wschema, avro_schema_t rschema, + avro_schema_t root_rschema) +{ + /* + * Enum schemas have to have the same name — but not the same + * list of symbols — to be compatible. + */ + + if (is_avro_enum(rschema)) { + const char *wname = avro_schema_name(wschema); + const char *rname = avro_schema_name(rschema); + + if (strcmp(wname, rname) == 0) { + *self = avro_resolved_writer_create(wschema, root_rschema); + avro_memoize_set(&state->mem, wschema, root_rschema, *self); + (*self)->parent.set_enum = avro_resolved_writer_set_enum; + } + } + return 0; +} + + +/*----------------------------------------------------------------------- + * fixed + */ + +static int +avro_resolved_writer_set_fixed(const avro_value_iface_t *viface, + void *vself, void *buf, size_t size) +{ + int rval; + const avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + avro_value_t *self = (avro_value_t *) vself; + avro_value_t dest; + check(rval, avro_resolved_writer_get_real_dest(iface, self, &dest)); + DEBUG("Storing <%p:%" PRIsz "> into (fixed) %p", buf, size, dest.self); + return avro_value_set_fixed(&dest, buf, size); +} + +static int +avro_resolved_writer_give_fixed(const avro_value_iface_t *viface, + void *vself, avro_wrapped_buffer_t *buf) +{ + int rval; + const avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + avro_value_t *self = (avro_value_t *) vself; + avro_value_t dest; + check(rval, avro_resolved_writer_get_real_dest(iface, self, &dest)); + DEBUG("Storing [%p] into (fixed) %p", buf, dest.self); + return avro_value_give_fixed(&dest, buf); +} + +static int +try_fixed(memoize_state_t *state, avro_resolved_writer_t **self, + avro_schema_t wschema, avro_schema_t rschema, + avro_schema_t root_rschema) +{ + /* + * Fixed schemas need the same name and size to be compatible. + */ + + if (avro_schema_equal(wschema, rschema)) { + *self = avro_resolved_writer_create(wschema, root_rschema); + avro_memoize_set(&state->mem, wschema, root_rschema, *self); + (*self)->parent.set_fixed = avro_resolved_writer_set_fixed; + (*self)->parent.give_fixed = avro_resolved_writer_give_fixed; + } + return 0; +} + + +/*----------------------------------------------------------------------- + * map + */ + +typedef struct avro_resolved_map_writer { + avro_resolved_writer_t parent; + avro_resolved_writer_t *child_resolver; +} avro_resolved_map_writer_t; + +typedef struct avro_resolved_map_value { + avro_value_t wrapped; + avro_raw_array_t children; +} avro_resolved_map_value_t; + +static void +avro_resolved_map_writer_calculate_size(avro_resolved_writer_t *iface) +{ + avro_resolved_map_writer_t *miface = + container_of(iface, avro_resolved_map_writer_t, parent); + + /* Only calculate the size for any resolver once */ + iface->calculate_size = NULL; + + DEBUG("Calculating size for %s->%s", + avro_schema_type_name((iface)->wschema), + avro_schema_type_name((iface)->rschema)); + iface->instance_size = sizeof(avro_resolved_map_value_t); + + avro_resolved_writer_calculate_size(miface->child_resolver); +} + +static void +avro_resolved_map_writer_free_iface(avro_resolved_writer_t *iface, st_table *freeing) +{ + avro_resolved_map_writer_t *miface = + container_of(iface, avro_resolved_map_writer_t, parent); + free_resolver(miface->child_resolver, freeing); + avro_schema_decref(iface->wschema); + avro_schema_decref(iface->rschema); + avro_freet(avro_resolved_map_writer_t, iface); +} + +static int +avro_resolved_map_writer_init(const avro_resolved_writer_t *iface, void *vself) +{ + const avro_resolved_map_writer_t *miface = + container_of(iface, avro_resolved_map_writer_t, parent); + avro_resolved_map_value_t *self = (avro_resolved_map_value_t *) vself; + size_t child_instance_size = miface->child_resolver->instance_size; + DEBUG("Initializing child array for map (child_size=%" PRIsz ")", child_instance_size); + avro_raw_array_init(&self->children, child_instance_size); + return 0; +} + +static void +avro_resolved_map_writer_free_elements(const avro_resolved_writer_t *child_iface, + avro_resolved_map_value_t *self) +{ + size_t i; + for (i = 0; i < avro_raw_array_size(&self->children); i++) { + void *child_self = avro_raw_array_get_raw(&self->children, i); + avro_resolved_writer_done(child_iface, child_self); + } +} + +static void +avro_resolved_map_writer_done(const avro_resolved_writer_t *iface, void *vself) +{ + const avro_resolved_map_writer_t *miface = + container_of(iface, avro_resolved_map_writer_t, parent); + avro_resolved_map_value_t *self = (avro_resolved_map_value_t *) vself; + avro_resolved_map_writer_free_elements(miface->child_resolver, self); + avro_raw_array_done(&self->children); +} + +static int +avro_resolved_map_writer_reset(const avro_resolved_writer_t *iface, void *vself) +{ + const avro_resolved_map_writer_t *miface = + container_of(iface, avro_resolved_map_writer_t, parent); + avro_resolved_map_value_t *self = (avro_resolved_map_value_t *) vself; + + /* Clear out our cache of wrapped children */ + avro_resolved_map_writer_free_elements(miface->child_resolver, self); + return 0; +} + +static int +avro_resolved_map_writer_get_size(const avro_value_iface_t *viface, + const void *vself, size_t *size) +{ + int rval; + const avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + const avro_resolved_map_value_t *self = (const avro_resolved_map_value_t *) vself; + avro_value_t dest; + check(rval, avro_resolved_writer_get_real_dest(iface, &self->wrapped, &dest)); + return avro_value_get_size(&dest, size); +} + +static int +avro_resolved_map_writer_add(const avro_value_iface_t *viface, + void *vself, const char *key, + avro_value_t *child, size_t *index, int *is_new) +{ + int rval; + const avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + const avro_resolved_map_writer_t *miface = + container_of(iface, avro_resolved_map_writer_t, parent); + avro_resolved_map_value_t *self = (avro_resolved_map_value_t *) vself; + avro_value_t dest; + check(rval, avro_resolved_writer_get_real_dest(iface, &self->wrapped, &dest)); + + /* + * This is a bit convoluted. We need to stash the wrapped child + * value somewhere in our children array. But we don't know + * where to put it until the wrapped map tells us whether this + * is a new value, and if not, which index the value should go + * in. + */ + + avro_value_t real_child; + size_t real_index; + int real_is_new; + + DEBUG("Adding %s to map %p", key, dest.self); + check(rval, avro_value_add(&dest, key, &real_child, &real_index, &real_is_new)); + + child->iface = &miface->child_resolver->parent; + if (real_is_new) { + child->self = avro_raw_array_append(&self->children); + DEBUG("Element is new (child resolver=%p)", child->self); + if (child->self == NULL) { + avro_set_error("Couldn't expand map"); + return ENOMEM; + } + check(rval, avro_resolved_writer_init + (miface->child_resolver, child->self)); + } else { + child->self = avro_raw_array_get_raw(&self->children, real_index); + DEBUG("Element is old (child resolver=%p)", child->self); + } + avro_value_t *child_vself = (avro_value_t *) child->self; + *child_vself = real_child; + + if (index != NULL) { + *index = real_index; + } + if (is_new != NULL) { + *is_new = real_is_new; + } + return 0; +} + +static avro_resolved_map_writer_t * +avro_resolved_map_writer_create(avro_schema_t wschema, avro_schema_t rschema) +{ + avro_resolved_writer_t *self = (avro_resolved_writer_t *) avro_new(avro_resolved_map_writer_t); + memset(self, 0, sizeof(avro_resolved_map_writer_t)); + + self->parent.incref_iface = avro_resolved_writer_incref_iface; + self->parent.decref_iface = avro_resolved_writer_decref_iface; + self->parent.incref = avro_resolved_writer_incref; + self->parent.decref = avro_resolved_writer_decref; + self->parent.reset = avro_resolved_writer_reset; + self->parent.get_type = avro_resolved_writer_get_type; + self->parent.get_schema = avro_resolved_writer_get_schema; + self->parent.get_size = avro_resolved_map_writer_get_size; + self->parent.add = avro_resolved_map_writer_add; + + self->refcount = 1; + self->wschema = avro_schema_incref(wschema); + self->rschema = avro_schema_incref(rschema); + self->reader_union_branch = -1; + self->calculate_size = avro_resolved_map_writer_calculate_size; + self->free_iface = avro_resolved_map_writer_free_iface; + self->init = avro_resolved_map_writer_init; + self->done = avro_resolved_map_writer_done; + self->reset_wrappers = avro_resolved_map_writer_reset; + return container_of(self, avro_resolved_map_writer_t, parent); +} + +static int +try_map(memoize_state_t *state, avro_resolved_writer_t **self, + avro_schema_t wschema, avro_schema_t rschema, + avro_schema_t root_rschema) +{ + /* + * First verify that the reader is an map. + */ + + if (!is_avro_map(rschema)) { + return 0; + } + + /* + * Map schemas have to have compatible element schemas to be + * compatible themselves. Try to create an resolver to check + * the compatibility. + */ + + avro_resolved_map_writer_t *mself = + avro_resolved_map_writer_create(wschema, root_rschema); + avro_memoize_set(&state->mem, wschema, root_rschema, mself); + + avro_schema_t witems = avro_schema_map_values(wschema); + avro_schema_t ritems = avro_schema_map_values(rschema); + + avro_resolved_writer_t *item_resolver = + avro_resolved_writer_new_memoized(state, witems, ritems); + if (item_resolver == NULL) { + avro_memoize_delete(&state->mem, wschema, root_rschema); + avro_value_iface_decref(&mself->parent.parent); + avro_prefix_error("Map values aren't compatible: "); + return EINVAL; + } + + /* + * The two schemas are compatible. Store the item schema's + * resolver into the child_resolver field. + */ + + mself->child_resolver = item_resolver; + *self = &mself->parent; + return 0; +} + + +/*----------------------------------------------------------------------- + * record + */ + +typedef struct avro_resolved_record_writer { + avro_resolved_writer_t parent; + size_t field_count; + size_t *field_offsets; + avro_resolved_writer_t **field_resolvers; + size_t *index_mapping; +} avro_resolved_record_writer_t; + +typedef struct avro_resolved_record_value { + avro_value_t wrapped; + /* The rest of the struct is taken up by the inline storage + * needed for each field. */ +} avro_resolved_record_value_t; + +/** Return a pointer to the given field within a record struct. */ +#define avro_resolved_record_field(iface, rec, index) \ + (((char *) (rec)) + (iface)->field_offsets[(index)]) + + +static void +avro_resolved_record_writer_calculate_size(avro_resolved_writer_t *iface) +{ + avro_resolved_record_writer_t *riface = + container_of(iface, avro_resolved_record_writer_t, parent); + + /* Only calculate the size for any resolver once */ + iface->calculate_size = NULL; + + DEBUG("Calculating size for %s->%s", + avro_schema_type_name((iface)->wschema), + avro_schema_type_name((iface)->rschema)); + + /* + * Once we've figured out which writer fields we actually need, + * calculate an offset for each one. + */ + + size_t wi; + size_t next_offset = sizeof(avro_resolved_record_value_t); + for (wi = 0; wi < riface->field_count; wi++) { + riface->field_offsets[wi] = next_offset; + if (riface->field_resolvers[wi] != NULL) { + avro_resolved_writer_calculate_size + (riface->field_resolvers[wi]); + size_t field_size = + riface->field_resolvers[wi]->instance_size; + DEBUG("Field %" PRIsz " has size %" PRIsz, wi, field_size); + next_offset += field_size; + } else { + DEBUG("Field %" PRIsz " is being skipped", wi); + } + } + + DEBUG("Record has size %" PRIsz, next_offset); + iface->instance_size = next_offset; +} + +static void +avro_resolved_record_writer_free_iface(avro_resolved_writer_t *iface, st_table *freeing) +{ + avro_resolved_record_writer_t *riface = + container_of(iface, avro_resolved_record_writer_t, parent); + + if (riface->field_offsets != NULL) { + avro_free(riface->field_offsets, + riface->field_count * sizeof(size_t)); + } + + if (riface->field_resolvers != NULL) { + size_t i; + for (i = 0; i < riface->field_count; i++) { + if (riface->field_resolvers[i] != NULL) { + DEBUG("Freeing field %" PRIsz " %p", i, + riface->field_resolvers[i]); + free_resolver(riface->field_resolvers[i], freeing); + } + } + avro_free(riface->field_resolvers, + riface->field_count * sizeof(avro_resolved_writer_t *)); + } + + if (riface->index_mapping != NULL) { + avro_free(riface->index_mapping, + riface->field_count * sizeof(size_t)); + } + + avro_schema_decref(iface->wschema); + avro_schema_decref(iface->rschema); + avro_freet(avro_resolved_record_writer_t, iface); +} + +static int +avro_resolved_record_writer_init(const avro_resolved_writer_t *iface, void *vself) +{ + int rval; + const avro_resolved_record_writer_t *riface = + container_of(iface, avro_resolved_record_writer_t, parent); + avro_resolved_record_value_t *self = (avro_resolved_record_value_t *) vself; + + /* Initialize each field */ + size_t i; + for (i = 0; i < riface->field_count; i++) { + if (riface->field_resolvers[i] != NULL) { + check(rval, avro_resolved_writer_init + (riface->field_resolvers[i], + avro_resolved_record_field(riface, self, i))); + } + } + + return 0; +} + +static void +avro_resolved_record_writer_done(const avro_resolved_writer_t *iface, void *vself) +{ + const avro_resolved_record_writer_t *riface = + container_of(iface, avro_resolved_record_writer_t, parent); + avro_resolved_record_value_t *self = (avro_resolved_record_value_t *) vself; + + /* Finalize each field */ + size_t i; + for (i = 0; i < riface->field_count; i++) { + if (riface->field_resolvers[i] != NULL) { + avro_resolved_writer_done + (riface->field_resolvers[i], + avro_resolved_record_field(riface, self, i)); + } + } +} + +static int +avro_resolved_record_writer_reset(const avro_resolved_writer_t *iface, void *vself) +{ + int rval; + const avro_resolved_record_writer_t *riface = + container_of(iface, avro_resolved_record_writer_t, parent); + avro_resolved_record_value_t *self = (avro_resolved_record_value_t *) vself; + + /* Reset each field */ + size_t i; + for (i = 0; i < riface->field_count; i++) { + if (riface->field_resolvers[i] != NULL) { + check(rval, avro_resolved_writer_reset_wrappers + (riface->field_resolvers[i], + avro_resolved_record_field(riface, self, i))); + } + } + + return 0; +} + +static int +avro_resolved_record_writer_get_size(const avro_value_iface_t *viface, + const void *vself, size_t *size) +{ + AVRO_UNUSED(vself); + const avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + const avro_resolved_record_writer_t *riface = + container_of(iface, avro_resolved_record_writer_t, parent); + *size = riface->field_count; + return 0; +} + +static int +avro_resolved_record_writer_get_by_index(const avro_value_iface_t *viface, + const void *vself, size_t index, + avro_value_t *child, const char **name) +{ + int rval; + const avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + const avro_resolved_record_writer_t *riface = + container_of(iface, avro_resolved_record_writer_t, parent); + const avro_resolved_record_value_t *self = (const avro_resolved_record_value_t *) vself; + avro_value_t dest; + + DEBUG("Getting writer field %" PRIsz " from record %p", index, self); + if (riface->field_resolvers[index] == NULL) { + DEBUG("Reader doesn't have field, skipping"); + child->iface = NULL; + child->self = NULL; + return 0; + } + + check(rval, avro_resolved_writer_get_real_dest(iface, &self->wrapped, &dest)); + size_t reader_index = riface->index_mapping[index]; + DEBUG(" Reader field is %" PRIsz, reader_index); + child->iface = &riface->field_resolvers[index]->parent; + child->self = avro_resolved_record_field(riface, self, index); + + return avro_value_get_by_index(&dest, reader_index, (avro_value_t *) child->self, name); +} + +static int +avro_resolved_record_writer_get_by_name(const avro_value_iface_t *viface, + const void *vself, const char *name, + avro_value_t *child, size_t *index) +{ + const avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + + int wi = avro_schema_record_field_get_index(iface->wschema, name); + if (wi == -1) { + avro_set_error("Record doesn't have field named %s", name); + return EINVAL; + } + + DEBUG("Writer field %s is at index %d", name, wi); + if (index != NULL) { + *index = wi; + } + return avro_resolved_record_writer_get_by_index(viface, vself, wi, child, NULL); +} + +static avro_resolved_record_writer_t * +avro_resolved_record_writer_create(avro_schema_t wschema, avro_schema_t rschema) +{ + avro_resolved_writer_t *self = (avro_resolved_writer_t *) avro_new(avro_resolved_record_writer_t); + memset(self, 0, sizeof(avro_resolved_record_writer_t)); + + self->parent.incref_iface = avro_resolved_writer_incref_iface; + self->parent.decref_iface = avro_resolved_writer_decref_iface; + self->parent.incref = avro_resolved_writer_incref; + self->parent.decref = avro_resolved_writer_decref; + self->parent.reset = avro_resolved_writer_reset; + self->parent.get_type = avro_resolved_writer_get_type; + self->parent.get_schema = avro_resolved_writer_get_schema; + self->parent.get_size = avro_resolved_record_writer_get_size; + self->parent.get_by_index = avro_resolved_record_writer_get_by_index; + self->parent.get_by_name = avro_resolved_record_writer_get_by_name; + + self->refcount = 1; + self->wschema = avro_schema_incref(wschema); + self->rschema = avro_schema_incref(rschema); + self->reader_union_branch = -1; + self->calculate_size = avro_resolved_record_writer_calculate_size; + self->free_iface = avro_resolved_record_writer_free_iface; + self->init = avro_resolved_record_writer_init; + self->done = avro_resolved_record_writer_done; + self->reset_wrappers = avro_resolved_record_writer_reset; + return container_of(self, avro_resolved_record_writer_t, parent); +} + +static int +try_record(memoize_state_t *state, avro_resolved_writer_t **self, + avro_schema_t wschema, avro_schema_t rschema, + avro_schema_t root_rschema) +{ + /* + * First verify that the reader is also a record, and has the + * same name as the writer. + */ + + if (!is_avro_record(rschema)) { + return 0; + } + + const char *wname = avro_schema_name(wschema); + const char *rname = avro_schema_name(rschema); + + if (strcmp(wname, rname) != 0) { + return 0; + } + + /* + * Categorize the fields in the record schemas. Fields that are + * only in the writer are ignored. Fields that are only in the + * reader raise a schema mismatch error, unless the field has a + * default value. Fields that are in both are resolved + * recursively. + * + * The field_resolvers array will contain an avro_value_iface_t + * for each field in the writer schema. To build this array, we + * loop through the fields of the reader schema. If that field + * is also in the writer schema, we resolve them recursively, + * and store the resolver into the array. If the field isn't in + * the writer schema, we raise an error. (TODO: Eventually, + * we'll handle default values here.) After this loop finishes, + * any NULLs in the field_resolvers array will represent fields + * in the writer but not the reader; these fields will be + * skipped when processing the input. + */ + + avro_resolved_record_writer_t *rself = + avro_resolved_record_writer_create(wschema, root_rschema); + avro_memoize_set(&state->mem, wschema, root_rschema, rself); + + size_t wfields = avro_schema_record_size(wschema); + size_t rfields = avro_schema_record_size(rschema); + + DEBUG("Checking writer record schema %s", wname); + + avro_resolved_writer_t **field_resolvers = + (avro_resolved_writer_t **) avro_calloc(wfields, sizeof(avro_resolved_writer_t *)); + size_t *field_offsets = (size_t *) avro_calloc(wfields, sizeof(size_t)); + size_t *index_mapping = (size_t *) avro_calloc(wfields, sizeof(size_t)); + + size_t ri; + for (ri = 0; ri < rfields; ri++) { + avro_schema_t rfield = + avro_schema_record_field_get_by_index(rschema, ri); + const char *field_name = + avro_schema_record_field_name(rschema, ri); + + DEBUG("Resolving reader record field %" PRIsz " (%s)", ri, field_name); + + /* + * See if this field is also in the writer schema. + */ + + int wi = avro_schema_record_field_get_index(wschema, field_name); + + if (wi == -1) { + /* + * This field isn't in the writer schema — + * that's an error! TODO: Handle default + * values! + */ + + DEBUG("Field %s isn't in writer", field_name); + + /* Allow missing fields in the writer. They + * will default to zero. So skip over the + * missing field, and continue building the + * resolver. Note also that all missing values + * are zero because avro_generic_value_new() + * initializes all values of the reader to 0 + * on creation. This is a work-around because + * default values are not implemented yet. + */ + #ifdef AVRO_ALLOW_MISSING_FIELDS_IN_RESOLVED_WRITER + continue; + #else + avro_set_error("Reader field %s doesn't appear in writer", + field_name); + goto error; + #endif + } + + /* + * Try to recursively resolve the schemas for this + * field. If they're not compatible, that's an error. + */ + + avro_schema_t wfield = + avro_schema_record_field_get_by_index(wschema, wi); + avro_resolved_writer_t *field_resolver = + avro_resolved_writer_new_memoized(state, wfield, rfield); + + if (field_resolver == NULL) { + avro_prefix_error("Field %s isn't compatible: ", field_name); + goto error; + } + + /* + * Save the details for this field. + */ + + DEBUG("Found match for field %s (%" PRIsz " in reader, %d in writer)", + field_name, ri, wi); + field_resolvers[wi] = field_resolver; + index_mapping[wi] = ri; + } + + /* + * We might not have found matches for all of the writer fields, + * but that's okay — any extras will be ignored. + */ + + rself->field_count = wfields; + rself->field_offsets = field_offsets; + rself->field_resolvers = field_resolvers; + rself->index_mapping = index_mapping; + *self = &rself->parent; + return 0; + +error: + /* + * Clean up any resolver we might have already created. + */ + + avro_memoize_delete(&state->mem, wschema, root_rschema); + avro_value_iface_decref(&rself->parent.parent); + + { + unsigned int i; + for (i = 0; i < wfields; i++) { + if (field_resolvers[i]) { + avro_value_iface_decref(&field_resolvers[i]->parent); + } + } + } + + avro_free(field_resolvers, wfields * sizeof(avro_resolved_writer_t *)); + avro_free(field_offsets, wfields * sizeof(size_t)); + avro_free(index_mapping, wfields * sizeof(size_t)); + return EINVAL; +} + + +/*----------------------------------------------------------------------- + * union + */ + +typedef struct avro_resolved_union_writer { + avro_resolved_writer_t parent; + size_t branch_count; + avro_resolved_writer_t **branch_resolvers; +} avro_resolved_union_writer_t; + +typedef struct avro_resolved_union_value { + avro_value_t wrapped; + + /** The currently active branch of the union. -1 if no branch + * is selected. */ + int discriminant; + + /* The rest of the struct is taken up by the inline storage + * needed for the active branch. */ +} avro_resolved_union_value_t; + +/** Return a pointer to the active branch within a union struct. */ +#define avro_resolved_union_branch(_union) \ + (((char *) (_union)) + sizeof(avro_resolved_union_value_t)) + + +static void +avro_resolved_union_writer_calculate_size(avro_resolved_writer_t *iface) +{ + avro_resolved_union_writer_t *uiface = + container_of(iface, avro_resolved_union_writer_t, parent); + + /* Only calculate the size for any resolver once */ + iface->calculate_size = NULL; + + DEBUG("Calculating size for %s->%s", + avro_schema_type_name((iface)->wschema), + avro_schema_type_name((iface)->rschema)); + + size_t i; + size_t max_branch_size = 0; + for (i = 0; i < uiface->branch_count; i++) { + if (uiface->branch_resolvers[i] == NULL) { + DEBUG("No match for writer union branch %" PRIsz, i); + } else { + avro_resolved_writer_calculate_size + (uiface->branch_resolvers[i]); + size_t branch_size = + uiface->branch_resolvers[i]->instance_size; + DEBUG("Writer branch %" PRIsz " has size %" PRIsz, i, branch_size); + if (branch_size > max_branch_size) { + max_branch_size = branch_size; + } + } + } + + DEBUG("Maximum branch size is %" PRIsz, max_branch_size); + iface->instance_size = + sizeof(avro_resolved_union_value_t) + max_branch_size; + DEBUG("Total union size is %" PRIsz, iface->instance_size); +} + +static void +avro_resolved_union_writer_free_iface(avro_resolved_writer_t *iface, st_table *freeing) +{ + avro_resolved_union_writer_t *uiface = + container_of(iface, avro_resolved_union_writer_t, parent); + + if (uiface->branch_resolvers != NULL) { + size_t i; + for (i = 0; i < uiface->branch_count; i++) { + if (uiface->branch_resolvers[i] != NULL) { + free_resolver(uiface->branch_resolvers[i], freeing); + } + } + avro_free(uiface->branch_resolvers, + uiface->branch_count * sizeof(avro_resolved_writer_t *)); + } + + avro_schema_decref(iface->wschema); + avro_schema_decref(iface->rschema); + avro_freet(avro_resolved_union_writer_t, iface); +} + +static int +avro_resolved_union_writer_init(const avro_resolved_writer_t *iface, void *vself) +{ + AVRO_UNUSED(iface); + avro_resolved_union_value_t *self = (avro_resolved_union_value_t *) vself; + self->discriminant = -1; + return 0; +} + +static void +avro_resolved_union_writer_done(const avro_resolved_writer_t *iface, void *vself) +{ + const avro_resolved_union_writer_t *uiface = + container_of(iface, avro_resolved_union_writer_t, parent); + avro_resolved_union_value_t *self = (avro_resolved_union_value_t *) vself; + if (self->discriminant >= 0) { + avro_resolved_writer_done + (uiface->branch_resolvers[self->discriminant], + avro_resolved_union_branch(self)); + self->discriminant = -1; + } +} + +static int +avro_resolved_union_writer_reset(const avro_resolved_writer_t *iface, void *vself) +{ + const avro_resolved_union_writer_t *uiface = + container_of(iface, avro_resolved_union_writer_t, parent); + avro_resolved_union_value_t *self = (avro_resolved_union_value_t *) vself; + + /* Keep the same branch selected, for the common case that we're + * about to reuse it. */ + if (self->discriminant >= 0) { + return avro_resolved_writer_reset_wrappers + (uiface->branch_resolvers[self->discriminant], + avro_resolved_union_branch(self)); + } + + return 0; +} + +static int +avro_resolved_union_writer_set_branch(const avro_value_iface_t *viface, + void *vself, int discriminant, + avro_value_t *branch) +{ + int rval; + const avro_resolved_writer_t *iface = + container_of(viface, avro_resolved_writer_t, parent); + const avro_resolved_union_writer_t *uiface = + container_of(iface, avro_resolved_union_writer_t, parent); + avro_resolved_union_value_t *self = (avro_resolved_union_value_t *) vself; + + DEBUG("Getting writer branch %d from union %p", discriminant, vself); + avro_resolved_writer_t *branch_resolver = + uiface->branch_resolvers[discriminant]; + if (branch_resolver == NULL) { + DEBUG("Reader doesn't have branch, skipping"); + avro_set_error("Writer union branch %d is incompatible " + "with reader schema \"%s\"", + discriminant, avro_schema_type_name(iface->rschema)); + return EINVAL; + } + + if (self->discriminant == discriminant) { + DEBUG("Writer branch %d already selected", discriminant); + } else { + if (self->discriminant >= 0) { + DEBUG("Finalizing old writer branch %d", self->discriminant); + avro_resolved_writer_done + (uiface->branch_resolvers[self->discriminant], + avro_resolved_union_branch(self)); + } + DEBUG("Initializing writer branch %d", discriminant); + check(rval, avro_resolved_writer_init + (uiface->branch_resolvers[discriminant], + avro_resolved_union_branch(self))); + self->discriminant = discriminant; + } + + branch->iface = &branch_resolver->parent; + branch->self = avro_resolved_union_branch(self); + avro_value_t *branch_vself = (avro_value_t *) branch->self; + *branch_vself = self->wrapped; + return 0; +} + +static avro_resolved_union_writer_t * +avro_resolved_union_writer_create(avro_schema_t wschema, avro_schema_t rschema) +{ + avro_resolved_writer_t *self = (avro_resolved_writer_t *) avro_new(avro_resolved_union_writer_t); + memset(self, 0, sizeof(avro_resolved_union_writer_t)); + + self->parent.incref_iface = avro_resolved_writer_incref_iface; + self->parent.decref_iface = avro_resolved_writer_decref_iface; + self->parent.incref = avro_resolved_writer_incref; + self->parent.decref = avro_resolved_writer_decref; + self->parent.reset = avro_resolved_writer_reset; + self->parent.get_type = avro_resolved_writer_get_type; + self->parent.get_schema = avro_resolved_writer_get_schema; + self->parent.set_branch = avro_resolved_union_writer_set_branch; + + self->refcount = 1; + self->wschema = avro_schema_incref(wschema); + self->rschema = avro_schema_incref(rschema); + self->reader_union_branch = -1; + self->calculate_size = avro_resolved_union_writer_calculate_size; + self->free_iface = avro_resolved_union_writer_free_iface; + self->init = avro_resolved_union_writer_init; + self->done = avro_resolved_union_writer_done; + self->reset_wrappers = avro_resolved_union_writer_reset; + return container_of(self, avro_resolved_union_writer_t, parent); +} + +static avro_resolved_writer_t * +try_union(memoize_state_t *state, + avro_schema_t wschema, avro_schema_t rschema) +{ + /* + * For a writer union, we recursively try to resolve each branch + * against the reader schema. This will work correctly whether + * or not the reader is also a union — if the reader is a union, + * then we'll resolve each (non-union) writer branch against the + * reader union, which will be checked in our calls to + * check_simple_writer below. The net result is that we might + * end up trying every combination of writer and reader + * branches, when looking for compatible schemas. + * + * Regardless of what the reader schema is, for each writer + * branch, we stash away the recursive resolver into the + * branch_resolvers array. A NULL entry in this array means + * that that branch isn't compatible with the reader. This + * isn't an immediate schema resolution error, since we allow + * incompatible branches in the types as long as that branch + * never appears in the actual data. We only return an error if + * there are *no* branches that are compatible. + */ + + size_t branch_count = avro_schema_union_size(wschema); + DEBUG("Checking %" PRIsz "-branch writer union schema", branch_count); + + avro_resolved_union_writer_t *uself = + avro_resolved_union_writer_create(wschema, rschema); + avro_memoize_set(&state->mem, wschema, rschema, uself); + + avro_resolved_writer_t **branch_resolvers = + (avro_resolved_writer_t **) avro_calloc(branch_count, sizeof(avro_resolved_writer_t *)); + int some_branch_compatible = 0; + + size_t i; + for (i = 0; i < branch_count; i++) { + avro_schema_t branch_schema = + avro_schema_union_branch(wschema, i); + + DEBUG("Resolving writer union branch %" PRIsz " (%s)", i, + avro_schema_type_name(branch_schema)); + + /* + * Try to recursively resolve this branch of the writer + * union. Don't raise an error if this fails — it's + * okay for some of the branches to not be compatible + * with the reader, as long as those branches never + * appear in the input. + */ + + branch_resolvers[i] = + avro_resolved_writer_new_memoized(state, branch_schema, rschema); + if (branch_resolvers[i] == NULL) { + DEBUG("No match for writer union branch %" PRIsz, i); + } else { + DEBUG("Found match for writer union branch %" PRIsz, i); + some_branch_compatible = 1; + } + } + + /* + * As long as there's at least one branch that's compatible with + * the reader, then we consider this schema resolution a + * success. + */ + + if (!some_branch_compatible) { + DEBUG("No writer union branches match"); + avro_set_error("No branches in the writer are compatible " + "with reader schema %s", + avro_schema_type_name(rschema)); + goto error; + } + + uself->branch_count = branch_count; + uself->branch_resolvers = branch_resolvers; + return &uself->parent; + +error: + /* + * Clean up any resolver we might have already created. + */ + + avro_memoize_delete(&state->mem, wschema, rschema); + avro_value_iface_decref(&uself->parent.parent); + + { + unsigned int i; + for (i = 0; i < branch_count; i++) { + if (branch_resolvers[i]) { + avro_value_iface_decref(&branch_resolvers[i]->parent); + } + } + } + + avro_free(branch_resolvers, branch_count * sizeof(avro_resolved_writer_t *)); + return NULL; +} + + +/*----------------------------------------------------------------------- + * Schema type dispatcher + */ + +static avro_resolved_writer_t * +avro_resolved_writer_new_memoized(memoize_state_t *state, + avro_schema_t wschema, avro_schema_t rschema) +{ + check_param(NULL, is_avro_schema(wschema), "writer schema"); + check_param(NULL, is_avro_schema(rschema), "reader schema"); + + skip_links(rschema); + + /* + * First see if we've already matched these two schemas. If so, + * just return that resolver. + */ + + avro_resolved_writer_t *saved = NULL; + if (avro_memoize_get(&state->mem, wschema, rschema, (void **) &saved)) { + DEBUG("Already resolved %s%s%s->%s", + is_avro_link(wschema)? "[": "", + avro_schema_type_name(wschema), + is_avro_link(wschema)? "]": "", + avro_schema_type_name(rschema)); + avro_value_iface_incref(&saved->parent); + return saved; + } else { + DEBUG("Resolving %s%s%s->%s", + is_avro_link(wschema)? "[": "", + avro_schema_type_name(wschema), + is_avro_link(wschema)? "]": "", + avro_schema_type_name(rschema)); + } + + /* + * Otherwise we have some work to do. + */ + + switch (avro_typeof(wschema)) + { + case AVRO_BOOLEAN: + check_simple_writer(state, wschema, rschema, boolean); + return NULL; + + case AVRO_BYTES: + check_simple_writer(state, wschema, rschema, bytes); + return NULL; + + case AVRO_DOUBLE: + check_simple_writer(state, wschema, rschema, double); + return NULL; + + case AVRO_FLOAT: + check_simple_writer(state, wschema, rschema, float); + return NULL; + + case AVRO_INT32: + check_simple_writer(state, wschema, rschema, int); + return NULL; + + case AVRO_INT64: + check_simple_writer(state, wschema, rschema, long); + return NULL; + + case AVRO_NULL: + check_simple_writer(state, wschema, rschema, null); + return NULL; + + case AVRO_STRING: + check_simple_writer(state, wschema, rschema, string); + return NULL; + + case AVRO_ARRAY: + check_simple_writer(state, wschema, rschema, array); + return NULL; + + case AVRO_ENUM: + check_simple_writer(state, wschema, rschema, enum); + return NULL; + + case AVRO_FIXED: + check_simple_writer(state, wschema, rschema, fixed); + return NULL; + + case AVRO_MAP: + check_simple_writer(state, wschema, rschema, map); + return NULL; + + case AVRO_RECORD: + check_simple_writer(state, wschema, rschema, record); + return NULL; + + case AVRO_UNION: + return try_union(state, wschema, rschema); + + case AVRO_LINK: + check_simple_writer(state, wschema, rschema, link); + return NULL; + + default: + avro_set_error("Unknown schema type"); + return NULL; + } + + return NULL; +} + + +avro_value_iface_t * +avro_resolved_writer_new(avro_schema_t wschema, avro_schema_t rschema) +{ + /* + * Create a state to keep track of the value implementations + * that we create for each subschema. + */ + + memoize_state_t state; + avro_memoize_init(&state.mem); + state.links = NULL; + + /* + * Create the value implementations. + */ + + avro_resolved_writer_t *result = + avro_resolved_writer_new_memoized(&state, wschema, rschema); + if (result == NULL) { + avro_memoize_done(&state.mem); + return NULL; + } + + /* + * Fix up any link schemas so that their value implementations + * point to their target schemas' implementations. + */ + + avro_resolved_writer_calculate_size(result); + while (state.links != NULL) { + avro_resolved_link_writer_t *liface = state.links; + avro_resolved_writer_calculate_size(liface->target_resolver); + state.links = liface->next; + liface->next = NULL; + } + + /* + * And now we can return. + */ + + avro_memoize_done(&state.mem); + return &result->parent; +} diff --git a/fluent-bit/lib/avro/src/resolver.c b/fluent-bit/lib/avro/src/resolver.c new file mode 100644 index 00000000..f0256c26 --- /dev/null +++ b/fluent-bit/lib/avro/src/resolver.c @@ -0,0 +1,1338 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <avro/platform.h> +#include <stdlib.h> +#include <string.h> + +#include "avro/allocation.h" +#include "avro/consumer.h" +#include "avro/data.h" +#include "avro/errors.h" +#include "avro/legacy.h" +#include "avro/schema.h" +#include "avro_private.h" +#include "st.h" + + +#if !defined(DEBUG_RESOLVER) +#define DEBUG_RESOLVER 0 +#endif + +#if DEBUG_RESOLVER +#include <stdio.h> +#define debug(...) { fprintf(stderr, __VA_ARGS__); fprintf(stderr, "\n"); } +#else +#define debug(...) /* no debug output */ +#endif + + +typedef struct avro_resolver_t avro_resolver_t; + +struct avro_resolver_t { + avro_consumer_t parent; + + /* The reader schema for this resolver. */ + avro_schema_t rschema; + + /* An array of any child resolvers needed for the subschemas of + * wschema */ + avro_consumer_t **child_resolvers; + + /* If the reader and writer schemas are records, this field + * contains a mapping from writer field indices to reader field + * indices. */ + int *index_mapping; + + /* The number of elements in the child_resolvers and + * index_mapping arrays. */ + size_t num_children; + + /* If the reader schema is a union, but the writer schema is + * not, this field indicates which branch of the reader union + * should be selected. */ + int reader_union_branch; +}; + + +/** + * Frees a resolver object, while ensuring that all of the resolvers in + * a graph of resolvers is only freed once. + */ + +static void +avro_resolver_free_cycles(avro_consumer_t *consumer, st_table *freeing) +{ + avro_resolver_t *resolver = (avro_resolver_t *) consumer; + + /* + * First check if we've already started freeing this resolver. + */ + + if (st_lookup(freeing, (st_data_t) resolver, NULL)) { + return; + } + + /* + * Otherwise add this resolver to the freeing set, and then + * actually free the thing. + */ + + st_insert(freeing, (st_data_t) resolver, (st_data_t) NULL); + + avro_schema_decref(resolver->parent.schema); + avro_schema_decref(resolver->rschema); + if (resolver->child_resolvers) { + unsigned int i; + for (i = 0; i < resolver->num_children; i++) { + avro_consumer_t *child = resolver->child_resolvers[i]; + if (child) { + avro_resolver_free_cycles(child, freeing); + } + } + avro_free(resolver->child_resolvers, + sizeof(avro_resolver_t *) * resolver->num_children); + } + if (resolver->index_mapping) { + avro_free(resolver->index_mapping, + sizeof(int) * resolver->num_children); + } + avro_freet(avro_resolver_t, resolver); +} + + +static void +avro_resolver_free(avro_consumer_t *consumer) +{ + st_table *freeing = st_init_numtable(); + avro_resolver_free_cycles(consumer, freeing); + st_free_table(freeing); +} + +/** + * Create a new avro_resolver_t instance. You must fill in the callback + * pointers that are appropriate for the writer schema after this + * function returns. + */ + +static avro_resolver_t * +avro_resolver_create(avro_schema_t wschema, + avro_schema_t rschema) +{ + avro_resolver_t *resolver = (avro_resolver_t *) avro_new(avro_resolver_t); + memset(resolver, 0, sizeof(avro_resolver_t)); + + resolver->parent.free = avro_resolver_free; + resolver->parent.schema = avro_schema_incref(wschema); + resolver->rschema = avro_schema_incref(rschema); + resolver->reader_union_branch = -1; + return resolver; +} + + +static avro_datum_t +avro_resolver_get_real_dest(avro_resolver_t *resolver, avro_datum_t dest) +{ + if (resolver->reader_union_branch < 0) { + /* + * The reader schema isn't a union, so use the dest + * field as-is. + */ + + return dest; + } + + debug("Retrieving union branch %d for %s value", + resolver->reader_union_branch, + avro_schema_type_name(resolver->parent.schema)); + + avro_datum_t branch = NULL; + avro_union_set_discriminant + (dest, resolver->reader_union_branch, &branch); + return branch; +} + + +#define skip_links(schema) \ + while (is_avro_link(schema)) { \ + schema = avro_schema_link_target(schema); \ + } + + +/*----------------------------------------------------------------------- + * Memoized resolvers + */ + +static avro_consumer_t * +avro_resolver_new_memoized(avro_memoize_t *mem, + avro_schema_t wschema, avro_schema_t rschema); + + +/*----------------------------------------------------------------------- + * Reader unions + */ + +/* + * For each Avro type, we have to check whether the reader schema on its + * own is compatible, and whether the reader is a union that contains a + * compatible type. The macros in this section help us perform both of + * these checks with less code. + */ + + +/** + * A helper macro that handles the case where neither writer nor reader + * are unions. Uses @ref check_func to see if the two schemas are + * compatible. + */ + +#define check_non_union(saved, wschema, rschema, check_func) \ +do { \ + avro_resolver_t *self = NULL; \ + int rc = check_func(saved, &self, wschema, rschema, \ + rschema); \ + if (self) { \ + debug("Non-union schemas %s (writer) " \ + "and %s (reader) match", \ + avro_schema_type_name(wschema), \ + avro_schema_type_name(rschema)); \ + \ + self->reader_union_branch = -1; \ + return &self->parent; \ + } \ + \ + if (rc) { \ + return NULL; \ + } \ +} while (0) + + +/** + * Helper macro that handles the case where the reader is a union, and + * the writer is not. Checks each branch of the reader union schema, + * looking for the first branch that is compatible with the writer + * schema. The @ref check_func argument should be a function that can + * check the compatiblity of each branch schema. + */ + +#define check_reader_union(saved, wschema, rschema, check_func) \ +do { \ + if (!is_avro_union(rschema)) { \ + break; \ + } \ + \ + debug("Checking reader union schema"); \ + size_t num_branches = avro_schema_union_size(rschema); \ + unsigned int i; \ + \ + for (i = 0; i < num_branches; i++) { \ + avro_schema_t branch_schema = \ + avro_schema_union_branch(rschema, i); \ + skip_links(branch_schema); \ + avro_resolver_t *self = NULL; \ + int rc = check_func(saved, &self, \ + wschema, branch_schema, \ + rschema); \ + if (self) { \ + debug("Reader union branch %d (%s) " \ + "and writer %s match", \ + i, avro_schema_type_name(branch_schema), \ + avro_schema_type_name(wschema)); \ + self->reader_union_branch = i; \ + return &self->parent; \ + } else { \ + debug("Reader union branch %d (%s) " \ + "doesn't match", \ + i, avro_schema_type_name(branch_schema)); \ + } \ + \ + if (rc) { \ + return NULL; \ + } \ + } \ + \ + debug("No reader union branches match"); \ +} while (0) + +/** + * A helper macro that defines wraps together check_non_union and + * check_reader_union for a simple (non-union) writer schema type. + */ + +#define check_simple_writer(saved, wschema, rschema, type_name) \ +do { \ + check_non_union(saved, wschema, rschema, try_##type_name); \ + check_reader_union(saved, wschema, rschema, try_##type_name); \ + debug("Writer %s doesn't match reader %s", \ + avro_schema_type_name(wschema), \ + avro_schema_type_name(rschema)); \ + avro_set_error("Cannot store " #type_name " into %s", \ + avro_schema_type_name(rschema)); \ + return NULL; \ +} while (0) + + +/*----------------------------------------------------------------------- + * primitives + */ + +static int +avro_resolver_boolean_value(avro_consumer_t *consumer, int value, + void *user_data) +{ + avro_resolver_t *resolver = (avro_resolver_t *) consumer; + avro_datum_t ud_dest = (avro_datum_t) user_data; + avro_datum_t dest = avro_resolver_get_real_dest(resolver, ud_dest); + debug("Storing %s into %p", value? "TRUE": "FALSE", dest); + return avro_boolean_set(dest, value); +} + +static int +try_boolean(avro_memoize_t *mem, avro_resolver_t **resolver, + avro_schema_t wschema, avro_schema_t rschema, + avro_schema_t root_rschema) +{ + if (is_avro_boolean(rschema)) { + *resolver = avro_resolver_create(wschema, root_rschema); + avro_memoize_set(mem, wschema, root_rschema, *resolver); + (*resolver)->parent.boolean_value = avro_resolver_boolean_value; + } + return 0; +} + + +static void +free_bytes(void *ptr, size_t sz) +{ + /* + * The binary encoder class allocates bytes values with an extra + * byte, so that they're NUL terminated. + */ + avro_free(ptr, sz+1); +} + +static int +avro_resolver_bytes_value(avro_consumer_t *consumer, + const void *value, size_t value_len, + void *user_data) +{ + avro_resolver_t *resolver = (avro_resolver_t *) consumer; + avro_datum_t ud_dest = (avro_datum_t) user_data; + avro_datum_t dest = avro_resolver_get_real_dest(resolver, ud_dest); + debug("Storing %" PRIsz " bytes into %p", value_len, dest); + return avro_givebytes_set(dest, (const char *) value, value_len, free_bytes); +} + +static int +try_bytes(avro_memoize_t *mem, avro_resolver_t **resolver, + avro_schema_t wschema, avro_schema_t rschema, + avro_schema_t root_rschema) +{ + if (is_avro_bytes(rschema)) { + *resolver = avro_resolver_create(wschema, root_rschema); + avro_memoize_set(mem, wschema, root_rschema, *resolver); + (*resolver)->parent.bytes_value = avro_resolver_bytes_value; + } + return 0; +} + + +static int +avro_resolver_double_value(avro_consumer_t *consumer, double value, + void *user_data) +{ + avro_resolver_t *resolver = (avro_resolver_t *) consumer; + avro_datum_t ud_dest = (avro_datum_t) user_data; + avro_datum_t dest = avro_resolver_get_real_dest(resolver, ud_dest); + debug("Storing %le into %p", value, dest); + return avro_double_set(dest, value); +} + +static int +try_double(avro_memoize_t *mem, avro_resolver_t **resolver, + avro_schema_t wschema, avro_schema_t rschema, + avro_schema_t root_rschema) +{ + if (is_avro_double(rschema)) { + *resolver = avro_resolver_create(wschema, root_rschema); + avro_memoize_set(mem, wschema, root_rschema, *resolver); + (*resolver)->parent.double_value = avro_resolver_double_value; + } + return 0; +} + + +static int +avro_resolver_float_value(avro_consumer_t *consumer, float value, + void *user_data) +{ + avro_resolver_t *resolver = (avro_resolver_t *) consumer; + avro_datum_t ud_dest = (avro_datum_t) user_data; + avro_datum_t dest = avro_resolver_get_real_dest(resolver, ud_dest); + debug("Storing %e into %p", value, dest); + return avro_float_set(dest, value); +} + +static int +avro_resolver_float_double_value(avro_consumer_t *consumer, float value, + void *user_data) +{ + avro_resolver_t *resolver = (avro_resolver_t *) consumer; + avro_datum_t ud_dest = (avro_datum_t) user_data; + avro_datum_t dest = avro_resolver_get_real_dest(resolver, ud_dest); + debug("Storing %e into %p (promoting float to double)", value, dest); + return avro_double_set(dest, value); +} + +static int +try_float(avro_memoize_t *mem, avro_resolver_t **resolver, + avro_schema_t wschema, avro_schema_t rschema, + avro_schema_t root_rschema) +{ + if (is_avro_float(rschema)) { + *resolver = avro_resolver_create(wschema, root_rschema); + avro_memoize_set(mem, wschema, root_rschema, *resolver); + (*resolver)->parent.float_value = avro_resolver_float_value; + } + else if (is_avro_double(rschema)) { + *resolver = avro_resolver_create(wschema, root_rschema); + avro_memoize_set(mem, wschema, root_rschema, *resolver); + (*resolver)->parent.float_value = avro_resolver_float_double_value; + } + return 0; +} + + +static int +avro_resolver_int_value(avro_consumer_t *consumer, int32_t value, + void *user_data) +{ + avro_resolver_t *resolver = (avro_resolver_t *) consumer; + avro_datum_t ud_dest = (avro_datum_t) user_data; + avro_datum_t dest = avro_resolver_get_real_dest(resolver, ud_dest); + debug("Storing %" PRId32 " into %p", value, dest); + return avro_int32_set(dest, value); +} + +static int +avro_resolver_int_long_value(avro_consumer_t *consumer, int32_t value, + void *user_data) +{ + avro_resolver_t *resolver = (avro_resolver_t *) consumer; + avro_datum_t ud_dest = (avro_datum_t) user_data; + avro_datum_t dest = avro_resolver_get_real_dest(resolver, ud_dest); + debug("Storing %" PRId32 " into %p (promoting int to long)", value, dest); + return avro_int64_set(dest, value); +} + +static int +avro_resolver_int_double_value(avro_consumer_t *consumer, int32_t value, + void *user_data) +{ + avro_resolver_t *resolver = (avro_resolver_t *) consumer; + avro_datum_t ud_dest = (avro_datum_t) user_data; + avro_datum_t dest = avro_resolver_get_real_dest(resolver, ud_dest); + debug("Storing %" PRId32 " into %p (promoting int to double)", value, dest); + return avro_double_set(dest, value); +} + +static int +avro_resolver_int_float_value(avro_consumer_t *consumer, int32_t value, + void *user_data) +{ + avro_resolver_t *resolver = (avro_resolver_t *) consumer; + avro_datum_t ud_dest = (avro_datum_t) user_data; + avro_datum_t dest = avro_resolver_get_real_dest(resolver, ud_dest); + debug("Storing %" PRId32 " into %p (promoting int to float)", value, dest); + return avro_float_set(dest, (const float) value); +} + +static int +try_int(avro_memoize_t *mem, avro_resolver_t **resolver, + avro_schema_t wschema, avro_schema_t rschema, + avro_schema_t root_rschema) +{ + if (is_avro_int32(rschema)) { + *resolver = avro_resolver_create(wschema, root_rschema); + avro_memoize_set(mem, wschema, root_rschema, *resolver); + (*resolver)->parent.int_value = avro_resolver_int_value; + } + else if (is_avro_int64(rschema)) { + *resolver = avro_resolver_create(wschema, root_rschema); + avro_memoize_set(mem, wschema, root_rschema, *resolver); + (*resolver)->parent.int_value = avro_resolver_int_long_value; + } + else if (is_avro_double(rschema)) { + *resolver = avro_resolver_create(wschema, root_rschema); + avro_memoize_set(mem, wschema, root_rschema, *resolver); + (*resolver)->parent.int_value = avro_resolver_int_double_value; + } + else if (is_avro_float(rschema)) { + *resolver = avro_resolver_create(wschema, root_rschema); + avro_memoize_set(mem, wschema, root_rschema, *resolver); + (*resolver)->parent.int_value = avro_resolver_int_float_value; + } + return 0; +} + + +static int +avro_resolver_long_value(avro_consumer_t *consumer, int64_t value, + void *user_data) +{ + avro_resolver_t *resolver = (avro_resolver_t *) consumer; + avro_datum_t ud_dest = (avro_datum_t) user_data; + avro_datum_t dest = avro_resolver_get_real_dest(resolver, ud_dest); + debug("Storing %" PRId64 " into %p", value, dest); + return avro_int64_set(dest, value); +} + +static int +avro_resolver_long_float_value(avro_consumer_t *consumer, int64_t value, + void *user_data) +{ + avro_resolver_t *resolver = (avro_resolver_t *) consumer; + avro_datum_t ud_dest = (avro_datum_t) user_data; + avro_datum_t dest = avro_resolver_get_real_dest(resolver, ud_dest); + debug("Storing %" PRId64 " into %p (promoting long to float)", value, dest); + return avro_float_set(dest, (const float) value); +} + +static int +avro_resolver_long_double_value(avro_consumer_t *consumer, int64_t value, + void *user_data) +{ + avro_resolver_t *resolver = (avro_resolver_t *) consumer; + avro_datum_t ud_dest = (avro_datum_t) user_data; + avro_datum_t dest = avro_resolver_get_real_dest(resolver, ud_dest); + debug("Storing %" PRId64 " into %p (promoting long to double)", value, dest); + return avro_double_set(dest, (const double) value); +} + +static int +try_long(avro_memoize_t *mem, avro_resolver_t **resolver, + avro_schema_t wschema, avro_schema_t rschema, + avro_schema_t root_rschema) +{ + if (is_avro_int64(rschema)) { + *resolver = avro_resolver_create(wschema, root_rschema); + avro_memoize_set(mem, wschema, root_rschema, *resolver); + (*resolver)->parent.long_value = avro_resolver_long_value; + } + else if (is_avro_double(rschema)) { + *resolver = avro_resolver_create(wschema, root_rschema); + avro_memoize_set(mem, wschema, root_rschema, *resolver); + (*resolver)->parent.long_value = avro_resolver_long_double_value; + } + else if (is_avro_float(rschema)) { + *resolver = avro_resolver_create(wschema, root_rschema); + avro_memoize_set(mem, wschema, root_rschema, *resolver); + (*resolver)->parent.long_value = avro_resolver_long_float_value; + } + return 0; +} + + +static int +avro_resolver_null_value(avro_consumer_t *consumer, void *user_data) +{ + avro_resolver_t *resolver = (avro_resolver_t *) consumer; + avro_datum_t ud_dest = (avro_datum_t) user_data; + avro_datum_t dest = avro_resolver_get_real_dest(resolver, ud_dest); + + AVRO_UNUSED(dest); + debug("Storing null into %p", dest); + return 0; +} + +static int +try_null(avro_memoize_t *mem, avro_resolver_t **resolver, + avro_schema_t wschema, avro_schema_t rschema, + avro_schema_t root_rschema) +{ + if (is_avro_null(rschema)) { + *resolver = avro_resolver_create(wschema, root_rschema); + avro_memoize_set(mem, wschema, root_rschema, *resolver); + (*resolver)->parent.null_value = avro_resolver_null_value; + } + return 0; +} + + +static int +avro_resolver_string_value(avro_consumer_t *consumer, + const void *value, size_t value_len, + void *user_data) +{ + AVRO_UNUSED(value_len); + avro_resolver_t *resolver = (avro_resolver_t *) consumer; + avro_datum_t ud_dest = (avro_datum_t) user_data; + avro_datum_t dest = avro_resolver_get_real_dest(resolver, ud_dest); + debug("Storing \"%s\" into %p", (const char *) value, dest); + return avro_givestring_set(dest, (const char *) value, avro_alloc_free_func); +} + +static int +try_string(avro_memoize_t *mem, avro_resolver_t **resolver, + avro_schema_t wschema, avro_schema_t rschema, + avro_schema_t root_rschema) +{ + if (is_avro_string(rschema)) { + *resolver = avro_resolver_create(wschema, root_rschema); + avro_memoize_set(mem, wschema, root_rschema, *resolver); + (*resolver)->parent.string_value = avro_resolver_string_value; + } + return 0; +} + + +/*----------------------------------------------------------------------- + * arrays + */ + +static int +avro_resolver_array_start_block(avro_consumer_t *consumer, + int is_first_block, + unsigned int block_count, + void *user_data) +{ + if (is_first_block) { + avro_resolver_t *resolver = (avro_resolver_t *) consumer; + avro_datum_t ud_dest = (avro_datum_t) user_data; + avro_datum_t dest = avro_resolver_get_real_dest(resolver, ud_dest); + AVRO_UNUSED(dest); + + debug("Starting array %p", dest); + } + + AVRO_UNUSED(block_count); + return 0; +} + +static int +avro_resolver_array_element(avro_consumer_t *consumer, + unsigned int index, + avro_consumer_t **element_consumer, + void **element_user_data, + void *user_data) +{ + AVRO_UNUSED(index); + + avro_resolver_t *resolver = (avro_resolver_t *) consumer; + avro_datum_t ud_dest = (avro_datum_t) user_data; + avro_datum_t dest = avro_resolver_get_real_dest(resolver, ud_dest); + debug("Adding element to array %p", dest); + + /* + * Allocate a new element datum and add it to the array. + */ + + avro_schema_t array_schema = avro_datum_get_schema(dest); + avro_schema_t item_schema = avro_schema_array_items(array_schema); + avro_datum_t element = avro_datum_from_schema(item_schema); + avro_array_append_datum(dest, element); + avro_datum_decref(element); + + /* + * Return the consumer that we allocated to process the array's + * children. + */ + + *element_consumer = resolver->child_resolvers[0]; + *element_user_data = element; + return 0; +} + +static int +try_array(avro_memoize_t *mem, avro_resolver_t **resolver, + avro_schema_t wschema, avro_schema_t rschema, + avro_schema_t root_rschema) +{ + /* + * First verify that the reader is an array. + */ + + if (!is_avro_array(rschema)) { + return 0; + } + + /* + * Array schemas have to have compatible element schemas to be + * compatible themselves. Try to create an avro_resolver_t to + * check the compatibility. + */ + + *resolver = avro_resolver_create(wschema, root_rschema); + avro_memoize_set(mem, wschema, root_rschema, *resolver); + + avro_schema_t witems = avro_schema_array_items(wschema); + avro_schema_t ritems = avro_schema_array_items(rschema); + + avro_consumer_t *item_consumer = + avro_resolver_new_memoized(mem, witems, ritems); + if (!item_consumer) { + avro_memoize_delete(mem, wschema, root_rschema); + avro_consumer_free(&(*resolver)->parent); + avro_prefix_error("Array values aren't compatible: "); + return EINVAL; + } + + /* + * The two schemas are compatible, so go ahead and create a + * GavroResolver for the array. Store the item schema's + * resolver into the child_resolvers field. + */ + + (*resolver)->num_children = 1; + (*resolver)->child_resolvers = (avro_consumer_t **) avro_calloc(1, sizeof(avro_consumer_t *)); + (*resolver)->child_resolvers[0] = item_consumer; + (*resolver)->parent.array_start_block = avro_resolver_array_start_block; + (*resolver)->parent.array_element = avro_resolver_array_element; + + return 0; +} + + +/*----------------------------------------------------------------------- + * enums + */ + +static int +avro_resolver_enum_value(avro_consumer_t *consumer, int value, + void *user_data) +{ + AVRO_UNUSED(value); + + avro_resolver_t *resolver = (avro_resolver_t *) consumer; + avro_datum_t ud_dest = (avro_datum_t) user_data; + avro_datum_t dest = avro_resolver_get_real_dest(resolver, ud_dest); + + const char *symbol_name = avro_schema_enum_get(resolver->parent.schema, value); + debug("Storing symbol %s into %p", symbol_name, dest); + return avro_enum_set_name(dest, symbol_name); +} + +static int +try_enum(avro_memoize_t *mem, avro_resolver_t **resolver, + avro_schema_t wschema, avro_schema_t rschema, + avro_schema_t root_rschema) +{ + /* + * Enum schemas have to have the same name — but not the same + * list of symbols — to be compatible. + */ + + if (is_avro_enum(rschema)) { + const char *wname = avro_schema_name(wschema); + const char *rname = avro_schema_name(rschema); + + if (!strcmp(wname, rname)) { + *resolver = avro_resolver_create(wschema, root_rschema); + avro_memoize_set(mem, wschema, root_rschema, *resolver); + (*resolver)->parent.enum_value = avro_resolver_enum_value; + } + } + return 0; +} + + +/*----------------------------------------------------------------------- + * fixed + */ + +static int +avro_resolver_fixed_value(avro_consumer_t *consumer, + const void *value, size_t value_len, + void *user_data) +{ + avro_resolver_t *resolver = (avro_resolver_t *) consumer; + avro_datum_t ud_dest = (avro_datum_t) user_data; + avro_datum_t dest = avro_resolver_get_real_dest(resolver, ud_dest); + debug("Storing (fixed) %" PRIsz " bytes into %p", value_len, dest); + return avro_givefixed_set(dest, (const char *) value, value_len, avro_alloc_free_func); +} + +static int +try_fixed(avro_memoize_t *mem, avro_resolver_t **resolver, + avro_schema_t wschema, avro_schema_t rschema, + avro_schema_t root_rschema) +{ + /* + * Fixed schemas need the same name and size to be compatible. + */ + + if (avro_schema_equal(wschema, rschema)) { + *resolver = avro_resolver_create(wschema, root_rschema); + avro_memoize_set(mem, wschema, root_rschema, *resolver); + (*resolver)->parent.fixed_value = avro_resolver_fixed_value; + } + return 0; +} + + +/*----------------------------------------------------------------------- + * maps + */ + +static int +avro_resolver_map_start_block(avro_consumer_t *consumer, + int is_first_block, + unsigned int block_count, + void *user_data) +{ + if (is_first_block) { + avro_resolver_t *resolver = (avro_resolver_t *) consumer; + avro_datum_t ud_dest = (avro_datum_t) user_data; + avro_datum_t dest = avro_resolver_get_real_dest(resolver, ud_dest); + AVRO_UNUSED(dest); + + debug("Starting map %p", dest); + } + + AVRO_UNUSED(block_count); + return 0; +} + +static int +avro_resolver_map_element(avro_consumer_t *consumer, + unsigned int index, + const char *key, + avro_consumer_t **value_consumer, + void **value_user_data, + void *user_data) +{ + AVRO_UNUSED(index); + + avro_resolver_t *resolver = (avro_resolver_t *) consumer; + avro_datum_t ud_dest = (avro_datum_t) user_data; + avro_datum_t dest = avro_resolver_get_real_dest(resolver, ud_dest); + debug("Adding element to map %p", dest); + + /* + * Allocate a new element datum and add it to the map. + */ + + avro_schema_t map_schema = avro_datum_get_schema(dest); + avro_schema_t value_schema = avro_schema_map_values(map_schema); + avro_datum_t value = avro_datum_from_schema(value_schema); + avro_map_set(dest, key, value); + avro_datum_decref(value); + + /* + * Return the consumer that we allocated to process the map's + * children. + */ + + *value_consumer = resolver->child_resolvers[0]; + *value_user_data = value; + return 0; +} + +static int +try_map(avro_memoize_t *mem, avro_resolver_t **resolver, + avro_schema_t wschema, avro_schema_t rschema, + avro_schema_t root_rschema) +{ + /* + * First verify that the reader is an map. + */ + + if (!is_avro_map(rschema)) { + return 0; + } + + /* + * Array schemas have to have compatible element schemas to be + * compatible themselves. Try to create an avro_resolver_t to + * check the compatibility. + */ + + *resolver = avro_resolver_create(wschema, root_rschema); + avro_memoize_set(mem, wschema, root_rschema, *resolver); + + avro_schema_t wvalues = avro_schema_map_values(wschema); + avro_schema_t rvalues = avro_schema_map_values(rschema); + + avro_consumer_t *value_consumer = + avro_resolver_new_memoized(mem, wvalues, rvalues); + if (!value_consumer) { + avro_memoize_delete(mem, wschema, root_rschema); + avro_consumer_free(&(*resolver)->parent); + avro_prefix_error("Map values aren't compatible: "); + return EINVAL; + } + + /* + * The two schemas are compatible, so go ahead and create a + * GavroResolver for the map. Store the value schema's + * resolver into the child_resolvers field. + */ + + (*resolver)->num_children = 1; + (*resolver)->child_resolvers = (avro_consumer_t **) avro_calloc(1, sizeof(avro_consumer_t *)); + (*resolver)->child_resolvers[0] = value_consumer; + (*resolver)->parent.map_start_block = avro_resolver_map_start_block; + (*resolver)->parent.map_element = avro_resolver_map_element; + + return 0; +} + + +/*----------------------------------------------------------------------- + * records + */ + +static int +avro_resolver_record_start(avro_consumer_t *consumer, + void *user_data) +{ + avro_resolver_t *resolver = (avro_resolver_t *) consumer; + avro_datum_t ud_dest = (avro_datum_t) user_data; + avro_datum_t dest = avro_resolver_get_real_dest(resolver, ud_dest); + AVRO_UNUSED(dest); + + debug("Starting record at %p", dest); + + /* + * TODO: Eventually, we'll fill in default values for the extra + * reader fields here. + */ + + return 0; +} + +static int +avro_resolver_record_field(avro_consumer_t *consumer, + unsigned int index, + avro_consumer_t **field_consumer, + void **field_user_data, + void *user_data) +{ + avro_resolver_t *resolver = (avro_resolver_t *) consumer; + avro_datum_t ud_dest = (avro_datum_t) user_data; + avro_datum_t dest = avro_resolver_get_real_dest(resolver, ud_dest); + + const char *field_name = + avro_schema_record_field_name(consumer->schema, index); + + /* + * Grab the resolver for this field of the writer record. If + * it's NULL, this this field doesn't exist in the reader + * record, and should be skipped. + */ + + debug("Retrieving resolver for writer field %i (%s)", + index, field_name); + + if (!resolver->child_resolvers[index]) { + debug("Reader doesn't have field %s, skipping", field_name); + return 0; + } + + /* + * TODO: Once we can retrieve record fields by index (quickly), + * use the index_mapping. + */ + + avro_datum_t field = NULL; + avro_record_get(dest, field_name, &field); + + *field_consumer = resolver->child_resolvers[index]; + *field_user_data = field; + return 0; +} + +static int +try_record(avro_memoize_t *mem, avro_resolver_t **resolver, + avro_schema_t wschema, avro_schema_t rschema, + avro_schema_t root_rschema) +{ + /* + * First verify that the reader is also a record, and has the + * same name as the writer. + */ + + if (!is_avro_record(rschema)) { + return 0; + } + + const char *wname = avro_schema_name(wschema); + const char *rname = avro_schema_name(rschema); + + if (strcmp(wname, rname)) { + return 0; + } + + /* + * Categorize the fields in the record schemas. Fields that are + * only in the writer are ignored. Fields that are only in the + * reader raise a schema mismatch error, unless the field has a + * default value. Fields that are in both are resolved + * recursively. + * + * The child_resolver array will contain an avro_resolver_t for + * each field in the writer schema. To build this array, we + * loop through the fields of the reader schema. If that field + * is also in the writer schema, we resolve them recursively, + * and store the resolver into the array. If the field isn't in + * the writer schema, we raise an error. (TODO: Eventually, + * we'll handle default values here.) After this loop finishes, + * any NULLs in the child_resolver array will represent fields + * in the writer but not the reader; these fields will be + * skipped when processing the input. + */ + + *resolver = avro_resolver_create(wschema, root_rschema); + avro_memoize_set(mem, wschema, root_rschema, *resolver); + + size_t wfields = avro_schema_record_size(wschema); + size_t rfields = avro_schema_record_size(rschema); + + debug("Checking writer record schema %s", wname); + + avro_consumer_t **child_resolvers = + (avro_consumer_t **) avro_calloc(wfields, sizeof(avro_consumer_t *)); + int *index_mapping = (int *) avro_calloc(wfields, sizeof(int)); + + unsigned int ri; + for (ri = 0; ri < rfields; ri++) { + avro_schema_t rfield = + avro_schema_record_field_get_by_index(rschema, ri); + const char *field_name = + avro_schema_record_field_name(rschema, ri); + + debug("Resolving reader record field %u (%s)", ri, field_name); + + /* + * See if this field is also in the writer schema. + */ + + int wi = avro_schema_record_field_get_index(wschema, field_name); + + if (wi == -1) { + /* + * This field isn't in the writer schema — + * that's an error! TODO: Handle default + * values! + */ + + debug("Field %s isn't in writer", field_name); + avro_set_error("Reader field %s doesn't appear in writer", + field_name); + goto error; + } + + /* + * Try to recursively resolve the schemas for this + * field. If they're not compatible, that's an error. + */ + + avro_schema_t wfield = + avro_schema_record_field_get_by_index(wschema, wi); + avro_consumer_t *field_resolver = + avro_resolver_new_memoized(mem, wfield, rfield); + + if (!field_resolver) { + avro_prefix_error("Field %s isn't compatible: ", field_name); + goto error; + } + + /* + * Save the details for this field. + */ + + debug("Found match for field %s (%u in reader, %d in writer)", + field_name, ri, wi); + child_resolvers[wi] = field_resolver; + index_mapping[wi] = ri; + } + + /* + * We might not have found matches for all of the writer fields, + * but that's okay — any extras will be ignored. + */ + + (*resolver)->num_children = wfields; + (*resolver)->child_resolvers = child_resolvers; + (*resolver)->index_mapping = index_mapping; + (*resolver)->parent.record_start = avro_resolver_record_start; + (*resolver)->parent.record_field = avro_resolver_record_field; + return 0; + +error: + /* + * Clean up any consumer we might have already created. + */ + + avro_memoize_delete(mem, wschema, root_rschema); + avro_consumer_free(&(*resolver)->parent); + + { + unsigned int i; + for (i = 0; i < wfields; i++) { + if (child_resolvers[i]) { + avro_consumer_free(child_resolvers[i]); + } + } + } + + avro_free(child_resolvers, wfields * sizeof(avro_consumer_t *)); + avro_free(index_mapping, wfields * sizeof(int)); + return EINVAL; +} + + +/*----------------------------------------------------------------------- + * union + */ + +static int +avro_resolver_union_branch(avro_consumer_t *consumer, + unsigned int discriminant, + avro_consumer_t **branch_consumer, + void **branch_user_data, + void *user_data) +{ + avro_resolver_t *resolver = (avro_resolver_t *) consumer; + + /* + * Grab the resolver for this branch of the writer union. If + * it's NULL, then this branch is incompatible with the reader. + */ + + debug("Retrieving resolver for writer branch %u", discriminant); + + if (!resolver->child_resolvers[discriminant]) { + avro_set_error("Writer union branch %u is incompatible " + "with reader schema \"%s\"", + discriminant, avro_schema_type_name(resolver->rschema)); + return EINVAL; + } + + /* + * Return the branch's resolver. + */ + + *branch_consumer = resolver->child_resolvers[discriminant]; + *branch_user_data = user_data; + return 0; +} + +static avro_consumer_t * +try_union(avro_memoize_t *mem, avro_schema_t wschema, avro_schema_t rschema) +{ + /* + * For a writer union, we recursively try to resolve each branch + * against the reader schema. This will work correctly whether + * or not the reader is also a union — if the reader is a union, + * then we'll resolve each (non-union) writer branch against the + * reader union, which will be checked in our calls to + * check_simple_writer below. The net result is that we might + * end up trying every combination of writer and reader + * branches, when looking for compatible schemas. + * + * Regardless of what the reader schema is, for each writer + * branch, we stash away the recursive avro_resolver_t into the + * child_resolvers array. A NULL entry in this array means that + * that branch isn't compatible with the reader. This isn't an + * immediate schema resolution error, since we allow + * incompatible branches in the types as long as that branch + * never appears in the actual data. We only return an error if + * there are *no* branches that are compatible. + */ + + size_t num_branches = avro_schema_union_size(wschema); + debug("Checking %" PRIsz "-branch writer union schema", num_branches); + + avro_resolver_t *resolver = avro_resolver_create(wschema, rschema); + avro_memoize_set(mem, wschema, rschema, resolver); + + avro_consumer_t **child_resolvers = + (avro_consumer_t **) avro_calloc(num_branches, sizeof(avro_consumer_t *)); + int some_branch_compatible = 0; + + unsigned int i; + for (i = 0; i < num_branches; i++) { + avro_schema_t branch_schema = + avro_schema_union_branch(wschema, i); + + debug("Resolving writer union branch %u (%s)", + i, avro_schema_type_name(branch_schema)); + + /* + * Try to recursively resolve this branch of the writer + * union. Don't raise an error if this fails — it's + * okay for some of the branches to not be compatible + * with the reader, as long as those branches never + * appear in the input. + */ + + child_resolvers[i] = + avro_resolver_new_memoized(mem, branch_schema, rschema); + if (child_resolvers[i]) { + debug("Found match for writer union branch %u", i); + some_branch_compatible = 1; + } else { + debug("No match for writer union branch %u", i); + } + } + + /* + * As long as there's at least one branch that's compatible with + * the reader, then we consider this schema resolution a + * success. + */ + + if (!some_branch_compatible) { + debug("No writer union branches match"); + avro_set_error("No branches in the writer are compatible " + "with reader schema %s", + avro_schema_type_name(rschema)); + goto error; + } + + resolver->num_children = num_branches; + resolver->child_resolvers = child_resolvers; + resolver->parent.union_branch = avro_resolver_union_branch; + return &resolver->parent; + +error: + /* + * Clean up any consumer we might have already created. + */ + + avro_memoize_delete(mem, wschema, rschema); + avro_consumer_free(&resolver->parent); + + for (i = 0; i < num_branches; i++) { + if (child_resolvers[i]) { + avro_consumer_free(child_resolvers[i]); + } + } + + avro_free(child_resolvers, num_branches * sizeof(avro_consumer_t *)); + return NULL; +} + + +/*----------------------------------------------------------------------- + * schema type dispatcher + */ + +static avro_consumer_t * +avro_resolver_new_memoized(avro_memoize_t *mem, + avro_schema_t wschema, avro_schema_t rschema) +{ + check_param(NULL, is_avro_schema(wschema), "writer schema"); + check_param(NULL, is_avro_schema(rschema), "reader schema"); + + skip_links(wschema); + skip_links(rschema); + + /* + * First see if we've already matched these two schemas. If so, + * just return that resolver. + */ + + avro_resolver_t *saved = NULL; + if (avro_memoize_get(mem, wschema, rschema, (void **) &saved)) { + debug("Already resolved %s and %s", + avro_schema_type_name(wschema), + avro_schema_type_name(rschema)); + return &saved->parent; + } + + /* + * Otherwise we have some work to do. + */ + + switch (avro_typeof(wschema)) + { + case AVRO_BOOLEAN: + check_simple_writer(mem, wschema, rschema, boolean); + return NULL; + + case AVRO_BYTES: + check_simple_writer(mem, wschema, rschema, bytes); + return NULL; + + case AVRO_DOUBLE: + check_simple_writer(mem, wschema, rschema, double); + return NULL; + + case AVRO_FLOAT: + check_simple_writer(mem, wschema, rschema, float); + return NULL; + + case AVRO_INT32: + check_simple_writer(mem, wschema, rschema, int); + return NULL; + + case AVRO_INT64: + check_simple_writer(mem, wschema, rschema, long); + return NULL; + + case AVRO_NULL: + check_simple_writer(mem, wschema, rschema, null); + return NULL; + + case AVRO_STRING: + check_simple_writer(mem, wschema, rschema, string); + return NULL; + + case AVRO_ARRAY: + check_simple_writer(mem, wschema, rschema, array); + return NULL; + + case AVRO_ENUM: + check_simple_writer(mem, wschema, rschema, enum); + return NULL; + + case AVRO_FIXED: + check_simple_writer(mem, wschema, rschema, fixed); + return NULL; + + case AVRO_MAP: + check_simple_writer(mem, wschema, rschema, map); + return NULL; + + case AVRO_RECORD: + check_simple_writer(mem, wschema, rschema, record); + return NULL; + + case AVRO_UNION: + return try_union(mem, wschema, rschema); + + default: + avro_set_error("Unknown schema type"); + return NULL; + } + + return NULL; +} + + +avro_consumer_t * +avro_resolver_new(avro_schema_t wschema, avro_schema_t rschema) +{ + avro_memoize_t mem; + avro_memoize_init(&mem); + avro_consumer_t *result = + avro_resolver_new_memoized(&mem, wschema, rschema); + avro_memoize_done(&mem); + return result; +} diff --git a/fluent-bit/lib/avro/src/schema.c b/fluent-bit/lib/avro/src/schema.c new file mode 100644 index 00000000..7b389002 --- /dev/null +++ b/fluent-bit/lib/avro/src/schema.c @@ -0,0 +1,1897 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include "avro/allocation.h" +#include "avro/refcount.h" +#include "avro/errors.h" +#include "avro/io.h" +#include "avro/legacy.h" +#include "avro/schema.h" +#include "avro_private.h" +#include <avro/platform.h> +#include <stdlib.h> +#include <string.h> +#include <errno.h> +#include <ctype.h> + +#include "jansson.h" +#include "st.h" +#include "schema.h" + +#define DEFAULT_TABLE_SIZE 32 + +/* forward declaration */ +static int +avro_schema_to_json2(const avro_schema_t schema, avro_writer_t out, + const char *parent_namespace); + +static void avro_schema_init(avro_schema_t schema, avro_type_t type) +{ + schema->type = type; + schema->class_type = AVRO_SCHEMA; + avro_refcount_set(&schema->refcount, 1); +} + +static int is_avro_id(const char *name) +{ + size_t i, len; + if (name) { + len = strlen(name); + if (len < 1) { + return 0; + } + for (i = 0; i < len; i++) { + if (!(isalpha(name[i]) + || name[i] == '_' || (i && isdigit(name[i])))) { + return 0; + } + } + /* + * starts with [A-Za-z_] subsequent [A-Za-z0-9_] + */ + return 1; + } + return 0; +} + +/* Splits a qualified name by the last period, e.g. fullname "foo.bar.Baz" into + * name "Baz" and namespace "foo.bar". Sets name_out to the name part (pointing + * to a later position in the buffer that was passed in), and returns the + * namespace (as a newly allocated buffer using Avro's allocator). */ +static char *split_namespace_name(const char *fullname, const char **name_out) +{ + char *last_dot = strrchr(fullname, '.'); + if (last_dot == NULL) { + *name_out = fullname; + return NULL; + } else { + *name_out = last_dot + 1; + return avro_strndup(fullname, last_dot - fullname); + } +} + +static int record_free_foreach(int i, struct avro_record_field_t *field, + void *arg) +{ + AVRO_UNUSED(i); + AVRO_UNUSED(arg); + + avro_str_free(field->name); + avro_schema_decref(field->type); + avro_freet(struct avro_record_field_t, field); + return ST_DELETE; +} + +static int enum_free_foreach(int i, char *sym, void *arg) +{ + AVRO_UNUSED(i); + AVRO_UNUSED(arg); + + avro_str_free(sym); + return ST_DELETE; +} + +static int union_free_foreach(int i, avro_schema_t schema, void *arg) +{ + AVRO_UNUSED(i); + AVRO_UNUSED(arg); + + avro_schema_decref(schema); + return ST_DELETE; +} + +static void avro_schema_free(avro_schema_t schema) +{ + if (is_avro_schema(schema)) { + switch (avro_typeof(schema)) { + case AVRO_STRING: + case AVRO_BYTES: + case AVRO_INT32: + case AVRO_INT64: + case AVRO_FLOAT: + case AVRO_DOUBLE: + case AVRO_BOOLEAN: + case AVRO_NULL: + /* no memory allocated for primitives */ + return; + + case AVRO_RECORD:{ + struct avro_record_schema_t *record; + record = avro_schema_to_record(schema); + avro_str_free(record->name); + if (record->space) { + avro_str_free(record->space); + } + st_foreach(record->fields, HASH_FUNCTION_CAST record_free_foreach, + 0); + st_free_table(record->fields_byname); + st_free_table(record->fields); + avro_freet(struct avro_record_schema_t, record); + } + break; + + case AVRO_ENUM:{ + struct avro_enum_schema_t *enump; + enump = avro_schema_to_enum(schema); + avro_str_free(enump->name); + if (enump->space) { + avro_str_free(enump->space); + } + st_foreach(enump->symbols, HASH_FUNCTION_CAST enum_free_foreach, + 0); + st_free_table(enump->symbols); + st_free_table(enump->symbols_byname); + avro_freet(struct avro_enum_schema_t, enump); + } + break; + + case AVRO_FIXED:{ + struct avro_fixed_schema_t *fixed; + fixed = avro_schema_to_fixed(schema); + avro_str_free((char *) fixed->name); + if (fixed->space) { + avro_str_free((char *) fixed->space); + } + avro_freet(struct avro_fixed_schema_t, fixed); + } + break; + + case AVRO_MAP:{ + struct avro_map_schema_t *map; + map = avro_schema_to_map(schema); + avro_schema_decref(map->values); + avro_freet(struct avro_map_schema_t, map); + } + break; + + case AVRO_ARRAY:{ + struct avro_array_schema_t *array; + array = avro_schema_to_array(schema); + avro_schema_decref(array->items); + avro_freet(struct avro_array_schema_t, array); + } + break; + case AVRO_UNION:{ + struct avro_union_schema_t *unionp; + unionp = avro_schema_to_union(schema); + st_foreach(unionp->branches, HASH_FUNCTION_CAST union_free_foreach, + 0); + st_free_table(unionp->branches); + st_free_table(unionp->branches_byname); + avro_freet(struct avro_union_schema_t, unionp); + } + break; + + case AVRO_LINK:{ + struct avro_link_schema_t *link; + link = avro_schema_to_link(schema); + /* Since we didn't increment the + * reference count of the target + * schema when we created the link, we + * should not decrement the reference + * count of the target schema when we + * free the link. + */ + avro_freet(struct avro_link_schema_t, link); + } + break; + } + } +} + +avro_schema_t avro_schema_incref(avro_schema_t schema) +{ + if (schema) { + avro_refcount_inc(&schema->refcount); + } + return schema; +} + +int +avro_schema_decref(avro_schema_t schema) +{ + if (schema && avro_refcount_dec(&schema->refcount)) { + avro_schema_free(schema); + return 0; + } + return 1; +} + +avro_schema_t avro_schema_string(void) +{ + static struct avro_obj_t obj = { + AVRO_STRING, + AVRO_SCHEMA, + 1 + }; + return avro_schema_incref(&obj); +} + +avro_schema_t avro_schema_bytes(void) +{ + static struct avro_obj_t obj = { + AVRO_BYTES, + AVRO_SCHEMA, + 1 + }; + return avro_schema_incref(&obj); +} + +avro_schema_t avro_schema_int(void) +{ + static struct avro_obj_t obj = { + AVRO_INT32, + AVRO_SCHEMA, + 1 + }; + return avro_schema_incref(&obj); +} + +avro_schema_t avro_schema_long(void) +{ + static struct avro_obj_t obj = { + AVRO_INT64, + AVRO_SCHEMA, + 1 + }; + return avro_schema_incref(&obj); +} + +avro_schema_t avro_schema_float(void) +{ + static struct avro_obj_t obj = { + AVRO_FLOAT, + AVRO_SCHEMA, + 1 + }; + return avro_schema_incref(&obj); +} + +avro_schema_t avro_schema_double(void) +{ + static struct avro_obj_t obj = { + AVRO_DOUBLE, + AVRO_SCHEMA, + 1 + }; + return avro_schema_incref(&obj); +} + +avro_schema_t avro_schema_boolean(void) +{ + static struct avro_obj_t obj = { + AVRO_BOOLEAN, + AVRO_SCHEMA, + 1 + }; + return avro_schema_incref(&obj); +} + +avro_schema_t avro_schema_null(void) +{ + static struct avro_obj_t obj = { + AVRO_NULL, + AVRO_SCHEMA, + 1 + }; + return avro_schema_incref(&obj); +} + +avro_schema_t avro_schema_fixed(const char *name, const int64_t size) +{ + return avro_schema_fixed_ns(name, NULL, size); +} + +avro_schema_t avro_schema_fixed_ns(const char *name, const char *space, + const int64_t size) +{ + if (!is_avro_id(name)) { + avro_set_error("Invalid Avro identifier"); + return NULL; + } + + struct avro_fixed_schema_t *fixed = + (struct avro_fixed_schema_t *) avro_new(struct avro_fixed_schema_t); + if (!fixed) { + avro_set_error("Cannot allocate new fixed schema"); + return NULL; + } + fixed->name = avro_strdup(name); + if (!fixed->name) { + avro_set_error("Cannot allocate new fixed schema"); + avro_freet(struct avro_fixed_schema_t, fixed); + return NULL; + } + fixed->space = space ? avro_strdup(space) : NULL; + if (space && !fixed->space) { + avro_set_error("Cannot allocate new fixed schema"); + avro_str_free((char *) fixed->name); + avro_freet(struct avro_fixed_schema_t, fixed); + return NULL; + } + fixed->size = size; + avro_schema_init(&fixed->obj, AVRO_FIXED); + return &fixed->obj; +} + +int64_t avro_schema_fixed_size(const avro_schema_t fixed) +{ + return avro_schema_to_fixed(fixed)->size; +} + +avro_schema_t avro_schema_union(void) +{ + struct avro_union_schema_t *schema = + (struct avro_union_schema_t *) avro_new(struct avro_union_schema_t); + if (!schema) { + avro_set_error("Cannot allocate new union schema"); + return NULL; + } + schema->branches = st_init_numtable_with_size(DEFAULT_TABLE_SIZE); + if (!schema->branches) { + avro_set_error("Cannot allocate new union schema"); + avro_freet(struct avro_union_schema_t, schema); + return NULL; + } + schema->branches_byname = + st_init_strtable_with_size(DEFAULT_TABLE_SIZE); + if (!schema->branches_byname) { + avro_set_error("Cannot allocate new union schema"); + st_free_table(schema->branches); + avro_freet(struct avro_union_schema_t, schema); + return NULL; + } + + avro_schema_init(&schema->obj, AVRO_UNION); + return &schema->obj; +} + +int +avro_schema_union_append(const avro_schema_t union_schema, + const avro_schema_t schema) +{ + check_param(EINVAL, is_avro_schema(union_schema), "union schema"); + check_param(EINVAL, is_avro_union(union_schema), "union schema"); + check_param(EINVAL, is_avro_schema(schema), "schema"); + + struct avro_union_schema_t *unionp = avro_schema_to_union(union_schema); + int new_index = unionp->branches->num_entries; + st_insert(unionp->branches, new_index, (st_data_t) schema); + const char *name = avro_schema_type_name(schema); + st_insert(unionp->branches_byname, (st_data_t) name, + (st_data_t) new_index); + avro_schema_incref(schema); + return 0; +} + +size_t avro_schema_union_size(const avro_schema_t union_schema) +{ + check_param(EINVAL, is_avro_schema(union_schema), "union schema"); + check_param(EINVAL, is_avro_union(union_schema), "union schema"); + struct avro_union_schema_t *unionp = avro_schema_to_union(union_schema); + return unionp->branches->num_entries; +} + +avro_schema_t avro_schema_union_branch(avro_schema_t unionp, + int branch_index) +{ + union { + st_data_t data; + avro_schema_t schema; + } val; + if (st_lookup(avro_schema_to_union(unionp)->branches, + branch_index, &val.data)) { + return val.schema; + } else { + avro_set_error("No union branch for discriminant %d", + branch_index); + return NULL; + } +} + +avro_schema_t avro_schema_union_branch_by_name +(avro_schema_t unionp, int *branch_index, const char *name) +{ + union { + st_data_t data; + int branch_index; + } val; + + if (!st_lookup(avro_schema_to_union(unionp)->branches_byname, + (st_data_t) name, &val.data)) { + avro_set_error("No union branch named %s", name); + return NULL; + } + + if (branch_index != NULL) { + *branch_index = val.branch_index; + } + return avro_schema_union_branch(unionp, val.branch_index); +} + +avro_schema_t avro_schema_array(const avro_schema_t items) +{ + struct avro_array_schema_t *array = + (struct avro_array_schema_t *) avro_new(struct avro_array_schema_t); + if (!array) { + avro_set_error("Cannot allocate new array schema"); + return NULL; + } + array->items = avro_schema_incref(items); + avro_schema_init(&array->obj, AVRO_ARRAY); + return &array->obj; +} + +avro_schema_t avro_schema_array_items(avro_schema_t array) +{ + return avro_schema_to_array(array)->items; +} + +avro_schema_t avro_schema_map(const avro_schema_t values) +{ + struct avro_map_schema_t *map = + (struct avro_map_schema_t *) avro_new(struct avro_map_schema_t); + if (!map) { + avro_set_error("Cannot allocate new map schema"); + return NULL; + } + map->values = avro_schema_incref(values); + avro_schema_init(&map->obj, AVRO_MAP); + return &map->obj; +} + +avro_schema_t avro_schema_map_values(avro_schema_t map) +{ + return avro_schema_to_map(map)->values; +} + +avro_schema_t avro_schema_enum(const char *name) +{ + return avro_schema_enum_ns(name, NULL); +} + +avro_schema_t avro_schema_enum_ns(const char *name, const char *space) +{ + if (!is_avro_id(name)) { + avro_set_error("Invalid Avro identifier"); + return NULL; + } + + struct avro_enum_schema_t *enump = (struct avro_enum_schema_t *) avro_new(struct avro_enum_schema_t); + if (!enump) { + avro_set_error("Cannot allocate new enum schema"); + return NULL; + } + enump->name = avro_strdup(name); + if (!enump->name) { + avro_set_error("Cannot allocate new enum schema"); + avro_freet(struct avro_enum_schema_t, enump); + return NULL; + } + enump->space = space ? avro_strdup(space) : NULL; + if (space && !enump->space) { + avro_set_error("Cannot allocate new enum schema"); + avro_str_free(enump->name); + avro_freet(struct avro_enum_schema_t, enump); + return NULL; + } + enump->symbols = st_init_numtable_with_size(DEFAULT_TABLE_SIZE); + if (!enump->symbols) { + avro_set_error("Cannot allocate new enum schema"); + if (enump->space) avro_str_free(enump->space); + avro_str_free(enump->name); + avro_freet(struct avro_enum_schema_t, enump); + return NULL; + } + enump->symbols_byname = st_init_strtable_with_size(DEFAULT_TABLE_SIZE); + if (!enump->symbols_byname) { + avro_set_error("Cannot allocate new enum schema"); + st_free_table(enump->symbols); + if (enump->space) avro_str_free(enump->space); + avro_str_free(enump->name); + avro_freet(struct avro_enum_schema_t, enump); + return NULL; + } + avro_schema_init(&enump->obj, AVRO_ENUM); + return &enump->obj; +} + +const char *avro_schema_enum_get(const avro_schema_t enump, + int index) +{ + union { + st_data_t data; + char *sym; + } val; + st_lookup(avro_schema_to_enum(enump)->symbols, index, &val.data); + return val.sym; +} + +int avro_schema_enum_get_by_name(const avro_schema_t enump, + const char *symbol_name) +{ + union { + st_data_t data; + long idx; + } val; + + if (st_lookup(avro_schema_to_enum(enump)->symbols_byname, + (st_data_t) symbol_name, &val.data)) { + return val.idx; + } else { + avro_set_error("No enum symbol named %s", symbol_name); + return -1; + } +} + +int +avro_schema_enum_symbol_append(const avro_schema_t enum_schema, + const char *symbol) +{ + check_param(EINVAL, is_avro_schema(enum_schema), "enum schema"); + check_param(EINVAL, is_avro_enum(enum_schema), "enum schema"); + check_param(EINVAL, symbol, "symbol"); + + char *sym; + long idx; + struct avro_enum_schema_t *enump = avro_schema_to_enum(enum_schema); + sym = avro_strdup(symbol); + if (!sym) { + avro_set_error("Cannot create copy of symbol name"); + return ENOMEM; + } + idx = enump->symbols->num_entries; + st_insert(enump->symbols, (st_data_t) idx, (st_data_t) sym); + st_insert(enump->symbols_byname, (st_data_t) sym, (st_data_t) idx); + return 0; +} + +int +avro_schema_enum_number_of_symbols(const avro_schema_t enum_schema) +{ + check_param(EINVAL, is_avro_schema(enum_schema), "enum schema"); + check_param(EINVAL, is_avro_enum(enum_schema), "enum schema"); + + struct avro_enum_schema_t *enump = avro_schema_to_enum(enum_schema); + return enump->symbols->num_entries; +} + +int +avro_schema_record_field_append(const avro_schema_t record_schema, + const char *field_name, + const avro_schema_t field_schema) +{ + check_param(EINVAL, is_avro_schema(record_schema), "record schema"); + check_param(EINVAL, is_avro_record(record_schema), "record schema"); + check_param(EINVAL, field_name, "field name"); + check_param(EINVAL, is_avro_schema(field_schema), "field schema"); + + if (!is_avro_id(field_name)) { + avro_set_error("Invalid Avro identifier"); + return EINVAL; + } + + if (record_schema == field_schema) { + avro_set_error("Cannot create a circular schema"); + return EINVAL; + } + + struct avro_record_schema_t *record = avro_schema_to_record(record_schema); + struct avro_record_field_t *new_field = (struct avro_record_field_t *) avro_new(struct avro_record_field_t); + if (!new_field) { + avro_set_error("Cannot allocate new record field"); + return ENOMEM; + } + new_field->index = record->fields->num_entries; + new_field->name = avro_strdup(field_name); + new_field->type = avro_schema_incref(field_schema); + st_insert(record->fields, record->fields->num_entries, + (st_data_t) new_field); + st_insert(record->fields_byname, (st_data_t) new_field->name, + (st_data_t) new_field); + return 0; +} + +avro_schema_t avro_schema_record(const char *name, const char *space) +{ + if (!is_avro_id(name)) { + avro_set_error("Invalid Avro identifier"); + return NULL; + } + + struct avro_record_schema_t *record = (struct avro_record_schema_t *) avro_new(struct avro_record_schema_t); + if (!record) { + avro_set_error("Cannot allocate new record schema"); + return NULL; + } + record->name = avro_strdup(name); + if (!record->name) { + avro_set_error("Cannot allocate new record schema"); + avro_freet(struct avro_record_schema_t, record); + return NULL; + } + record->space = space ? avro_strdup(space) : NULL; + if (space && !record->space) { + avro_set_error("Cannot allocate new record schema"); + avro_str_free(record->name); + avro_freet(struct avro_record_schema_t, record); + return NULL; + } + record->fields = st_init_numtable_with_size(DEFAULT_TABLE_SIZE); + if (!record->fields) { + avro_set_error("Cannot allocate new record schema"); + if (record->space) { + avro_str_free(record->space); + } + avro_str_free(record->name); + avro_freet(struct avro_record_schema_t, record); + return NULL; + } + record->fields_byname = st_init_strtable_with_size(DEFAULT_TABLE_SIZE); + if (!record->fields_byname) { + avro_set_error("Cannot allocate new record schema"); + st_free_table(record->fields); + if (record->space) { + avro_str_free(record->space); + } + avro_str_free(record->name); + avro_freet(struct avro_record_schema_t, record); + return NULL; + } + + avro_schema_init(&record->obj, AVRO_RECORD); + return &record->obj; +} + +size_t avro_schema_record_size(const avro_schema_t record) +{ + return avro_schema_to_record(record)->fields->num_entries; +} + +avro_schema_t avro_schema_record_field_get(const avro_schema_t + record, const char *field_name) +{ + union { + st_data_t data; + struct avro_record_field_t *field; + } val; + st_lookup(avro_schema_to_record(record)->fields_byname, + (st_data_t) field_name, &val.data); + return val.field->type; +} + +int avro_schema_record_field_get_index(const avro_schema_t schema, + const char *field_name) +{ + union { + st_data_t data; + struct avro_record_field_t *field; + } val; + if (st_lookup(avro_schema_to_record(schema)->fields_byname, + (st_data_t) field_name, &val.data)) { + return val.field->index; + } + + avro_set_error("No field named %s in record", field_name); + return -1; +} + +const char *avro_schema_record_field_name(const avro_schema_t schema, int index) +{ + union { + st_data_t data; + struct avro_record_field_t *field; + } val; + st_lookup(avro_schema_to_record(schema)->fields, index, &val.data); + return val.field->name; +} + +avro_schema_t avro_schema_record_field_get_by_index +(const avro_schema_t record, int index) +{ + union { + st_data_t data; + struct avro_record_field_t *field; + } val; + st_lookup(avro_schema_to_record(record)->fields, index, &val.data); + return val.field->type; +} + +avro_schema_t avro_schema_link(avro_schema_t to) +{ + if (!is_avro_named_type(to)) { + avro_set_error("Can only link to named types"); + return NULL; + } + + struct avro_link_schema_t *link = (struct avro_link_schema_t *) avro_new(struct avro_link_schema_t); + if (!link) { + avro_set_error("Cannot allocate new link schema"); + return NULL; + } + + /* Do not increment the reference count of target schema + * pointed to by the AVRO_LINK. AVRO_LINKs are only valid + * internal to a schema. The target schema pointed to by a + * link will be valid as long as the top-level schema is + * valid. Similarly, the link will be valid as long as the + * top-level schema is valid. Therefore the validity of the + * link ensures the validity of its target, and we don't need + * an additional reference count on the target. This mechanism + * of an implied validity also breaks reference count cycles + * for recursive schemas, which result in memory leaks. + */ + link->to = to; + avro_schema_init(&link->obj, AVRO_LINK); + return &link->obj; +} + +avro_schema_t avro_schema_link_target(avro_schema_t schema) +{ + check_param(NULL, is_avro_schema(schema), "schema"); + check_param(NULL, is_avro_link(schema), "schema"); + + struct avro_link_schema_t *link = avro_schema_to_link(schema); + return link->to; +} + +static const char * +qualify_name(const char *name, const char *namespace) +{ + char *full_name; + if (namespace != NULL && strchr(name, '.') == NULL) { + full_name = avro_str_alloc(strlen(name) + strlen(namespace) + 2); + sprintf(full_name, "%s.%s", namespace, name); + } else { + full_name = avro_strdup(name); + } + return full_name; +} + +static int +save_named_schemas(const avro_schema_t schema, st_table *st) +{ + const char *name = avro_schema_name(schema); + const char *namespace = avro_schema_namespace(schema); + const char *full_name = qualify_name(name, namespace); + int rval = st_insert(st, (st_data_t) full_name, (st_data_t) schema); + return rval; +} + +static avro_schema_t +find_named_schemas(const char *name, const char *namespace, st_table *st) +{ + union { + avro_schema_t schema; + st_data_t data; + } val; + const char *full_name = qualify_name(name, namespace); + int rval = st_lookup(st, (st_data_t) full_name, &(val.data)); + avro_str_free((char *)full_name); + if (rval) { + return val.schema; + } + avro_set_error("No schema type named %s", name); + return NULL; +}; + +static int +avro_type_from_json_t(json_t *json, avro_type_t *type, + st_table *named_schemas, avro_schema_t *named_type, + const char *namespace) +{ + json_t *json_type; + const char *type_str; + + if (json_is_array(json)) { + *type = AVRO_UNION; + return 0; + } else if (json_is_object(json)) { + json_type = json_object_get(json, "type"); + } else { + json_type = json; + } + if (!json_is_string(json_type)) { + avro_set_error("\"type\" field must be a string"); + return EINVAL; + } + type_str = json_string_value(json_type); + if (!type_str) { + avro_set_error("\"type\" field must be a string"); + return EINVAL; + } + /* + * TODO: gperf/re2c this + */ + if (strcmp(type_str, "string") == 0) { + *type = AVRO_STRING; + } else if (strcmp(type_str, "bytes") == 0) { + *type = AVRO_BYTES; + } else if (strcmp(type_str, "int") == 0) { + *type = AVRO_INT32; + } else if (strcmp(type_str, "long") == 0) { + *type = AVRO_INT64; + } else if (strcmp(type_str, "float") == 0) { + *type = AVRO_FLOAT; + } else if (strcmp(type_str, "double") == 0) { + *type = AVRO_DOUBLE; + } else if (strcmp(type_str, "boolean") == 0) { + *type = AVRO_BOOLEAN; + } else if (strcmp(type_str, "null") == 0) { + *type = AVRO_NULL; + } else if (strcmp(type_str, "record") == 0) { + *type = AVRO_RECORD; + } else if (strcmp(type_str, "enum") == 0) { + *type = AVRO_ENUM; + } else if (strcmp(type_str, "array") == 0) { + *type = AVRO_ARRAY; + } else if (strcmp(type_str, "map") == 0) { + *type = AVRO_MAP; + } else if (strcmp(type_str, "fixed") == 0) { + *type = AVRO_FIXED; + } else if ((*named_type = find_named_schemas(type_str, namespace, named_schemas))) { + *type = AVRO_LINK; + } else { + avro_set_error("Unknown Avro \"type\": %s", type_str); + return EINVAL; + } + return 0; +} + +static int +avro_schema_from_json_t(json_t *json, avro_schema_t *schema, + st_table *named_schemas, const char *parent_namespace) +{ +#ifdef _WIN32 + #pragma message("#warning: Bug: '0' is not of type avro_type_t.") +#else + #warning "Bug: '0' is not of type avro_type_t." +#endif + /* We should really have an "AVRO_INVALID" type in + * avro_type_t. Suppress warning below in which we set type to 0. + */ + avro_type_t type = (avro_type_t) 0; + unsigned int i; + avro_schema_t named_type = NULL; + + if (avro_type_from_json_t(json, &type, named_schemas, &named_type, parent_namespace)) { + return EINVAL; + } + + switch (type) { + case AVRO_LINK: + *schema = avro_schema_link(named_type); + break; + + case AVRO_STRING: + *schema = avro_schema_string(); + break; + + case AVRO_BYTES: + *schema = avro_schema_bytes(); + break; + + case AVRO_INT32: + *schema = avro_schema_int(); + break; + + case AVRO_INT64: + *schema = avro_schema_long(); + break; + + case AVRO_FLOAT: + *schema = avro_schema_float(); + break; + + case AVRO_DOUBLE: + *schema = avro_schema_double(); + break; + + case AVRO_BOOLEAN: + *schema = avro_schema_boolean(); + break; + + case AVRO_NULL: + *schema = avro_schema_null(); + break; + + case AVRO_RECORD: + { + json_t *json_name = json_object_get(json, "name"); + json_t *json_namespace = + json_object_get(json, "namespace"); + json_t *json_fields = json_object_get(json, "fields"); + unsigned int num_fields; + const char *fullname, *name; + + if (!json_is_string(json_name)) { + avro_set_error("Record type must have a \"name\""); + return EINVAL; + } + if (!json_is_array(json_fields)) { + avro_set_error("Record type must have \"fields\""); + return EINVAL; + } + num_fields = json_array_size(json_fields); + fullname = json_string_value(json_name); + if (!fullname) { + avro_set_error("Record type must have a \"name\""); + return EINVAL; + } + + if (strchr(fullname, '.')) { + char *namespace = split_namespace_name(fullname, &name); + *schema = avro_schema_record(name, namespace); + avro_str_free(namespace); + } else if (json_is_string(json_namespace)) { + const char *namespace = json_string_value(json_namespace); + if (strlen(namespace) == 0) { + namespace = NULL; + } + *schema = avro_schema_record(fullname, namespace); + } else { + *schema = avro_schema_record(fullname, parent_namespace); + } + + if (*schema == NULL) { + return ENOMEM; + } + if (save_named_schemas(*schema, named_schemas)) { + avro_set_error("Cannot save record schema"); + return ENOMEM; + } + for (i = 0; i < num_fields; i++) { + json_t *json_field = + json_array_get(json_fields, i); + json_t *json_field_name; + json_t *json_field_type; + avro_schema_t json_field_type_schema; + int field_rval; + + if (!json_is_object(json_field)) { + avro_set_error("Record field %d must be an array", i); + avro_schema_decref(*schema); + return EINVAL; + } + json_field_name = + json_object_get(json_field, "name"); + if (!json_field_name) { + avro_set_error("Record field %d must have a \"name\"", i); + avro_schema_decref(*schema); + return EINVAL; + } + json_field_type = + json_object_get(json_field, "type"); + if (!json_field_type) { + avro_set_error("Record field %d must have a \"type\"", i); + avro_schema_decref(*schema); + return EINVAL; + } + field_rval = + avro_schema_from_json_t(json_field_type, + &json_field_type_schema, + named_schemas, + avro_schema_namespace(*schema)); + if (field_rval) { + avro_schema_decref(*schema); + return field_rval; + } + field_rval = + avro_schema_record_field_append(*schema, + json_string_value + (json_field_name), + json_field_type_schema); + avro_schema_decref(json_field_type_schema); + if (field_rval != 0) { + avro_schema_decref(*schema); + return field_rval; + } + } + } + break; + + case AVRO_ENUM: + { + json_t *json_name = json_object_get(json, "name"); + json_t *json_symbols = json_object_get(json, "symbols"); + json_t *json_namespace = json_object_get(json, "namespace"); + const char *fullname, *name; + unsigned int num_symbols; + + if (!json_is_string(json_name)) { + avro_set_error("Enum type must have a \"name\""); + return EINVAL; + } + if (!json_is_array(json_symbols)) { + avro_set_error("Enum type must have \"symbols\""); + return EINVAL; + } + + fullname = json_string_value(json_name); + if (!fullname) { + avro_set_error("Enum type must have a \"name\""); + return EINVAL; + } + num_symbols = json_array_size(json_symbols); + if (num_symbols == 0) { + avro_set_error("Enum type must have at least one symbol"); + return EINVAL; + } + + if (strchr(fullname, '.')) { + char *namespace; + namespace = split_namespace_name(fullname, &name); + *schema = avro_schema_enum_ns(name, namespace); + avro_str_free(namespace); + } else if (json_is_string(json_namespace)) { + const char *namespace = json_string_value(json_namespace); + if (strlen(namespace) == 0) { + namespace = NULL; + } + *schema = avro_schema_enum_ns(fullname, namespace); + } else { + *schema = avro_schema_enum_ns(fullname, parent_namespace); + } + + if (*schema == NULL) { + return ENOMEM; + } + if (save_named_schemas(*schema, named_schemas)) { + avro_set_error("Cannot save enum schema"); + return ENOMEM; + } + for (i = 0; i < num_symbols; i++) { + int enum_rval; + json_t *json_symbol = + json_array_get(json_symbols, i); + const char *symbol; + if (!json_is_string(json_symbol)) { + avro_set_error("Enum symbol %d must be a string", i); + avro_schema_decref(*schema); + return EINVAL; + } + symbol = json_string_value(json_symbol); + enum_rval = + avro_schema_enum_symbol_append(*schema, + symbol); + if (enum_rval != 0) { + avro_schema_decref(*schema); + return enum_rval; + } + } + } + break; + + case AVRO_ARRAY: + { + int items_rval; + json_t *json_items = json_object_get(json, "items"); + avro_schema_t items_schema; + if (!json_items) { + avro_set_error("Array type must have \"items\""); + return EINVAL; + } + items_rval = + avro_schema_from_json_t(json_items, &items_schema, + named_schemas, parent_namespace); + if (items_rval) { + return items_rval; + } + *schema = avro_schema_array(items_schema); + avro_schema_decref(items_schema); + } + break; + + case AVRO_MAP: + { + int values_rval; + json_t *json_values = json_object_get(json, "values"); + avro_schema_t values_schema; + + if (!json_values) { + avro_set_error("Map type must have \"values\""); + return EINVAL; + } + values_rval = + avro_schema_from_json_t(json_values, &values_schema, + named_schemas, parent_namespace); + if (values_rval) { + return values_rval; + } + *schema = avro_schema_map(values_schema); + avro_schema_decref(values_schema); + } + break; + + case AVRO_UNION: + { + unsigned int num_schemas = json_array_size(json); + avro_schema_t s; + if (num_schemas == 0) { + avro_set_error("Union type must have at least one branch"); + return EINVAL; + } + *schema = avro_schema_union(); + for (i = 0; i < num_schemas; i++) { + int schema_rval; + json_t *schema_json = json_array_get(json, i); + if (!schema_json) { + avro_set_error("Cannot retrieve branch JSON"); + return EINVAL; + } + schema_rval = + avro_schema_from_json_t(schema_json, &s, + named_schemas, parent_namespace); + if (schema_rval != 0) { + avro_schema_decref(*schema); + return schema_rval; + } + schema_rval = + avro_schema_union_append(*schema, s); + avro_schema_decref(s); + if (schema_rval != 0) { + avro_schema_decref(*schema); + return schema_rval; + } + } + } + break; + + case AVRO_FIXED: + { + json_t *json_size = json_object_get(json, "size"); + json_t *json_name = json_object_get(json, "name"); + json_t *json_namespace = json_object_get(json, "namespace"); + json_int_t size; + const char *fullname, *name; + if (!json_is_integer(json_size)) { + avro_set_error("Fixed type must have a \"size\""); + return EINVAL; + } + if (!json_is_string(json_name)) { + avro_set_error("Fixed type must have a \"name\""); + return EINVAL; + } + size = json_integer_value(json_size); + fullname = json_string_value(json_name); + + if (strchr(fullname, '.')) { + char *namespace; + namespace = split_namespace_name(fullname, &name); + *schema = avro_schema_fixed_ns(name, namespace, (int64_t) size); + avro_str_free(namespace); + } else if (json_is_string(json_namespace)) { + const char *namespace = json_string_value(json_namespace); + if (strlen(namespace) == 0) { + namespace = NULL; + } + *schema = avro_schema_fixed_ns(fullname, namespace, (int64_t) size); + } else { + *schema = avro_schema_fixed_ns(fullname, parent_namespace, (int64_t) size); + } + + if (*schema == NULL) { + return ENOMEM; + } + if (save_named_schemas(*schema, named_schemas)) { + avro_set_error("Cannot save fixed schema"); + return ENOMEM; + } + } + break; + + default: + avro_set_error("Unknown schema type"); + return EINVAL; + } + return 0; +} + +static int named_schema_free_foreach(char *full_name, st_data_t value, st_data_t arg) +{ + AVRO_UNUSED(value); + AVRO_UNUSED(arg); + + avro_str_free(full_name); + return ST_DELETE; +} + +static int +avro_schema_from_json_root(json_t *root, avro_schema_t *schema) +{ + int rval; + st_table *named_schemas; + + named_schemas = st_init_strtable_with_size(DEFAULT_TABLE_SIZE); + if (!named_schemas) { + avro_set_error("Cannot allocate named schema map"); + json_decref(root); + return ENOMEM; + } + + /* json_dumpf(root, stderr, 0); */ + rval = avro_schema_from_json_t(root, schema, named_schemas, NULL); + json_decref(root); + st_foreach(named_schemas, HASH_FUNCTION_CAST named_schema_free_foreach, 0); + st_free_table(named_schemas); + return rval; +} + +int +avro_schema_from_json(const char *jsontext, const int32_t len, + avro_schema_t *schema, avro_schema_error_t *e) +{ + check_param(EINVAL, jsontext, "JSON text"); + check_param(EINVAL, schema, "schema pointer"); + + json_t *root; + json_error_t json_error; + + AVRO_UNUSED(len); + AVRO_UNUSED(e); + + root = json_loads(jsontext, JSON_DECODE_ANY, &json_error); + if (!root) { + avro_set_error("Error parsing JSON: %s", json_error.text); + return EINVAL; + } + + return avro_schema_from_json_root(root, schema); +} + +int +avro_schema_from_json_length(const char *jsontext, size_t length, + avro_schema_t *schema) +{ + check_param(EINVAL, jsontext, "JSON text"); + check_param(EINVAL, schema, "schema pointer"); + + json_t *root; + json_error_t json_error; + + root = json_loadb(jsontext, length, JSON_DECODE_ANY, &json_error); + if (!root) { + avro_set_error("Error parsing JSON: %s", json_error.text); + return EINVAL; + } + + return avro_schema_from_json_root(root, schema); +} + +avro_schema_t avro_schema_copy_root(avro_schema_t schema, st_table *named_schemas) +{ + long i; + avro_schema_t new_schema = NULL; + if (!schema) { + return NULL; + } + switch (avro_typeof(schema)) { + case AVRO_STRING: + case AVRO_BYTES: + case AVRO_INT32: + case AVRO_INT64: + case AVRO_FLOAT: + case AVRO_DOUBLE: + case AVRO_BOOLEAN: + case AVRO_NULL: + /* + * No need to copy primitives since they're static + */ + new_schema = schema; + break; + + case AVRO_RECORD: + { + struct avro_record_schema_t *record_schema = + avro_schema_to_record(schema); + new_schema = + avro_schema_record(record_schema->name, + record_schema->space); + if (save_named_schemas(new_schema, named_schemas)) { + avro_set_error("Cannot save enum schema"); + return NULL; + } + for (i = 0; i < record_schema->fields->num_entries; i++) { + union { + st_data_t data; + struct avro_record_field_t *field; + } val; + st_lookup(record_schema->fields, i, &val.data); + avro_schema_t type_copy = + avro_schema_copy_root(val.field->type, named_schemas); + avro_schema_record_field_append(new_schema, + val.field->name, + type_copy); + avro_schema_decref(type_copy); + } + } + break; + + case AVRO_ENUM: + { + struct avro_enum_schema_t *enum_schema = + avro_schema_to_enum(schema); + new_schema = avro_schema_enum_ns(enum_schema->name, + enum_schema->space); + if (save_named_schemas(new_schema, named_schemas)) { + avro_set_error("Cannot save enum schema"); + return NULL; + } + for (i = 0; i < enum_schema->symbols->num_entries; i++) { + union { + st_data_t data; + char *sym; + } val; + st_lookup(enum_schema->symbols, i, &val.data); + avro_schema_enum_symbol_append(new_schema, + val.sym); + } + } + break; + + case AVRO_FIXED: + { + struct avro_fixed_schema_t *fixed_schema = + avro_schema_to_fixed(schema); + new_schema = + avro_schema_fixed_ns(fixed_schema->name, + fixed_schema->space, + fixed_schema->size); + if (save_named_schemas(new_schema, named_schemas)) { + avro_set_error("Cannot save fixed schema"); + return NULL; + } + } + break; + + case AVRO_MAP: + { + struct avro_map_schema_t *map_schema = + avro_schema_to_map(schema); + avro_schema_t values_copy = + avro_schema_copy_root(map_schema->values, named_schemas); + if (!values_copy) { + return NULL; + } + new_schema = avro_schema_map(values_copy); + avro_schema_decref(values_copy); + } + break; + + case AVRO_ARRAY: + { + struct avro_array_schema_t *array_schema = + avro_schema_to_array(schema); + avro_schema_t items_copy = + avro_schema_copy_root(array_schema->items, named_schemas); + if (!items_copy) { + return NULL; + } + new_schema = avro_schema_array(items_copy); + avro_schema_decref(items_copy); + } + break; + + case AVRO_UNION: + { + struct avro_union_schema_t *union_schema = + avro_schema_to_union(schema); + + new_schema = avro_schema_union(); + for (i = 0; i < union_schema->branches->num_entries; + i++) { + avro_schema_t schema_copy; + union { + st_data_t data; + avro_schema_t schema; + } val; + st_lookup(union_schema->branches, i, &val.data); + schema_copy = avro_schema_copy_root(val.schema, named_schemas); + if (avro_schema_union_append + (new_schema, schema_copy)) { + avro_schema_decref(new_schema); + return NULL; + } + avro_schema_decref(schema_copy); + } + } + break; + + case AVRO_LINK: + { + struct avro_link_schema_t *link_schema = + avro_schema_to_link(schema); + avro_schema_t to; + + to = find_named_schemas(avro_schema_name(link_schema->to), + avro_schema_namespace(link_schema->to), + named_schemas); + new_schema = avro_schema_link(to); + } + break; + + default: + return NULL; + } + return new_schema; +} + +avro_schema_t avro_schema_copy(avro_schema_t schema) +{ + avro_schema_t new_schema; + st_table *named_schemas; + + named_schemas = st_init_strtable_with_size(DEFAULT_TABLE_SIZE); + if (!named_schemas) { + avro_set_error("Cannot allocate named schema map"); + return NULL; + } + + new_schema = avro_schema_copy_root(schema, named_schemas); + st_foreach(named_schemas, HASH_FUNCTION_CAST named_schema_free_foreach, 0); + st_free_table(named_schemas); + return new_schema; +} + +avro_schema_t avro_schema_get_subschema(const avro_schema_t schema, + const char *name) +{ + if (is_avro_record(schema)) { + const struct avro_record_schema_t *rschema = + avro_schema_to_record(schema); + union { + st_data_t data; + struct avro_record_field_t *field; + } field; + + if (st_lookup(rschema->fields_byname, + (st_data_t) name, &field.data)) + { + return field.field->type; + } + + avro_set_error("No record field named %s", name); + return NULL; + } else if (is_avro_union(schema)) { + const struct avro_union_schema_t *uschema = + avro_schema_to_union(schema); + long i; + + for (i = 0; i < uschema->branches->num_entries; i++) { + union { + st_data_t data; + avro_schema_t schema; + } val; + st_lookup(uschema->branches, i, &val.data); + if (strcmp(avro_schema_type_name(val.schema), + name) == 0) + { + return val.schema; + } + } + + avro_set_error("No union branch named %s", name); + return NULL; + } else if (is_avro_array(schema)) { + if (strcmp(name, "[]") == 0) { + const struct avro_array_schema_t *aschema = + avro_schema_to_array(schema); + return aschema->items; + } + + avro_set_error("Array subschema must be called \"[]\""); + return NULL; + } else if (is_avro_map(schema)) { + if (strcmp(name, "{}") == 0) { + const struct avro_map_schema_t *mschema = + avro_schema_to_map(schema); + return mschema->values; + } + + avro_set_error("Map subschema must be called \"{}\""); + return NULL; + } + + avro_set_error("Can only retrieve subschemas from record, union, array, or map"); + return NULL; +} + +const char *avro_schema_name(const avro_schema_t schema) +{ + if (is_avro_record(schema)) { + return (avro_schema_to_record(schema))->name; + } else if (is_avro_enum(schema)) { + return (avro_schema_to_enum(schema))->name; + } else if (is_avro_fixed(schema)) { + return (avro_schema_to_fixed(schema))->name; + } + avro_set_error("Schema has no name"); + return NULL; +} + +const char *avro_schema_namespace(const avro_schema_t schema) +{ + if (is_avro_record(schema)) { + return (avro_schema_to_record(schema))->space; + } else if (is_avro_enum(schema)) { + return (avro_schema_to_enum(schema))->space; + } else if (is_avro_fixed(schema)) { + return (avro_schema_to_fixed(schema))->space; + } + return NULL; +} + +const char *avro_schema_type_name(const avro_schema_t schema) +{ + if (is_avro_record(schema)) { + return (avro_schema_to_record(schema))->name; + } else if (is_avro_enum(schema)) { + return (avro_schema_to_enum(schema))->name; + } else if (is_avro_fixed(schema)) { + return (avro_schema_to_fixed(schema))->name; + } else if (is_avro_union(schema)) { + return "union"; + } else if (is_avro_array(schema)) { + return "array"; + } else if (is_avro_map(schema)) { + return "map"; + } else if (is_avro_int32(schema)) { + return "int"; + } else if (is_avro_int64(schema)) { + return "long"; + } else if (is_avro_float(schema)) { + return "float"; + } else if (is_avro_double(schema)) { + return "double"; + } else if (is_avro_boolean(schema)) { + return "boolean"; + } else if (is_avro_null(schema)) { + return "null"; + } else if (is_avro_string(schema)) { + return "string"; + } else if (is_avro_bytes(schema)) { + return "bytes"; + } else if (is_avro_link(schema)) { + avro_schema_t target = avro_schema_link_target(schema); + return avro_schema_type_name(target); + } + avro_set_error("Unknown schema type"); + return NULL; +} + +avro_datum_t avro_datum_from_schema(const avro_schema_t schema) +{ + check_param(NULL, is_avro_schema(schema), "schema"); + + switch (avro_typeof(schema)) { + case AVRO_STRING: + return avro_givestring("", NULL); + + case AVRO_BYTES: + return avro_givebytes("", 0, NULL); + + case AVRO_INT32: + return avro_int32(0); + + case AVRO_INT64: + return avro_int64(0); + + case AVRO_FLOAT: + return avro_float(0); + + case AVRO_DOUBLE: + return avro_double(0); + + case AVRO_BOOLEAN: + return avro_boolean(0); + + case AVRO_NULL: + return avro_null(); + + case AVRO_RECORD: + { + const struct avro_record_schema_t *record_schema = + avro_schema_to_record(schema); + + avro_datum_t rec = avro_record(schema); + + int i; + for (i = 0; i < record_schema->fields->num_entries; i++) { + union { + st_data_t data; + struct avro_record_field_t *field; + } val; + st_lookup(record_schema->fields, i, &val.data); + + avro_datum_t field = + avro_datum_from_schema(val.field->type); + avro_record_set(rec, val.field->name, field); + avro_datum_decref(field); + } + + return rec; + } + + case AVRO_ENUM: + return avro_enum(schema, 0); + + case AVRO_FIXED: + { + const struct avro_fixed_schema_t *fixed_schema = + avro_schema_to_fixed(schema); + return avro_givefixed(schema, NULL, fixed_schema->size, NULL); + } + + case AVRO_MAP: + return avro_map(schema); + + case AVRO_ARRAY: + return avro_array(schema); + + case AVRO_UNION: + return avro_union(schema, -1, NULL); + + case AVRO_LINK: + { + const struct avro_link_schema_t *link_schema = + avro_schema_to_link(schema); + return avro_datum_from_schema(link_schema->to); + } + + default: + avro_set_error("Unknown schema type"); + return NULL; + } +} + +/* simple helper for writing strings */ +static int avro_write_str(avro_writer_t out, const char *str) +{ + return avro_write(out, (char *)str, strlen(str)); +} + +static int write_field(avro_writer_t out, const struct avro_record_field_t *field, + const char *parent_namespace) +{ + int rval; + check(rval, avro_write_str(out, "{\"name\":\"")); + check(rval, avro_write_str(out, field->name)); + check(rval, avro_write_str(out, "\",\"type\":")); + check(rval, avro_schema_to_json2(field->type, out, parent_namespace)); + return avro_write_str(out, "}"); +} + +static int write_record(avro_writer_t out, const struct avro_record_schema_t *record, + const char *parent_namespace) +{ + int rval; + long i; + + check(rval, avro_write_str(out, "{\"type\":\"record\",\"name\":\"")); + check(rval, avro_write_str(out, record->name)); + check(rval, avro_write_str(out, "\",")); + if (nullstrcmp(record->space, parent_namespace)) { + check(rval, avro_write_str(out, "\"namespace\":\"")); + if (record->space) { + check(rval, avro_write_str(out, record->space)); + } + check(rval, avro_write_str(out, "\",")); + } + check(rval, avro_write_str(out, "\"fields\":[")); + for (i = 0; i < record->fields->num_entries; i++) { + union { + st_data_t data; + struct avro_record_field_t *field; + } val; + st_lookup(record->fields, i, &val.data); + if (i) { + check(rval, avro_write_str(out, ",")); + } + check(rval, write_field(out, val.field, record->space)); + } + return avro_write_str(out, "]}"); +} + +static int write_enum(avro_writer_t out, const struct avro_enum_schema_t *enump, + const char *parent_namespace) +{ + int rval; + long i; + check(rval, avro_write_str(out, "{\"type\":\"enum\",\"name\":\"")); + check(rval, avro_write_str(out, enump->name)); + check(rval, avro_write_str(out, "\",")); + if (nullstrcmp(enump->space, parent_namespace)) { + check(rval, avro_write_str(out, "\"namespace\":\"")); + if (enump->space) { + check(rval, avro_write_str(out, enump->space)); + } + check(rval, avro_write_str(out, "\",")); + } + check(rval, avro_write_str(out, "\"symbols\":[")); + + for (i = 0; i < enump->symbols->num_entries; i++) { + union { + st_data_t data; + char *sym; + } val; + st_lookup(enump->symbols, i, &val.data); + if (i) { + check(rval, avro_write_str(out, ",")); + } + check(rval, avro_write_str(out, "\"")); + check(rval, avro_write_str(out, val.sym)); + check(rval, avro_write_str(out, "\"")); + } + return avro_write_str(out, "]}"); +} + +static int write_fixed(avro_writer_t out, const struct avro_fixed_schema_t *fixed, + const char *parent_namespace) +{ + int rval; + char size[16]; + check(rval, avro_write_str(out, "{\"type\":\"fixed\",\"name\":\"")); + check(rval, avro_write_str(out, fixed->name)); + check(rval, avro_write_str(out, "\",")); + if (nullstrcmp(fixed->space, parent_namespace)) { + check(rval, avro_write_str(out, "\"namespace\":\"")); + if (fixed->space) { + check(rval, avro_write_str(out, fixed->space)); + } + check(rval, avro_write_str(out, "\",")); + } + check(rval, avro_write_str(out, "\"size\":")); + snprintf(size, sizeof(size), "%" PRId64, fixed->size); + check(rval, avro_write_str(out, size)); + return avro_write_str(out, "}"); +} + +static int write_map(avro_writer_t out, const struct avro_map_schema_t *map, + const char *parent_namespace) +{ + int rval; + check(rval, avro_write_str(out, "{\"type\":\"map\",\"values\":")); + check(rval, avro_schema_to_json2(map->values, out, parent_namespace)); + return avro_write_str(out, "}"); +} +static int write_array(avro_writer_t out, const struct avro_array_schema_t *array, + const char *parent_namespace) +{ + int rval; + check(rval, avro_write_str(out, "{\"type\":\"array\",\"items\":")); + check(rval, avro_schema_to_json2(array->items, out, parent_namespace)); + return avro_write_str(out, "}"); +} +static int write_union(avro_writer_t out, const struct avro_union_schema_t *unionp, + const char *parent_namespace) +{ + int rval; + long i; + check(rval, avro_write_str(out, "[")); + + for (i = 0; i < unionp->branches->num_entries; i++) { + union { + st_data_t data; + avro_schema_t schema; + } val; + st_lookup(unionp->branches, i, &val.data); + if (i) { + check(rval, avro_write_str(out, ",")); + } + check(rval, avro_schema_to_json2(val.schema, out, parent_namespace)); + } + return avro_write_str(out, "]"); +} +static int write_link(avro_writer_t out, const struct avro_link_schema_t *link, + const char *parent_namespace) +{ + int rval; + check(rval, avro_write_str(out, "\"")); + const char *namespace = avro_schema_namespace(link->to); + if (namespace && nullstrcmp(namespace, parent_namespace)) { + check(rval, avro_write_str(out, namespace)); + check(rval, avro_write_str(out, ".")); + } + check(rval, avro_write_str(out, avro_schema_name(link->to))); + return avro_write_str(out, "\""); +} + +static int +avro_schema_to_json2(const avro_schema_t schema, avro_writer_t out, + const char *parent_namespace) +{ + check_param(EINVAL, is_avro_schema(schema), "schema"); + check_param(EINVAL, out, "writer"); + + int rval; + + if (is_avro_primitive(schema)) { + check(rval, avro_write_str(out, "{\"type\":\"")); + } + + switch (avro_typeof(schema)) { + case AVRO_STRING: + check(rval, avro_write_str(out, "string")); + break; + case AVRO_BYTES: + check(rval, avro_write_str(out, "bytes")); + break; + case AVRO_INT32: + check(rval, avro_write_str(out, "int")); + break; + case AVRO_INT64: + check(rval, avro_write_str(out, "long")); + break; + case AVRO_FLOAT: + check(rval, avro_write_str(out, "float")); + break; + case AVRO_DOUBLE: + check(rval, avro_write_str(out, "double")); + break; + case AVRO_BOOLEAN: + check(rval, avro_write_str(out, "boolean")); + break; + case AVRO_NULL: + check(rval, avro_write_str(out, "null")); + break; + case AVRO_RECORD: + return write_record(out, avro_schema_to_record(schema), parent_namespace); + case AVRO_ENUM: + return write_enum(out, avro_schema_to_enum(schema), parent_namespace); + case AVRO_FIXED: + return write_fixed(out, avro_schema_to_fixed(schema), parent_namespace); + case AVRO_MAP: + return write_map(out, avro_schema_to_map(schema), parent_namespace); + case AVRO_ARRAY: + return write_array(out, avro_schema_to_array(schema), parent_namespace); + case AVRO_UNION: + return write_union(out, avro_schema_to_union(schema), parent_namespace); + case AVRO_LINK: + return write_link(out, avro_schema_to_link(schema), parent_namespace); + } + + if (is_avro_primitive(schema)) { + return avro_write_str(out, "\"}"); + } + avro_set_error("Unknown schema type"); + return EINVAL; +} + +int avro_schema_to_json(const avro_schema_t schema, avro_writer_t out) +{ + return avro_schema_to_json2(schema, out, NULL); +} diff --git a/fluent-bit/lib/avro/src/schema.h b/fluent-bit/lib/avro/src/schema.h new file mode 100644 index 00000000..3c99ee63 --- /dev/null +++ b/fluent-bit/lib/avro/src/schema.h @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +#ifndef AVRO_SCHEMA_PRIV_H +#define AVRO_SCHEMA_PRIV_H + +#include <avro/platform.h> +#include "avro/basics.h" +#include "avro/schema.h" +#include "avro_private.h" +#include "st.h" + +struct avro_record_field_t { + int index; + char *name; + avro_schema_t type; + /* + * TODO: default values + */ +}; + +struct avro_record_schema_t { + struct avro_obj_t obj; + char *name; + char *space; + st_table *fields; + st_table *fields_byname; +}; + +struct avro_enum_schema_t { + struct avro_obj_t obj; + char *name; + char *space; + st_table *symbols; + st_table *symbols_byname; +}; + +struct avro_array_schema_t { + struct avro_obj_t obj; + avro_schema_t items; +}; + +struct avro_map_schema_t { + struct avro_obj_t obj; + avro_schema_t values; +}; + +struct avro_union_schema_t { + struct avro_obj_t obj; + st_table *branches; + st_table *branches_byname; +}; + +struct avro_fixed_schema_t { + struct avro_obj_t obj; + const char *name; + const char *space; + int64_t size; +}; + +struct avro_link_schema_t { + struct avro_obj_t obj; + avro_schema_t to; +}; + +#define avro_schema_to_record(schema_) (container_of(schema_, struct avro_record_schema_t, obj)) +#define avro_schema_to_enum(schema_) (container_of(schema_, struct avro_enum_schema_t, obj)) +#define avro_schema_to_array(schema_) (container_of(schema_, struct avro_array_schema_t, obj)) +#define avro_schema_to_map(schema_) (container_of(schema_, struct avro_map_schema_t, obj)) +#define avro_schema_to_union(schema_) (container_of(schema_, struct avro_union_schema_t, obj)) +#define avro_schema_to_fixed(schema_) (container_of(schema_, struct avro_fixed_schema_t, obj)) +#define avro_schema_to_link(schema_) (container_of(schema_, struct avro_link_schema_t, obj)) + +#endif diff --git a/fluent-bit/lib/avro/src/schema_equal.c b/fluent-bit/lib/avro/src/schema_equal.c new file mode 100644 index 00000000..419ed127 --- /dev/null +++ b/fluent-bit/lib/avro/src/schema_equal.c @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include "avro_private.h" +#include "schema.h" +#include <string.h> + +static int +schema_record_equal(struct avro_record_schema_t *a, + struct avro_record_schema_t *b) +{ + long i; + if (strcmp(a->name, b->name)) { + /* + * They have different names + */ + return 0; + } + if (nullstrcmp(a->space, b->space)) { + return 0; + } + if (a->fields->num_entries != b->fields->num_entries) { + /* They have different numbers of fields */ + return 0; + } + for (i = 0; i < a->fields->num_entries; i++) { + union { + st_data_t data; + struct avro_record_field_t *f; + } fa, fb; + st_lookup(a->fields, i, &fa.data); + if (!st_lookup(b->fields, i, &fb.data)) { + return 0; + } + if (strcmp(fa.f->name, fb.f->name)) { + /* + * They have fields with different names + */ + return 0; + } + if (!avro_schema_equal(fa.f->type, fb.f->type)) { + /* + * They have fields with different schemas + */ + return 0; + } + } + return 1; +} + +static int +schema_enum_equal(struct avro_enum_schema_t *a, struct avro_enum_schema_t *b) +{ + long i; + if (strcmp(a->name, b->name)) { + /* + * They have different names + */ + return 0; + } + if (nullstrcmp(a->space, b->space)) { + return 0; + } + for (i = 0; i < a->symbols->num_entries; i++) { + union { + st_data_t data; + char *sym; + } sa, sb; + st_lookup(a->symbols, i, &sa.data); + if (!st_lookup(b->symbols, i, &sb.data)) { + return 0; + } + if (strcmp(sa.sym, sb.sym) != 0) { + /* + * They have different symbol names + */ + return 0; + } + } + return 1; +} + +static int +schema_fixed_equal(struct avro_fixed_schema_t *a, struct avro_fixed_schema_t *b) +{ + if (strcmp(a->name, b->name)) { + /* + * They have different names + */ + return 0; + } + if (nullstrcmp(a->space, b->space)) { + return 0; + } + return (a->size == b->size); +} + +static int +schema_map_equal(struct avro_map_schema_t *a, struct avro_map_schema_t *b) +{ + return avro_schema_equal(a->values, b->values); +} + +static int +schema_array_equal(struct avro_array_schema_t *a, struct avro_array_schema_t *b) +{ + return avro_schema_equal(a->items, b->items); +} + +static int +schema_union_equal(struct avro_union_schema_t *a, struct avro_union_schema_t *b) +{ + long i; + for (i = 0; i < a->branches->num_entries; i++) { + union { + st_data_t data; + avro_schema_t schema; + } ab, bb; + st_lookup(a->branches, i, &ab.data); + if (!st_lookup(b->branches, i, &bb.data)) { + return 0; + } + if (!avro_schema_equal(ab.schema, bb.schema)) { + /* + * They don't have the same schema types + */ + return 0; + } + } + return 1; +} + +static int +schema_link_equal(struct avro_link_schema_t *a, struct avro_link_schema_t *b) +{ + /* + * NOTE: links can only be used for named types. They are used in + * recursive schemas so we just check the name of the schema pointed + * to instead of a deep check. Otherwise, we recurse forever... + */ + if (is_avro_record(a->to)) { + if (!is_avro_record(b->to)) { + return 0; + } + if (nullstrcmp(avro_schema_to_record(a->to)->space, + avro_schema_to_record(b->to)->space)) { + return 0; + } + } + return (strcmp(avro_schema_name(a->to), avro_schema_name(b->to)) == 0); +} + +int avro_schema_equal(avro_schema_t a, avro_schema_t b) +{ + if (!a || !b) { + /* + * this is an error. protecting from segfault. + */ + return 0; + } else if (a == b) { + /* + * an object is equal to itself + */ + return 1; + } else if (avro_typeof(a) != avro_typeof(b)) { + return 0; + } else if (is_avro_record(a)) { + return schema_record_equal(avro_schema_to_record(a), + avro_schema_to_record(b)); + } else if (is_avro_enum(a)) { + return schema_enum_equal(avro_schema_to_enum(a), + avro_schema_to_enum(b)); + } else if (is_avro_fixed(a)) { + return schema_fixed_equal(avro_schema_to_fixed(a), + avro_schema_to_fixed(b)); + } else if (is_avro_map(a)) { + return schema_map_equal(avro_schema_to_map(a), + avro_schema_to_map(b)); + } else if (is_avro_array(a)) { + return schema_array_equal(avro_schema_to_array(a), + avro_schema_to_array(b)); + } else if (is_avro_union(a)) { + return schema_union_equal(avro_schema_to_union(a), + avro_schema_to_union(b)); + } else if (is_avro_link(a)) { + return schema_link_equal(avro_schema_to_link(a), + avro_schema_to_link(b)); + } + return 1; +} diff --git a/fluent-bit/lib/avro/src/schema_specific.c b/fluent-bit/lib/avro/src/schema_specific.c new file mode 100644 index 00000000..7a0150c5 --- /dev/null +++ b/fluent-bit/lib/avro/src/schema_specific.c @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include "avro_private.h" +#include <stdio.h> +#include <errno.h> +#include <string.h> + +#include "schema.h" + +enum specific_state { + START_STATE, +}; +typedef enum specific_state specific_state; + +struct specific_ctx { + FILE *header; + FILE *source; + int depth; + specific_state state; +}; +typedef struct specific_ctx specific_ctx; + +static void indent(specific_ctx * ctx, FILE * fp) +{ + int i; + for (i = 0; i < ctx->depth; i++) { + fprintf(fp, " "); + } +} + +static int avro_schema_to_source(avro_schema_t schema, specific_ctx * ctx) +{ + switch (schema->type) { + default: + return 0; + } + return EINVAL; +} + +static int avro_schema_to_header(avro_schema_t schema, specific_ctx * ctx) +{ + size_t i; + FILE *fp = ctx->header; + + indent(ctx, fp); + ctx->depth++; + + if (is_avro_primitive(schema) && !ctx->name) { + return 0; + } + + switch (schema->type) { + case AVRO_STRING: + fprintf(fp, "char *%s;\n", ctx->name); + break; + + case AVRO_BYTES: + fprintf(fp, "struct %s { size_t %s_len; char *%s_val } %s;\n", + ctx->name, ctx->name, ctx->name, ctx->name); + break; + + case AVRO_INT: + fprintf(fp, "int %s;\n", ctx->name); + break; + + case AVRO_LONG: + fprintf(fp, "long %s;\n", ctx->name); + break; + + case AVRO_FLOAT: + fprintf(fp, "float %s;\n", ctx->name); + break; + + case AVRO_DOUBLE: + fprintf(fp, "double %s;\n", ctx->name); + break; + + case AVRO_BOOLEAN: + fprintf(fp, "int %s; /* boolean */\n", ctx->name); + break; + + case AVRO_NULL: + break; + + case AVRO_RECORD: + { + struct schema_record_t *record_schema = + avro_schema_to_record(schema); + fprintf(fp, "struct %s {\n", record_schema->name); + for (i = 0; i < record_schema->num_fields; i++) { + struct record_field_t *field = + record_schema->fields[i]; + ctx->name = field->name; + avro_schema_to_header(field->type, ctx); + ctx->name = NULL; + } + fprintf(fp, "};\n"); + fprintf(fp, "typedef struct %s %s;\n\n", + record_schema->name, record_schema->name); + } + break; + + case AVRO_ENUM: + { + struct schema_enum_t *enum_schema = + avro_schema_to_enum(schema); + fprintf(fp, "enum %s {\n", enum_schema->name); + ctx->depth++; + for (i = 0; i < enum_schema->num_symbols; i++) { + indent(ctx, fp); + fprintf(fp, "%s = %ld,\n", + enum_schema->symbols[i], i); + } + ctx->depth--; + fprintf(fp, "};\n"); + fprintf(fp, "typedef enum %s %s;\n\n", + enum_schema->name, enum_schema->name); + } + break; + + case AVRO_FIXED: + { + struct schema_fixed_t *fixed_schema = + avro_schema_to_fixed(schema); + fprintf(fp, "char %s[%ld];\n", fixed_schema->name, + fixed_schema->size); + } + break; + + case AVRO_MAP: + { + + } + break; + + case AVRO_ARRAY: + { + struct schema_array_t *array_schema = + avro_schema_to_array(schema); + if (!ctx->name) { + break; + } + fprintf(fp, "struct { size_t %s_len; ", ctx->name); + if (is_avro_named_type(array_schema->items)) { + fprintf(fp, "%s", + avro_schema_name(array_schema->items)); + } else if (is_avro_link(array_schema->items)) { + struct schema_link_t *link_schema = + avro_schema_to_link(array_schema->items); + fprintf(fp, "struct %s", + avro_schema_name(link_schema->to)); + } else { + avro_schema_to_header(array_schema->items, ctx); + } + fprintf(fp, " *%s_val;} %s;\n", ctx->name, ctx->name); + } + break; + case AVRO_UNION: + { + struct schema_union_t *union_schema = + avro_schema_to_array(schema); + if (!ctx->name) { + break; + } + fprintf(fp, "union {\n"); + for (i = 0; i < union_schema->num_schemas; i++) { + avro_schema_to_header(union_schema->schemas[i], + ctx); + } + fprintf(fp, "%s_u;\n"); + } + break; + case AVRO_LINK: + break; + default: + return EINVAL; + } + + ctx->depth--; + return 0; +} + +int avro_schema_to_specific(avro_schema_t schema, const char *prefix) +{ + specific_ctx ctx; + char buf[1024]; + int rval; + + if (!schema) { + return EINVAL; + } + + memset(&ctx, 0, sizeof(ctx)); + snprintf(buf, sizeof(buf), "%s_avro.h", prefix); + ctx.header = fopen(buf, "w"); + if (!ctx.header) { + return errno; + } + snprintf(buf, sizeof(buf), "%s_avro.c", prefix); + ctx.source = fopen(buf, "w"); + if (!ctx.source) { + fclose(ctx.header); + return errno; + } + + rval = avro_schema_to_header(schema, &ctx); + if (rval) { + goto out; + } + + rval = avro_schema_to_source(schema, &ctx); + + out: + fclose(ctx.header); + fclose(ctx.source); + return rval; +} diff --git a/fluent-bit/lib/avro/src/st.c b/fluent-bit/lib/avro/src/st.c new file mode 100644 index 00000000..27578289 --- /dev/null +++ b/fluent-bit/lib/avro/src/st.c @@ -0,0 +1,543 @@ +/* + * This is a public domain general purpose hash table package written by + * Peter Moore @ UCB. + */ + +/* + * static char sccsid[] = "@(#) st.c 5.1 89/12/14 Crucible"; + */ + +#include "avro_private.h" +#include "avro/allocation.h" +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "st.h" + +typedef struct st_table_entry st_table_entry; + +struct st_table_entry { + unsigned int hash; + st_data_t key; + st_data_t record; + st_table_entry *next; +}; + +#define ST_DEFAULT_MAX_DENSITY 5 +#define ST_DEFAULT_INIT_TABLE_SIZE 11 + + /* + * DEFAULT_MAX_DENSITY is the default for the largest we allow the + * average number of items per bin before increasing the number of + * bins + * + * DEFAULT_INIT_TABLE_SIZE is the default for the number of bins + * allocated initially + * + */ +static int numcmp(long, long); +static int numhash(long); +static struct st_hash_type type_numhash = { + HASH_FUNCTION_CAST numcmp, + HASH_FUNCTION_CAST numhash +}; + +/* + * extern int strcmp(const char *, const char *); + */ +static int strhash(const char *); +static struct st_hash_type type_strhash = { + HASH_FUNCTION_CAST strcmp, + HASH_FUNCTION_CAST strhash +}; + +static void rehash(st_table *); + +#ifdef RUBY +#define malloc xmalloc +#define calloc xcalloc +#endif + +#define Calloc(n,s) (char*)avro_calloc((n),(s)) + +#define free_bins(tbl) \ + avro_free(tbl->bins, tbl->num_bins * sizeof(st_table_entry *)) + +#define EQUAL(table,x,y) ((x)==(y) || (*table->type->compare)((x),(y)) == 0) + +#define do_hash(key,table) (unsigned int)(*(table)->type->hash)((key)) +#define do_hash_bin(key,table) (do_hash(key, table)%(table)->num_bins) + +/* + * MINSIZE is the minimum size of a dictionary. + */ + +#define MINSIZE 8 + +/* + * Table of prime numbers 2^n+a, 2<=n<=30. + */ +static long primes[] = { + 8 + 3, + 16 + 3, + 32 + 5, + 64 + 3, + 128 + 3, + 256 + 27, + 512 + 9, + 1024 + 9, + 2048 + 5, + 4096 + 3, + 8192 + 27, + 16384 + 43, + 32768 + 3, + 65536 + 45, + 131072 + 29, + 262144 + 3, + 524288 + 21, + 1048576 + 7, + 2097152 + 17, + 4194304 + 15, + 8388608 + 9, + 16777216 + 43, + 33554432 + 35, + 67108864 + 15, + 134217728 + 29, + 268435456 + 3, + 536870912 + 11, + 1073741824 + 85, + 0 +}; + +static int new_size(int size) +{ + unsigned int i; + +#if 0 + for (i = 3; i < 31; i++) { + if ((1 << i) > size) + return 1 << i; + } + return -1; +#else + int newsize; + + for (i = 0, newsize = MINSIZE; + i < sizeof(primes) / sizeof(primes[0]); i++, newsize <<= 1) { + if (newsize > size) + return primes[i]; + } + /* + * Ran out of polynomials + */ + return -1; /* should raise exception */ +#endif +} + +#ifdef HASH_LOG +static int collision = 0; +static int init_st = 0; + +static void stat_col() +{ + FILE *f = fopen("/tmp/col", "w"); + fprintf(f, "collision: %d\n", collision); + fclose(f); +} +#endif + +st_table *st_init_table_with_size(struct st_hash_type *type, int size) +{ + st_table *tbl; + +#ifdef HASH_LOG + if (init_st == 0) { + init_st = 1; + atexit(stat_col); + } +#endif + + size = new_size(size); /* round up to prime number */ + + tbl = (st_table *) avro_new(st_table); + tbl->type = type; + tbl->num_entries = 0; + tbl->num_bins = size; + tbl->bins = (st_table_entry **) Calloc(size, sizeof(st_table_entry *)); + + return tbl; +} + +st_table *st_init_table(struct st_hash_type *type) +{ + return st_init_table_with_size(type, 0); +} + +st_table *st_init_numtable(void) +{ + return st_init_table(&type_numhash); +} + +st_table *st_init_numtable_with_size(int size) +{ + return st_init_table_with_size(&type_numhash, size); +} + +st_table *st_init_strtable(void) +{ + return st_init_table(&type_strhash); +} + +st_table *st_init_strtable_with_size(int size) +{ + return st_init_table_with_size(&type_strhash, size); +} + +void st_free_table(st_table *table) +{ + register st_table_entry *ptr, *next; + int i; + + for (i = 0; i < table->num_bins; i++) { + ptr = table->bins[i]; + while (ptr != 0) { + next = ptr->next; + avro_freet(st_table_entry, ptr); + ptr = next; + } + } + free_bins(table); + avro_freet(st_table, table); +} + +#define PTR_NOT_EQUAL(table, ptr, hash_val, key) \ +((ptr) != 0 && (ptr->hash != (hash_val) || !EQUAL((table), (key), (ptr)->key))) + +#ifdef HASH_LOG +#define COLLISION collision++ +#else +#define COLLISION +#endif + +#define FIND_ENTRY(table, ptr, hash_val, bin_pos) do {\ + bin_pos = hash_val%(table)->num_bins;\ + ptr = (table)->bins[bin_pos];\ + if (PTR_NOT_EQUAL(table, ptr, hash_val, key)) {\ + COLLISION;\ + while (PTR_NOT_EQUAL(table, ptr->next, hash_val, key)) {\ + ptr = ptr->next;\ + }\ + ptr = ptr->next;\ + }\ +} while (0) + +int st_lookup(st_table *table, register st_data_t key, st_data_t *value) +{ + unsigned int hash_val, bin_pos; + register st_table_entry *ptr; + + hash_val = do_hash(key, table); + FIND_ENTRY(table, ptr, hash_val, bin_pos); + + if (ptr == 0) { + return 0; + } else { + if (value != 0) + *value = ptr->record; + return 1; + } +} + +#define ADD_DIRECT(table, key, value, hash_val, bin_pos)\ +do {\ + st_table_entry *entry;\ + if (table->num_entries/(table->num_bins) > ST_DEFAULT_MAX_DENSITY) {\ + rehash(table);\ + bin_pos = hash_val % table->num_bins;\ + }\ + \ + entry = (st_table_entry *) avro_new(st_table_entry);\ + \ + entry->hash = hash_val;\ + entry->key = key;\ + entry->record = value;\ + entry->next = table->bins[bin_pos];\ + table->bins[bin_pos] = entry;\ + table->num_entries++;\ +} while (0) + +int st_insert(register st_table *table, register st_data_t key, st_data_t value) +{ + unsigned int hash_val, bin_pos; + register st_table_entry *ptr; + + hash_val = do_hash(key, table); + FIND_ENTRY(table, ptr, hash_val, bin_pos); + + if (ptr == 0) { + ADD_DIRECT(table, key, value, hash_val, bin_pos); + return 0; + } else { + ptr->record = value; + return 1; + } +} + +void st_add_direct(st_table *table,st_data_t key,st_data_t value) +{ + unsigned int hash_val, bin_pos; + + hash_val = do_hash(key, table); + bin_pos = hash_val % table->num_bins; + ADD_DIRECT(table, key, value, hash_val, bin_pos); +} + +static void rehash(register st_table *table) +{ + register st_table_entry *ptr, *next, **new_bins; + int i, old_num_bins = table->num_bins, new_num_bins; + unsigned int hash_val; + + new_num_bins = new_size(old_num_bins + 1); + new_bins = + (st_table_entry **) Calloc(new_num_bins, sizeof(st_table_entry *)); + + for (i = 0; i < old_num_bins; i++) { + ptr = table->bins[i]; + while (ptr != 0) { + next = ptr->next; + hash_val = ptr->hash % new_num_bins; + ptr->next = new_bins[hash_val]; + new_bins[hash_val] = ptr; + ptr = next; + } + } + free_bins(table); + table->num_bins = new_num_bins; + table->bins = new_bins; +} + +st_table *st_copy(st_table *old_table) +{ + st_table *new_table; + st_table_entry *ptr, *entry; + int i, num_bins = old_table->num_bins; + + new_table = (st_table *) avro_new(st_table); + if (new_table == 0) { + return 0; + } + + *new_table = *old_table; + new_table->bins = (st_table_entry **) + Calloc((unsigned)num_bins, sizeof(st_table_entry *)); + + if (new_table->bins == 0) { + avro_freet(st_table, new_table); + return 0; + } + + for (i = 0; i < num_bins; i++) { + new_table->bins[i] = 0; + ptr = old_table->bins[i]; + while (ptr != 0) { + entry = (st_table_entry *) avro_new(st_table_entry); + if (entry == 0) { + free_bins(new_table); + avro_freet(st_table, new_table); + return 0; + } + *entry = *ptr; + entry->next = new_table->bins[i]; + new_table->bins[i] = entry; + ptr = ptr->next; + } + } + return new_table; +} + +int st_delete(register st_table *table,register st_data_t *key,st_data_t *value) +{ + unsigned int hash_val; + st_table_entry *tmp; + register st_table_entry *ptr; + + hash_val = do_hash_bin(*key, table); + ptr = table->bins[hash_val]; + + if (ptr == 0) { + if (value != 0) + *value = 0; + return 0; + } + + if (EQUAL(table, *key, ptr->key)) { + table->bins[hash_val] = ptr->next; + table->num_entries--; + if (value != 0) + *value = ptr->record; + *key = ptr->key; + avro_freet(st_table_entry, ptr); + return 1; + } + + for (; ptr->next != 0; ptr = ptr->next) { + if (EQUAL(table, ptr->next->key, *key)) { + tmp = ptr->next; + ptr->next = ptr->next->next; + table->num_entries--; + if (value != 0) + *value = tmp->record; + *key = tmp->key; + avro_freet(st_table_entry, tmp); + return 1; + } + } + + return 0; +} + +int st_delete_safe(register st_table *table,register st_data_t *key,st_data_t *value,st_data_t never) +{ + unsigned int hash_val; + register st_table_entry *ptr; + + hash_val = do_hash_bin(*key, table); + ptr = table->bins[hash_val]; + + if (ptr == 0) { + if (value != 0) + *value = 0; + return 0; + } + + for (; ptr != 0; ptr = ptr->next) { + if ((ptr->key != never) && EQUAL(table, ptr->key, *key)) { + table->num_entries--; + *key = ptr->key; + if (value != 0) + *value = ptr->record; + ptr->key = ptr->record = never; + return 1; + } + } + + return 0; +} + +static int delete_never(st_data_t key, st_data_t value, st_data_t never) +{ + AVRO_UNUSED(key); + + if (value == never) + return ST_DELETE; + return ST_CONTINUE; +} + +void st_cleanup_safe(st_table *table,st_data_t never) +{ + int num_entries = table->num_entries; + + st_foreach(table, HASH_FUNCTION_CAST delete_never, never); + table->num_entries = num_entries; +} + +int st_foreach(st_table *table,int (*func) (ANYARGS),st_data_t arg) +{ + st_table_entry *ptr, *last, *tmp; + enum st_retval retval; + int i; + + for (i = 0; i < table->num_bins; i++) { + last = 0; + for (ptr = table->bins[i]; ptr != 0;) { + retval = (enum st_retval) (*func) (ptr->key, ptr->record, arg); + switch (retval) { + case ST_CHECK: /* check if hash is modified during + * iteration */ + tmp = 0; + if (i < table->num_bins) { + for (tmp = table->bins[i]; tmp; + tmp = tmp->next) { + if (tmp == ptr) + break; + } + } + if (!tmp) { + /* + * call func with error notice + */ + return 1; + } + /* + * fall through + */ + case ST_CONTINUE: + last = ptr; + ptr = ptr->next; + break; + case ST_STOP: + return 0; + case ST_DELETE: + tmp = ptr; + if (last == 0) { + table->bins[i] = ptr->next; + } else { + last->next = ptr->next; + } + ptr = ptr->next; + avro_freet(st_table_entry, tmp); + table->num_entries--; + } + } + } + return 0; +} + +static int strhash(register const char *string) +{ + register int c; + +#ifdef HASH_ELFHASH + register unsigned int h = 0, g; + + while ((c = *string++) != '\0') { + h = (h << 4) + c; + if (g = h & 0xF0000000) + h ^= g >> 24; + h &= ~g; + } + return h; +#elif defined(HASH_PERL) + register int val = 0; + + while ((c = *string++) != '\0') { + val += c; + val += (val << 10); + val ^= (val >> 6); + } + val += (val << 3); + val ^= (val >> 11); + + return val + (val << 15); +#else + register int val = 0; + + while ((c = *string++) != '\0') { + val = val * 997 + c; + } + + return val + (val >> 5); +#endif +} + +static int numcmp(long x, long y) +{ + return x != y; +} + +static int numhash(long n) +{ + return n; +} diff --git a/fluent-bit/lib/avro/src/st.h b/fluent-bit/lib/avro/src/st.h new file mode 100644 index 00000000..cf8a2249 --- /dev/null +++ b/fluent-bit/lib/avro/src/st.h @@ -0,0 +1,87 @@ +/* + * This is a public domain general purpose hash table package written by + * Peter Moore @ UCB. + */ + +/* + * @(#) st.h 5.1 89/12/14 + */ + +#ifndef ST_INCLUDED +#define ST_INCLUDED +#ifdef __cplusplus +extern "C" { +#define CLOSE_EXTERN } +#else +#define CLOSE_EXTERN +#endif + +#include <avro/platform.h> /* for uintptr_t */ + +#pragma GCC visibility push(hidden) + +#ifndef ANYARGS + #ifdef __cplusplus + #define ANYARGS ... + #else + #define ANYARGS + #endif +#endif + +#ifdef _WIN32 + #define HASH_FUNCTION_CAST (int (__cdecl *)(ANYARGS)) +#else + #define HASH_FUNCTION_CAST +#endif + +typedef uintptr_t st_data_t; +typedef struct st_table st_table; + +struct st_hash_type { + int (*compare) (ANYARGS); + int (*hash) (ANYARGS); +}; + +struct st_table { + struct st_hash_type *type; + int num_bins; + int num_entries; + struct st_table_entry **bins; +}; + +#define st_is_member(table,key) st_lookup(table,key,(st_data_t *)0) + +enum st_retval { ST_CONTINUE, ST_STOP, ST_DELETE, ST_CHECK }; + +#ifndef _ +# define _(args) args +#endif + +st_table *st_init_table _((struct st_hash_type *)); +st_table *st_init_table_with_size _((struct st_hash_type *, int)); +st_table *st_init_numtable _((void)); +st_table *st_init_numtable_with_size _((int)); +st_table *st_init_strtable _((void)); +st_table *st_init_strtable_with_size _((int)); +int st_delete _((st_table *, st_data_t *, st_data_t *)); +int st_delete_safe _((st_table *, st_data_t *, st_data_t *, st_data_t)); +int st_insert _((st_table *, st_data_t, st_data_t)); +int st_lookup _((st_table *, st_data_t, st_data_t *)); +int st_foreach _((st_table *, int (*)(ANYARGS), st_data_t)); +void st_add_direct _((st_table *, st_data_t, st_data_t)); +void st_free_table _((st_table *)); +void st_cleanup_safe _((st_table *, st_data_t)); +st_table *st_copy _((st_table *)); + +#define ST_NUMCMP ((int (*)()) 0) +#define ST_NUMHASH ((int (*)()) -2) + +#define st_numcmp ST_NUMCMP +#define st_numhash ST_NUMHASH + +int st_strhash(); + +#pragma GCC visibility pop + +CLOSE_EXTERN +#endif /* ST_INCLUDED */ diff --git a/fluent-bit/lib/avro/src/string.c b/fluent-bit/lib/avro/src/string.c new file mode 100644 index 00000000..f5cde949 --- /dev/null +++ b/fluent-bit/lib/avro/src/string.c @@ -0,0 +1,304 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <stddef.h> +#include <stdlib.h> +#include <string.h> + +#include "avro_private.h" +#include "avro/data.h" +#include "avro/allocation.h" +#include "avro/errors.h" + +#ifndef AVRO_STRING_DEBUG +#define AVRO_STRING_DEBUG 0 +#endif + +#if AVRO_STRING_DEBUG +#include <stdio.h> +#define DEBUG(...) \ + do { \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, "\n"); \ + } while (0) +#else +#define DEBUG(...) /* don't print messages */ +#endif + + +/* + * A resizable wrapped buffer implementation. This implementation makes + * actual copies in its copy method; if we wanted a zero-copy solution + * here, then we'd have to keep track of all copies of the buffer, so + * that we can update pointers whenever the buffer is resized (since + * this might change the location of the memory region). + */ + +struct avro_wrapped_resizable { + size_t buf_size; +}; + +#define avro_wrapped_resizable_size(sz) \ + (sizeof(struct avro_wrapped_resizable) + (sz)) + +static void +avro_wrapped_resizable_free(avro_wrapped_buffer_t *self) +{ + DEBUG("--- Freeing resizable <%p:%" PRIsz "> (%p)", self->buf, self->size, self->user_data); + struct avro_wrapped_resizable *resizable = (struct avro_wrapped_resizable *) self->user_data; + avro_free(resizable, avro_wrapped_resizable_size(resizable->buf_size)); +} + +static int +avro_wrapped_resizable_resize(avro_wrapped_buffer_t *self, size_t desired) +{ + struct avro_wrapped_resizable *resizable = (struct avro_wrapped_resizable *) self->user_data; + + /* + * If we've already allocated enough memory for the desired + * size, there's nothing to do. + */ + + if (resizable->buf_size >= desired) { + return 0; + } + + size_t new_buf_size = resizable->buf_size * 2; + if (desired > new_buf_size) { + new_buf_size = desired; + } + + DEBUG("--- Resizing <%p:%" PRIsz "> (%p) -> %" PRIsz, + self->buf, self->buf_size, self->user_data, new_buf_size); + + struct avro_wrapped_resizable *new_resizable = + (struct avro_wrapped_resizable *) avro_realloc(resizable, + avro_wrapped_resizable_size(resizable->buf_size), + avro_wrapped_resizable_size(new_buf_size)); + if (new_resizable == NULL) { + return ENOMEM; + } + DEBUG("--- New buffer <%p:%" PRIsz ">", new_buf, new_buf_size); + + new_resizable->buf_size = new_buf_size; + + char *old_buf = (char *) resizable; + char *new_buf = (char *) new_resizable; + + ptrdiff_t offset = (char *) self->buf - old_buf; + DEBUG("--- Old data pointer is %p", self->buf); + self->buf = new_buf + offset; + self->user_data = new_resizable; + DEBUG("--- New data pointer is %p", self->buf); + return 0; +} + +static int +avro_wrapped_resizable_new(avro_wrapped_buffer_t *dest, size_t buf_size) +{ + size_t allocated_size = avro_wrapped_resizable_size(buf_size); + struct avro_wrapped_resizable *resizable = + (struct avro_wrapped_resizable *) avro_malloc(allocated_size); + if (resizable == NULL) { + return ENOMEM; + } + + resizable->buf_size = buf_size; + + dest->buf = ((char *) resizable) + sizeof(struct avro_wrapped_resizable); + DEBUG("--- Creating resizable <%p:%" PRIsz "> (%p)", dest->buf, buf_size, resizable); + dest->size = buf_size; + dest->user_data = resizable; + dest->free = avro_wrapped_resizable_free; + dest->copy = NULL; + dest->slice = NULL; + return 0; +} + +#define is_resizable(buf) \ + ((buf).free == avro_wrapped_resizable_free) + + + +void +avro_raw_string_init(avro_raw_string_t *str) +{ + memset(str, 0, sizeof(avro_raw_string_t)); +} + + +void +avro_raw_string_clear(avro_raw_string_t *str) +{ + /* + * If the string's buffer is one that we control, then we don't + * free it; that lets us reuse the storage on the next call to + * avro_raw_string_set[_length]. + */ + + if (is_resizable(str->wrapped)) { + DEBUG("--- Clearing resizable buffer"); + str->wrapped.size = 0; + } else { + DEBUG("--- Freeing wrapped buffer"); + avro_wrapped_buffer_free(&str->wrapped); + avro_raw_string_init(str); + } +} + + +void +avro_raw_string_done(avro_raw_string_t *str) +{ + avro_wrapped_buffer_free(&str->wrapped); + avro_raw_string_init(str); +} + + +/** + * Makes sure that the string's buffer is one that we allocated + * ourselves, and that the buffer is big enough to hold a string of the + * given length. + */ + +static int +avro_raw_string_ensure_buf(avro_raw_string_t *str, size_t length) +{ + int rval; + + DEBUG("--- Ensuring resizable buffer of size %" PRIsz, length); + if (is_resizable(str->wrapped)) { + /* + * If we've already got a resizable buffer, just have it + * resize itself. + */ + + return avro_wrapped_resizable_resize(&str->wrapped, length); + } else { + /* + * Stash a copy of the old wrapped buffer, and then + * create a new resizable buffer to store our content + * in. + */ + + avro_wrapped_buffer_t orig = str->wrapped; + check(rval, avro_wrapped_resizable_new(&str->wrapped, length)); + + /* + * If there was any content in the old wrapped buffer, + * copy it into the new resizable one. + */ + + if (orig.size > 0) { + size_t to_copy = + (orig.size < length)? orig.size: length; + memcpy((void *) str->wrapped.buf, orig.buf, to_copy); + } + avro_wrapped_buffer_free(&orig); + + return 0; + } +} + + +void +avro_raw_string_set_length(avro_raw_string_t *str, + const void *src, size_t length) +{ + avro_raw_string_ensure_buf(str, length+1); + memcpy((void *) str->wrapped.buf, src, length); + ((char *) str->wrapped.buf)[length] = '\0'; + str->wrapped.size = length; +} + + +void avro_raw_string_append_length(avro_raw_string_t *str, + const void *src, + size_t length) +{ + if (avro_raw_string_length(str) == 0) { + return avro_raw_string_set_length(str, src, length); + } + + avro_raw_string_ensure_buf(str, str->wrapped.size + length); + memcpy((char *) str->wrapped.buf + str->wrapped.size, src, length); + str->wrapped.size += length; +} + + +void +avro_raw_string_set(avro_raw_string_t *str, const char *src) +{ + size_t length = strlen(src); + avro_raw_string_ensure_buf(str, length+1); + memcpy((void *) str->wrapped.buf, src, length+1); + str->wrapped.size = length+1; +} + + +void +avro_raw_string_append(avro_raw_string_t *str, const char *src) +{ + if (avro_raw_string_length(str) == 0) { + return avro_raw_string_set(str, src); + } + + /* Assume that str->wrapped.size includes a NUL terminator */ + size_t length = strlen(src); + avro_raw_string_ensure_buf(str, str->wrapped.size + length); + memcpy((char *) str->wrapped.buf + str->wrapped.size - 1, src, length+1); + str->wrapped.size += length; +} + + +void +avro_raw_string_give(avro_raw_string_t *str, + avro_wrapped_buffer_t *src) +{ + DEBUG("--- Giving control of <%p:%" PRIsz "> (%p) to string", + src->buf, src->size, src); + avro_wrapped_buffer_free(&str->wrapped); + avro_wrapped_buffer_move(&str->wrapped, src); +} + +int +avro_raw_string_grab(const avro_raw_string_t *str, + avro_wrapped_buffer_t *dest) +{ + return avro_wrapped_buffer_copy(dest, &str->wrapped, 0, str->wrapped.size); +} + + +int +avro_raw_string_equals(const avro_raw_string_t *str1, + const avro_raw_string_t *str2) +{ + if (str1 == str2) { + return 1; + } + + if (!str1 || !str2) { + return 0; + } + + if (str1->wrapped.size != str2->wrapped.size) { + return 0; + } + + return (memcmp(str1->wrapped.buf, str2->wrapped.buf, + str1->wrapped.size) == 0); +} diff --git a/fluent-bit/lib/avro/src/value-hash.c b/fluent-bit/lib/avro/src/value-hash.c new file mode 100644 index 00000000..c717924b --- /dev/null +++ b/fluent-bit/lib/avro/src/value-hash.c @@ -0,0 +1,294 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <avro/platform.h> +#include <stdlib.h> +#include <string.h> + +#include "avro/allocation.h" +#include "avro/data.h" +#include "avro/errors.h" +#include "avro/value.h" +#include "avro_private.h" + +#define check_return(retval, call) \ + do { \ + int rval = call; \ + if (rval != 0) { return (retval); } \ + } while (0) + +/* + * We currently use MurmurHash3 [1], which is public domain, as our hash + * implementation. + * + * [1] https://code.google.com/p/smhasher/ + */ + +/* Our seed is the MurmurHash3 of the string "avro.value" */ +#define SEED 0xaf4c78df + +#define ROTL32(a,b) (((a) << ((b) & 0x1f)) | ((a) >> (32 - ((b) & 0x1f)))) + +static inline uint32_t +fmix(uint32_t h) +{ + h ^= h >> 16; + h *= 0x85ebca6b; + h ^= h >> 13; + h *= 0xc2b2ae35; + h ^= h >> 16; + return h; +} + +static const uint32_t c1 = 0xcc9e2d51; +static const uint32_t c2 = 0x1b873593; + +static inline uint32_t +add_hash(uint32_t start, uint32_t current) +{ + current *= c1; + current = ROTL32(current, 15); + current *= c2; + + start ^= current; + start = ROTL32(start, 13); + start = start * 5 + 0xe6546b64; + + return start; +} + +static inline uint32_t +hash_buffer(uint32_t start, const void *src, size_t len) +{ + const uint8_t *data = (const uint8_t *) src; + const int nblocks = len / 4; + + uint32_t h1 = start; + + //---------- + // body + + const uint32_t *blocks = (const uint32_t *) (data + nblocks*4); + int i; + + for (i = -nblocks; i != 0; i++) { + uint32_t k1 = blocks[i]; + + k1 *= c1; + k1 = ROTL32(k1,15); + k1 *= c2; + + h1 ^= k1; + h1 = ROTL32(h1,13); + h1 = h1*5+0xe6546b64; + } + + //---------- + // tail + + const uint8_t *tail = (const uint8_t *) (data + nblocks*4); + + uint32_t k1 = 0; + + switch (len & 3) + { + case 3: k1 ^= tail[2] << 16; + case 2: k1 ^= tail[1] << 8; + case 1: k1 ^= tail[0]; + k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1; + }; + + //---------- + // finalization + + h1 ^= len; + return h1; +} + +static uint32_t +avro_value_hash_fast(avro_value_t *value, uint32_t start) +{ + avro_type_t type = avro_value_get_type(value); + + switch (type) { + case AVRO_BOOLEAN: + { + int v; + check_return(0, avro_value_get_boolean(value, &v)); + return add_hash(start, v); + } + + case AVRO_BYTES: + { + const void *buf; + size_t size; + check_return(0, avro_value_get_bytes(value, &buf, &size)); + return hash_buffer(start, buf, size); + } + + case AVRO_DOUBLE: + { + union { + double d; + uint32_t u32[2]; + } v; + check_return(0, avro_value_get_double(value, &v.d)); + return add_hash(add_hash(start, v.u32[0]), v.u32[1]); + } + + case AVRO_FLOAT: + { + union { + float f; + uint32_t u32; + } v; + check_return(0, avro_value_get_float(value, &v.f)); + return add_hash(start, v.u32); + } + + case AVRO_INT32: + { + int32_t v; + check_return(0, avro_value_get_int(value, &v)); + return add_hash(start, v); + } + + case AVRO_INT64: + { + union { + int64_t u64; + uint32_t u32[2]; + } v; + check_return(0, avro_value_get_long(value, &v.u64)); + return add_hash(add_hash(start, v.u32[0]), v.u32[1]); + } + + case AVRO_NULL: + { + check_return(0, avro_value_get_null(value)); + return add_hash(start, 0); + } + + case AVRO_STRING: + { + const char *buf; + size_t size; + check_return(0, avro_value_get_string(value, &buf, &size)); + return hash_buffer(start, buf, size); + } + + case AVRO_ARRAY: + { + size_t count; + size_t i; + check_return(0, avro_value_get_size(value, &count)); + + for (i = 0; i < count; i++) { + avro_value_t child; + check_return(0, avro_value_get_by_index + (value, i, &child, NULL)); + start = avro_value_hash_fast(&child, start); + } + + start ^= count; + return start; + } + + case AVRO_ENUM: + { + int v; + check_return(0, avro_value_get_enum(value, &v)); + return add_hash(start, v); + } + + case AVRO_FIXED: + { + const void *buf; + size_t size; + check_return(0, avro_value_get_fixed(value, &buf, &size)); + return hash_buffer(start, buf, size); + } + + case AVRO_MAP: + { + size_t count; + size_t i; + check_return(0, avro_value_get_size(value, &count)); + + /* + * The hash for a map must be built up without + * taking into account the order of the elements + */ + uint32_t map_hash = 0; + for (i = 0; i < count; i++) { + avro_value_t child; + const char *key; + check_return(0, avro_value_get_by_index + (value, i, &child, &key)); + + uint32_t element = SEED; + element = hash_buffer(element, key, strlen(key)); + element = avro_value_hash_fast(&child, element); + element = fmix(element); + + map_hash ^= element; + } + map_hash ^= count; + + return add_hash(start, map_hash); + } + + case AVRO_RECORD: + { + size_t count; + size_t i; + check_return(0, avro_value_get_size(value, &count)); + + for (i = 0; i < count; i++) { + avro_value_t child; + check_return(0, avro_value_get_by_index + (value, i, &child, NULL)); + start = avro_value_hash_fast(&child, start); + } + + start ^= count; + return start; + } + + case AVRO_UNION: + { + int disc; + avro_value_t branch; + check_return(0, avro_value_get_discriminant(value, &disc)); + check_return(0, avro_value_get_current_branch(value, &branch)); + + start = add_hash(start, disc); + start = avro_value_hash_fast(&branch, start); + return start; + } + + default: + return 0; + } +} + +uint32_t +avro_value_hash(avro_value_t *value) +{ + uint32_t hash = avro_value_hash_fast(value, SEED); + return (hash == 0)? hash: fmix(hash); +} diff --git a/fluent-bit/lib/avro/src/value-json.c b/fluent-bit/lib/avro/src/value-json.c new file mode 100644 index 00000000..53c2b3d3 --- /dev/null +++ b/fluent-bit/lib/avro/src/value-json.c @@ -0,0 +1,417 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <errno.h> +#include <avro/platform.h> +#include <stdlib.h> +#include <string.h> + +#include "avro/allocation.h" +#include "avro/errors.h" +#include "avro/legacy.h" +#include "avro/schema.h" +#include "avro/value.h" +#include "avro_private.h" +#include "jansson.h" + +/* + * Converts a binary buffer into a NUL-terminated JSON UTF-8 string. + * Avro bytes and fixed values are encoded in JSON as a string, and JSON + * strings must be in UTF-8. For these Avro types, the JSON string is + * restricted to the characters U+0000..U+00FF, which corresponds to the + * ISO-8859-1 character set. This function performs this conversion. + * The resulting string must be freed using avro_free when you're done + * with it. + */ + +static int +encode_utf8_bytes(const void *src, size_t src_len, + void **dest, size_t *dest_len) +{ + check_param(EINVAL, src, "source"); + check_param(EINVAL, dest, "dest"); + check_param(EINVAL, dest_len, "dest_len"); + + // First, determine the size of the resulting UTF-8 buffer. + // Bytes in the range 0x00..0x7f will take up one byte; bytes in + // the range 0x80..0xff will take up two. + const uint8_t *src8 = (const uint8_t *) src; + + size_t utf8_len = src_len + 1; // +1 for NUL terminator + size_t i; + for (i = 0; i < src_len; i++) { + if (src8[i] & 0x80) { + utf8_len++; + } + } + + // Allocate a new buffer for the UTF-8 string and fill it in. + uint8_t *dest8 = (uint8_t *) avro_malloc(utf8_len); + if (dest8 == NULL) { + avro_set_error("Cannot allocate JSON bytes buffer"); + return ENOMEM; + } + + uint8_t *curr = dest8; + for (i = 0; i < src_len; i++) { + if (src8[i] & 0x80) { + *curr++ = (0xc0 | (src8[i] >> 6)); + *curr++ = (0x80 | (src8[i] & 0x3f)); + } else { + *curr++ = src8[i]; + } + } + + *curr = '\0'; + + // And we're good. + *dest = dest8; + *dest_len = utf8_len; + return 0; +} + +#define return_json(type, exp) \ + { \ + json_t *result = exp; \ + if (result == NULL) { \ + avro_set_error("Cannot allocate JSON " type); \ + } \ + return result; \ + } + +#define check_return(retval, call) \ + do { \ + int __rc; \ + __rc = call; \ + if (__rc != 0) { \ + return retval; \ + } \ + } while (0) + +static json_t * +avro_value_to_json_t(const avro_value_t *value) +{ + switch (avro_value_get_type(value)) { + case AVRO_BOOLEAN: + { + int val; + check_return(NULL, avro_value_get_boolean(value, &val)); + return_json("boolean", + val? json_true(): json_false()); + } + + case AVRO_BYTES: + { + const void *val; + size_t size; + void *encoded = NULL; + size_t encoded_size = 0; + + check_return(NULL, avro_value_get_bytes(value, &val, &size)); + + if (encode_utf8_bytes(val, size, &encoded, &encoded_size)) { + return NULL; + } + + json_t *result = json_string_nocheck((const char *) encoded); + avro_free(encoded, encoded_size); + if (result == NULL) { + avro_set_error("Cannot allocate JSON bytes"); + } + return result; + } + + case AVRO_DOUBLE: + { + double val; + check_return(NULL, avro_value_get_double(value, &val)); + return_json("double", json_real(val)); + } + + case AVRO_FLOAT: + { + float val; + check_return(NULL, avro_value_get_float(value, &val)); + return_json("float", json_real(val)); + } + + case AVRO_INT32: + { + int32_t val; + check_return(NULL, avro_value_get_int(value, &val)); + return_json("int", json_integer(val)); + } + + case AVRO_INT64: + { + int64_t val; + check_return(NULL, avro_value_get_long(value, &val)); + return_json("long", json_integer(val)); + } + + case AVRO_NULL: + { + check_return(NULL, avro_value_get_null(value)); + return_json("null", json_null()); + } + + case AVRO_STRING: + { + const char *val; + size_t size; + check_return(NULL, avro_value_get_string(value, &val, &size)); + return_json("string", json_string(val)); + } + + case AVRO_ARRAY: + { + int rc; + size_t element_count, i; + json_t *result = json_array(); + if (result == NULL) { + avro_set_error("Cannot allocate JSON array"); + return NULL; + } + + rc = avro_value_get_size(value, &element_count); + if (rc != 0) { + json_decref(result); + return NULL; + } + + for (i = 0; i < element_count; i++) { + avro_value_t element; + rc = avro_value_get_by_index(value, i, &element, NULL); + if (rc != 0) { + json_decref(result); + return NULL; + } + + json_t *element_json = avro_value_to_json_t(&element); + if (element_json == NULL) { + json_decref(result); + return NULL; + } + + if (json_array_append_new(result, element_json)) { + avro_set_error("Cannot append element to array"); + json_decref(result); + return NULL; + } + } + + return result; + } + + case AVRO_ENUM: + { + avro_schema_t enum_schema; + int symbol_value; + const char *symbol_name; + + check_return(NULL, avro_value_get_enum(value, &symbol_value)); + enum_schema = avro_value_get_schema(value); + symbol_name = avro_schema_enum_get(enum_schema, symbol_value); + return_json("enum", json_string(symbol_name)); + } + + case AVRO_FIXED: + { + const void *val; + size_t size; + void *encoded = NULL; + size_t encoded_size = 0; + + check_return(NULL, avro_value_get_fixed(value, &val, &size)); + + if (encode_utf8_bytes(val, size, &encoded, &encoded_size)) { + return NULL; + } + + json_t *result = json_string_nocheck((const char *) encoded); + avro_free(encoded, encoded_size); + if (result == NULL) { + avro_set_error("Cannot allocate JSON fixed"); + } + return result; + } + + case AVRO_MAP: + { + int rc; + size_t element_count, i; + json_t *result = json_object(); + if (result == NULL) { + avro_set_error("Cannot allocate JSON map"); + return NULL; + } + + rc = avro_value_get_size(value, &element_count); + if (rc != 0) { + json_decref(result); + return NULL; + } + + for (i = 0; i < element_count; i++) { + const char *key; + avro_value_t element; + + rc = avro_value_get_by_index(value, i, &element, &key); + if (rc != 0) { + json_decref(result); + return NULL; + } + + json_t *element_json = avro_value_to_json_t(&element); + if (element_json == NULL) { + json_decref(result); + return NULL; + } + + if (json_object_set_new(result, key, element_json)) { + avro_set_error("Cannot append element to map"); + json_decref(result); + return NULL; + } + } + + return result; + } + + case AVRO_RECORD: + { + int rc; + size_t field_count, i; + json_t *result = json_object(); + if (result == NULL) { + avro_set_error("Cannot allocate new JSON record"); + return NULL; + } + + rc = avro_value_get_size(value, &field_count); + if (rc != 0) { + json_decref(result); + return NULL; + } + + for (i = 0; i < field_count; i++) { + const char *field_name; + avro_value_t field; + + rc = avro_value_get_by_index(value, i, &field, &field_name); + if (rc != 0) { + json_decref(result); + return NULL; + } + + json_t *field_json = avro_value_to_json_t(&field); + if (field_json == NULL) { + json_decref(result); + return NULL; + } + + if (json_object_set_new(result, field_name, field_json)) { + avro_set_error("Cannot append field to record"); + json_decref(result); + return NULL; + } + } + + return result; + } + + case AVRO_UNION: + { + int disc; + avro_value_t branch; + avro_schema_t union_schema; + avro_schema_t branch_schema; + const char *branch_name; + + check_return(NULL, avro_value_get_current_branch(value, &branch)); + + if (avro_value_get_type(&branch) == AVRO_NULL) { + return_json("null", json_null()); + } + + check_return(NULL, avro_value_get_discriminant(value, &disc)); + union_schema = avro_value_get_schema(value); + branch_schema = + avro_schema_union_branch(union_schema, disc); + branch_name = avro_schema_type_name(branch_schema); + + json_t *result = json_object(); + if (result == NULL) { + avro_set_error("Cannot allocate JSON union"); + return NULL; + } + + json_t *branch_json = avro_value_to_json_t(&branch); + if (branch_json == NULL) { + json_decref(result); + return NULL; + } + + if (json_object_set_new(result, branch_name, branch_json)) { + avro_set_error("Cannot append branch to union"); + json_decref(result); + return NULL; + } + + return result; + } + + default: + return NULL; + } +} + +int +avro_value_to_json(const avro_value_t *value, + int one_line, char **json_str) +{ + check_param(EINVAL, value, "value"); + check_param(EINVAL, json_str, "string buffer"); + + json_t *json = avro_value_to_json_t(value); + if (json == NULL) { + return ENOMEM; + } + + /* + * Jansson will only encode an object or array as the root + * element. + */ + + *json_str = json_dumps + (json, + JSON_ENCODE_ANY | + JSON_INDENT(one_line? 0: 2) | + JSON_ENSURE_ASCII | + JSON_PRESERVE_ORDER); + json_decref(json); + return 0; +} + +int +avro_datum_to_json(const avro_datum_t datum, + int one_line, char **json_str) +{ + avro_value_t value; + avro_datum_as_value(&value, datum); + return avro_value_to_json(&value, one_line, json_str); +} diff --git a/fluent-bit/lib/avro/src/value-read.c b/fluent-bit/lib/avro/src/value-read.c new file mode 100644 index 00000000..b6b6e79f --- /dev/null +++ b/fluent-bit/lib/avro/src/value-read.c @@ -0,0 +1,392 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <avro/platform.h> +#include <stdlib.h> +#include <string.h> + +#include "avro/allocation.h" +#include "avro/basics.h" +#include "avro/data.h" +#include "avro/io.h" +#include "avro/value.h" +#include "avro_private.h" +#include "encoding.h" + + +/* + * Forward declaration; this is basically the same as avro_value_read, + * but it doesn't reset dest first. (Since it will have already been + * reset in avro_value_read itself). + */ + +static int +read_value(avro_reader_t reader, avro_value_t *dest); + + +static int +read_array_value(avro_reader_t reader, avro_value_t *dest) +{ + int rval; + size_t i; /* index within the current block */ + size_t index = 0; /* index within the entire array */ + int64_t block_count; + int64_t block_size; + + check_prefix(rval, avro_binary_encoding. + read_long(reader, &block_count), + "Cannot read array block count: "); + + while (block_count != 0) { + if (block_count < 0) { + block_count = block_count * -1; + check_prefix(rval, avro_binary_encoding. + read_long(reader, &block_size), + "Cannot read array block size: "); + } + + for (i = 0; i < (size_t) block_count; i++, index++) { + avro_value_t child; + + check(rval, avro_value_append(dest, &child, NULL)); + check(rval, read_value(reader, &child)); + } + + check_prefix(rval, avro_binary_encoding. + read_long(reader, &block_count), + "Cannot read array block count: "); + } + + return 0; +} + + +static int +read_map_value(avro_reader_t reader, avro_value_t *dest) +{ + int rval; + size_t i; /* index within the current block */ + size_t index = 0; /* index within the entire array */ + int64_t block_count; + int64_t block_size; + + check_prefix(rval, avro_binary_encoding.read_long(reader, &block_count), + "Cannot read map block count: "); + + while (block_count != 0) { + if (block_count < 0) { + block_count = block_count * -1; + check_prefix(rval, avro_binary_encoding. + read_long(reader, &block_size), + "Cannot read map block size: "); + } + + for (i = 0; i < (size_t) block_count; i++, index++) { + char *key; + int64_t key_size; + avro_value_t child; + + check_prefix(rval, avro_binary_encoding. + read_string(reader, &key, &key_size), + "Cannot read map key: "); + + rval = avro_value_add(dest, key, &child, NULL, NULL); + if (rval) { + avro_free(key, key_size); + return rval; + } + + rval = read_value(reader, &child); + if (rval) { + avro_free(key, key_size); + return rval; + } + + avro_free(key, key_size); + } + + check_prefix(rval, avro_binary_encoding. + read_long(reader, &block_count), + "Cannot read map block count: "); + } + + return 0; +} + + +static int +read_record_value(avro_reader_t reader, avro_value_t *dest) +{ + int rval; + size_t field_count; + size_t i; + + avro_schema_t record_schema = avro_value_get_schema(dest); + + check(rval, avro_value_get_size(dest, &field_count)); + for (i = 0; i < field_count; i++) { + avro_value_t field; + + check(rval, avro_value_get_by_index(dest, i, &field, NULL)); + if (field.iface != NULL) { + check(rval, read_value(reader, &field)); + } else { + avro_schema_t field_schema = + avro_schema_record_field_get_by_index(record_schema, i); + check(rval, avro_skip_data(reader, field_schema)); + } + } + + return 0; +} + + +static int +read_union_value(avro_reader_t reader, avro_value_t *dest) +{ + int rval; + int64_t discriminant; + avro_schema_t union_schema; + int64_t branch_count; + avro_value_t branch; + + check_prefix(rval, avro_binary_encoding. + read_long(reader, &discriminant), + "Cannot read union discriminant: "); + + union_schema = avro_value_get_schema(dest); + branch_count = avro_schema_union_size(union_schema); + + if (discriminant < 0 || discriminant >= branch_count) { + avro_set_error("Invalid union discriminant value: (%d)", + discriminant); + return 1; + } + + check(rval, avro_value_set_branch(dest, discriminant, &branch)); + check(rval, read_value(reader, &branch)); + return 0; +} + + +/* + * A wrapped buffer implementation that takes control of a buffer + * allocated using avro_malloc. + */ + +struct avro_wrapped_alloc { + const void *original; + size_t allocated_size; +}; + +static void +avro_wrapped_alloc_free(avro_wrapped_buffer_t *self) +{ + struct avro_wrapped_alloc *alloc = (struct avro_wrapped_alloc *) self->user_data; + avro_free((void *) alloc->original, alloc->allocated_size); + avro_freet(struct avro_wrapped_alloc, alloc); +} + +static int +avro_wrapped_alloc_new(avro_wrapped_buffer_t *dest, + const void *buf, size_t length) +{ + struct avro_wrapped_alloc *alloc = (struct avro_wrapped_alloc *) avro_new(struct avro_wrapped_alloc); + if (alloc == NULL) { + return ENOMEM; + } + + dest->buf = buf; + dest->size = length; + dest->user_data = alloc; + dest->free = avro_wrapped_alloc_free; + dest->copy = NULL; + dest->slice = NULL; + + alloc->original = buf; + alloc->allocated_size = length; + return 0; +} + + +static int +read_value(avro_reader_t reader, avro_value_t *dest) +{ + int rval; + + switch (avro_value_get_type(dest)) { + case AVRO_BOOLEAN: + { + int8_t val; + check_prefix(rval, avro_binary_encoding. + read_boolean(reader, &val), + "Cannot read boolean value: "); + return avro_value_set_boolean(dest, val); + } + + case AVRO_BYTES: + { + char *bytes; + int64_t len; + check_prefix(rval, avro_binary_encoding. + read_bytes(reader, &bytes, &len), + "Cannot read bytes value: "); + + /* + * read_bytes allocates an extra byte to always + * ensure that the data is NUL terminated, but + * that byte isn't included in the length. We + * include that extra byte in the allocated + * size, but not in the length of the buffer. + */ + + avro_wrapped_buffer_t buf; + check(rval, avro_wrapped_alloc_new(&buf, bytes, len+1)); + buf.size--; + return avro_value_give_bytes(dest, &buf); + } + + case AVRO_DOUBLE: + { + double val; + check_prefix(rval, avro_binary_encoding. + read_double(reader, &val), + "Cannot read double value: "); + return avro_value_set_double(dest, val); + } + + case AVRO_FLOAT: + { + float val; + check_prefix(rval, avro_binary_encoding. + read_float(reader, &val), + "Cannot read float value: "); + return avro_value_set_float(dest, val); + } + + case AVRO_INT32: + { + int32_t val; + check_prefix(rval, avro_binary_encoding. + read_int(reader, &val), + "Cannot read int value: "); + return avro_value_set_int(dest, val); + } + + case AVRO_INT64: + { + int64_t val; + check_prefix(rval, avro_binary_encoding. + read_long(reader, &val), + "Cannot read long value: "); + return avro_value_set_long(dest, val); + } + + case AVRO_NULL: + { + check_prefix(rval, avro_binary_encoding. + read_null(reader), + "Cannot read null value: "); + return avro_value_set_null(dest); + } + + case AVRO_STRING: + { + char *str; + int64_t size; + + /* + * read_string returns a size that includes the + * NUL terminator, and the free function will be + * called with a size that also includes the NUL + */ + + check_prefix(rval, avro_binary_encoding. + read_string(reader, &str, &size), + "Cannot read string value: "); + + avro_wrapped_buffer_t buf; + check(rval, avro_wrapped_alloc_new(&buf, str, size)); + return avro_value_give_string_len(dest, &buf); + } + + case AVRO_ARRAY: + return read_array_value(reader, dest); + + case AVRO_ENUM: + { + int64_t val; + check_prefix(rval, avro_binary_encoding. + read_long(reader, &val), + "Cannot read enum value: "); + return avro_value_set_enum(dest, val); + } + + case AVRO_FIXED: + { + avro_schema_t schema = avro_value_get_schema(dest); + char *bytes; + int64_t size = avro_schema_fixed_size(schema); + + bytes = (char *) avro_malloc(size); + if (!bytes) { + avro_prefix_error("Cannot allocate new fixed value"); + return ENOMEM; + } + rval = avro_read(reader, bytes, size); + if (rval) { + avro_prefix_error("Cannot read fixed value: "); + avro_free(bytes, size); + return rval; + } + + avro_wrapped_buffer_t buf; + rval = avro_wrapped_alloc_new(&buf, bytes, size); + if (rval != 0) { + avro_free(bytes, size); + return rval; + } + + return avro_value_give_fixed(dest, &buf); + } + + case AVRO_MAP: + return read_map_value(reader, dest); + + case AVRO_RECORD: + return read_record_value(reader, dest); + + case AVRO_UNION: + return read_union_value(reader, dest); + + default: + { + avro_set_error("Unknown schema type"); + return EINVAL; + } + } + + return 0; +} + +int +avro_value_read(avro_reader_t reader, avro_value_t *dest) +{ + int rval; + check(rval, avro_value_reset(dest)); + return read_value(reader, dest); +} diff --git a/fluent-bit/lib/avro/src/value-sizeof.c b/fluent-bit/lib/avro/src/value-sizeof.c new file mode 100644 index 00000000..bcbffb5b --- /dev/null +++ b/fluent-bit/lib/avro/src/value-sizeof.c @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <avro/platform.h> +#include <stdlib.h> + +#include "avro/basics.h" +#include "avro/io.h" +#include "avro/value.h" +#include "avro_private.h" +#include "encoding.h" + + +/* + * Forward declaration; this is basically the same as avro_value_sizeof, + * but it doesn't initialize size first. (Since it will have already + * been initialized in avro_value_sizeof itself). + */ + +static int +sizeof_value(avro_value_t *src, size_t *size); + + +static int +sizeof_array_value(avro_value_t *src, size_t *size) +{ + int rval; + size_t element_count; + check(rval, avro_value_get_size(src, &element_count)); + + if (element_count > 0) { + *size += avro_binary_encoding.size_long(NULL, element_count); + + size_t i; + for (i = 0; i < element_count; i++) { + avro_value_t child; + check(rval, avro_value_get_by_index(src, i, &child, NULL)); + check(rval, sizeof_value(&child, size)); + } + } + + *size += avro_binary_encoding.size_long(NULL, 0); + return 0; +} + + +static int +sizeof_map_value(avro_value_t *src, size_t *size) +{ + int rval; + size_t element_count; + check(rval, avro_value_get_size(src, &element_count)); + + if (element_count > 0) { + *size += avro_binary_encoding.size_long(NULL, element_count); + + size_t i; + for (i = 0; i < element_count; i++) { + avro_value_t child; + const char *key; + check(rval, avro_value_get_by_index(src, i, &child, &key)); + *size += avro_binary_encoding.size_string(NULL, key); + check(rval, sizeof_value(&child, size)); + } + } + + *size += avro_binary_encoding.size_long(NULL, 0); + return 0; +} + +static int +sizeof_record_value(avro_value_t *src, size_t *size) +{ + int rval; + size_t field_count; + check(rval, avro_value_get_size(src, &field_count)); + + size_t i; + for (i = 0; i < field_count; i++) { + avro_value_t field; + check(rval, avro_value_get_by_index(src, i, &field, NULL)); + check(rval, sizeof_value(&field, size)); + } + + return 0; +} + +static int +sizeof_union_value(avro_value_t *src, size_t *size) +{ + int rval; + int discriminant; + avro_value_t branch; + + check(rval, avro_value_get_discriminant(src, &discriminant)); + check(rval, avro_value_get_current_branch(src, &branch)); + *size += avro_binary_encoding.size_long(NULL, discriminant); + return sizeof_value(&branch, size); +} + +static int +sizeof_value(avro_value_t *src, size_t *size) +{ + int rval; + + switch (avro_value_get_type(src)) { + case AVRO_BOOLEAN: + { + int val; + check(rval, avro_value_get_boolean(src, &val)); + *size += avro_binary_encoding.size_boolean(NULL, val); + return 0; + } + + case AVRO_BYTES: + { + const void *buf; + size_t sz; + check(rval, avro_value_get_bytes(src, &buf, &sz)); + *size += avro_binary_encoding.size_bytes(NULL, (const char *) buf, sz); + return 0; + } + + case AVRO_DOUBLE: + { + double val; + check(rval, avro_value_get_double(src, &val)); + *size += avro_binary_encoding.size_double(NULL, val); + return 0; + } + + case AVRO_FLOAT: + { + float val; + check(rval, avro_value_get_float(src, &val)); + *size += avro_binary_encoding.size_float(NULL, val); + return 0; + } + + case AVRO_INT32: + { + int32_t val; + check(rval, avro_value_get_int(src, &val)); + *size += avro_binary_encoding.size_long(NULL, val); + return 0; + } + + case AVRO_INT64: + { + int64_t val; + check(rval, avro_value_get_long(src, &val)); + *size += avro_binary_encoding.size_long(NULL, val); + return 0; + } + + case AVRO_NULL: + { + check(rval, avro_value_get_null(src)); + *size += avro_binary_encoding.size_null(NULL); + return 0; + } + + case AVRO_STRING: + { + const char *str; + size_t sz; + check(rval, avro_value_get_string(src, &str, &sz)); + *size += avro_binary_encoding.size_bytes(NULL, str, sz-1); + return 0; + } + + case AVRO_ARRAY: + return sizeof_array_value(src, size); + + case AVRO_ENUM: + { + int val; + check(rval, avro_value_get_enum(src, &val)); + *size += avro_binary_encoding.size_long(NULL, val); + return 0; + } + + case AVRO_FIXED: + { + size_t sz; + check(rval, avro_value_get_fixed(src, NULL, &sz)); + *size += sz; + return 0; + } + + case AVRO_MAP: + return sizeof_map_value(src, size); + + case AVRO_RECORD: + return sizeof_record_value(src, size); + + case AVRO_UNION: + return sizeof_union_value(src, size); + + default: + { + avro_set_error("Unknown schema type"); + return EINVAL; + } + } + + return 0; +} + +int +avro_value_sizeof(avro_value_t *src, size_t *size) +{ + check_param(EINVAL, size, "size pointer"); + *size = 0; + return sizeof_value(src, size); +} diff --git a/fluent-bit/lib/avro/src/value-write.c b/fluent-bit/lib/avro/src/value-write.c new file mode 100644 index 00000000..bcd0fb0a --- /dev/null +++ b/fluent-bit/lib/avro/src/value-write.c @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <avro/platform.h> +#include <stdlib.h> + +#include "avro/basics.h" +#include "avro/io.h" +#include "avro/value.h" +#include "avro_private.h" +#include "encoding.h" + + +static int +write_array_value(avro_writer_t writer, avro_value_t *src) +{ + int rval; + size_t element_count; + check(rval, avro_value_get_size(src, &element_count)); + + if (element_count > 0) { + check_prefix(rval, avro_binary_encoding.write_long + (writer, element_count), + "Cannot write array block count: "); + + size_t i; + for (i = 0; i < element_count; i++) { + avro_value_t child; + check(rval, avro_value_get_by_index(src, i, &child, NULL)); + check(rval, avro_value_write(writer, &child)); + } + } + + check_prefix(rval, avro_binary_encoding.write_long(writer, 0), + "Cannot write array block count: "); + return 0; +} + + +static int +write_map_value(avro_writer_t writer, avro_value_t *src) +{ + int rval; + size_t element_count; + check(rval, avro_value_get_size(src, &element_count)); + + if (element_count > 0) { + check_prefix(rval, avro_binary_encoding.write_long + (writer, element_count), + "Cannot write map block count: "); + + size_t i; + for (i = 0; i < element_count; i++) { + avro_value_t child; + const char *key; + check(rval, avro_value_get_by_index(src, i, &child, &key)); + check(rval, avro_binary_encoding.write_string(writer, key)); + check(rval, avro_value_write(writer, &child)); + } + } + + check_prefix(rval, avro_binary_encoding.write_long(writer, 0), + "Cannot write map block count: "); + return 0; +} + +static int +write_record_value(avro_writer_t writer, avro_value_t *src) +{ + int rval; + size_t field_count; + check(rval, avro_value_get_size(src, &field_count)); + + size_t i; + for (i = 0; i < field_count; i++) { + avro_value_t field; + check(rval, avro_value_get_by_index(src, i, &field, NULL)); + check(rval, avro_value_write(writer, &field)); + } + + return 0; +} + +static int +write_union_value(avro_writer_t writer, avro_value_t *src) +{ + int rval; + int discriminant; + avro_value_t branch; + + check(rval, avro_value_get_discriminant(src, &discriminant)); + check(rval, avro_value_get_current_branch(src, &branch)); + check(rval, avro_binary_encoding.write_long(writer, discriminant)); + return avro_value_write(writer, &branch); +} + +int +avro_value_write(avro_writer_t writer, avro_value_t *src) +{ + int rval; + + switch (avro_value_get_type(src)) { + case AVRO_BOOLEAN: + { + int val; + check(rval, avro_value_get_boolean(src, &val)); + return avro_binary_encoding.write_boolean(writer, val); + } + + case AVRO_BYTES: + { + const void *buf; + size_t size; + check(rval, avro_value_get_bytes(src, &buf, &size)); + return avro_binary_encoding.write_bytes(writer, (const char *) buf, size); + } + + case AVRO_DOUBLE: + { + double val; + check(rval, avro_value_get_double(src, &val)); + return avro_binary_encoding.write_double(writer, val); + } + + case AVRO_FLOAT: + { + float val; + check(rval, avro_value_get_float(src, &val)); + return avro_binary_encoding.write_float(writer, val); + } + + case AVRO_INT32: + { + int32_t val; + check(rval, avro_value_get_int(src, &val)); + return avro_binary_encoding.write_long(writer, val); + } + + case AVRO_INT64: + { + int64_t val; + check(rval, avro_value_get_long(src, &val)); + return avro_binary_encoding.write_long(writer, val); + } + + case AVRO_NULL: + { + check(rval, avro_value_get_null(src)); + return avro_binary_encoding.write_null(writer); + } + + case AVRO_STRING: + { + const char *str; + size_t size; + check(rval, avro_value_get_string(src, &str, &size)); + return avro_binary_encoding.write_bytes(writer, str, size-1); + } + + case AVRO_ARRAY: + return write_array_value(writer, src); + + case AVRO_ENUM: + { + int val; + check(rval, avro_value_get_enum(src, &val)); + return avro_binary_encoding.write_long(writer, val); + } + + case AVRO_FIXED: + { + const void *buf; + size_t size; + check(rval, avro_value_get_fixed(src, &buf, &size)); + return avro_write(writer, (void *) buf, size); + } + + case AVRO_MAP: + return write_map_value(writer, src); + + case AVRO_RECORD: + return write_record_value(writer, src); + + case AVRO_UNION: + return write_union_value(writer, src); + + default: + { + avro_set_error("Unknown schema type"); + return EINVAL; + } + } + + return 0; +} diff --git a/fluent-bit/lib/avro/src/value.c b/fluent-bit/lib/avro/src/value.c new file mode 100644 index 00000000..b177504e --- /dev/null +++ b/fluent-bit/lib/avro/src/value.c @@ -0,0 +1,690 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <errno.h> +#include <stdlib.h> +#include <string.h> + +#include "avro/allocation.h" +#include "avro/data.h" +#include "avro/errors.h" +#include "avro/value.h" +#include "avro_private.h" + + +#define check_return(retval, call) \ + do { \ + int rval = call; \ + if (rval != 0) { return (retval); } \ + } while (0) + + +void +avro_value_incref(avro_value_t *value) +{ + value->iface->incref(value); +} + +void +avro_value_decref(avro_value_t *value) +{ + value->iface->decref(value); + avro_value_iface_decref(value->iface); + value->iface = NULL; + value->self = NULL; +} + +void +avro_value_copy_ref(avro_value_t *dest, const avro_value_t *src) +{ + dest->iface = src->iface; + dest->self = src->self; + avro_value_iface_incref(dest->iface); + dest->iface->incref(dest); +} + +void +avro_value_move_ref(avro_value_t *dest, avro_value_t *src) +{ + dest->iface = src->iface; + dest->self = src->self; + src->iface = NULL; + src->self = NULL; +} + + +int +avro_value_equal_fast(avro_value_t *val1, avro_value_t *val2) +{ + avro_type_t type1 = avro_value_get_type(val1); + avro_type_t type2 = avro_value_get_type(val2); + if (type1 != type2) { + return 0; + } + + switch (type1) { + case AVRO_BOOLEAN: + { + int v1; + int v2; + check_return(0, avro_value_get_boolean(val1, &v1)); + check_return(0, avro_value_get_boolean(val2, &v2)); + return (v1 == v2); + } + + case AVRO_BYTES: + { + const void *buf1; + const void *buf2; + size_t size1; + size_t size2; + check_return(0, avro_value_get_bytes(val1, &buf1, &size1)); + check_return(0, avro_value_get_bytes(val2, &buf2, &size2)); + if (size1 != size2) { + return 0; + } + return (memcmp(buf1, buf2, size1) == 0); + } + + case AVRO_DOUBLE: + { + double v1; + double v2; + check_return(0, avro_value_get_double(val1, &v1)); + check_return(0, avro_value_get_double(val2, &v2)); + return (v1 == v2); + } + + case AVRO_FLOAT: + { + float v1; + float v2; + check_return(0, avro_value_get_float(val1, &v1)); + check_return(0, avro_value_get_float(val2, &v2)); + return (v1 == v2); + } + + case AVRO_INT32: + { + int32_t v1; + int32_t v2; + check_return(0, avro_value_get_int(val1, &v1)); + check_return(0, avro_value_get_int(val2, &v2)); + return (v1 == v2); + } + + case AVRO_INT64: + { + int64_t v1; + int64_t v2; + check_return(0, avro_value_get_long(val1, &v1)); + check_return(0, avro_value_get_long(val2, &v2)); + return (v1 == v2); + } + + case AVRO_NULL: + { + check_return(0, avro_value_get_null(val1)); + check_return(0, avro_value_get_null(val2)); + return 1; + } + + case AVRO_STRING: + { + const char *buf1; + const char *buf2; + size_t size1; + size_t size2; + check_return(0, avro_value_get_string(val1, &buf1, &size1)); + check_return(0, avro_value_get_string(val2, &buf2, &size2)); + if (size1 != size2) { + return 0; + } + return (memcmp(buf1, buf2, size1) == 0); + } + + case AVRO_ARRAY: + { + size_t count1; + size_t count2; + check_return(0, avro_value_get_size(val1, &count1)); + check_return(0, avro_value_get_size(val2, &count2)); + if (count1 != count2) { + return 0; + } + + size_t i; + for (i = 0; i < count1; i++) { + avro_value_t child1; + avro_value_t child2; + check_return(0, avro_value_get_by_index + (val1, i, &child1, NULL)); + check_return(0, avro_value_get_by_index + (val2, i, &child2, NULL)); + if (!avro_value_equal_fast(&child1, &child2)) { + return 0; + } + } + + return 1; + } + + case AVRO_ENUM: + { + int v1; + int v2; + check_return(0, avro_value_get_enum(val1, &v1)); + check_return(0, avro_value_get_enum(val2, &v2)); + return (v1 == v2); + } + + case AVRO_FIXED: + { + const void *buf1; + const void *buf2; + size_t size1; + size_t size2; + check_return(0, avro_value_get_fixed(val1, &buf1, &size1)); + check_return(0, avro_value_get_fixed(val2, &buf2, &size2)); + if (size1 != size2) { + return 0; + } + return (memcmp(buf1, buf2, size1) == 0); + } + + case AVRO_MAP: + { + size_t count1; + size_t count2; + check_return(0, avro_value_get_size(val1, &count1)); + check_return(0, avro_value_get_size(val2, &count2)); + if (count1 != count2) { + return 0; + } + + size_t i; + for (i = 0; i < count1; i++) { + avro_value_t child1; + avro_value_t child2; + const char *key1; + check_return(0, avro_value_get_by_index + (val1, i, &child1, &key1)); + check_return(0, avro_value_get_by_name + (val2, key1, &child2, NULL)); + if (!avro_value_equal_fast(&child1, &child2)) { + return 0; + } + } + + return 1; + } + + case AVRO_RECORD: + { + size_t count1; + check_return(0, avro_value_get_size(val1, &count1)); + + size_t i; + for (i = 0; i < count1; i++) { + avro_value_t child1; + avro_value_t child2; + check_return(0, avro_value_get_by_index + (val1, i, &child1, NULL)); + check_return(0, avro_value_get_by_index + (val2, i, &child2, NULL)); + if (!avro_value_equal_fast(&child1, &child2)) { + return 0; + } + } + + return 1; + } + + case AVRO_UNION: + { + int disc1; + int disc2; + check_return(0, avro_value_get_discriminant(val1, &disc1)); + check_return(0, avro_value_get_discriminant(val2, &disc2)); + if (disc1 != disc2) { + return 0; + } + + avro_value_t branch1; + avro_value_t branch2; + check_return(0, avro_value_get_current_branch(val1, &branch1)); + check_return(0, avro_value_get_current_branch(val2, &branch2)); + return avro_value_equal_fast(&branch1, &branch2); + } + + default: + return 0; + } +} + +int +avro_value_equal(avro_value_t *val1, avro_value_t *val2) +{ + avro_schema_t schema1 = avro_value_get_schema(val1); + avro_schema_t schema2 = avro_value_get_schema(val2); + if (!avro_schema_equal(schema1, schema2)) { + return 0; + } + + return avro_value_equal_fast(val1, val2); +} + + +#define cmp(v1, v2) \ + (((v1) == (v2))? 0: \ + ((v1) < (v2))? -1: 1) +int +avro_value_cmp_fast(avro_value_t *val1, avro_value_t *val2) +{ + avro_type_t type1 = avro_value_get_type(val1); + avro_type_t type2 = avro_value_get_type(val2); + if (type1 != type2) { + return -1; + } + + switch (type1) { + case AVRO_BOOLEAN: + { + int v1; + int v2; + check_return(0, avro_value_get_boolean(val1, &v1)); + check_return(0, avro_value_get_boolean(val2, &v2)); + return cmp(!!v1, !!v2); + } + + case AVRO_BYTES: + { + const void *buf1; + const void *buf2; + size_t size1; + size_t size2; + size_t min_size; + int result; + + check_return(0, avro_value_get_bytes(val1, &buf1, &size1)); + check_return(0, avro_value_get_bytes(val2, &buf2, &size2)); + + min_size = (size1 < size2)? size1: size2; + result = memcmp(buf1, buf2, min_size); + if (result != 0) { + return result; + } else { + return cmp(size1, size2); + } + } + + case AVRO_DOUBLE: + { + double v1; + double v2; + check_return(0, avro_value_get_double(val1, &v1)); + check_return(0, avro_value_get_double(val2, &v2)); + return cmp(v1, v2); + } + + case AVRO_FLOAT: + { + float v1; + float v2; + check_return(0, avro_value_get_float(val1, &v1)); + check_return(0, avro_value_get_float(val2, &v2)); + return cmp(v1, v2); + } + + case AVRO_INT32: + { + int32_t v1; + int32_t v2; + check_return(0, avro_value_get_int(val1, &v1)); + check_return(0, avro_value_get_int(val2, &v2)); + return cmp(v1, v2); + } + + case AVRO_INT64: + { + int64_t v1; + int64_t v2; + check_return(0, avro_value_get_long(val1, &v1)); + check_return(0, avro_value_get_long(val2, &v2)); + return cmp(v1, v2); + } + + case AVRO_NULL: + { + check_return(0, avro_value_get_null(val1)); + check_return(0, avro_value_get_null(val2)); + return 0; + } + + case AVRO_STRING: + { + const char *buf1; + const char *buf2; + size_t size1; + size_t size2; + size_t min_size; + int result; + check_return(0, avro_value_get_string(val1, &buf1, &size1)); + check_return(0, avro_value_get_string(val2, &buf2, &size2)); + + min_size = (size1 < size2)? size1: size2; + result = memcmp(buf1, buf2, min_size); + if (result != 0) { + return result; + } else { + return cmp(size1, size2); + } + } + + case AVRO_ARRAY: + { + size_t count1; + size_t count2; + size_t min_count; + size_t i; + check_return(0, avro_value_get_size(val1, &count1)); + check_return(0, avro_value_get_size(val2, &count2)); + + min_count = (count1 < count2)? count1: count2; + for (i = 0; i < min_count; i++) { + avro_value_t child1; + avro_value_t child2; + int result; + check_return(0, avro_value_get_by_index + (val1, i, &child1, NULL)); + check_return(0, avro_value_get_by_index + (val2, i, &child2, NULL)); + result = avro_value_cmp_fast(&child1, &child2); + if (result != 0) { + return result; + } + } + + return cmp(count1, count2); + } + + case AVRO_ENUM: + { + int v1; + int v2; + check_return(0, avro_value_get_enum(val1, &v1)); + check_return(0, avro_value_get_enum(val2, &v2)); + return cmp(v1, v2); + } + + case AVRO_FIXED: + { + const void *buf1; + const void *buf2; + size_t size1; + size_t size2; + check_return(0, avro_value_get_fixed(val1, &buf1, &size1)); + check_return(0, avro_value_get_fixed(val2, &buf2, &size2)); + if (size1 != size2) { + return -1; + } + return memcmp(buf1, buf2, size1); + } + + case AVRO_MAP: + { + return -1; + } + + case AVRO_RECORD: + { + size_t count1; + check_return(0, avro_value_get_size(val1, &count1)); + + size_t i; + for (i = 0; i < count1; i++) { + avro_value_t child1; + avro_value_t child2; + int result; + + check_return(0, avro_value_get_by_index + (val1, i, &child1, NULL)); + check_return(0, avro_value_get_by_index + (val2, i, &child2, NULL)); + result = avro_value_cmp_fast(&child1, &child2); + if (result != 0) { + return result; + } + } + + return 0; + } + + case AVRO_UNION: + { + int disc1; + int disc2; + check_return(0, avro_value_get_discriminant(val1, &disc1)); + check_return(0, avro_value_get_discriminant(val2, &disc2)); + + if (disc1 == disc2) { + avro_value_t branch1; + avro_value_t branch2; + check_return(0, avro_value_get_current_branch(val1, &branch1)); + check_return(0, avro_value_get_current_branch(val2, &branch2)); + return avro_value_cmp_fast(&branch1, &branch2); + } else { + return cmp(disc1, disc2); + } + } + + default: + return 0; + } +} + +int +avro_value_cmp(avro_value_t *val1, avro_value_t *val2) +{ + avro_schema_t schema1 = avro_value_get_schema(val1); + avro_schema_t schema2 = avro_value_get_schema(val2); + if (!avro_schema_equal(schema1, schema2)) { + return 0; + } + + return avro_value_cmp_fast(val1, val2); +} + + +int +avro_value_copy_fast(avro_value_t *dest, const avro_value_t *src) +{ + avro_type_t dest_type = avro_value_get_type(dest); + avro_type_t src_type = avro_value_get_type(src); + if (dest_type != src_type) { + return 0; + } + + int rval; + check(rval, avro_value_reset(dest)); + + switch (dest_type) { + case AVRO_BOOLEAN: + { + int val; + check(rval, avro_value_get_boolean(src, &val)); + return avro_value_set_boolean(dest, val); + } + + case AVRO_BYTES: + { + avro_wrapped_buffer_t val; + check(rval, avro_value_grab_bytes(src, &val)); + return avro_value_give_bytes(dest, &val); + } + + case AVRO_DOUBLE: + { + double val; + check(rval, avro_value_get_double(src, &val)); + return avro_value_set_double(dest, val); + } + + case AVRO_FLOAT: + { + float val; + check(rval, avro_value_get_float(src, &val)); + return avro_value_set_float(dest, val); + } + + case AVRO_INT32: + { + int32_t val; + check(rval, avro_value_get_int(src, &val)); + return avro_value_set_int(dest, val); + } + + case AVRO_INT64: + { + int64_t val; + check(rval, avro_value_get_long(src, &val)); + return avro_value_set_long(dest, val); + } + + case AVRO_NULL: + { + check(rval, avro_value_get_null(src)); + return avro_value_set_null(dest); + } + + case AVRO_STRING: + { + avro_wrapped_buffer_t val; + check(rval, avro_value_grab_string(src, &val)); + return avro_value_give_string_len(dest, &val); + } + + case AVRO_ARRAY: + { + size_t count; + check(rval, avro_value_get_size(src, &count)); + + size_t i; + for (i = 0; i < count; i++) { + avro_value_t src_child; + avro_value_t dest_child; + + check(rval, avro_value_get_by_index + (src, i, &src_child, NULL)); + check(rval, avro_value_append + (dest, &dest_child, NULL)); + check(rval, avro_value_copy_fast + (&dest_child, &src_child)); + } + + return 0; + } + + case AVRO_ENUM: + { + int val; + check(rval, avro_value_get_enum(src, &val)); + return avro_value_set_enum(dest, val); + } + + case AVRO_FIXED: + { + avro_wrapped_buffer_t val; + check(rval, avro_value_grab_fixed(src, &val)); + return avro_value_give_fixed(dest, &val); + } + + case AVRO_MAP: + { + size_t count; + check(rval, avro_value_get_size(src, &count)); + + size_t i; + for (i = 0; i < count; i++) { + avro_value_t src_child; + avro_value_t dest_child; + const char *key; + + check(rval, avro_value_get_by_index + (src, i, &src_child, &key)); + check(rval, avro_value_add + (dest, key, &dest_child, NULL, NULL)); + check(rval, avro_value_copy_fast + (&dest_child, &src_child)); + } + + return 0; + } + + case AVRO_RECORD: + { + size_t count; + check(rval, avro_value_get_size(src, &count)); + + size_t i; + for (i = 0; i < count; i++) { + avro_value_t src_child; + avro_value_t dest_child; + + check(rval, avro_value_get_by_index + (src, i, &src_child, NULL)); + check(rval, avro_value_get_by_index + (dest, i, &dest_child, NULL)); + check(rval, avro_value_copy_fast + (&dest_child, &src_child)); + } + + return 0; + } + + case AVRO_UNION: + { + int disc; + check(rval, avro_value_get_discriminant(src, &disc)); + + avro_value_t src_branch; + avro_value_t dest_branch; + + check(rval, avro_value_get_current_branch(src, &src_branch)); + check(rval, avro_value_set_branch(dest, disc, &dest_branch)); + + return avro_value_copy_fast(&dest_branch, &src_branch); + } + + default: + return 0; + } +} + + +int +avro_value_copy(avro_value_t *dest, const avro_value_t *src) +{ + avro_schema_t dest_schema = avro_value_get_schema(dest); + avro_schema_t src_schema = avro_value_get_schema(src); + if (!avro_schema_equal(dest_schema, src_schema)) { + avro_set_error("Schemas don't match"); + return EINVAL; + } + + return avro_value_copy_fast(dest, src); +} diff --git a/fluent-bit/lib/avro/src/wrapped-buffer.c b/fluent-bit/lib/avro/src/wrapped-buffer.c new file mode 100644 index 00000000..e7496b42 --- /dev/null +++ b/fluent-bit/lib/avro/src/wrapped-buffer.c @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <errno.h> +#include <stdlib.h> +#include <string.h> + +#include "avro_private.h" +#include "avro/allocation.h" +#include "avro/data.h" +#include "avro/refcount.h" + +struct avro_wrapped_copy { + volatile int refcount; + size_t allocated_size; +}; + +static void +avro_wrapped_copy_free(avro_wrapped_buffer_t *self) +{ + struct avro_wrapped_copy *copy = (struct avro_wrapped_copy *) self->user_data; + if (avro_refcount_dec(©->refcount)) { + avro_free(copy, copy->allocated_size); + } +} + +static int +avro_wrapped_copy_copy(avro_wrapped_buffer_t *dest, + const avro_wrapped_buffer_t *src, + size_t offset, size_t length) +{ + struct avro_wrapped_copy *copy = (struct avro_wrapped_copy *) src->user_data; + avro_refcount_inc(©->refcount); + dest->buf = (char *) src->buf + offset; + dest->size = length; + dest->user_data = copy; + dest->free = avro_wrapped_copy_free; + dest->copy = avro_wrapped_copy_copy; + dest->slice = NULL; + return 0; +} + +int +avro_wrapped_buffer_new_copy(avro_wrapped_buffer_t *dest, + const void *buf, size_t length) +{ + size_t allocated_size = sizeof(struct avro_wrapped_copy) + length; + struct avro_wrapped_copy *copy = (struct avro_wrapped_copy *) avro_malloc(allocated_size); + if (copy == NULL) { + return ENOMEM; + } + + dest->buf = ((char *) copy) + sizeof(struct avro_wrapped_copy); + dest->size = length; + dest->user_data = copy; + dest->free = avro_wrapped_copy_free; + dest->copy = avro_wrapped_copy_copy; + dest->slice = NULL; + + avro_refcount_set(©->refcount, 1); + copy->allocated_size = allocated_size; + memcpy((void *) dest->buf, buf, length); + return 0; +} + +int +avro_wrapped_buffer_new(avro_wrapped_buffer_t *dest, + const void *buf, size_t length) +{ + dest->buf = buf; + dest->size = length; + dest->user_data = NULL; + dest->free = NULL; + dest->copy = NULL; + dest->slice = NULL; + return 0; +} + + +void +avro_wrapped_buffer_move(avro_wrapped_buffer_t *dest, + avro_wrapped_buffer_t *src) +{ + memcpy(dest, src, sizeof(avro_wrapped_buffer_t)); + memset(src, 0, sizeof(avro_wrapped_buffer_t)); +} + +int +avro_wrapped_buffer_copy(avro_wrapped_buffer_t *dest, + const avro_wrapped_buffer_t *src, + size_t offset, size_t length) +{ + if (offset > src->size) { + avro_set_error("Invalid offset when slicing buffer"); + return EINVAL; + } + + if ((offset+length) > src->size) { + avro_set_error("Invalid length when slicing buffer"); + return EINVAL; + } + + if (src->copy == NULL) { + return avro_wrapped_buffer_new_copy(dest, (char *) src->buf + offset, length); + } else { + return src->copy(dest, src, offset, length); + } +} + +int +avro_wrapped_buffer_slice(avro_wrapped_buffer_t *self, + size_t offset, size_t length) +{ + if (offset > self->size) { + avro_set_error("Invalid offset when slicing buffer"); + return EINVAL; + } + + if ((offset+length) > self->size) { + avro_set_error("Invalid length when slicing buffer"); + return EINVAL; + } + + if (self->slice == NULL) { + self->buf = (char *) self->buf + offset; + self->size = length; + return 0; + } else { + return self->slice(self, offset, length); + } +} diff --git a/fluent-bit/lib/avro/tests/.gitignore b/fluent-bit/lib/avro/tests/.gitignore new file mode 100644 index 00000000..534eb068 --- /dev/null +++ b/fluent-bit/lib/avro/tests/.gitignore @@ -0,0 +1,9 @@ +generate_interop_data +performance +test_avro_data +test_avro_schema +test_avro_schema_names +test_avro_values +test_cpp +test_data_structures +test_interop_data diff --git a/fluent-bit/lib/avro/tests/CMakeLists.txt b/fluent-bit/lib/avro/tests/CMakeLists.txt new file mode 100644 index 00000000..2e84a06a --- /dev/null +++ b/fluent-bit/lib/avro/tests/CMakeLists.txt @@ -0,0 +1,87 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +macro(add_avro_executable name) + set(source "${ARGV1}") + if (NOT source) + set(source "${name}.c") + endif (NOT source) + add_executable(${name} ${source}) + target_link_libraries(${name} avro-static) +endmacro(add_avro_executable) + +macro(add_avro_test name) + add_avro_executable(${name} ${ARGN}) + if (WIN32) + set(exec_name ${CMAKE_CURRENT_BINARY_DIR}/Debug/${name}.exe) + else (WIN32) + set(exec_name ${CMAKE_CURRENT_BINARY_DIR}/${name}) + endif (WIN32) + + add_test(${name} + ${CMAKE_COMMAND} -E chdir ${AvroC_SOURCE_DIR}/tests + ${exec_name} + ) +endmacro(add_avro_test) + +macro(add_avro_test_checkmem name) + add_avro_test(${name} ${ARGN}) + if(UNIX) + find_program(MEMORYCHECK_COMMAND valgrind ) + if(MEMORYCHECK_COMMAND) + add_test(memcheck_${name} + ${CMAKE_COMMAND} -E chdir ${AvroC_SOURCE_DIR}/tests + ${MEMORYCHECK_COMMAND} + --log-file=${CMAKE_CURRENT_BINARY_DIR}/memcheck_${name}.log + --leak-check=full + --show-reachable=yes + --error-exitcode=1 + ${exec_name} + ) + endif(MEMORYCHECK_COMMAND) + endif (UNIX) +endmacro(add_avro_test_checkmem) + +add_avro_executable(generate_interop_data) +add_avro_executable(performance) +add_avro_executable(test_interop_data) + +add_avro_test_checkmem(test_data_structures) +add_avro_test_checkmem(test_avro_schema) +add_avro_test_checkmem(test_avro_schema_names) +add_avro_test_checkmem(test_avro_values) +add_avro_test_checkmem(test_avro_766) +add_avro_test_checkmem(test_avro_968) +add_avro_test_checkmem(test_avro_984) +add_avro_test_checkmem(test_avro_1034) +add_avro_test_checkmem(test_avro_1084) +add_avro_test_checkmem(test_avro_1087) +add_avro_test_checkmem(test_avro_1165) +add_avro_test_checkmem(test_avro_1167) +add_avro_test_checkmem(test_avro_1237) +add_avro_test_checkmem(test_avro_1238) +add_avro_test_checkmem(test_avro_1279) +add_avro_test_checkmem(test_avro_1405) +add_avro_test_checkmem(test_avro_1572) +add_avro_test(test_avro_data) # Skip memory check for datum. Deprecated and has a lot of memory issues +add_avro_test_checkmem(test_refcount) +add_avro_test_checkmem(test_avro_1379) +add_avro_test_checkmem(test_avro_1691) +add_avro_test_checkmem(test_avro_1906) +add_avro_test_checkmem(test_avro_1904) diff --git a/fluent-bit/lib/avro/tests/avro-1237-bad-union-discriminant.avro b/fluent-bit/lib/avro/tests/avro-1237-bad-union-discriminant.avro Binary files differnew file mode 100644 index 00000000..6dc539ee --- /dev/null +++ b/fluent-bit/lib/avro/tests/avro-1237-bad-union-discriminant.avro diff --git a/fluent-bit/lib/avro/tests/avro-1237-good.avro b/fluent-bit/lib/avro/tests/avro-1237-good.avro Binary files differnew file mode 100644 index 00000000..336dc289 --- /dev/null +++ b/fluent-bit/lib/avro/tests/avro-1237-good.avro diff --git a/fluent-bit/lib/avro/tests/avro-1238-good.avro b/fluent-bit/lib/avro/tests/avro-1238-good.avro Binary files differnew file mode 100644 index 00000000..336dc289 --- /dev/null +++ b/fluent-bit/lib/avro/tests/avro-1238-good.avro diff --git a/fluent-bit/lib/avro/tests/avro-1238-truncated.avro b/fluent-bit/lib/avro/tests/avro-1238-truncated.avro Binary files differnew file mode 100644 index 00000000..f48d54d7 --- /dev/null +++ b/fluent-bit/lib/avro/tests/avro-1238-truncated.avro diff --git a/fluent-bit/lib/avro/tests/avro-1279-codec.avro b/fluent-bit/lib/avro/tests/avro-1279-codec.avro Binary files differnew file mode 100644 index 00000000..dd242305 --- /dev/null +++ b/fluent-bit/lib/avro/tests/avro-1279-codec.avro diff --git a/fluent-bit/lib/avro/tests/avro-1279-no-codec.avro b/fluent-bit/lib/avro/tests/avro-1279-no-codec.avro Binary files differnew file mode 100644 index 00000000..4099de55 --- /dev/null +++ b/fluent-bit/lib/avro/tests/avro-1279-no-codec.avro diff --git a/fluent-bit/lib/avro/tests/generate_interop_data.c b/fluent-bit/lib/avro/tests/generate_interop_data.c new file mode 100644 index 00000000..e7f1365a --- /dev/null +++ b/fluent-bit/lib/avro/tests/generate_interop_data.c @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include "avro.h" +#include "avro_private.h" +#include <stdio.h> +#include <stdlib.h> + +int main(int argc, char *argv[]) +{ + int rval; + avro_file_writer_t file_writer; + avro_file_reader_t file_reader; + char outpath[128]; + FILE *fp; + char jsontext[16 * 1024]; + avro_schema_t schema; + avro_schema_error_t schema_error; + avro_datum_t interop; + avro_datum_t array_datum; + avro_datum_t node_datum; + avro_datum_t union_datum; + avro_datum_t out_datum; + enum Kind { + KIND_A, + KIND_B, + KIND_C + }; + + if (argc != 3) { + exit(EXIT_FAILURE); + } + snprintf(outpath, sizeof(outpath), "%s/c.avro", argv[2]); + fprintf(stderr, "Writing to %s\n", outpath); + + fp = fopen(argv[1], "r"); + rval = fread(jsontext, 1, sizeof(jsontext) - 1, fp); + jsontext[rval] = '\0'; + + check(rval, + avro_schema_from_json(jsontext, rval, &schema, &schema_error)); + check(rval, avro_file_writer_create(outpath, schema, &file_writer)); + + /* TODO: create a method for generating random data from schema */ + interop = avro_record(schema); + avro_record_set(interop, "intField", avro_int32(42)); + avro_record_set(interop, "longField", avro_int64(4242)); + avro_record_set(interop, "stringField", + avro_givestring("Follow your bliss.", NULL)); + avro_record_set(interop, "boolField", avro_boolean(1)); + avro_record_set(interop, "floatField", avro_float(3.14159265)); + avro_record_set(interop, "doubleField", avro_double(2.71828183)); + avro_record_set(interop, "bytesField", avro_bytes("abcd", 4)); + avro_record_set(interop, "nullField", avro_null()); + + avro_schema_t array_schema = avro_schema_get_subschema(schema, "arrayField"); + array_datum = avro_array(array_schema); + avro_array_append_datum(array_datum, avro_double(1.0)); + avro_array_append_datum(array_datum, avro_double(2.0)); + avro_array_append_datum(array_datum, avro_double(3.0)); + avro_record_set(interop, "arrayField", array_datum); + + avro_schema_t map_schema = avro_schema_get_subschema(schema, "mapField"); + avro_record_set(interop, "mapField", avro_map(map_schema)); + + avro_schema_t union_schema = avro_schema_get_subschema(schema, "unionField"); + union_datum = avro_union(union_schema, 1, avro_double(1.61803399)); + avro_record_set(interop, "unionField", union_datum); + + avro_schema_t enum_schema = avro_schema_get_subschema(schema, "enumField"); + avro_record_set(interop, "enumField", avro_enum(enum_schema, KIND_A)); + + avro_schema_t fixed_schema = avro_schema_get_subschema(schema, "fixedField"); + avro_record_set(interop, "fixedField", + avro_fixed(fixed_schema, "1234567890123456", 16)); + + avro_schema_t node_schema = avro_schema_get_subschema(schema, "recordField"); + node_datum = avro_record(node_schema); + avro_record_set(node_datum, "label", + avro_givestring("If you label me, you negate me.", NULL)); + avro_schema_t children_schema = avro_schema_get_subschema(node_schema, "children"); + avro_record_set(node_datum, "children", avro_array(children_schema)); + avro_record_set(interop, "recordField", node_datum); + + rval = avro_file_writer_append(file_writer, interop); + if (rval) { + fprintf(stderr, "Unable to append data to interop file!\n"); + exit(EXIT_FAILURE); + } else { + fprintf(stderr, "Successfully appended datum to file\n"); + } + + check(rval, avro_file_writer_close(file_writer)); + fprintf(stderr, "Closed writer.\n"); + + check(rval, avro_file_reader(outpath, &file_reader)); + fprintf(stderr, "Re-reading datum to verify\n"); + check(rval, avro_file_reader_read(file_reader, NULL, &out_datum)); + fprintf(stderr, "Verifying datum..."); + if (!avro_datum_equal(interop, out_datum)) { + fprintf(stderr, "fail!\n"); + exit(EXIT_FAILURE); + } + fprintf(stderr, "ok\n"); + check(rval, avro_file_reader_close(file_reader)); + fprintf(stderr, "Closed reader.\n"); + return 0; +} diff --git a/fluent-bit/lib/avro/tests/msdirent.h b/fluent-bit/lib/avro/tests/msdirent.h new file mode 100644 index 00000000..445d040d --- /dev/null +++ b/fluent-bit/lib/avro/tests/msdirent.h @@ -0,0 +1,372 @@ +/***************************************************************************** + * dirent.h - dirent API for Microsoft Visual Studio + * + * Copyright (C) 2006 Toni Ronkko + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * ``Software''), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL TONI RONKKO BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Mar 15, 2011, Toni Ronkko + * Defined FILE_ATTRIBUTE_DEVICE for MSVC 6.0. + * + * Aug 11, 2010, Toni Ronkko + * Added d_type and d_namlen fields to dirent structure. The former is + * especially useful for determining whether directory entry represents a + * file or a directory. For more information, see + * http://www.delorie.com/gnu/docs/glibc/libc_270.html + * + * Aug 11, 2010, Toni Ronkko + * Improved conformance to the standards. For example, errno is now set + * properly on failure and assert() is never used. Thanks to Peter Brockam + * for suggestions. + * + * Aug 11, 2010, Toni Ronkko + * Fixed a bug in rewinddir(): when using relative directory names, change + * of working directory no longer causes rewinddir() to fail. + * + * Dec 15, 2009, John Cunningham + * Added rewinddir member function + * + * Jan 18, 2008, Toni Ronkko + * Using FindFirstFileA and WIN32_FIND_DATAA to avoid converting string + * between multi-byte and unicode representations. This makes the + * code simpler and also allows the code to be compiled under MingW. Thanks + * to Azriel Fasten for the suggestion. + * + * Mar 4, 2007, Toni Ronkko + * Bug fix: due to the strncpy_s() function this file only compiled in + * Visual Studio 2005. Using the new string functions only when the + * compiler version allows. + * + * Nov 2, 2006, Toni Ronkko + * Major update: removed support for Watcom C, MS-DOS and Turbo C to + * simplify the file, updated the code to compile cleanly on Visual + * Studio 2005 with both unicode and multi-byte character strings, + * removed rewinddir() as it had a bug. + * + * Aug 20, 2006, Toni Ronkko + * Removed all remarks about MSVC 1.0, which is antiqued now. Simplified + * comments by removing SGML tags. + * + * May 14 2002, Toni Ronkko + * Embedded the function definitions directly to the header so that no + * source modules need to be included in the Visual Studio project. Removed + * all the dependencies to other projects so that this very header can be + * used independently. + * + * May 28 1998, Toni Ronkko + * First version. + *****************************************************************************/ +#ifndef DIRENT_H +#define DIRENT_H + +#define WIN32_LEAN_AND_MEAN +#include <windows.h> +#include <string.h> +#include <stdlib.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <errno.h> + +/* Entries missing from MSVC 6.0 */ +#if !defined(FILE_ATTRIBUTE_DEVICE) +# define FILE_ATTRIBUTE_DEVICE 0x40 +#endif + +/* File type and permission flags for stat() */ +#if defined(_MSC_VER) && !defined(S_IREAD) +# define S_IFMT _S_IFMT /* file type mask */ +# define S_IFDIR _S_IFDIR /* directory */ +# define S_IFCHR _S_IFCHR /* character device */ +# define S_IFFIFO _S_IFFIFO /* pipe */ +# define S_IFREG _S_IFREG /* regular file */ +# define S_IREAD _S_IREAD /* read permission */ +# define S_IWRITE _S_IWRITE /* write permission */ +# define S_IEXEC _S_IEXEC /* execute permission */ +#endif +#define S_IFBLK 0 /* block device */ +#define S_IFLNK 0 /* link */ +#define S_IFSOCK 0 /* socket */ + +#if defined(_MSC_VER) +# define S_IRUSR S_IREAD /* read, user */ +# define S_IWUSR S_IWRITE /* write, user */ +# define S_IXUSR 0 /* execute, user */ +# define S_IRGRP 0 /* read, group */ +# define S_IWGRP 0 /* write, group */ +# define S_IXGRP 0 /* execute, group */ +# define S_IROTH 0 /* read, others */ +# define S_IWOTH 0 /* write, others */ +# define S_IXOTH 0 /* execute, others */ +#endif + +/* Indicates that d_type field is available in dirent structure */ +#define _DIRENT_HAVE_D_TYPE + +/* File type flags for d_type */ +#define DT_UNKNOWN 0 +#define DT_REG S_IFREG +#define DT_DIR S_IFDIR +#define DT_FIFO S_IFFIFO +#define DT_SOCK S_IFSOCK +#define DT_CHR S_IFCHR +#define DT_BLK S_IFBLK + +/* Macros for converting between st_mode and d_type */ +#define IFTODT(mode) ((mode) & S_IFMT) +#define DTTOIF(type) (type) + +/* + * File type macros. Note that block devices, sockets and links cannot be + * distinguished on Windows and the macros S_ISBLK, S_ISSOCK and S_ISLNK are + * only defined for compatibility. These macros should always return false + * on Windows. + */ +#define S_ISFIFO(mode) (((mode) & S_IFMT) == S_IFFIFO) +#define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR) +#define S_ISREG(mode) (((mode) & S_IFMT) == S_IFREG) +#define S_ISLNK(mode) (((mode) & S_IFMT) == S_IFLNK) +#define S_ISSOCK(mode) (((mode) & S_IFMT) == S_IFSOCK) +#define S_ISCHR(mode) (((mode) & S_IFMT) == S_IFCHR) +#define S_ISBLK(mode) (((mode) & S_IFMT) == S_IFBLK) + +#ifdef __cplusplus +extern "C" { +#endif + + +typedef struct dirent +{ + char d_name[MAX_PATH + 1]; /* File name */ + size_t d_namlen; /* Length of name without \0 */ + int d_type; /* File type */ +} dirent; + + +typedef struct DIR +{ + dirent curentry; /* Current directory entry */ + WIN32_FIND_DATAA find_data; /* Private file data */ + int cached; /* True if data is valid */ + HANDLE search_handle; /* Win32 search handle */ + char patt[MAX_PATH + 3]; /* Initial directory name */ +} DIR; + + +/* Forward declarations */ +static DIR *opendir(const char *dirname); +static struct dirent *readdir(DIR *dirp); +static int closedir(DIR *dirp); +static void rewinddir(DIR* dirp); + + +/* Use the new safe string functions introduced in Visual Studio 2005 */ +#if defined(_MSC_VER) && _MSC_VER >= 1400 +# define DIRENT_STRNCPY(dest,src,size) strncpy_s((dest),(size),(src),_TRUNCATE) +#else +# define DIRENT_STRNCPY(dest,src,size) strncpy((dest),(src),(size)) +#endif + +/* Set errno variable */ +#if defined(_MSC_VER) +#define DIRENT_SET_ERRNO(x) _set_errno (x) +#else +#define DIRENT_SET_ERRNO(x) (errno = (x)) +#endif + + +/***************************************************************************** + * Open directory stream DIRNAME for read and return a pointer to the + * internal working area that is used to retrieve individual directory + * entries. + */ +static DIR *opendir(const char *dirname) +{ + DIR *dirp; + + /* ensure that the resulting search pattern will be a valid file name */ + if (dirname == NULL) { + DIRENT_SET_ERRNO (ENOENT); + return NULL; + } + if (strlen (dirname) + 3 >= MAX_PATH) { + DIRENT_SET_ERRNO (ENAMETOOLONG); + return NULL; + } + + /* construct new DIR structure */ + dirp = (DIR*) malloc (sizeof (struct DIR)); + if (dirp != NULL) { + int error; + + /* + * Convert relative directory name to an absolute one. This + * allows rewinddir() to function correctly when the current working + * directory is changed between opendir() and rewinddir(). + */ + if (GetFullPathNameA (dirname, MAX_PATH, dirp->patt, NULL)) { + char *p; + + /* append the search pattern "\\*\0" to the directory name */ + p = strchr (dirp->patt, '\0'); + if (dirp->patt < p && *(p-1) != '\\' && *(p-1) != ':') { + *p++ = '\\'; + } + *p++ = '*'; + *p = '\0'; + + /* open directory stream and retrieve the first entry */ + dirp->search_handle = FindFirstFileA (dirp->patt, &dirp->find_data); + if (dirp->search_handle != INVALID_HANDLE_VALUE) { + /* a directory entry is now waiting in memory */ + dirp->cached = 1; + error = 0; + } else { + /* search pattern is not a directory name? */ + DIRENT_SET_ERRNO (ENOENT); + error = 1; + } + } else { + /* buffer too small */ + DIRENT_SET_ERRNO (ENOMEM); + error = 1; + } + + if (error) { + free (dirp); + dirp = NULL; + } + } + + return dirp; +} + + +/***************************************************************************** + * Read a directory entry, and return a pointer to a dirent structure + * containing the name of the entry in d_name field. Individual directory + * entries returned by this very function include regular files, + * sub-directories, pseudo-directories "." and "..", but also volume labels, + * hidden files and system files may be returned. + */ +static struct dirent *readdir(DIR *dirp) +{ + DWORD attr; + if (dirp == NULL) { + /* directory stream did not open */ + DIRENT_SET_ERRNO (EBADF); + return NULL; + } + + /* get next directory entry */ + if (dirp->cached != 0) { + /* a valid directory entry already in memory */ + dirp->cached = 0; + } else { + /* get the next directory entry from stream */ + if (dirp->search_handle == INVALID_HANDLE_VALUE) { + return NULL; + } + if (FindNextFileA (dirp->search_handle, &dirp->find_data) == FALSE) { + /* the very last entry has been processed or an error occured */ + FindClose (dirp->search_handle); + dirp->search_handle = INVALID_HANDLE_VALUE; + return NULL; + } + } + + /* copy as a multibyte character string */ + DIRENT_STRNCPY ( dirp->curentry.d_name, + dirp->find_data.cFileName, + sizeof(dirp->curentry.d_name) ); + dirp->curentry.d_name[MAX_PATH] = '\0'; + + /* compute the length of name */ + dirp->curentry.d_namlen = strlen (dirp->curentry.d_name); + + /* determine file type */ + attr = dirp->find_data.dwFileAttributes; + if ((attr & FILE_ATTRIBUTE_DEVICE) != 0) { + dirp->curentry.d_type = DT_CHR; + } else if ((attr & FILE_ATTRIBUTE_DIRECTORY) != 0) { + dirp->curentry.d_type = DT_DIR; + } else { + dirp->curentry.d_type = DT_REG; + } + return &dirp->curentry; +} + + +/***************************************************************************** + * Close directory stream opened by opendir() function. Close of the + * directory stream invalidates the DIR structure as well as any previously + * read directory entry. + */ +static int closedir(DIR *dirp) +{ + if (dirp == NULL) { + /* invalid directory stream */ + DIRENT_SET_ERRNO (EBADF); + return -1; + } + + /* release search handle */ + if (dirp->search_handle != INVALID_HANDLE_VALUE) { + FindClose (dirp->search_handle); + dirp->search_handle = INVALID_HANDLE_VALUE; + } + + /* release directory structure */ + free (dirp); + return 0; +} + + +/***************************************************************************** + * Resets the position of the directory stream to which dirp refers to the + * beginning of the directory. It also causes the directory stream to refer + * to the current state of the corresponding directory, as a call to opendir() + * would have done. If dirp does not refer to a directory stream, the effect + * is undefined. + */ +static void rewinddir(DIR* dirp) +{ + if (dirp != NULL) { + /* release search handle */ + if (dirp->search_handle != INVALID_HANDLE_VALUE) { + FindClose (dirp->search_handle); + } + + /* open new search handle and retrieve the first entry */ + dirp->search_handle = FindFirstFileA (dirp->patt, &dirp->find_data); + if (dirp->search_handle != INVALID_HANDLE_VALUE) { + /* a directory entry is now waiting in memory */ + dirp->cached = 1; + } else { + /* failed to re-open directory: no directory entry in memory */ + dirp->cached = 0; + } + } +} + + +#ifdef __cplusplus +} +#endif +#endif /*DIRENT_H*/ diff --git a/fluent-bit/lib/avro/tests/performance.c b/fluent-bit/lib/avro/tests/performance.c new file mode 100644 index 00000000..a6f50427 --- /dev/null +++ b/fluent-bit/lib/avro/tests/performance.c @@ -0,0 +1,848 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <limits.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> + +#include "avro.h" +#include "avro_private.h" + + +/* The following definitions can be used as bitflags. They can also be + * passed in as the resolution_mode flags to the helper functions. + */ +#define USE_MATCHED_SCHEMAS (0x00) +#define USE_RESOLVED_READER (0x01) +#define USE_RESOLVED_WRITER (0x02) +#define USE_BOTH_RESOLVED (0x03) + + +/* + * A series of performance tests. + */ + +typedef void +(*test_func_t)(unsigned long); + + +void init_rand(void) +{ + srand(time(NULL)); +} + +double rand_number(double from, double to) +{ + double range = to - from; + return from + ((double)rand() / (RAND_MAX + 1.0)) * range; +} + +int64_t rand_int64(void) +{ + return (int64_t) rand_number(LONG_MIN, LONG_MAX); +} + +int32_t rand_int32(void) +{ + return (int32_t) rand_number(INT_MIN, INT_MAX); +} + + +/** + * Tests the single-threaded performance of our reference counting + * mechanism. We create a single datum, and then reference and + * deference it many many times. + */ + +static void +test_refcount(unsigned long num_tests) +{ + unsigned long i; + + avro_datum_t datum = avro_int32(42); + for (i = 0; i < num_tests; i++) { + avro_datum_incref(datum); + avro_datum_decref(datum); + } + avro_datum_decref(datum); +} + + +/** + * Tests the performance of serializing and deserializing a somewhat + * complex record type using the legacy datum API. + */ + +static void +test_nested_record_datum(unsigned long num_tests) +{ + static const char *schema_json = + "{" + " \"type\": \"record\"," + " \"name\": \"test\"," + " \"fields\": [" + " { \"name\": \"i\", \"type\": \"int\" }," + " { \"name\": \"l\", \"type\": \"long\" }," + " { \"name\": \"s\", \"type\": \"string\" }," + " {" + " \"name\": \"subrec\"," + " \"type\": {" + " \"type\": \"record\"," + " \"name\": \"sub\"," + " \"fields\": [" + " { \"name\": \"f\", \"type\": \"float\" }," + " { \"name\": \"d\", \"type\": \"double\" }" + " ]" + " }" + " }" + " ]" + "}"; + + static const char *strings[] = { + "Four score and seven years ago", + "our father brought forth on this continent", + "a new nation", "conceived in Liberty", + "and dedicated to the proposition that all men are created equal." + }; + static const unsigned int NUM_STRINGS = + sizeof(strings) / sizeof(strings[0]); + + int rc; + static char buf[4096]; + avro_reader_t reader = avro_reader_memory(buf, sizeof(buf)); + avro_writer_t writer = avro_writer_memory(buf, sizeof(buf)); + + avro_schema_t schema = NULL; + avro_schema_error_t error = NULL; + avro_schema_from_json(schema_json, strlen(schema_json), + &schema, &error); + + unsigned long i; + + avro_datum_t in = avro_datum_from_schema(schema); + + for (i = 0; i < num_tests; i++) { + avro_record_set_field_value(rc, in, int32, "i", rand_int32()); + avro_record_set_field_value(rc, in, int64, "l", rand_int64()); + avro_record_set_field_value(rc, in, givestring, "s", + strings[i % NUM_STRINGS], NULL); + + avro_datum_t subrec = NULL; + avro_record_get(in, "subrec", &subrec); + avro_record_set_field_value(rc, in, float, "f", rand_number(-1e10, 1e10)); + avro_record_set_field_value(rc, in, double, "d", rand_number(-1e10, 1e10)); + + avro_writer_reset(writer); + avro_write_data(writer, schema, in); + + avro_datum_t out = NULL; + + avro_reader_reset(reader); + avro_read_data(reader, schema, schema, &out); + + avro_datum_equal(in, out); + avro_datum_decref(out); + } + + avro_datum_decref(in); + avro_schema_decref(schema); + avro_writer_free(writer); + avro_reader_free(reader); +} + + +/** + * Tests the performance of serializing and deserializing a somewhat + * complex record type using the new value API, retrieving record fields + * by index. + */ + +static void +test_nested_record_value_by_index(unsigned long num_tests) +{ + static const char *schema_json = + "{" + " \"type\": \"record\"," + " \"name\": \"test\"," + " \"fields\": [" + " { \"name\": \"i\", \"type\": \"int\" }," + " { \"name\": \"l\", \"type\": \"long\" }," + " { \"name\": \"s\", \"type\": \"string\" }," + " {" + " \"name\": \"subrec\"," + " \"type\": {" + " \"type\": \"record\"," + " \"name\": \"sub\"," + " \"fields\": [" + " { \"name\": \"f\", \"type\": \"float\" }," + " { \"name\": \"d\", \"type\": \"double\" }" + " ]" + " }" + " }" + " ]" + "}"; + + static char *strings[] = { + "Four score and seven years ago", + "our father brought forth on this continent", + "a new nation", "conceived in Liberty", + "and dedicated to the proposition that all men are created equal." + }; + static const unsigned int NUM_STRINGS = + sizeof(strings) / sizeof(strings[0]); + + static char buf[4096]; + avro_reader_t reader = avro_reader_memory(buf, sizeof(buf)); + avro_writer_t writer = avro_writer_memory(buf, sizeof(buf)); + + avro_schema_t schema = NULL; + avro_schema_error_t error = NULL; + avro_schema_from_json(schema_json, strlen(schema_json), + &schema, &error); + + unsigned long i; + + avro_value_iface_t *iface = avro_generic_class_from_schema(schema); + + avro_value_t val; + avro_generic_value_new(iface, &val); + + avro_value_t out; + avro_generic_value_new(iface, &out); + + for (i = 0; i < num_tests; i++) { + avro_value_t field; + + avro_value_get_by_index(&val, 0, &field, NULL); + avro_value_set_int(&field, rand_int32()); + + avro_value_get_by_index(&val, 1, &field, NULL); + avro_value_set_long(&field, rand_int64()); + + avro_wrapped_buffer_t wbuf; + avro_wrapped_buffer_new_string(&wbuf, strings[i % NUM_STRINGS]); + avro_value_get_by_index(&val, 2, &field, NULL); + avro_value_give_string_len(&field, &wbuf); + + avro_value_t subrec; + avro_value_get_by_index(&val, 3, &subrec, NULL); + + avro_value_get_by_index(&subrec, 0, &field, NULL); + avro_value_set_float(&field, rand_number(-1e10, 1e10)); + + avro_value_get_by_index(&subrec, 1, &field, NULL); + avro_value_set_double(&field, rand_number(-1e10, 1e10)); + + avro_writer_reset(writer); + avro_value_write(writer, &val); + + avro_reader_reset(reader); + avro_value_read(reader, &out); + + if (! avro_value_equal_fast(&val, &out) ) { + printf("Broken\n"); + exit (1); + } + } + + avro_value_decref(&val); + avro_value_decref(&out); + avro_value_iface_decref(iface); + avro_schema_decref(schema); + avro_writer_free(writer); + avro_reader_free(reader); +} + + + +/** + * Tests the performance of serializing and deserializing a somewhat + * complex record type using the new value API, retrieving record fields + * by name. + */ + +static void +test_nested_record_value_by_name(unsigned long num_tests) +{ + static const char *schema_json = + "{" + " \"type\": \"record\"," + " \"name\": \"test\"," + " \"fields\": [" + " { \"name\": \"i\", \"type\": \"int\" }," + " { \"name\": \"l\", \"type\": \"long\" }," + " { \"name\": \"s\", \"type\": \"string\" }," + " {" + " \"name\": \"subrec\"," + " \"type\": {" + " \"type\": \"record\"," + " \"name\": \"sub\"," + " \"fields\": [" + " { \"name\": \"f\", \"type\": \"float\" }," + " { \"name\": \"d\", \"type\": \"double\" }" + " ]" + " }" + " }" + " ]" + "}"; + + static char *strings[] = { + "Four score and seven years ago", + "our father brought forth on this continent", + "a new nation", "conceived in Liberty", + "and dedicated to the proposition that all men are created equal." + }; + static const unsigned int NUM_STRINGS = + sizeof(strings) / sizeof(strings[0]); + + static char buf[4096]; + avro_reader_t reader = avro_reader_memory(buf, sizeof(buf)); + avro_writer_t writer = avro_writer_memory(buf, sizeof(buf)); + + avro_schema_t schema = NULL; + avro_schema_error_t error = NULL; + avro_schema_from_json(schema_json, strlen(schema_json), + &schema, &error); + + unsigned long i; + + avro_value_iface_t *iface = avro_generic_class_from_schema(schema); + + avro_value_t val; + avro_generic_value_new(iface, &val); + + avro_value_t out; + avro_generic_value_new(iface, &out); + + for (i = 0; i < num_tests; i++) { + avro_value_t field; + + avro_value_get_by_name(&val, "i", &field, NULL); + avro_value_set_int(&field, rand_int32()); + + avro_value_get_by_name(&val, "l", &field, NULL); + avro_value_set_long(&field, rand_int64()); + + avro_wrapped_buffer_t wbuf; + avro_wrapped_buffer_new_string(&wbuf, strings[i % NUM_STRINGS]); + avro_value_get_by_name(&val, "s", &field, NULL); + avro_value_give_string_len(&field, &wbuf); + + avro_value_t subrec; + avro_value_get_by_name(&val, "subrec", &subrec, NULL); + + avro_value_get_by_name(&subrec, "f", &field, NULL); + avro_value_set_float(&field, rand_number(-1e10, 1e10)); + + avro_value_get_by_name(&subrec, "d", &field, NULL); + avro_value_set_double(&field, rand_number(-1e10, 1e10)); + + avro_writer_reset(writer); + avro_value_write(writer, &val); + + avro_reader_reset(reader); + avro_value_read(reader, &out); + + if (! avro_value_equal_fast(&val, &out) ) { + printf("Broken\n"); + exit (1); + } + } + + avro_value_decref(&val); + avro_value_decref(&out); + avro_value_iface_decref(iface); + avro_schema_decref(schema); + avro_writer_free(writer); + avro_reader_free(reader); +} + + + +/** + * Helper function to test the performance of serializing and + * deserializing a given avro value using the provided function to + * populate avro value using the new value API. Allows testing using + * matching schemas or using schema resolution. + */ + +static void +test_generic_helper( unsigned long num_tests, + int resolution_type, + const char *schema_json, + void (*populate_value_func)(avro_value_t *, + unsigned long) + ) +{ + static char buf[4096]; + + avro_reader_t reader = avro_reader_memory(buf, sizeof(buf)); + avro_writer_t writer = avro_writer_memory(buf, sizeof(buf)); + + avro_schema_t schema = NULL; + avro_schema_error_t error = NULL; + avro_schema_from_json(schema_json, strlen(schema_json), + &schema, &error); + + unsigned long i; + + avro_value_iface_t *writer_iface = avro_generic_class_from_schema(schema); + avro_value_iface_t *reader_iface = avro_generic_class_from_schema(schema); + + avro_value_t val; + avro_generic_value_new(writer_iface, &val); + + avro_value_t out; + avro_generic_value_new(reader_iface, &out); + + /* Use resolved reader to resolve schemas while writing data to memory */ + avro_value_iface_t *resolved_reader_iface = NULL; + avro_value_t resolved_reader_value; + if ( resolution_type & USE_RESOLVED_READER ) { + resolved_reader_iface = avro_resolved_reader_new( schema, schema ); + avro_resolved_reader_new_value( resolved_reader_iface, + &resolved_reader_value ); + avro_resolved_reader_set_source( &resolved_reader_value, &val ); + } + + /* Use resolved writer to resolve schemas while reading data from memory */ + avro_value_iface_t *resolved_writer_iface = NULL; + avro_value_t resolved_writer_value; + if ( resolution_type & USE_RESOLVED_WRITER ) { + resolved_writer_iface = avro_resolved_writer_new( schema, schema ); + avro_resolved_writer_new_value( resolved_writer_iface, + &resolved_writer_value ); + avro_resolved_writer_set_dest( &resolved_writer_value, &out ); + } + + /* Set up pointers */ + avro_value_t *p_value_to_write_to_memory = NULL; + avro_value_t *p_value_to_read_from_memory = NULL; + + if ( resolution_type == USE_MATCHED_SCHEMAS ) { + p_value_to_write_to_memory = &val; + p_value_to_read_from_memory = &out; + } + else if ( resolution_type == USE_RESOLVED_READER ) { + p_value_to_write_to_memory = &resolved_reader_value; + p_value_to_read_from_memory = &out; + } + else if ( resolution_type == USE_RESOLVED_WRITER ) { + p_value_to_write_to_memory = &val; + p_value_to_read_from_memory = &resolved_writer_value; + } + else if ( resolution_type == USE_BOTH_RESOLVED ) { + p_value_to_write_to_memory = &resolved_reader_value; + p_value_to_read_from_memory = &resolved_writer_value; + } + + /* Perform the tests */ + for (i = 0; i < num_tests; i++) { + + avro_value_reset(&val); + + /* Execute the function to populate the Avro Value */ + (*populate_value_func)(&val, i); + + avro_writer_reset(writer); + avro_value_write(writer, p_value_to_write_to_memory); + + avro_reader_reset(reader); + avro_value_read(reader, p_value_to_read_from_memory); + + if (! avro_value_equal_fast(&val, &out) ) { + printf("Broken\n"); + exit (1); + } + } + + avro_value_decref(&val); + avro_value_decref(&out); + if ( resolution_type & USE_RESOLVED_READER ) { + avro_value_decref(&resolved_reader_value); + avro_value_iface_decref(resolved_reader_iface); + } + if ( resolution_type & USE_RESOLVED_WRITER ) { + avro_value_decref(&resolved_writer_value); + avro_value_iface_decref(resolved_writer_iface); + } + avro_value_iface_decref(writer_iface); + avro_value_iface_decref(reader_iface); + avro_schema_decref(schema); + avro_writer_free(writer); + avro_reader_free(reader); +} + + + + +/** + * Helper function to populate a somewhat complex record type using + * the new value API, retrieving record fields by index. + */ + +static const char *complex_record_schema_json = + "{" + " \"type\": \"record\"," + " \"name\": \"test\"," + " \"fields\": [" + " { \"name\": \"i\", \"type\": \"int\" }," + " { \"name\": \"l\", \"type\": \"long\" }," + " { \"name\": \"s\", \"type\": \"string\" }," + " {" + " \"name\": \"subrec\"," + " \"type\": {" + " \"type\": \"record\"," + " \"name\": \"sub\"," + " \"fields\": [" + " { \"name\": \"f\", \"type\": \"float\" }," + " { \"name\": \"d\", \"type\": \"double\" }" + " ]" + " }" + " }" + " ]" + "}"; + + + +static void +populate_complex_record(avro_value_t *p_val, unsigned long i) +{ + static char *strings[] = { + "Four score and seven years ago", + "our father brought forth on this continent", + "a new nation", "conceived in Liberty", + "and dedicated to the proposition that all men are created equal." + }; + static const unsigned int NUM_STRINGS = + sizeof(strings) / sizeof(strings[0]); + + avro_value_t field; + + avro_value_get_by_index(p_val, 0, &field, NULL); + avro_value_set_int(&field, rand_int32()); + + avro_value_get_by_index(p_val, 1, &field, NULL); + avro_value_set_long(&field, rand_int64()); + + avro_wrapped_buffer_t wbuf; + avro_wrapped_buffer_new_string(&wbuf, strings[i % NUM_STRINGS]); + avro_value_get_by_index(p_val, 2, &field, NULL); + avro_value_give_string_len(&field, &wbuf); + + avro_value_t subrec; + avro_value_get_by_index(p_val, 3, &subrec, NULL); + + avro_value_get_by_index(&subrec, 0, &field, NULL); + avro_value_set_float(&field, rand_number(-1e10, 1e10)); + + avro_value_get_by_index(&subrec, 1, &field, NULL); + avro_value_set_double(&field, rand_number(-1e10, 1e10)); + +} + + + +/** + * Tests the performance of serializing and deserializing a somewhat + * complex record type using the new value API, retrieving record + * fields by index. The functionality is almost identical to + * test_nested_record_value_by_index(), however, there may be some + * overhead of using function calls instead of inline code, and + * running some additional "if" statements.. + */ + +static void +test_nested_record_value_by_index_matched_schemas(unsigned long num_tests) +{ + test_generic_helper(num_tests, + USE_MATCHED_SCHEMAS, + complex_record_schema_json, + populate_complex_record); +} + + +/** + * Tests the performance of serializing and deserializing a somewhat + * complex record type using the new value API, retrieving record + * fields by index. Uses a resolved_writer to resolve between two + * (identical) schemas when reading the array. + */ + +static void +test_nested_record_value_by_index_resolved_writer(unsigned long num_tests) +{ + test_generic_helper(num_tests, + USE_RESOLVED_WRITER, + complex_record_schema_json, + populate_complex_record); +} + + + +/** + * Tests the performance of serializing and deserializing a somewhat + * complex record type using the new value API, retrieving record + * fields by index. Uses a resolved_reader to resolve between two + * (identical) schemas when writing the array. + */ + +static void +test_nested_record_value_by_index_resolved_reader(unsigned long num_tests) +{ + test_generic_helper(num_tests, + USE_RESOLVED_READER, + complex_record_schema_json, + populate_complex_record); +} + + + +/** + * Helper function to test the performance of serializing and + * deserializing a simple array using the new value API. Allows + * testing using matching schemas or using schema resolution. + */ + +static const char *simple_array_schema_json = + "{\"name\": \"a\", \"type\": \"array\", \"items\":\"long\"}"; + +static void +populate_simple_array(avro_value_t *p_val, unsigned long i) +{ + const size_t array_length = 21; + avro_value_t field; + size_t idx; + size_t dummy_index; + (void) i; + + for ( idx = 0; idx < array_length; idx++ ) { + avro_value_append(p_val, &field, &dummy_index); + avro_value_set_long(&field, rand_int64()); + } +} + + + +/** + * Tests the performance of serializing and deserializing a simple + * array using the new value API. + */ + +static void +test_simple_array(unsigned long num_tests) +{ + test_generic_helper(num_tests, + USE_MATCHED_SCHEMAS, + simple_array_schema_json, + populate_simple_array); +} + + + +/** + * Tests the performance of serializing and deserializing a simple + * array using the new value API, using a resolved writer to resolve + * between (identical) reader and writer schemas, when reading the + * array. + */ +static void +test_simple_array_resolved_writer(unsigned long num_tests) +{ + test_generic_helper(num_tests, + USE_RESOLVED_WRITER, + simple_array_schema_json, + populate_simple_array); +} + + + +/** + * Tests the performance of serializing and deserializing a simple + * array using the new value API, using a resolved reader to resolve + * between (identical) reader and writer schemas, when writing the + * array. + */ + +static void +test_simple_array_resolved_reader(unsigned long num_tests) +{ + test_generic_helper(num_tests, + USE_RESOLVED_READER, + simple_array_schema_json, + populate_simple_array); +} + + + + +/** + * Helper function to test the performance of serializing and + * deserializing a nested array using the new value API. Allows + * testing using matching schemas or using schema resolution. + */ + +static const char *nested_array_schema_json = + "{\"type\":\"array\", \"items\": {\"type\": \"array\", \"items\": \"long\"}}"; + + +static void +populate_nested_array(avro_value_t *p_val, unsigned long i) +{ + + const size_t array_length = 7; + const size_t subarray_length = 3; + avro_value_t subarray; + avro_value_t field; + size_t idx; + size_t jdx; + size_t dummy_index; + (void) i; + + for ( idx = 0; idx < array_length; idx++ ) { + avro_value_append(p_val, &subarray, &dummy_index); + for ( jdx = 0; jdx < subarray_length; jdx ++ ) { + avro_value_append(&subarray, &field, &dummy_index); + avro_value_set_long(&field, rand_int64()); + } + } +} + + +/** + * Tests the performance of serializing and deserializing a nested + * array using the new value API. + */ + +static void +test_nested_array(unsigned long num_tests) +{ + test_generic_helper(num_tests, + USE_MATCHED_SCHEMAS, + nested_array_schema_json, + populate_nested_array); +} + + +/** + * Tests the performance of serializing and deserializing a nested + * array using the new value API, using a resolved writer to resolve + * between (identical) reader and writer schemas, when reading the + * array. + */ + +static void +test_nested_array_resolved_writer(unsigned long num_tests) +{ + test_generic_helper(num_tests, + USE_RESOLVED_WRITER, + nested_array_schema_json, + populate_nested_array); +} + + +/** + * Tests the performance of serializing and deserializing a nested + * array using the new value API, using a resolved reader to resolve + * between (identical) reader and writer schemas, when writing the + * array. + */ + +static void +test_nested_array_resolved_reader(unsigned long num_tests) +{ + test_generic_helper(num_tests, + USE_RESOLVED_READER, + nested_array_schema_json, + populate_nested_array); +} + + + +/** + * Test harness + */ + +#define NUM_RUNS 3 + +int +main(int argc, char **argv) +{ + AVRO_UNUSED(argc); + AVRO_UNUSED(argv); + + init_rand(); + + unsigned int i; + struct avro_tests { + const char *name; + unsigned long num_tests; + test_func_t func; + } tests[] = { + { "refcount", 100000000, + test_refcount }, + { "nested record (legacy)", 100000, + test_nested_record_datum }, + { "nested record (value by index)", 1000000, + test_nested_record_value_by_index }, + { "nested record (value by name)", 1000000, + test_nested_record_value_by_name }, + { "nested record (value by index) matched schemas", 1000000, + test_nested_record_value_by_index_matched_schemas }, + { "nested record (value by index) resolved writer", 1000000, + test_nested_record_value_by_index_resolved_writer }, + { "nested record (value by index) resolved reader", 1000000, + test_nested_record_value_by_index_resolved_reader }, + { "simple array matched schemas", 250000, + test_simple_array }, + { "simple array resolved writer", 250000, + test_simple_array_resolved_writer }, + { "simple array resolved reader", 250000, + test_simple_array_resolved_reader }, + { "nested array matched schemas", 250000, + test_nested_array }, + { "nested array resolved writer", 250000, + test_nested_array_resolved_writer }, + { "nested array resolved reader", 250000, + test_nested_array_resolved_reader }, + }; + + for (i = 0; i < sizeof(tests) / sizeof(tests[0]); i++) { + fprintf(stderr, "**** Running %s ****\n %lu tests per run\n", + tests[i].name, tests[i].num_tests); + unsigned int run; + + double sum = 0.0; + + for (run = 1; run <= NUM_RUNS; run++) { + fprintf(stderr, " Run %u\n", run); + + clock_t before = clock(); + tests[i].func(tests[i].num_tests); + clock_t after = clock(); + double secs = ((double) after-before) / CLOCKS_PER_SEC; + sum += secs; + } + + fprintf(stderr, " Average time: %.03lfs\n", sum / NUM_RUNS); + fprintf(stderr, " Tests/sec: %.0lf\n", + tests[i].num_tests / (sum / NUM_RUNS)); + } + + return EXIT_SUCCESS; +} diff --git a/fluent-bit/lib/avro/tests/schema_tests/fail/enum_nonarray_symbols b/fluent-bit/lib/avro/tests/schema_tests/fail/enum_nonarray_symbols new file mode 100644 index 00000000..f4dae950 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/fail/enum_nonarray_symbols @@ -0,0 +1,3 @@ +{"type": "enum", + "name": "Status", + "symbols": "Normal Caution Critical"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/fail/enum_nonstring_name b/fluent-bit/lib/avro/tests/schema_tests/fail/enum_nonstring_name new file mode 100644 index 00000000..baa13d95 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/fail/enum_nonstring_name @@ -0,0 +1,3 @@ +{"type": "enum", + "name": [ 0, 1, 1, 2, 3, 5, 8 ], + "symbols": ["Golden", "Mean"]} diff --git a/fluent-bit/lib/avro/tests/schema_tests/fail/enum_without_name b/fluent-bit/lib/avro/tests/schema_tests/fail/enum_without_name new file mode 100644 index 00000000..57f68532 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/fail/enum_without_name @@ -0,0 +1,3 @@ +{"type": "enum" + "symbols" : ["I", "will", "fail", "no", "name"] +} diff --git a/fluent-bit/lib/avro/tests/schema_tests/fail/fixed_without_name b/fluent-bit/lib/avro/tests/schema_tests/fail/fixed_without_name new file mode 100644 index 00000000..fbf96abe --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/fail/fixed_without_name @@ -0,0 +1,2 @@ +{"type": "fixed", + "size": 314} diff --git a/fluent-bit/lib/avro/tests/schema_tests/fail/fixed_without_size b/fluent-bit/lib/avro/tests/schema_tests/fail/fixed_without_size new file mode 100644 index 00000000..15c5e789 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/fail/fixed_without_size @@ -0,0 +1,2 @@ +{"type": "fixed", + "name": "Missing size"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/fail/illegal_type b/fluent-bit/lib/avro/tests/schema_tests/fail/illegal_type new file mode 100644 index 00000000..e65c0463 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/fail/illegal_type @@ -0,0 +1 @@ +{"type":"panther"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/fail/invalid_avro_id b/fluent-bit/lib/avro/tests/schema_tests/fail/invalid_avro_id new file mode 100644 index 00000000..c684e7de --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/fail/invalid_avro_id @@ -0,0 +1,3 @@ +{ "name" : "2d2", + "type": "enum", + "symbols" : [ "c3po" ] } diff --git a/fluent-bit/lib/avro/tests/schema_tests/fail/record_with_field_missing_name b/fluent-bit/lib/avro/tests/schema_tests/fail/record_with_field_missing_name new file mode 100644 index 00000000..ba62d52e --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/fail/record_with_field_missing_name @@ -0,0 +1,5 @@ +{"type": "record", + "name": "Address", + "fields": [ + {"type": "string"}, + {"type": "string", "name": "City"}]} diff --git a/fluent-bit/lib/avro/tests/schema_tests/fail/record_with_field_missing_type b/fluent-bit/lib/avro/tests/schema_tests/fail/record_with_field_missing_type new file mode 100644 index 00000000..b449f3b4 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/fail/record_with_field_missing_type @@ -0,0 +1,5 @@ +{"type": "record", + "name": "Event", + "fields": [ + { "name": "Sponsor"}, + { "name": "City", "type": "string"}]} diff --git a/fluent-bit/lib/avro/tests/schema_tests/fail/record_with_invalid_reference b/fluent-bit/lib/avro/tests/schema_tests/fail/record_with_invalid_reference new file mode 100644 index 00000000..49b35902 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/fail/record_with_invalid_reference @@ -0,0 +1,7 @@ +{ "type": "record", + "name": "recursive", + "fields": [ + { "name": "label", "type": "string" }, + { "name": "children", "type": {"type": "array", "items": "foobar"} } + ] +} diff --git a/fluent-bit/lib/avro/tests/schema_tests/fail/record_with_nonarray_fields b/fluent-bit/lib/avro/tests/schema_tests/fail/record_with_nonarray_fields new file mode 100644 index 00000000..b81fbe32 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/fail/record_with_nonarray_fields @@ -0,0 +1,3 @@ +{ "type": "record", + "fields": "His vision, from the constantly passing bars," + "name", "Rainer" } diff --git a/fluent-bit/lib/avro/tests/schema_tests/fail/record_with_nonstring_name b/fluent-bit/lib/avro/tests/schema_tests/fail/record_with_nonstring_name new file mode 100644 index 00000000..0ded9c55 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/fail/record_with_nonstring_name @@ -0,0 +1,3 @@ +{"name": ["Tom", "Jerry"], + "type": "record", + "fields": [ {"name": "name", "type": "string"} ]} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/array b/fluent-bit/lib/avro/tests/schema_tests/pass/array new file mode 100644 index 00000000..d6950491 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/array @@ -0,0 +1 @@ +{"type": "array", "items": "long"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/boolean_full b/fluent-bit/lib/avro/tests/schema_tests/pass/boolean_full new file mode 100644 index 00000000..69d35799 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/boolean_full @@ -0,0 +1 @@ +{"type":"boolean"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/bytes_full b/fluent-bit/lib/avro/tests/schema_tests/pass/bytes_full new file mode 100644 index 00000000..3b91ef01 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/bytes_full @@ -0,0 +1 @@ +{"type":"bytes"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/double_full b/fluent-bit/lib/avro/tests/schema_tests/pass/double_full new file mode 100644 index 00000000..dbd22f78 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/double_full @@ -0,0 +1 @@ +{"type":"double"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/enum b/fluent-bit/lib/avro/tests/schema_tests/pass/enum new file mode 100644 index 00000000..749b0a3c --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/enum @@ -0,0 +1,4 @@ +{ "type": "enum", + "name": "three_stooges", + "symbols" : [ "Moe", "Larry", "Curly" ] +} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/extra_attributes b/fluent-bit/lib/avro/tests/schema_tests/pass/extra_attributes new file mode 100644 index 00000000..49885b9d --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/extra_attributes @@ -0,0 +1 @@ +{"type":"string", "ignored": "value"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/fixed b/fluent-bit/lib/avro/tests/schema_tests/pass/fixed new file mode 100644 index 00000000..0449ebca --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/fixed @@ -0,0 +1 @@ +{"type": "fixed", "size": 16, "name": "md5"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/float_full b/fluent-bit/lib/avro/tests/schema_tests/pass/float_full new file mode 100644 index 00000000..fbd11642 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/float_full @@ -0,0 +1 @@ +{"type":"float"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/int_full b/fluent-bit/lib/avro/tests/schema_tests/pass/int_full new file mode 100644 index 00000000..92b134da --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/int_full @@ -0,0 +1 @@ +{"type":"int"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/interop.avsc b/fluent-bit/lib/avro/tests/schema_tests/pass/interop.avsc new file mode 100644 index 00000000..8cfbba22 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/interop.avsc @@ -0,0 +1,28 @@ +{"type": "record", "name":"Interop", "namespace": "org.apache.avro", + "fields": [ + {"name": "intField", "type": "int"}, + {"name": "longField", "type": "long"}, + {"name": "stringField", "type": "string"}, + {"name": "boolField", "type": "boolean"}, + {"name": "floatField", "type": "float"}, + {"name": "doubleField", "type": "double"}, + {"name": "bytesField", "type": "bytes"}, + {"name": "nullField", "type": "null"}, + {"name": "arrayField", "type": {"type": "array", "items": "double"}}, + {"name": "mapField", "type": + {"type": "map", "values": + {"type": "record", "name": "Foo", + "fields": [{"name": "label", "type": "string"}]}}}, + {"name": "unionField", "type": + ["boolean", "double", {"type": "array", "items": "bytes"}]}, + {"name": "enumField", "type": + {"type": "enum", "name": "Kind", "symbols": ["A","B","C"]}}, + {"name": "fixedField", "type": + {"type": "fixed", "name": "MD5", "size": 16}}, + {"name": "recordField", "type": + {"type": "record", "name": "Node", + "fields": [ + {"name": "label", "type": "string"}, + {"name": "children", "type": {"type": "array", "items": "Node"}}]}} + ] +} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/long_full b/fluent-bit/lib/avro/tests/schema_tests/pass/long_full new file mode 100644 index 00000000..ccfd9170 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/long_full @@ -0,0 +1 @@ +{"type":"long"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/map b/fluent-bit/lib/avro/tests/schema_tests/pass/map new file mode 100644 index 00000000..436d961d --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/map @@ -0,0 +1 @@ +{"type" : "map", "values": "long"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_enum b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_enum new file mode 100644 index 00000000..5b77b353 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_enum @@ -0,0 +1,9 @@ +{"type": "record", "name": "org.apache.avro.tests.Hello", "fields": [ + {"name": "f1", "type": {"type": "enum", "name": "MyEnum", "symbols": ["Foo", "Bar", "Baz"]}}, + {"name": "f2", "type": "org.apache.avro.tests.MyEnum"}, + {"name": "f3", "type": "MyEnum"}, + {"name": "f4", "type": {"type": "enum", "name": "other.namespace.OtherEnum", "symbols": ["one", "two", "three"]}}, + {"name": "f5", "type": "other.namespace.OtherEnum"}, + {"name": "f6", "type": {"type": "enum", "name": "ThirdEnum", "namespace": "some.other", "symbols": ["Alice", "Bob"]}}, + {"name": "f7", "type": "some.other.ThirdEnum"} +]} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_fixed b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_fixed new file mode 100644 index 00000000..f621e797 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_fixed @@ -0,0 +1,9 @@ +{"type": "record", "name": "org.apache.avro.tests.Hello", "fields": [ + {"name": "f1", "type": {"type": "fixed", "name": "MyFixed", "size": 16}}, + {"name": "f2", "type": "org.apache.avro.tests.MyFixed"}, + {"name": "f3", "type": "MyFixed"}, + {"name": "f4", "type": {"type": "fixed", "name": "other.namespace.OtherFixed", "size": 18}}, + {"name": "f5", "type": "other.namespace.OtherFixed"}, + {"name": "f6", "type": {"type": "fixed", "name": "ThirdFixed", "namespace": "some.other", "size": 20}}, + {"name": "f7", "type": "some.other.ThirdFixed"} +]} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_fullname b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_fullname new file mode 100644 index 00000000..0cc24564 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_fullname @@ -0,0 +1,8 @@ +{"type": "record", "name": "x.Y", "fields": [ + {"name": "e", "type": + {"type": "record", "name": "Z", "fields": [ + {"name": "f", "type": "x.Y"}, + {"name": "g", "type": "x.Z"} + ]} + } +]} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_null_enum b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_null_enum new file mode 100644 index 00000000..3c3a7452 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_null_enum @@ -0,0 +1,8 @@ +{"type": "record", "name": "R", "fields": [ + {"name": "s", "type": {"type": "record", "namespace": "x", "name": "Y", "fields": [ + {"name": "e", "type": {"type": "enum", "namespace": "", "name": "Z", + "symbols": ["Foo", "Bar"]} + } + ]}}, + {"name": "t", "type": "Z"} +]} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_null_fixed b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_null_fixed new file mode 100644 index 00000000..a3aa5701 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_null_fixed @@ -0,0 +1,7 @@ +{"type": "record", "name": "R", "fields": [ + {"name": "s", "type": {"type": "record", "namespace": "x", "name": "Y", "fields": [ + {"name": "e", "type": {"type": "fixed", "namespace": "", "name": "Z", "size": 8} + } + ]}}, + {"name": "t", "type": "Z"} +]} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_null_record b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_null_record new file mode 100644 index 00000000..4b18dd54 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_null_record @@ -0,0 +1,8 @@ +{"type": "record", "name": "R", "fields": [ + {"name": "s", "type": {"type": "record", "namespace": "x", "name": "Y", "fields": [ + {"name": "e", "type": {"type": "record", "namespace": "", "name": "Z", "fields": [ + {"name": "f", "type": "Z"} + ]}} + ]}}, + {"name": "t", "type": "Z"} +]} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_recursive b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_recursive new file mode 100644 index 00000000..3c2d0eb7 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_recursive @@ -0,0 +1,28 @@ +{ "type": "record", + "name": "Container", + "namespace": "namespace1", + "fields": [ + { "name": "contained", + "type": { "type": "record", + "name": "MutuallyRecursive", + "fields": [ + { "name": "label", "type": "string" }, + { "name": "children", + "type": {"type": "array", "items": + {"type": "record", + "name": "MutuallyRecursive", + "namespace": "namespace2", + "fields": [ + { "name": "value", "type": "int" }, + { "name": "children", "type": {"type": "array", "items": "namespace1.MutuallyRecursive" }}, + { "name": "morechildren", "type": {"type": "array", "items": "MutuallyRecursive" }} + ] + } + } + }, + { "name": "anotherchild", "type": "namespace2.MutuallyRecursive"} + ] + } + } + ] +} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_simple b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_simple new file mode 100644 index 00000000..f5a117f4 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_simple @@ -0,0 +1,5 @@ +{"type": "record", "namespace": "x", "name": "Y", "fields": [ + {"name": "e", "type": {"type": "record", "name": "Z", "fields": [ + {"name": "f", "type": "x.Z"} + ]}} +]} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/null_full b/fluent-bit/lib/avro/tests/schema_tests/pass/null_full new file mode 100644 index 00000000..cae87673 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/null_full @@ -0,0 +1 @@ +{"type":"null"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/record b/fluent-bit/lib/avro/tests/schema_tests/pass/record new file mode 100644 index 00000000..43ac456f --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/record @@ -0,0 +1,5 @@ +{"name": "person", + "type": "record", + "fields": [ {"name": "height", "type": "long"}, + {"name": "weight", "type": "long"}, + {"name": "name", "type": "string"}]} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/record_fields_with_defaults b/fluent-bit/lib/avro/tests/schema_tests/pass/record_fields_with_defaults new file mode 100644 index 00000000..545ccbbc --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/record_fields_with_defaults @@ -0,0 +1,6 @@ +{"name": "person", + "type": "record", + "fields": [ {"name": "height", "type": "long"}, + {"name": "weight", "type": "long"}, + {"name": "name", "type": "string"}, + {"name": "hacker", "type": "boolean", "default": false}]} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/record_no_fields b/fluent-bit/lib/avro/tests/schema_tests/pass/record_no_fields new file mode 100644 index 00000000..142f4527 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/record_no_fields @@ -0,0 +1 @@ +{"type": "record", "name": "R", "fields": []} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/recursive_record b/fluent-bit/lib/avro/tests/schema_tests/pass/recursive_record new file mode 100644 index 00000000..0967bb4a --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/recursive_record @@ -0,0 +1,7 @@ +{ "type": "record", + "name": "recursive", + "fields": [ + { "name": "label", "type": "string" }, + { "name": "children", "type": {"type": "array", "items": "recursive"} } + ] +} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/string_extra_attributes b/fluent-bit/lib/avro/tests/schema_tests/pass/string_extra_attributes new file mode 100644 index 00000000..49885b9d --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/string_extra_attributes @@ -0,0 +1 @@ +{"type":"string", "ignored": "value"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/string_full b/fluent-bit/lib/avro/tests/schema_tests/pass/string_full new file mode 100644 index 00000000..5566b9f8 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/string_full @@ -0,0 +1 @@ +{"type": "string"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/union b/fluent-bit/lib/avro/tests/schema_tests/pass/union new file mode 100644 index 00000000..ef2b6ecf --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/union @@ -0,0 +1 @@ +["string", "long", "null"] diff --git a/fluent-bit/lib/avro/tests/test_avro_1034.c b/fluent-bit/lib/avro/tests/test_avro_1034.c new file mode 100644 index 00000000..b44d6e40 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_1034.c @@ -0,0 +1,395 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <avro.h> +#include <stdio.h> +#include <stdlib.h> + +/* Test code for JIRA Issue AVRO-1034. + * + * AVRO-1034: Resolved reader does not initialize children of arrays, + * resulting in seg faults + * + * This program tests schema resolution for nested arrays. For the + * purposes of this test, there are two schemas "old" and "new" which + * are created by reading the same JSON schema. + * + * The test creates and populates a nested array avro value, and + * serializes it to memory. The raw memory is written to a file. Note + * that the schema is not written to the file. The nested array is + * also printed to the screen. + * + * An identical nested array avro value is then created. A + * resolved_reader_class and a corresponding resolved_record instance + * is created (using identical "writer" and "reader" schemas for + * simplicity), and an attempt is made to "read" the resolved avro + * value. + * + * Once the resolved value has been read, the source value (nested) + * and the resolved value (resolved_record) are both reset using + * avro_value_reset(). Then the source value (nested) is populated + * with another (larger) nested array. Then an attempt is made to read + * the resolved avro value again. + * + * This second attempt to read the resolved value results in a + * segmentation fault under Linux, using the patch in + * https://issues.apache.org/jira/secure/attachment/12516487/0001-AVRO-1034.-C-Resolved-reader-initializes-child-array.patch. + * + * However, the program does not seg fault, using the patch in + * https://issues.apache.org/jira/secure/attachment/12515544/AVRO-1034.patch + * + * AVRO-C was compiled with CMAKE_INSTALL_PREFIX=avrolib + * The static library (libavro.a) was copied into a subdirectory of avrolib/lib/static + * + * This file was compiled under Linux using: + * gcc -g avro-1034-test-2.c -o test2 -I../../build/avrolib/include -L../../build/avrolib/lib/static -lavro + * + */ + + +// Encode the following json string in NESTED_ARRAY +// {"type":"array", "items": {"type": "array", "items": "long"}} +// +#define NESTED_ARRAY \ + "{\"type\":\"array\", \"items\": {\"type\": \"array\", \"items\": \"long\"}}" + +avro_schema_t schema_old = NULL; +avro_schema_t schema_new = NULL; + +/* Parse schema into a schema data structure */ +void init_schema(void) +{ + avro_schema_error_t error; + if (avro_schema_from_json(NESTED_ARRAY, sizeof(NESTED_ARRAY), + &schema_old, &error)) { + printf( "Unable to parse old schema\n"); + exit(EXIT_FAILURE); + } + + if (avro_schema_from_json(NESTED_ARRAY, sizeof(NESTED_ARRAY), + &schema_new, &error)) { + printf( "Unable to parse new schema\n"); + exit(EXIT_FAILURE); + } +} + +#define try(call, msg) \ + do { \ + if (call) { \ + printf( msg ":\n %s\n", avro_strerror()); \ + exit (EXIT_FAILURE); \ + } \ + } while (0) + + +/* The input avro_value_t p_array should contain a nested array. + * Print the fields of this nested array to the screen. + */ +int print_array_fields ( avro_value_t *p_array ) +{ + size_t idx; + size_t length; + avro_type_t val_type; + + val_type = avro_value_get_type( p_array ); + printf( "Main array type = %d\n", val_type ); + + try( avro_value_get_size( p_array, &length ), + "Couldn't get array size" ); + printf( "Main array length = %d\n", (int) length ); + + for ( idx = 0; idx < length; idx ++ ) + { + avro_value_t subarray; + size_t sublength; + size_t jdx; + const char *unused; + + try ( avro_value_get_by_index( p_array, idx, &subarray, &unused ), + "Couldn't get subarray" ); + + val_type = avro_value_get_type( &subarray ); + printf( "Subarray type = %d\n", val_type ); + + try( avro_value_get_size( &subarray, &sublength ), + "Couldn't get subarray size" ); + printf( "Subarray length = %d\n", (int) sublength ); + + for ( jdx = 0; jdx < sublength; jdx++ ) + { + avro_value_t element; + int64_t val; + + try ( avro_value_get_by_index( &subarray, jdx, &element, &unused ), + "Couldn't get subarray element" ); + + val_type = avro_value_get_type( &element ); + + try ( avro_value_get_long( &element, &val ), + "Couldn't get subarray element value" ); + + printf( "nested_array[%d][%d]: type = %d value = %lld\n", + (int) idx, (int) jdx, (int) val_type, (long long) val ); + + } + } + + return 0; +} + + +/* The input avro_value_t p_subarray should contain an array of long + * integers. Add "elements" number of long integers to this array. Set + * the values to be distinct based on the iteration parameter. + */ +int add_subarray( avro_value_t *p_subarray, + size_t elements, + int32_t iteration ) +{ + avro_value_t element; + size_t index; + size_t idx; + + for ( idx = 0; idx < elements; idx ++ ) + { + // Append avro array element to subarray + try ( avro_value_append( p_subarray, &element, &index ), + "Error appending element in subarray" ); + + try ( avro_value_set_long( &element, (iteration+1)*100 + (iteration+1) ), + "Error setting subarray element" ); + } + + return 0; +} + +int populate_array( avro_value_t *p_array, int32_t elements ) +{ + int32_t idx; + fprintf( stderr, "Elements = %d\n", elements); + for ( idx = 0; idx < elements; idx ++ ) + { + avro_value_t subarray; + size_t index; + + // Append avro array element for top level array + try ( avro_value_append( p_array, &subarray, &index ), + "Error appending subarray" ); + + // Populate array element with subarray of length 2 +#define SUBARRAY_LENGTH (2) + try ( add_subarray( &subarray, SUBARRAY_LENGTH, idx ), + "Error populating subarray" ); + } + return 0; +} + + +/* Create a nested array using the schema NESTED_ARRAY. Populate its + * elements with unique values. Serialize the nested array to the + * memory buffer in avro_writer_t. The number of elements in the first + * dimension of the nested array is "elements". The number of elements + * in the second dimension of the nested array is hardcoded to 2. + */ +int add_array( avro_writer_t writer, + int32_t elements, + int use_resolving_writer ) +{ + avro_schema_t chosen_schema; + avro_value_iface_t *nested_array_class; + avro_value_t nested; + + // Select (hardcode) schema to use + chosen_schema = schema_old; + + // Create avro class and value + nested_array_class = avro_generic_class_from_schema( chosen_schema ); + try ( avro_generic_value_new( nested_array_class, &nested ), + "Error creating instance of record" ); + + try ( populate_array( &nested, elements ), + "Error populating array" ); + + if ( use_resolving_writer ) + { + // Resolve schema differences + avro_value_iface_t *resolved_reader_class; + avro_value_iface_t *writer_class; + avro_value_t resolved_record; + + // Note - we will read values from the reader of "schema to write + // to file" and we will copy them into a writer of the same + // schema. + resolved_reader_class = avro_resolved_reader_new( schema_old,// schema populated above + schema_new // schema_to_write_to_file + ); + if ( resolved_reader_class == NULL ) + { + printf( "Failed avro_resolved_reader_new()\n"); + exit( EXIT_FAILURE ); + } + + try ( avro_resolved_reader_new_value( resolved_reader_class, &resolved_record ), + "Failed avro_resolved_reader_new_value" ); + + // Map the resolved reader to the record you want to get data from + avro_resolved_reader_set_source( &resolved_record, &nested ); + + // Now the resolved_record is mapped to read data from record. Now + // we need to copy the data from resolved_record into a + // writer_record, which is an instance of the same schema as + // resolved_record. + + // Create a writer of the schema you want to write using + writer_class = avro_generic_class_from_schema( schema_new ); + if ( writer_class == NULL ) + { + printf( "Failed avro_generic_class_from_schema()\n"); + exit( EXIT_FAILURE ); + } + + try ( avro_value_write( writer, &resolved_record ), + "Unable to write record into memory using writer_record" ); + + print_array_fields( &resolved_record ); + + avro_value_reset( &nested ); + + // Question: Is it permissible to call avro_value_reset() on a + // resolved_record? Set the #if 1 to #if 0 to disable the + // avro_value_reset(), to prevent the segmentation fault. + #if 1 + avro_value_reset( &resolved_record ); + #endif + + try ( populate_array( &nested, 2*elements ), + "Error populating array" ); + + try ( avro_value_write( writer, &resolved_record ), + "Unable to write record into memory using writer_record" ); + + print_array_fields( &resolved_record ); + + avro_value_decref( &resolved_record ); + avro_value_iface_decref( writer_class ); + avro_value_iface_decref( resolved_reader_class ); + } + else + { + // Write the value to memory + try ( avro_value_write( writer, &nested ), + "Unable to write nested into memory" ); + + print_array_fields( &nested ); + } + + + // Release the record + avro_value_decref( &nested ); + avro_value_iface_decref( nested_array_class ); + + return 0; +} + +/* Create a raw binary file containing a serialized version of a + * nested array. This file will later be read by + * read_nested_array_file(). + */ +int write_nested_array_file ( int64_t buf_len, + const char *raw_binary_file_name, + int use_resolving_writer ) +{ + char *buf; + avro_writer_t nested_writer; + FILE *fid = NULL; + + fprintf( stdout, "Create %s\n", raw_binary_file_name ); + + // Allocate a buffer + buf = (char *) malloc( buf_len * sizeof( char ) ); + if ( buf == NULL ) + { + printf( "There was an error creating the nested buffer %s.\n", raw_binary_file_name); + exit(EXIT_FAILURE); + } + + /* Create a new memory writer */ + nested_writer = avro_writer_memory( buf, buf_len ); + if ( nested_writer == NULL ) + { + printf( "There was an error creating the buffer for writing %s.\n", raw_binary_file_name); + exit(EXIT_FAILURE); + } + + /* Add an array containing 4 subarrays */ + printf( "before avro_writer_tell %d\n", (int) avro_writer_tell( nested_writer ) ); +#define ARRAY_LENGTH (4) + add_array( nested_writer, ARRAY_LENGTH, use_resolving_writer ); + printf( "after avro_writer_tell %d\n", (int) avro_writer_tell( nested_writer ) ); + + /* Serialize the nested array */ + printf( "Serialize the data to a file\n"); + + /* Delete the nested array if it exists, and create a new one */ + remove(raw_binary_file_name); + fid = fopen( raw_binary_file_name, "w+"); + if ( fid == NULL ) + { + printf( "There was an error creating the file %s.\n", raw_binary_file_name); + exit(EXIT_FAILURE); + } + fwrite( buf, 1, avro_writer_tell( nested_writer ), fid ); + fclose(fid); + avro_writer_free( nested_writer ); + free(buf); + return 0; +} + + +/* Top level function to impelement a test for the JIRA issue + * AVRO-1034. See detailed documentation at the top of this file. + */ +int main(void) +{ + const char *raw_binary_file_name = "nested_array.bin"; + const char *raw_binary_file_name_resolved = "nested_array_resolved.bin"; + int64_t buf_len = 2048; + int use_resolving_writer; + + /* Initialize the schema structure from JSON */ + init_schema(); + + printf( "Write the serialized nested array to %s\n", raw_binary_file_name ); + use_resolving_writer = 0; + write_nested_array_file( buf_len, raw_binary_file_name, use_resolving_writer ); + + printf( "\nWrite the serialized nested array after schema resolution to %s\n", + raw_binary_file_name_resolved ); + use_resolving_writer = 1; + write_nested_array_file( buf_len, raw_binary_file_name_resolved, use_resolving_writer ); + + // Close out schemas + avro_schema_decref(schema_old); + avro_schema_decref(schema_new); + + // Remove the binary files + remove(raw_binary_file_name); + remove(raw_binary_file_name_resolved); + + printf("\n"); + return 0; +} diff --git a/fluent-bit/lib/avro/tests/test_avro_1084.c b/fluent-bit/lib/avro/tests/test_avro_1084.c new file mode 100644 index 00000000..75d4c7a7 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_1084.c @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <avro.h> +#include <stdio.h> +#include <stdlib.h> + +const char PERSON_SCHEMA[] = +"{\"type\":\"record\",\ + \"name\":\"Person\",\ + \"fields\":[\ + {\"name\": \"ID\", \"type\": \"long\"}]}"; + +const char *dbname = "test.db"; +avro_schema_t schema; + +int main() +{ + avro_file_writer_t writer; + + // refcount == 1 + if (avro_schema_from_json_literal (PERSON_SCHEMA, &schema)) + { + printf ("Unable to parse schema\n"); + return EXIT_FAILURE; + } + + // BUG: refcount == 1 + if (avro_file_writer_create ("test.db", schema, &writer)) + { + printf ("There was an error creating db: %s\n", avro_strerror()); + return EXIT_FAILURE; + } + + // this is "unusual" behaviour + // refcount == 0 + avro_schema_decref (schema); + + // crash + avro_datum_t main_datum = avro_record(schema); + avro_datum_t id_datum = avro_int32(1); + + if (avro_record_set (main_datum, "ID", id_datum)) + { + printf ("Unable to create datum"); + return EXIT_FAILURE; + } + + avro_file_writer_append (writer, main_datum); + avro_file_writer_flush (writer); + avro_file_writer_close (writer); + remove (dbname); + + avro_datum_decref (id_datum); + avro_datum_decref (main_datum); + + return EXIT_SUCCESS; +} + diff --git a/fluent-bit/lib/avro/tests/test_avro_1087.c b/fluent-bit/lib/avro/tests/test_avro_1087.c new file mode 100644 index 00000000..b2e82aaa --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_1087.c @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <avro.h> +#include <stdio.h> +#include <stdlib.h> + +const char PERSON_SCHEMA[] = +"{\"type\":\"record\",\ + \"name\":\"Person\",\ + \"fields\":[\ + {\"name\": \"ID\", \"type\": \"int\"}]}"; + +const char *dbname = "test.db"; +avro_schema_t schema; + +void add_record (avro_file_writer_t writer) +{ + avro_datum_t main_datum = avro_record(schema); + avro_datum_t id_datum = avro_int32(1); + + if (avro_record_set (main_datum, "ID", id_datum)) + { + printf ("Unable to create datum"); + exit (EXIT_FAILURE); + } + + avro_file_writer_append (writer, main_datum); + + avro_datum_decref (id_datum); + avro_datum_decref (main_datum); +} + +void create_database() +{ + avro_file_writer_t writer; + + if (avro_schema_from_json_literal (PERSON_SCHEMA, &schema)) + { + printf ("Unable to parse schema\n"); + exit (EXIT_FAILURE); + } + + if (avro_file_writer_create ("test.db", schema, &writer)) + { + printf ("There was an error creating db: %s\n", avro_strerror()); + exit (EXIT_FAILURE); + } + + add_record (writer); + + avro_file_writer_flush (writer); + avro_file_writer_close (writer); +} + + +int main() +{ + avro_file_writer_t writer; + + create_database(); + + avro_file_writer_open (dbname, &writer); + add_record (writer); + + avro_file_writer_flush (writer); + avro_file_writer_close (writer); + + avro_schema_decref(schema); + + remove (dbname); + + return EXIT_SUCCESS; +} diff --git a/fluent-bit/lib/avro/tests/test_avro_1165.c b/fluent-bit/lib/avro/tests/test_avro_1165.c new file mode 100644 index 00000000..09f51726 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_1165.c @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <stdio.h> +#include <string.h> +#include <avro.h> + +/* To validate AVRO-1165, run this test program through valgrind + * before and after applying the AVRO-1165.patch. Before the patch + * valgrind will show memory leaks, and after the patch it will not. + * The specific valgrind commandline to use from the + * avro-trunk/lang/c/tests directory is: + * valgrind -v --track-origins=yes --leak-check=full + * --show-reachable = yes ../build/tests/test_avro_1165 + */ + +int main(int argc, char **argv) +{ + const char *json = + "{" + " \"name\": \"repeated_subrecord_array\"," + " \"type\": \"record\"," + " \"fields\": [" + " { \"name\": \"subrecord_one\"," + " \"type\": {" + " \"name\": \"SubrecordType\"," + " \"type\": \"record\"," + " \"fields\": [" + " { \"name\": \"x\", \"type\": \"int\" }," + " { \"name\": \"y\", \"type\": \"int\" }" + " ]" + " }" + " }," + " { \"name\": \"subrecord_two\", \"type\": \"SubrecordType\" }," + " { \"name\": \"subrecord_array\", \"type\": {" + " \"type\":\"array\"," + " \"items\": \"SubrecordType\"" + " }" + " }" + " ]" + "}"; + + int rval; + avro_schema_t schema = NULL; + avro_schema_error_t error; + avro_value_iface_t *p_reader_class; + + (void) argc; + (void) argv; + + rval = avro_schema_from_json(json, strlen(json), &schema, &error); + if ( rval ) + { + printf("Failed to read schema from JSON.\n"); + return 1; + } + else + { + printf("Successfully read schema from JSON.\n"); + } + + p_reader_class = avro_generic_class_from_schema(schema); + + avro_value_iface_decref(p_reader_class); + + avro_schema_decref(schema); + return 0; +} diff --git a/fluent-bit/lib/avro/tests/test_avro_1167.c b/fluent-bit/lib/avro/tests/test_avro_1167.c new file mode 100644 index 00000000..09ad787a --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_1167.c @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <stdio.h> +#include <string.h> +#include <avro.h> + +/* To see the AVRO-1167 memory leak, run this test program through + * valgrind. The specific valgrind commandline to use from the + * avro-trunk/lang/c/tests directory is: + * valgrind -v --track-origins=yes --leak-check=full + * --show-reachable = yes ../build/tests/test_avro_1167 + */ + +int main(int argc, char **argv) +{ + const char *json = + "{" + " \"name\": \"repeated_subrecord_array\"," + " \"type\": \"record\"," + " \"fields\": [" + " { \"name\": \"subrecord_one\"," + " \"type\": {" + " \"name\": \"SubrecordType\"," + " \"type\": \"record\"," + " \"fields\": [" + " { \"name\": \"x\", \"type\": \"int\" }," + " { \"name\": \"y\", \"type\": \"int\" }" + " ]" + " }" + " }," + " { \"name\": \"subrecord_two\", \"type\": \"SubrecordType\" }," + " { \"name\": \"subrecord_array\", \"type\": {" + " \"type\":\"array\"," + " \"items\": \"SubrecordType\"" + " }" + " }" + " ]" + "}"; + + int rval; + avro_schema_t schema = NULL; + avro_schema_t schema_copy = NULL; + avro_schema_error_t error; + + (void) argc; + (void) argv; + + rval = avro_schema_from_json(json, strlen(json), &schema, &error); + if ( rval ) + { + printf("Failed to read schema from JSON.\n"); + exit(EXIT_FAILURE); + } + else + { + printf("Successfully read schema from JSON.\n"); + } + + schema_copy = avro_schema_copy( schema ); + if ( ! avro_schema_equal(schema, schema_copy) ) + { + printf("Failed avro_schema_equal(schema, schema_copy)\n"); + exit(EXIT_FAILURE); + } + + avro_schema_decref(schema); + avro_schema_decref(schema_copy); + return 0; +} diff --git a/fluent-bit/lib/avro/tests/test_avro_1237.c b/fluent-bit/lib/avro/tests/test_avro_1237.c new file mode 100644 index 00000000..f382cc03 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_1237.c @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <avro.h> +#include <stdio.h> +#include <stdlib.h> + +#define check_exit(call) \ + do { \ + int __rc = call; \ + if (__rc != 0) { \ + fprintf(stderr, "Unexpected error:\n %s\n %s\n", \ + avro_strerror(), #call); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#define expect_error(call) \ + do { \ + int __rc = call; \ + if (__rc == 0) { \ + fprintf(stderr, "Expected an error:\n %s\n", #call); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#define check_expected_value(actual, expected) \ + do { \ + if (!avro_value_equal_fast((actual), (expected))) { \ + char *actual_json; \ + char *expected_json; \ + avro_value_to_json((actual), 1, &actual_json); \ + avro_value_to_json((expected), 1, &expected_json); \ + fprintf(stderr, "Expected %s\nGot %s\n", \ + expected_json, actual_json); \ + free(actual_json); \ + free(expected_json); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +int main(void) +{ + avro_schema_t schema; + avro_file_reader_t reader; + avro_value_iface_t *iface; + avro_value_t actual; + avro_value_t expected; + avro_value_t branch; + + schema = avro_schema_union(); + avro_schema_union_append(schema, avro_schema_null()); + avro_schema_union_append(schema, avro_schema_int()); + + iface = avro_generic_class_from_schema(schema); + avro_generic_value_new(iface, &actual); + avro_generic_value_new(iface, &expected); + + + /* First read the contents of the good file. */ + + check_exit(avro_file_reader("avro-1237-good.avro", &reader)); + + check_exit(avro_file_reader_read_value(reader, &actual)); + check_exit(avro_value_set_branch(&expected, 0, &branch)); + check_exit(avro_value_set_null(&branch)); + check_expected_value(&actual, &expected); + + check_exit(avro_file_reader_read_value(reader, &actual)); + check_exit(avro_value_set_branch(&expected, 1, &branch)); + check_exit(avro_value_set_int(&branch, 100)); + check_expected_value(&actual, &expected); + + check_exit(avro_file_reader_close(reader)); + + + /* Then read from the malformed file. */ + + check_exit(avro_file_reader + ("avro-1237-bad-union-discriminant.avro", &reader)); + + check_exit(avro_file_reader_read_value(reader, &actual)); + check_exit(avro_value_set_branch(&expected, 0, &branch)); + check_exit(avro_value_set_null(&branch)); + check_expected_value(&actual, &expected); + + expect_error(avro_file_reader_read_value(reader, &actual)); + + check_exit(avro_file_reader_close(reader)); + + + /* Clean up and exit */ + avro_value_decref(&actual); + avro_value_decref(&expected); + avro_value_iface_decref(iface); + avro_schema_decref(schema); + exit(EXIT_SUCCESS); +} diff --git a/fluent-bit/lib/avro/tests/test_avro_1238.c b/fluent-bit/lib/avro/tests/test_avro_1238.c new file mode 100644 index 00000000..eed8a807 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_1238.c @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <avro.h> +#include <stdio.h> +#include <stdlib.h> + +#define check_exit(call) \ + do { \ + int __rc = call; \ + if (__rc != 0) { \ + fprintf(stderr, "Unexpected error:\n %s\n %s\n", \ + avro_strerror(), #call); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#define expect_eof(call) \ + do { \ + int __rc = call; \ + if (__rc != EOF) { \ + fprintf(stderr, "Expected EOF:\n %s\n", #call); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#define expect_error(call) \ + do { \ + int __rc = call; \ + if (__rc == 0) { \ + fprintf(stderr, "Expected an error:\n %s\n", #call); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#define check_expected_value(actual, expected) \ + do { \ + if (!avro_value_equal_fast((actual), (expected))) { \ + char *actual_json; \ + char *expected_json; \ + avro_value_to_json((actual), 1, &actual_json); \ + avro_value_to_json((expected), 1, &expected_json); \ + fprintf(stderr, "Expected %s\nGot %s\n", \ + expected_json, actual_json); \ + free(actual_json); \ + free(expected_json); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +int main(void) +{ + avro_schema_t schema; + avro_file_reader_t reader; + avro_value_iface_t *iface; + avro_value_t actual; + avro_value_t expected; + avro_value_t branch; + + schema = avro_schema_union(); + avro_schema_union_append(schema, avro_schema_null()); + avro_schema_union_append(schema, avro_schema_int()); + + iface = avro_generic_class_from_schema(schema); + avro_generic_value_new(iface, &actual); + avro_generic_value_new(iface, &expected); + + + /* First read the contents of the good file. */ + + check_exit(avro_file_reader("avro-1238-good.avro", &reader)); + + check_exit(avro_file_reader_read_value(reader, &actual)); + check_exit(avro_value_set_branch(&expected, 0, &branch)); + check_exit(avro_value_set_null(&branch)); + check_expected_value(&actual, &expected); + + check_exit(avro_file_reader_read_value(reader, &actual)); + check_exit(avro_value_set_branch(&expected, 1, &branch)); + check_exit(avro_value_set_int(&branch, 100)); + check_expected_value(&actual, &expected); + + expect_eof(avro_file_reader_read_value(reader, &actual)); + check_exit(avro_file_reader_close(reader)); + + + /* Then read from the truncated file. */ + + check_exit(avro_file_reader("avro-1238-truncated.avro", &reader)); + + check_exit(avro_file_reader_read_value(reader, &actual)); + check_exit(avro_value_set_branch(&expected, 0, &branch)); + check_exit(avro_value_set_null(&branch)); + check_expected_value(&actual, &expected); + + check_exit(avro_file_reader_read_value(reader, &actual)); + check_exit(avro_value_set_branch(&expected, 1, &branch)); + check_exit(avro_value_set_int(&branch, 100)); + check_expected_value(&actual, &expected); + + expect_error(avro_file_reader_read_value(reader, &actual)); + check_exit(avro_file_reader_close(reader)); + + + /* Clean up and exit */ + avro_value_decref(&actual); + avro_value_decref(&expected); + avro_value_iface_decref(iface); + avro_schema_decref(schema); + exit(EXIT_SUCCESS); +} diff --git a/fluent-bit/lib/avro/tests/test_avro_1279.c b/fluent-bit/lib/avro/tests/test_avro_1279.c new file mode 100644 index 00000000..9d490d46 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_1279.c @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <avro.h> +#include <stdio.h> +#include <stdlib.h> + +#define check_exit(call) \ + do { \ + int __rc = call; \ + if (__rc != 0) { \ + fprintf(stderr, "Unexpected error:\n %s\n %s\n", \ + avro_strerror(), #call); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +int main(void) +{ + avro_file_reader_t reader; + + /* First open the file with the explicit codec. */ + check_exit(avro_file_reader("avro-1279-codec.avro", &reader)); + check_exit(avro_file_reader_close(reader)); + + + /* Then the file with no codec. */ + check_exit(avro_file_reader("avro-1279-no-codec.avro", &reader)); + check_exit(avro_file_reader_close(reader)); + + /* Clean up and exit */ + exit(EXIT_SUCCESS); +} diff --git a/fluent-bit/lib/avro/tests/test_avro_1379.c b/fluent-bit/lib/avro/tests/test_avro_1379.c new file mode 100644 index 00000000..bfd77dc8 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_1379.c @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include "avro.h" +#include "avro_private.h" +#include <stdio.h> +#include <stdlib.h> + +static const char *schema_json = + "{" + " \"type\": \"record\"," + " \"name\": \"test\"," + " \"fields\": [" + " { \"name\": \"i\", \"type\": \"int\" }," + " { \"name\": \"l\", \"type\": \"long\" }," + " { \"name\": \"s\", \"type\": \"string\" }," + " {" + " \"name\": \"subrec\"," + " \"type\": {" + " \"type\": \"record\"," + " \"name\": \"sub\"," + " \"fields\": [" + " { \"name\": \"f\", \"type\": \"float\" }," + " { \"name\": \"d\", \"type\": \"double\" }" + " ]" + " }" + " }" + " ]" + "}"; + +static void +populate_complex_record(avro_value_t *p_val) +{ + avro_value_t field; + + avro_value_get_by_index(p_val, 0, &field, NULL); + avro_value_set_int(&field, 42); + + avro_value_get_by_index(p_val, 1, &field, NULL); + avro_value_set_long(&field, 4242); + + avro_wrapped_buffer_t wbuf; + avro_wrapped_buffer_new_string(&wbuf, "Follow your bliss."); + avro_value_get_by_index(p_val, 2, &field, NULL); + avro_value_give_string_len(&field, &wbuf); + + avro_value_t subrec; + avro_value_get_by_index(p_val, 3, &subrec, NULL); + + avro_value_get_by_index(&subrec, 0, &field, NULL); + avro_value_set_float(&field, 3.14159265); + + avro_value_get_by_index(&subrec, 1, &field, NULL); + avro_value_set_double(&field, 2.71828183); +} + +int main(void) +{ + int rval = 0; + size_t len; + static char buf[4096]; + avro_writer_t writer; + avro_file_writer_t file_writer; + avro_file_reader_t file_reader; + const char *outpath = "test-1379.avro"; + + avro_schema_t schema = NULL; + avro_schema_error_t error = NULL; + check(rval, avro_schema_from_json(schema_json, strlen(schema_json), &schema, &error)); + + avro_value_iface_t *iface = avro_generic_class_from_schema(schema); + + avro_value_t val; + avro_generic_value_new(iface, &val); + + avro_value_t out; + avro_generic_value_new(iface, &out); + + /* create the val */ + avro_value_reset(&val); + populate_complex_record(&val); + + /* create the writers */ + writer = avro_writer_memory(buf, sizeof(buf)); + check(rval, avro_file_writer_create(outpath, schema, &file_writer)); + + fprintf(stderr, "Writing to buffer\n"); + check(rval, avro_value_write(writer, &val)); + + fprintf(stderr, "Writing buffer to %s " + "using avro_file_writer_append_encoded()\n", outpath); + len = avro_writer_tell(writer); + check(rval, avro_file_writer_append_encoded(file_writer, buf, len)); + check(rval, avro_file_writer_close(file_writer)); + + check(rval, avro_file_reader(outpath, &file_reader)); + fprintf(stderr, "Re-reading value to verify\n"); + check(rval, avro_file_reader_read_value(file_reader, &out)); + fprintf(stderr, "Verifying value..."); + if (!avro_value_equal(&val, &out)) { + fprintf(stderr, "fail!\n"); + exit(EXIT_FAILURE); + } + fprintf(stderr, "ok\n"); + check(rval, avro_file_reader_close(file_reader)); + remove(outpath); + + avro_writer_free(writer); + avro_value_decref(&out); + avro_value_decref(&val); + avro_value_iface_decref(iface); + avro_schema_decref(schema); + + exit(EXIT_SUCCESS); +} diff --git a/fluent-bit/lib/avro/tests/test_avro_1405.c b/fluent-bit/lib/avro/tests/test_avro_1405.c new file mode 100644 index 00000000..b853dae9 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_1405.c @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <stdio.h> +#include "avro.h" + +#define NUM_RECORDS 10 + +const char PERSON_SCHEMA[] = + "{" + " \"type\":\"record\"," + " \"name\":\"Person\"," + " \"fields\": [" + " {\"name\": \"ID\", \"type\": \"long\"}," + " {\"name\": \"First\", \"type\": \"string\"}," + " {\"name\": \"Last\", \"type\": \"string\"}," + " {\"name\": \"Phone\", \"type\": \"string\"}," + " {\"name\": \"Age\", \"type\": \"int\"}" + " ]" + "}"; + +const char *file = "avro_file.dat"; + +void print_avro_value(avro_value_t *value) { + char *json; + if (!avro_value_to_json(value, 1, &json)) { + printf("%s\n", json); + free(json); + } +} + +int read_data() { + int rval; + int records_read = 0; + + avro_file_reader_t reader; + avro_value_iface_t *iface; + avro_value_t value; + + avro_file_reader(file, &reader); + avro_schema_t schema = avro_file_reader_get_writer_schema(reader); + + iface = avro_generic_class_from_schema(schema); + avro_generic_value_new(iface, &value); + + printf("\nReading...\n"); + while ((rval = avro_file_reader_read_value(reader, &value)) == 0) { + char *json; + + if (avro_value_to_json(&value, 1, &json)) { + printf("Error converting value to JSON: %s\n",avro_strerror()); + } else { + printf("%s\n", json); + free(json); + records_read++; + } + + avro_value_reset(&value); + } + + avro_value_decref(&value); + avro_value_iface_decref(iface); + avro_schema_decref(schema); + avro_file_reader_close(reader); + + if (rval != EOF || records_read != NUM_RECORDS) { + fprintf(stderr, "Error: %s\n", avro_strerror()); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int write_data() { + int i; + avro_schema_t schema; + avro_schema_error_t error; + avro_file_writer_t writer; + avro_value_iface_t *iface; + avro_value_t value; + + if (avro_schema_from_json(PERSON_SCHEMA, 0, &schema, &error)) { + printf ("Unable to parse schema\n"); + return EXIT_FAILURE; + } + + iface = avro_generic_class_from_schema(schema); + avro_generic_value_new(iface, &value); + + if (avro_file_writer_create(file, schema, &writer)) { + printf ("There was an error creating file: %s\n", avro_strerror()); + return EXIT_FAILURE; + } + + printf("\nWriting...\n"); + for (i = 0; i < NUM_RECORDS; i++) { + avro_value_t field; + avro_value_get_by_name(&value, "ID", &field, NULL); + avro_value_set_long(&field, (int64_t) i); + + avro_value_get_by_name(&value, "Age", &field, NULL); + avro_value_set_int(&field, i); + + avro_value_get_by_name(&value, "First", &field, NULL); + avro_value_set_string(&field, "Firstname"); + + avro_value_get_by_name(&value, "Last", &field, NULL); + avro_value_set_string(&field, "Lastname"); + + + avro_value_get_by_name(&value, "Phone", &field, NULL); + avro_value_set_string(&field, "1234567"); + + print_avro_value(&value); + + avro_file_writer_append_value(writer, &value); + + // Writing multiple blocks + avro_file_writer_close(writer); + avro_file_writer_open(file, &writer); + + avro_value_reset(&value); + } + + avro_file_writer_close(writer); + avro_value_iface_decref(iface); + avro_value_decref(&value); + avro_schema_decref(schema); + + return EXIT_SUCCESS; +} + + +int main() +{ + int read_data_result; + + if (write_data()) { + return EXIT_FAILURE; + } + + read_data_result = read_data(); + remove(file); + + return read_data_result; +} diff --git a/fluent-bit/lib/avro/tests/test_avro_1572.c b/fluent-bit/lib/avro/tests/test_avro_1572.c new file mode 100644 index 00000000..eb94cc1a --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_1572.c @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <stdio.h> +#include <sys/stat.h> +#include "avro.h" + +#define BUFFER_LEN 4096 + +#define HEADER_LEN 116 +#define SYNC_LEN 16 +#define BLOCKINFO_LEN 4 +#define RECORD_LEN 1 + +static const int NUM_RECORDS1 = (BUFFER_LEN - HEADER_LEN - 2 * SYNC_LEN - BLOCKINFO_LEN) / RECORD_LEN; +static const int NUM_RECORDS2 = (BUFFER_LEN - SYNC_LEN - BLOCKINFO_LEN) / RECORD_LEN; + +static const char PERSON_SCHEMA[] = + "{" + " \"type\":\"record\"," + " \"name\":\"Person\"," + " \"fields\": [" + " {\"name\": \"ab\", \"type\": \"int\"}" + " ]" + "}"; + +static const char *filename = "avro_file.dat"; + +static int read_data() { + int rval; + int records_read = 0; + + avro_file_reader_t reader; + avro_value_iface_t *iface; + avro_value_t value; + + fprintf(stderr, "\nReading...\n"); + + avro_file_reader(filename, &reader); + avro_schema_t schema = avro_file_reader_get_writer_schema(reader); + + iface = avro_generic_class_from_schema(schema); + avro_generic_value_new(iface, &value); + + while ((rval = avro_file_reader_read_value(reader, &value)) == 0) { + records_read++; + avro_value_reset(&value); + } + + avro_value_decref(&value); + avro_value_iface_decref(iface); + avro_schema_decref(schema); + avro_file_reader_close(reader); + + fprintf(stderr, "wanted %d records, read %d records.\n", NUM_RECORDS1 + NUM_RECORDS2, records_read); + + if (rval != EOF || records_read != (NUM_RECORDS1 + NUM_RECORDS2)) { + fprintf(stderr, "Error: %s\n", avro_strerror()); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +static off_t fsize(const char *filename) { + struct stat st; + + if (stat(filename, &st) == 0) { + return st.st_size; + } + + return -1; +} + +static int write_data() { + int i; + avro_schema_t schema; + avro_schema_error_t error; + avro_file_writer_t writer; + avro_value_iface_t *iface; + avro_value_t value; + avro_value_t field; + + fprintf(stderr, "\nWriting...\n"); + + if (avro_schema_from_json(PERSON_SCHEMA, 0, &schema, &error)) { + fprintf(stderr, "Unable to parse schema\n"); + return EXIT_FAILURE; + } + + iface = avro_generic_class_from_schema(schema); + avro_generic_value_new(iface, &value); + + if (avro_file_writer_create(filename, schema, &writer)) { + fprintf(stderr, "There was an error creating file: %s\n", avro_strerror()); + return EXIT_FAILURE; + } + + avro_value_get_by_name(&value, "ab", &field, NULL); + avro_value_set_int(&field, 1); + + fprintf(stderr, "NUM_RECORDS1 = %d NUM_RECORDS2 = %d\n", NUM_RECORDS1, NUM_RECORDS2); + + for (i = 0; i < NUM_RECORDS1; i++) { + avro_file_writer_append_value(writer, &value); + } + + avro_file_writer_close(writer); + + /* Make sure the sync ends at a BUFFER_LEN boundary */ + if (fsize(filename) != BUFFER_LEN) { + fprintf(stderr, "internal error\n"); + return EXIT_FAILURE; + } + + avro_file_writer_open(filename, &writer); + + for (i = 0; i < NUM_RECORDS2; i++) { + avro_file_writer_append_value(writer, &value); + } + + avro_file_writer_close(writer); + + /* Make sure the whole file ends at a BUFFER_LEN boundary. */ + if (fsize(filename) != 2 * BUFFER_LEN) { + fprintf(stderr, "internal error\n"); + return EXIT_FAILURE; + } + + avro_value_iface_decref(iface); + avro_value_decref(&value); + avro_schema_decref(schema); + + return EXIT_SUCCESS; +} + + +int main() +{ + int read_data_result; + + if (write_data()) { + return EXIT_FAILURE; + } + + read_data_result = read_data(); + remove(filename); + + return read_data_result; +} diff --git a/fluent-bit/lib/avro/tests/test_avro_1691.c b/fluent-bit/lib/avro/tests/test_avro_1691.c new file mode 100644 index 00000000..bb45fc50 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_1691.c @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +/** + * AVRO-1691 test case - support of string-field JSON schemas. + */ +#include "avro.h" +#include "avro_private.h" +#include <stdio.h> +#include <stdlib.h> + +static const char *json_schemas[] = { + /* These two schemas are functionally equivalent. */ + "{ \"type\": \"string\" }", /* Object wrapped */ + "\"string\"", /* JSON string field */ + NULL +}; + + + + +int main(void) +{ + int pass; + + for (pass = 0 ; json_schemas[pass] ; pass++) { + int rval = 0; + size_t len; + static char buf[4096]; + avro_writer_t writer; + avro_file_writer_t file_writer; + avro_file_reader_t file_reader; + avro_schema_t schema = NULL; + avro_schema_error_t error = NULL; + char outpath[64]; + const char *json_schema = json_schemas[pass]; + + printf("pass %d with schema %s\n", pass, json_schema); + check(rval, avro_schema_from_json(json_schema, strlen(json_schema), + &schema, &error)); + + avro_value_iface_t *iface = avro_generic_class_from_schema(schema); + + avro_value_t val; + avro_generic_value_new(iface, &val); + + avro_value_t out; + avro_generic_value_new(iface, &out); + + /* create the val */ + avro_value_reset(&val); + avro_value_set_string(&val, "test-1691"); + + /* Write value to file */ + snprintf(outpath, sizeof(outpath), "test-1691-%d.avro", pass); + + /* create the writers */ + writer = avro_writer_memory(buf, sizeof(buf)); + check(rval, avro_file_writer_create(outpath, schema, &file_writer)); + + check(rval, avro_value_write(writer, &val)); + + len = avro_writer_tell(writer); + check(rval, avro_file_writer_append_encoded(file_writer, buf, len)); + check(rval, avro_file_writer_close(file_writer)); + + /* Read the value back */ + check(rval, avro_file_reader(outpath, &file_reader)); + check(rval, avro_file_reader_read_value(file_reader, &out)); + if (!avro_value_equal(&val, &out)) { + fprintf(stderr, "fail!\n"); + exit(EXIT_FAILURE); + } + fprintf(stderr, "pass %d: ok: schema %s\n", pass, json_schema); + check(rval, avro_file_reader_close(file_reader)); + remove(outpath); + + avro_writer_free(writer); + avro_value_decref(&out); + avro_value_decref(&val); + avro_value_iface_decref(iface); + avro_schema_decref(schema); + } + + exit(EXIT_SUCCESS); +} diff --git a/fluent-bit/lib/avro/tests/test_avro_1904.c b/fluent-bit/lib/avro/tests/test_avro_1904.c new file mode 100644 index 00000000..d13f2487 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_1904.c @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <stdio.h> +#include <sys/stat.h> +#include "avro.h" + +#define NUM_RECORDS 100 + +static const char *filename = "avro_file.dat"; + +static const char PERSON_SCHEMA[] = + "{" + " \"type\":\"record\"," + " \"name\":\"Person\"," + " \"fields\": [" + " ]" + "}"; + +static int read_data() { + int rval; + int records_read = 0; + + avro_file_reader_t reader; + avro_value_iface_t *iface; + avro_value_t value; + + fprintf(stderr, "\nReading...\n"); + + rval = avro_file_reader(filename, &reader); + + if (rval) { + fprintf(stderr, "Error: %s\n", avro_strerror()); + return EXIT_FAILURE; + } + + avro_schema_t schema = avro_file_reader_get_writer_schema(reader); + + iface = avro_generic_class_from_schema(schema); + avro_generic_value_new(iface, &value); + + while ((rval = avro_file_reader_read_value(reader, &value)) == 0) { + records_read++; + avro_value_reset(&value); + } + + avro_value_decref(&value); + avro_value_iface_decref(iface); + avro_schema_decref(schema); + avro_file_reader_close(reader); + + fprintf(stderr, "read %d records.\n", records_read); + + if (rval != EOF || records_read != NUM_RECORDS) { + fprintf(stderr, "Error: %s\n", avro_strerror()); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +static int write_data() { + int i; + avro_schema_t schema; + avro_schema_error_t error; + avro_file_writer_t writer; + avro_value_iface_t *iface; + avro_value_t value; + + fprintf(stderr, "\nWriting...\n"); + + if (avro_schema_from_json(PERSON_SCHEMA, 0, &schema, &error)) { + fprintf(stderr, "Unable to parse schema\n"); + return EXIT_FAILURE; + } + + iface = avro_generic_class_from_schema(schema); + avro_generic_value_new(iface, &value); + + if (avro_file_writer_create(filename, schema, &writer)) { + fprintf(stderr, "There was an error creating file: %s\n", avro_strerror()); + return EXIT_FAILURE; + } + + for (i = 0; i < NUM_RECORDS; i++) { + if (avro_file_writer_append_value(writer, &value)) { + fprintf(stderr, "There was an error creating file: %s\n", avro_strerror()); + return EXIT_FAILURE; + } + } + + if (avro_file_writer_close(writer)) { + fprintf(stderr, "There was an error creating file: %s\n", avro_strerror()); + return EXIT_FAILURE; + } + + avro_value_iface_decref(iface); + avro_value_decref(&value); + avro_schema_decref(schema); + + return EXIT_SUCCESS; +} + +int main() +{ + int read_data_result; + + if (write_data()) { + return EXIT_FAILURE; + } + + read_data_result = read_data(); + remove(filename); + + return read_data_result; +} diff --git a/fluent-bit/lib/avro/tests/test_avro_1906.c b/fluent-bit/lib/avro/tests/test_avro_1906.c new file mode 100644 index 00000000..7188aec0 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_1906.c @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <stdio.h> +#include <sys/stat.h> +#include "avro.h" + +static const char *filename = "avro_file.dat"; + +static const char PERSON_SCHEMA[] = + "{" + " \"type\":\"record\"," + " \"name\":\"Person\"," + " \"fields\": [" + " {\"name\": \"ab\", \"type\": \"int\"}" + " ]" + "}"; + +static int read_data() { + int rval; + int records_read = 0; + + avro_file_reader_t reader; + avro_value_iface_t *iface; + avro_value_t value; + + fprintf(stderr, "\nReading...\n"); + + rval = avro_file_reader(filename, &reader); + + if (rval) { + fprintf(stderr, "Error: %s\n", avro_strerror()); + return -1; + } + + avro_schema_t schema = avro_file_reader_get_writer_schema(reader); + + iface = avro_generic_class_from_schema(schema); + avro_generic_value_new(iface, &value); + + while ((rval = avro_file_reader_read_value(reader, &value)) == 0) { + avro_value_t field; + int32_t val; + avro_value_get_by_index(&value, 0, &field, NULL); + avro_value_get_int(&field, &val); + fprintf(stderr, "value = %d\n", val); + records_read++; + avro_value_reset(&value); + } + + avro_value_decref(&value); + avro_value_iface_decref(iface); + avro_schema_decref(schema); + avro_file_reader_close(reader); + + fprintf(stderr, "read %d records.\n", records_read); + + if (rval != EOF) { + fprintf(stderr, "Error: %s\n", avro_strerror()); + return -1; + } + + return records_read; +} + +static int read_data_datum() { + int rval; + int records_read = 0; + + avro_file_reader_t reader; + avro_datum_t datum; + + fprintf(stderr, "\nReading...\n"); + + rval = avro_file_reader(filename, &reader); + + if (rval) { + fprintf(stderr, "Error using 'datum': %s\n", avro_strerror()); + return -1; + } + + avro_schema_t schema = avro_file_reader_get_writer_schema(reader); + + while ((rval = avro_file_reader_read(reader, schema, &datum)) == 0) { + avro_datum_t val_datum; + int32_t val; + if (avro_record_get(datum, "ab", &val_datum)) { + fprintf(stderr, "Error getting value: %s\n", avro_strerror()); + return -1; + } + avro_int32_get(val_datum, &val); + fprintf(stderr, "value = %d\n", val); + records_read++; + avro_datum_decref(datum); + } + + avro_schema_decref(schema); + avro_file_reader_close(reader); + + fprintf(stderr, "read %d records using 'datum'.\n", records_read); + + if (rval != EOF) { + fprintf(stderr, "Error using 'datum': %s\n", avro_strerror()); + return -1; + } + + return records_read; +} + +static int write_data(int n_records) { + int i; + avro_schema_t schema; + avro_schema_error_t error; + avro_file_writer_t writer; + avro_value_iface_t *iface; + avro_value_t value; + + fprintf(stderr, "\nWriting...\n"); + + if (avro_schema_from_json(PERSON_SCHEMA, 0, &schema, &error)) { + fprintf(stderr, "Unable to parse schema\n"); + return -1; + } + + if (avro_file_writer_create(filename, schema, &writer)) { + fprintf(stderr, "There was an error creating file: %s\n", avro_strerror()); + return -1; + } + + iface = avro_generic_class_from_schema(schema); + avro_generic_value_new(iface, &value); + + avro_value_t field; + + avro_value_get_by_index(&value, 0, &field, NULL); + avro_value_set_int(&field, 123); + + for (i = 0; i < n_records; i++) { + if (avro_file_writer_append_value(writer, &value)) { + fprintf(stderr, "There was an error writing file: %s\n", avro_strerror()); + return -1; + } + } + + if (avro_file_writer_close(writer)) { + fprintf(stderr, "There was an error creating file: %s\n", avro_strerror()); + return -1; + } + + avro_value_decref(&value); + avro_value_iface_decref(iface); + avro_schema_decref(schema); + + return n_records; +} + +static int test_n_records(int n_records) { + int res = 0; + + if (write_data(n_records) != n_records) { + remove(filename); + return -1; + } + + if (read_data() != n_records) { + remove(filename); + return -1; + } + + if (read_data_datum() != n_records) { + remove(filename); + return -1; + } + + remove(filename); + return 0; +} + +int main() +{ + if (test_n_records(1) < 0) { + return EXIT_FAILURE; + } + + if (test_n_records(0) < 0) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} diff --git a/fluent-bit/lib/avro/tests/test_avro_766.c b/fluent-bit/lib/avro/tests/test_avro_766.c new file mode 100644 index 00000000..a0b13b79 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_766.c @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <stdio.h> +#include <string.h> +#include <avro.h> + +/* To see the AVRO-766 memory leak, run this test program through + * valgrind. The specific valgrind commandline to use from the + * avro-trunk/lang/c/tests directory is: + * valgrind -v --track-origins=yes --leak-check=full + * --show-reachable = yes ../build/tests/test_avro_766 + */ +int main(int argc, char **argv) +{ + const char *json = + "{" + " \"type\": \"record\"," + " \"name\": \"list\"," + " \"fields\": [" + " { \"name\": \"x\", \"type\": \"int\" }," + " { \"name\": \"y\", \"type\": \"int\" }," + " { \"name\": \"next\", \"type\": [\"null\",\"list\"]}," + " { \"name\": \"arraylist\", \"type\": { \"type\":\"array\", \"items\": \"list\" } }" + " ]" + "}"; + + int rval; + avro_schema_t schema = NULL; + avro_schema_error_t error; + + (void) argc; + (void) argv; + + rval = avro_schema_from_json(json, strlen(json), &schema, &error); + if ( rval ) + { + printf("Failed to read schema from JSON.\n"); + exit(EXIT_FAILURE); + } + else + { + printf("Successfully read schema from JSON.\n"); + } + +#define TEST_AVRO_1167 (1) + #if TEST_AVRO_1167 + { + avro_schema_t schema_copy = NULL; + schema_copy = avro_schema_copy( schema ); + if ( ! avro_schema_equal(schema, schema_copy) ) + { + printf("Failed avro_schema_equal(schema, schema_copy)\n"); + exit(EXIT_FAILURE); + } + avro_schema_decref(schema_copy); + } + #endif + + avro_schema_decref(schema); + return 0; +} diff --git a/fluent-bit/lib/avro/tests/test_avro_968.c b/fluent-bit/lib/avro/tests/test_avro_968.c new file mode 100644 index 00000000..87620014 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_968.c @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <stdio.h> + +#include "avro.h" +#include "avro_private.h" + +#define try(call, msg) \ + do { \ + if (call) { \ + fprintf(stderr, msg ":\n %s\n", avro_strerror()); \ + return EXIT_FAILURE; \ + } \ + } while (0) + +int +main(int argc, char **argv) +{ + AVRO_UNUSED(argc); + AVRO_UNUSED(argv); + + avro_value_t v1; + avro_value_t v2; + + try(avro_generic_string_new(&v1, "test string a"), + "Cannot create string value"); + try(avro_generic_string_new(&v2, "test string b"), + "Cannot create string value"); + + if (avro_value_equal(&v1, &v2)) { + fprintf(stderr, "Unexpected avro_value_equal\n"); + return EXIT_FAILURE; + } + + if (avro_value_equal_fast(&v1, &v2)) { + fprintf(stderr, "Unexpected avro_value_equal_fast\n"); + return EXIT_FAILURE; + } + + if (avro_value_cmp(&v1, &v2) >= 0) { + fprintf(stderr, "Unexpected avro_value_cmp\n"); + return EXIT_FAILURE; + } + + if (avro_value_cmp_fast(&v1, &v2) >= 0) { + fprintf(stderr, "Unexpected avro_value_cmp_fast\n"); + return EXIT_FAILURE; + } + + avro_value_decref(&v1); + avro_value_decref(&v2); + return 0; +} diff --git a/fluent-bit/lib/avro/tests/test_avro_984.c b/fluent-bit/lib/avro/tests/test_avro_984.c new file mode 100644 index 00000000..c89a5116 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_984.c @@ -0,0 +1,464 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <avro.h> +#include <stdio.h> +#include <stdlib.h> + + +/* Test code for JIRA Issue AVRO-984. + * + * AVRO-984: Avro-C schema resolution fails on nested array + * + * This program tests schema resolution for nested arrays. For the + * purposes of this test, there are two schemas "old" and "new" which + * are created by reading the same JSON schema. + * + * The test creates and populates a nested array, and serializes it to + * memory. The raw memory is written to a file, primarily to decouple + * writing and reading. Note that the schema is not written to the + * file. The nested array is also printed to the screen. + * + * The binary file is then read using two separate readers -- the + * matched reader and the resolved reader. + * + * In the matched reader case, the "old" and "new" schemas are known + * to match, and therefore no schema resolution is done. The binary + * buffer is deserialized into an avro value and the nested array + * encoded in the avro value is printed to the screen. + * + * In the resolved reader case, the "old" and "new" schemas are not + * known to match, and therefore schema resolution is performed. (Note + * that the schemas *do* match, but we perform schema resolution + * anyway, to test the resolution process). The schema resolution + * appears to succeed. However, once the code tries to perform an + * "avro_value_read()" the code fails to read the nested array into + * the avro value. + * + * Additionally valgrind indicates that conditional jumps are being + * performed based on uninitialized values. + * + * AVRO-C was compiled with CMAKE_INSTALL_PREFIX=avrolib + * The static library (libavro.a) was copied into a subdirectory of avrolib/lib/static + * + * This file was compiled under Linux using: + * gcc -g avro-984-test.c -o avro984 -I../../build/avrolib/include -L../../build/avrolib/lib/static -lavro + * + * The code was tested with valgrind using the command: + * valgrind -v --leak-check=full --track-origins=yes ./avro984 + * + */ + + +// Encode the following json string in NESTED_ARRAY +// {"type":"array", "items": {"type": "array", "items": "long"}} +// +#define NESTED_ARRAY \ + "{\"type\":\"array\", \"items\": {\"type\": \"array\", \"items\": \"long\"}}" + +avro_schema_t schema_old = NULL; +avro_schema_t schema_new = NULL; + +/* Parse schema into a schema data structure */ +void init_schema(void) +{ + avro_schema_error_t error; + if (avro_schema_from_json(NESTED_ARRAY, sizeof(NESTED_ARRAY), + &schema_old, &error)) { + printf( "Unable to parse old schema\n"); + exit(EXIT_FAILURE); + } + + if (avro_schema_from_json(NESTED_ARRAY, sizeof(NESTED_ARRAY), + &schema_new, &error)) { + printf( "Unable to parse new schema\n"); + exit(EXIT_FAILURE); + } +} + +#define try(call, msg) \ + do { \ + if (call) { \ + printf( msg ":\n %s\n", avro_strerror()); \ + exit (EXIT_FAILURE); \ + } \ + } while (0) + + +/* The input avro_value_t p_array should contain a nested array. + * Print the fields of this nested array to the screen. + */ +int print_array_fields ( avro_value_t *p_array ) +{ + size_t idx; + size_t length; + avro_type_t val_type; + + val_type = avro_value_get_type( p_array ); + printf( "Main array type = %d\n", val_type ); + + try( avro_value_get_size( p_array, &length ), + "Couldn't get array size" ); + printf( "Main array length = %d\n", (int) length ); + + for ( idx = 0; idx < length; idx ++ ) + { + avro_value_t subarray; + size_t sublength; + size_t jdx; + const char *unused; + + try ( avro_value_get_by_index( p_array, idx, &subarray, &unused ), + "Couldn't get subarray" ); + + val_type = avro_value_get_type( &subarray ); + printf( "Subarray type = %d\n", val_type ); + + try( avro_value_get_size( &subarray, &sublength ), + "Couldn't get subarray size" ); + printf( "Subarray length = %d\n", (int) sublength ); + + for ( jdx = 0; jdx < sublength; jdx++ ) + { + avro_value_t element; + int64_t val; + + try ( avro_value_get_by_index( &subarray, jdx, &element, &unused ), + "Couldn't get subarray element" ); + + val_type = avro_value_get_type( &element ); + + try ( avro_value_get_long( &element, &val ), + "Couldn't get subarray element value" ); + + printf( "nested_array[%d][%d]: type = %d value = %lld\n", + (int) idx, (int) jdx, (int) val_type, (long long) val ); + + } + } + + return 0; +} + + +/* The input avro_value_t p_subarray should contain an array of long + * integers. Add "elements" number of long integers to this array. Set + * the values to be distinct based on the iteration parameter. + */ +int add_subarray( avro_value_t *p_subarray, + int32_t elements, + int32_t iteration ) +{ + avro_value_t element; + size_t index; + size_t idx; + + for ( idx = 0; idx < (size_t) elements; idx ++ ) + { + // Append avro array element to subarray + try ( avro_value_append( p_subarray, &element, &index ), + "Error appending element in subarray" ); + + try ( avro_value_set_long( &element, (iteration+1)*100 + (iteration+1) ), + "Error setting subarray element" ); + } + + return 0; +} + + +/* Create a nested array using the schema NESTED_ARRAY. Populate its + * elements with unique values. Serialize the nested array to the + * memory buffer in avro_writer_t. The number of elements in the first + * dimension of the nested array is "elements". The number of elements + * in the second dimension of the nested array is hardcoded to 2. + */ +int add_array( avro_writer_t writer, + int32_t elements ) +{ + avro_schema_t chosen_schema; + avro_value_iface_t *nested_array_class; + avro_value_t nested; + int32_t idx; + + // Select (hardcode) schema to use + chosen_schema = schema_old; + + // Create avro class and value + nested_array_class = avro_generic_class_from_schema( chosen_schema ); + try ( avro_generic_value_new( nested_array_class, &nested ), + "Error creating instance of record" ); + + for ( idx = 0; idx < elements; idx ++ ) + { + avro_value_t subarray; + size_t index; + + // Append avro array element for top level array + try ( avro_value_append( &nested, &subarray, &index ), + "Error appending subarray" ); + + // Populate array element with subarray of length 2 +#define SUBARRAY_LENGTH (2) + try ( add_subarray( &subarray, SUBARRAY_LENGTH, idx ), + "Error populating subarray" ); + } + + // Write the value to memory + try ( avro_value_write( writer, &nested ), + "Unable to write nested into memory" ); + + print_array_fields( &nested ); + + // Release the record + avro_value_decref( &nested ); + avro_value_iface_decref( nested_array_class ); + + return 0; +} + +/* Create a raw binary file containing a serialized version of a + * nested array. This file will later be read by + * read_nested_array_file(). + */ +int write_nested_array_file ( int64_t buf_len, const char *raw_binary_file_name ) +{ + char *buf; + avro_writer_t nested_writer; + FILE *fid = NULL; + + fprintf( stdout, "Create %s\n", raw_binary_file_name ); + + // Allocate a buffer + buf = (char *) malloc( buf_len * sizeof( char ) ); + if ( buf == NULL ) + { + printf( "There was an error creating the nested buffer %s.\n", raw_binary_file_name); + exit(EXIT_FAILURE); + } + + /* Create a new memory writer */ + nested_writer = avro_writer_memory( buf, buf_len ); + if ( nested_writer == NULL ) + { + printf( "There was an error creating the buffer for writing %s.\n", raw_binary_file_name); + exit(EXIT_FAILURE); + } + + /* Add an array containing 4 subarrays */ + printf( "before avro_writer_tell %d\n", (int) avro_writer_tell( nested_writer ) ); +#define ARRAY_LENGTH (4) + add_array( nested_writer, ARRAY_LENGTH ); + printf( "after avro_writer_tell %d\n", (int) avro_writer_tell( nested_writer ) ); + + /* Serialize the nested array */ + printf( "Serialize the data to a file\n"); + + /* Delete the nested array if it exists, and create a new one */ + remove(raw_binary_file_name); + fid = fopen( raw_binary_file_name, "w+"); + if ( fid == NULL ) + { + printf( "There was an error creating the file %s.\n", raw_binary_file_name); + exit(EXIT_FAILURE); + } + fwrite( buf, 1, avro_writer_tell( nested_writer ), fid ); + fclose(fid); + avro_writer_free( nested_writer ); + free(buf); + return 0; +} + + +/* Read the raw binary file containing a serialized version of a + * nested array, written by write_nested_array_file() + */ +int read_nested_array_file ( int64_t buf_len, + const char *raw_binary_file_name, + avro_schema_t writer_schema, + avro_schema_t reader_schema, + int use_resolving_reader + ) +{ + + char *buf; + FILE *fid = NULL; + avro_reader_t nested_reader; + int64_t file_len; + + // For Matched Reader and Resolving Reader + avro_value_iface_t *reader_class; + avro_value_t nested; + + // For Resolving Reader + avro_value_iface_t *resolver; + avro_value_t resolved_value; + + fprintf( stdout, "Use %s reader\n", use_resolving_reader ? "Resolving":"Matched" ); + + // Allocate a buffer + buf = (char *) calloc( buf_len, sizeof( char ) ); + if ( buf == NULL ) + { + printf( "There was an error creating the buffer for reading %s.\n", raw_binary_file_name); + exit(EXIT_FAILURE); + } + // Start with a garbage buffer + memset(buf, 0xff, buf_len ); + + // Read the file into the buffer + fid = fopen( raw_binary_file_name, "r" ); + if ( fid == NULL ) + { + printf( "There was an error reading the file %s.\n", raw_binary_file_name); + exit(EXIT_FAILURE); + } + file_len = fread( buf, 1, buf_len, fid ); + printf( "Read %d bytes\n", (int) file_len ); + fclose(fid); + + if ( use_resolving_reader ) + { + // Resolving Reader + + /* First resolve the writer and reader schemas */ + resolver = avro_resolved_writer_new( writer_schema, reader_schema ); + if ( !resolver ) + { + printf( "Could not create resolver\n"); + free(buf); + exit(EXIT_FAILURE); + } + + /* Create a value that the resolver can write into. This is just + * an interface value, that is not directly read from. + */ + if ( avro_resolved_writer_new_value( resolver, &resolved_value ) ) + { + avro_value_iface_decref( resolver ); + free(buf); + exit(EXIT_FAILURE); + } + + /* Then create the value with the reader schema, that we are going + * to use to read from. + */ + reader_class = avro_generic_class_from_schema(reader_schema); + try ( avro_generic_value_new( reader_class, &nested ), + "Error creating instance of nested array" ); + + // When we read the memory using the resolved writer, we want to + // populate the instance of the value with the reader schema. This + // is done by set_dest. + avro_resolved_writer_set_dest(&resolved_value, &nested); + + // Create a memory reader + nested_reader = avro_reader_memory( buf, buf_len ); + + if ( avro_value_read( nested_reader, &resolved_value ) ) + { + printf( "Avro value read failed\n" ); + + avro_value_decref( &nested ); + avro_value_iface_decref( reader_class ); + avro_value_iface_decref( resolver ); + avro_value_decref( &resolved_value ); + + exit(EXIT_FAILURE); + } + } + else + { + // Matched Reader + reader_class = avro_generic_class_from_schema(reader_schema); + + try ( avro_generic_value_new( reader_class, &nested ), + "Error creating instance of nested array" ); + + // Send the memory in the buffer into the reader + nested_reader = avro_reader_memory( buf, buf_len ); + + try ( avro_value_read( nested_reader, &nested ), + "Could not read value from memory" ); + } + + + /* Now the resolved record has been read into "nested" which is + * a value of type reader_class + */ + print_array_fields( &nested ); + + if ( use_resolving_reader ) + { + // Resolving Reader + avro_value_decref( &nested ); + avro_value_iface_decref( reader_class ); + avro_value_iface_decref( resolver ); + avro_value_decref( &resolved_value ); + } + else + { + // Matched Reader + avro_value_decref( &nested ); + avro_value_iface_decref( reader_class ); + } + + fprintf( stdout, "Done.\n\n"); + avro_reader_free( nested_reader ); + free(buf); + return 0; +} + + +/* Top level function to impelement a test for the JIRA issue + * AVRO-984. See detailed documentation at the top of this file. + */ +int main(void) +{ + const char *raw_binary_file_name = "nested_array.bin"; + int64_t buf_len = 2048; + int use_resolving_reader; + + /* Initialize the schema structure from JSON */ + init_schema(); + + printf( "Write the serialized nested array to %s\n", raw_binary_file_name ); + + write_nested_array_file( buf_len, raw_binary_file_name ); + + printf("\nNow read all the array back out\n\n"); + + for ( use_resolving_reader = 0; use_resolving_reader < 2; use_resolving_reader++ ) + { + read_nested_array_file( buf_len, + raw_binary_file_name, + schema_old, + schema_new, + use_resolving_reader + ); + } + + // Close out schemas + avro_schema_decref(schema_old); + avro_schema_decref(schema_new); + + // Remove the binary file + remove(raw_binary_file_name); + + printf("\n"); + return 0; +} diff --git a/fluent-bit/lib/avro/tests/test_avro_data.c b/fluent-bit/lib/avro/tests/test_avro_data.c new file mode 100644 index 00000000..1da09e6d --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_data.c @@ -0,0 +1,684 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include "avro.h" +#include "avro_private.h" +#include <limits.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> + +char buf[4096]; +avro_reader_t reader; +avro_writer_t writer; + +typedef int (*avro_test) (void); + +/* + * Use a custom allocator that verifies that the size that we use to + * free an object matches the size that we use to allocate it. + */ + +static void * +test_allocator(void *ud, void *ptr, size_t osize, size_t nsize) +{ + AVRO_UNUSED(ud); + AVRO_UNUSED(osize); + + if (nsize == 0) { + size_t *size = ((size_t *) ptr) - 1; + if (osize != *size) { + fprintf(stderr, + "Error freeing %p:\n" + "Size passed to avro_free (%" PRIsz ") " + "doesn't match size passed to " + "avro_malloc (%" PRIsz ")\n", + ptr, osize, *size); + abort(); + //exit(EXIT_FAILURE); + } + free(size); + return NULL; + } else { + size_t real_size = nsize + sizeof(size_t); + size_t *old_size = ptr? ((size_t *) ptr)-1: NULL; + size_t *size = (size_t *) realloc(old_size, real_size); + *size = nsize; + return (size + 1); + } +} + +void init_rand(void) +{ + srand(time(NULL)); +} + +double rand_number(double from, double to) +{ + double range = to - from; + return from + ((double)rand() / (RAND_MAX + 1.0)) * range; +} + +int64_t rand_int64(void) +{ + return (int64_t) rand_number(LONG_MIN, LONG_MAX); +} + +int32_t rand_int32(void) +{ + return (int32_t) rand_number(INT_MIN, INT_MAX); +} + +void +write_read_check(avro_schema_t writers_schema, avro_datum_t datum, + avro_schema_t readers_schema, avro_datum_t expected, char *type) +{ + avro_datum_t datum_out; + int validate; + + for (validate = 0; validate <= 1; validate++) { + + reader = avro_reader_memory(buf, sizeof(buf)); + writer = avro_writer_memory(buf, sizeof(buf)); + + if (!expected) { + expected = datum; + } + + /* Validating read/write */ + if (avro_write_data + (writer, validate ? writers_schema : NULL, datum)) { + fprintf(stderr, "Unable to write %s validate=%d\n %s\n", + type, validate, avro_strerror()); + exit(EXIT_FAILURE); + } + int64_t size = + avro_size_data(writer, validate ? writers_schema : NULL, + datum); + if (size != avro_writer_tell(writer)) { + fprintf(stderr, + "Unable to calculate size %s validate=%d " + "(%"PRId64" != %"PRId64")\n %s\n", + type, validate, size, avro_writer_tell(writer), + avro_strerror()); + exit(EXIT_FAILURE); + } + if (avro_read_data + (reader, writers_schema, readers_schema, &datum_out)) { + fprintf(stderr, "Unable to read %s validate=%d\n %s\n", + type, validate, avro_strerror()); + fprintf(stderr, " %s\n", avro_strerror()); + exit(EXIT_FAILURE); + } + if (!avro_datum_equal(expected, datum_out)) { + fprintf(stderr, + "Unable to encode/decode %s validate=%d\n %s\n", + type, validate, avro_strerror()); + exit(EXIT_FAILURE); + } + + avro_reader_dump(reader, stderr); + avro_datum_decref(datum_out); + avro_reader_free(reader); + avro_writer_free(writer); + } +} + +static void test_json(avro_datum_t datum, const char *expected) +{ + char *json = NULL; + avro_datum_to_json(datum, 1, &json); + if (strcasecmp(json, expected) != 0) { + fprintf(stderr, "Unexpected JSON encoding: %s\n", json); + exit(EXIT_FAILURE); + } + free(json); +} + +static int test_string(void) +{ + unsigned int i; + const char *strings[] = { "Four score and seven years ago", + "our father brought forth on this continent", + "a new nation", "conceived in Liberty", + "and dedicated to the proposition that all men are created equal." + }; + avro_schema_t writer_schema = avro_schema_string(); + for (i = 0; i < sizeof(strings) / sizeof(strings[0]); i++) { + avro_datum_t datum = avro_givestring(strings[i], NULL); + write_read_check(writer_schema, datum, NULL, NULL, "string"); + avro_datum_decref(datum); + } + + avro_datum_t datum = avro_givestring(strings[0], NULL); + test_json(datum, "\"Four score and seven years ago\""); + avro_datum_decref(datum); + + // The following should bork if we don't copy the string value + // correctly (since we'll try to free a static string). + + datum = avro_string("this should be copied"); + avro_string_set(datum, "also this"); + avro_datum_decref(datum); + + avro_schema_decref(writer_schema); + return 0; +} + +static int test_bytes(void) +{ + char bytes[] = { 0xDE, 0xAD, 0xBE, 0xEF }; + avro_schema_t writer_schema = avro_schema_bytes(); + avro_datum_t datum; + avro_datum_t expected_datum; + + datum = avro_givebytes(bytes, sizeof(bytes), NULL); + write_read_check(writer_schema, datum, NULL, NULL, "bytes"); + test_json(datum, "\"\\u00de\\u00ad\\u00be\\u00ef\""); + avro_datum_decref(datum); + avro_schema_decref(writer_schema); + + datum = avro_givebytes(NULL, 0, NULL); + avro_givebytes_set(datum, bytes, sizeof(bytes), NULL); + expected_datum = avro_givebytes(bytes, sizeof(bytes), NULL); + if (!avro_datum_equal(datum, expected_datum)) { + fprintf(stderr, + "Expected equal bytes instances.\n"); + exit(EXIT_FAILURE); + } + avro_datum_decref(datum); + avro_datum_decref(expected_datum); + + // The following should bork if we don't copy the bytes value + // correctly (since we'll try to free a static string). + + datum = avro_bytes("original", 8); + avro_bytes_set(datum, "alsothis", 8); + avro_datum_decref(datum); + + avro_schema_decref(writer_schema); + return 0; +} + +static int test_int32(void) +{ + int i; + avro_schema_t writer_schema = avro_schema_int(); + avro_schema_t long_schema = avro_schema_long(); + avro_schema_t float_schema = avro_schema_float(); + avro_schema_t double_schema = avro_schema_double(); + for (i = 0; i < 100; i++) { + int32_t value = rand_int32(); + avro_datum_t datum = avro_int32(value); + avro_datum_t long_datum = avro_int64(value); + avro_datum_t float_datum = avro_float(value); + avro_datum_t double_datum = avro_double(value); + write_read_check(writer_schema, datum, NULL, NULL, "int"); + write_read_check(writer_schema, datum, + long_schema, long_datum, "int->long"); + write_read_check(writer_schema, datum, + float_schema, float_datum, "int->float"); + write_read_check(writer_schema, datum, + double_schema, double_datum, "int->double"); + avro_datum_decref(datum); + avro_datum_decref(long_datum); + avro_datum_decref(float_datum); + avro_datum_decref(double_datum); + } + + avro_datum_t datum = avro_int32(10000); + test_json(datum, "10000"); + avro_datum_decref(datum); + + avro_schema_decref(writer_schema); + avro_schema_decref(long_schema); + avro_schema_decref(float_schema); + avro_schema_decref(double_schema); + return 0; +} + +static int test_int64(void) +{ + int i; + avro_schema_t writer_schema = avro_schema_long(); + avro_schema_t float_schema = avro_schema_float(); + avro_schema_t double_schema = avro_schema_double(); + for (i = 0; i < 100; i++) { + int64_t value = rand_int64(); + avro_datum_t datum = avro_int64(value); + avro_datum_t float_datum = avro_float(value); + avro_datum_t double_datum = avro_double(value); + write_read_check(writer_schema, datum, NULL, NULL, "long"); + write_read_check(writer_schema, datum, + float_schema, float_datum, "long->float"); + write_read_check(writer_schema, datum, + double_schema, double_datum, "long->double"); + avro_datum_decref(datum); + avro_datum_decref(float_datum); + avro_datum_decref(double_datum); + } + + avro_datum_t datum = avro_int64(10000); + test_json(datum, "10000"); + avro_datum_decref(datum); + + avro_schema_decref(writer_schema); + avro_schema_decref(float_schema); + avro_schema_decref(double_schema); + return 0; +} + +static int test_double(void) +{ + int i; + avro_schema_t schema = avro_schema_double(); + for (i = 0; i < 100; i++) { + avro_datum_t datum = avro_double(rand_number(-1.0E10, 1.0E10)); + write_read_check(schema, datum, NULL, NULL, "double"); + avro_datum_decref(datum); + } + + avro_datum_t datum = avro_double(2000.0); + test_json(datum, "2000.0"); + avro_datum_decref(datum); + + avro_schema_decref(schema); + return 0; +} + +static int test_float(void) +{ + int i; + avro_schema_t schema = avro_schema_float(); + avro_schema_t double_schema = avro_schema_double(); + for (i = 0; i < 100; i++) { + float value = rand_number(-1.0E10, 1.0E10); + avro_datum_t datum = avro_float(value); + avro_datum_t double_datum = avro_double(value); + write_read_check(schema, datum, NULL, NULL, "float"); + write_read_check(schema, datum, + double_schema, double_datum, "float->double"); + avro_datum_decref(datum); + avro_datum_decref(double_datum); + } + + avro_datum_t datum = avro_float(2000.0); + test_json(datum, "2000.0"); + avro_datum_decref(datum); + + avro_schema_decref(schema); + avro_schema_decref(double_schema); + return 0; +} + +static int test_boolean(void) +{ + int i; + const char *expected_json[] = { "false", "true" }; + avro_schema_t schema = avro_schema_boolean(); + for (i = 0; i <= 1; i++) { + avro_datum_t datum = avro_boolean(i); + write_read_check(schema, datum, NULL, NULL, "boolean"); + test_json(datum, expected_json[i]); + avro_datum_decref(datum); + } + avro_schema_decref(schema); + return 0; +} + +static int test_null(void) +{ + avro_schema_t schema = avro_schema_null(); + avro_datum_t datum = avro_null(); + write_read_check(schema, datum, NULL, NULL, "null"); + test_json(datum, "null"); + avro_datum_decref(datum); + return 0; +} + +static int test_record(void) +{ + avro_schema_t schema = avro_schema_record("person", NULL); + avro_schema_record_field_append(schema, "name", avro_schema_string()); + avro_schema_record_field_append(schema, "age", avro_schema_int()); + + avro_datum_t datum = avro_record(schema); + avro_datum_t name_datum, age_datum; + + name_datum = avro_givestring("Joseph Campbell", NULL); + age_datum = avro_int32(83); + + avro_record_set(datum, "name", name_datum); + avro_record_set(datum, "age", age_datum); + + write_read_check(schema, datum, NULL, NULL, "record"); + test_json(datum, "{\"name\": \"Joseph Campbell\", \"age\": 83}"); + + int rc; + avro_record_set_field_value(rc, datum, int32, "age", 104); + + int32_t age = 0; + avro_record_get_field_value(rc, datum, int32, "age", &age); + if (age != 104) { + fprintf(stderr, "Incorrect age value\n"); + exit(EXIT_FAILURE); + } + + avro_datum_decref(name_datum); + avro_datum_decref(age_datum); + avro_datum_decref(datum); + avro_schema_decref(schema); + return 0; +} + +static int test_nested_record(void) +{ + const char *json = + "{" + " \"type\": \"record\"," + " \"name\": \"list\"," + " \"fields\": [" + " { \"name\": \"x\", \"type\": \"int\" }," + " { \"name\": \"y\", \"type\": \"int\" }," + " { \"name\": \"next\", \"type\": [\"null\",\"list\"]}" + " ]" + "}"; + + int rval; + + avro_schema_t schema = NULL; + avro_schema_error_t error; + avro_schema_from_json(json, strlen(json), &schema, &error); + + avro_datum_t head = avro_datum_from_schema(schema); + avro_record_set_field_value(rval, head, int32, "x", 10); + avro_record_set_field_value(rval, head, int32, "y", 10); + + avro_datum_t next = NULL; + avro_datum_t tail = NULL; + + avro_record_get(head, "next", &next); + avro_union_set_discriminant(next, 1, &tail); + avro_record_set_field_value(rval, tail, int32, "x", 20); + avro_record_set_field_value(rval, tail, int32, "y", 20); + + avro_record_get(tail, "next", &next); + avro_union_set_discriminant(next, 0, NULL); + + write_read_check(schema, head, NULL, NULL, "nested record"); + + avro_schema_decref(schema); + avro_datum_decref(head); + + return 0; +} + +static int test_enum(void) +{ + enum avro_languages { + AVRO_C, + AVRO_CPP, + AVRO_PYTHON, + AVRO_RUBY, + AVRO_JAVA + }; + avro_schema_t schema = avro_schema_enum("language"); + avro_datum_t datum = avro_enum(schema, AVRO_C); + + avro_schema_enum_symbol_append(schema, "C"); + avro_schema_enum_symbol_append(schema, "C++"); + avro_schema_enum_symbol_append(schema, "Python"); + avro_schema_enum_symbol_append(schema, "Ruby"); + avro_schema_enum_symbol_append(schema, "Java"); + + if (avro_enum_get(datum) != AVRO_C) { + fprintf(stderr, "Unexpected enum value AVRO_C\n"); + exit(EXIT_FAILURE); + } + + if (strcmp(avro_enum_get_name(datum), "C") != 0) { + fprintf(stderr, "Unexpected enum value name C\n"); + exit(EXIT_FAILURE); + } + + write_read_check(schema, datum, NULL, NULL, "enum"); + test_json(datum, "\"C\""); + + avro_enum_set(datum, AVRO_CPP); + if (strcmp(avro_enum_get_name(datum), "C++") != 0) { + fprintf(stderr, "Unexpected enum value name C++\n"); + exit(EXIT_FAILURE); + } + + write_read_check(schema, datum, NULL, NULL, "enum"); + test_json(datum, "\"C++\""); + + avro_enum_set_name(datum, "Python"); + if (avro_enum_get(datum) != AVRO_PYTHON) { + fprintf(stderr, "Unexpected enum value AVRO_PYTHON\n"); + exit(EXIT_FAILURE); + } + + write_read_check(schema, datum, NULL, NULL, "enum"); + test_json(datum, "\"Python\""); + + avro_datum_decref(datum); + avro_schema_decref(schema); + return 0; +} + +static int test_array(void) +{ + int i, rval; + avro_schema_t schema = avro_schema_array(avro_schema_int()); + avro_datum_t datum = avro_array(schema); + + for (i = 0; i < 10; i++) { + avro_datum_t i32_datum = avro_int32(i); + rval = avro_array_append_datum(datum, i32_datum); + avro_datum_decref(i32_datum); + if (rval) { + exit(EXIT_FAILURE); + } + } + + if (avro_array_size(datum) != 10) { + fprintf(stderr, "Unexpected array size"); + exit(EXIT_FAILURE); + } + + write_read_check(schema, datum, NULL, NULL, "array"); + test_json(datum, "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]"); + avro_datum_decref(datum); + avro_schema_decref(schema); + return 0; +} + +static int test_map(void) +{ + avro_schema_t schema = avro_schema_map(avro_schema_long()); + avro_datum_t datum = avro_map(schema); + int64_t i = 0; + char *nums[] = + { "zero", "one", "two", "three", "four", "five", "six", NULL }; + while (nums[i]) { + avro_datum_t i_datum = avro_int64(i); + avro_map_set(datum, nums[i], i_datum); + avro_datum_decref(i_datum); + i++; + } + + if (avro_array_size(datum) != 7) { + fprintf(stderr, "Unexpected map size\n"); + exit(EXIT_FAILURE); + } + + avro_datum_t value; + const char *key; + avro_map_get_key(datum, 2, &key); + avro_map_get(datum, key, &value); + int64_t val; + avro_int64_get(value, &val); + + if (val != 2) { + fprintf(stderr, "Unexpected map value 2\n"); + exit(EXIT_FAILURE); + } + + int index; + if (avro_map_get_index(datum, "two", &index)) { + fprintf(stderr, "Can't get index for key \"two\": %s\n", + avro_strerror()); + exit(EXIT_FAILURE); + } + if (index != 2) { + fprintf(stderr, "Unexpected index for key \"two\"\n"); + exit(EXIT_FAILURE); + } + if (!avro_map_get_index(datum, "foobar", &index)) { + fprintf(stderr, "Unexpected index for key \"foobar\"\n"); + exit(EXIT_FAILURE); + } + + write_read_check(schema, datum, NULL, NULL, "map"); + test_json(datum, + "{\"zero\": 0, \"one\": 1, \"two\": 2, \"three\": 3, " + "\"four\": 4, \"five\": 5, \"six\": 6}"); + avro_datum_decref(datum); + avro_schema_decref(schema); + return 0; +} + +static int test_union(void) +{ + avro_schema_t schema = avro_schema_union(); + avro_datum_t union_datum; + avro_datum_t datum; + avro_datum_t union_datum1; + avro_datum_t datum1; + + avro_schema_union_append(schema, avro_schema_string()); + avro_schema_union_append(schema, avro_schema_int()); + avro_schema_union_append(schema, avro_schema_null()); + + datum = avro_givestring("Follow your bliss.", NULL); + union_datum = avro_union(schema, 0, datum); + + if (avro_union_discriminant(union_datum) != 0) { + fprintf(stderr, "Unexpected union discriminant\n"); + exit(EXIT_FAILURE); + } + + if (avro_union_current_branch(union_datum) != datum) { + fprintf(stderr, "Unexpected union branch datum\n"); + exit(EXIT_FAILURE); + } + + union_datum1 = avro_datum_from_schema(schema); + avro_union_set_discriminant(union_datum1, 0, &datum1); + avro_givestring_set(datum1, "Follow your bliss.", NULL); + + if (!avro_datum_equal(datum, datum1)) { + fprintf(stderr, "Union values should be equal\n"); + exit(EXIT_FAILURE); + } + + write_read_check(schema, union_datum, NULL, NULL, "union"); + test_json(union_datum, "{\"string\": \"Follow your bliss.\"}"); + + avro_datum_decref(datum); + avro_union_set_discriminant(union_datum, 2, &datum); + test_json(union_datum, "null"); + + avro_datum_decref(union_datum); + avro_datum_decref(datum); + avro_datum_decref(union_datum1); + avro_schema_decref(schema); + return 0; +} + +static int test_fixed(void) +{ + char bytes[] = { 0xD, 0xA, 0xD, 0xA, 0xB, 0xA, 0xB, 0xA }; + avro_schema_t schema = avro_schema_fixed("msg", sizeof(bytes)); + avro_datum_t datum; + avro_datum_t expected_datum; + + datum = avro_givefixed(schema, bytes, sizeof(bytes), NULL); + write_read_check(schema, datum, NULL, NULL, "fixed"); + test_json(datum, "\"\\r\\n\\r\\n\\u000b\\n\\u000b\\n\""); + avro_datum_decref(datum); + + datum = avro_givefixed(schema, NULL, sizeof(bytes), NULL); + avro_givefixed_set(datum, bytes, sizeof(bytes), NULL); + expected_datum = avro_givefixed(schema, bytes, sizeof(bytes), NULL); + if (!avro_datum_equal(datum, expected_datum)) { + fprintf(stderr, + "Expected equal fixed instances.\n"); + exit(EXIT_FAILURE); + } + avro_datum_decref(datum); + avro_datum_decref(expected_datum); + + // The following should bork if we don't copy the fixed value + // correctly (since we'll try to free a static string). + + datum = avro_fixed(schema, "original", 8); + avro_fixed_set(datum, "alsothis", 8); + avro_datum_decref(datum); + + avro_schema_decref(schema); + return 0; +} + +int main(void) +{ + avro_set_allocator(test_allocator, NULL); + + unsigned int i; + struct avro_tests { + char *name; + avro_test func; + } tests[] = { + { + "string", test_string}, { + "bytes", test_bytes}, { + "int", test_int32}, { + "long", test_int64}, { + "float", test_float}, { + "double", test_double}, { + "boolean", test_boolean}, { + "null", test_null}, { + "record", test_record}, { + "nested_record", test_nested_record}, { + "enum", test_enum}, { + "array", test_array}, { + "map", test_map}, { + "fixed", test_fixed}, { + "union", test_union} + }; + + init_rand(); + for (i = 0; i < sizeof(tests) / sizeof(tests[0]); i++) { + struct avro_tests *test = tests + i; + fprintf(stderr, "**** Running %s tests ****\n", test->name); + if (test->func() != 0) { + return EXIT_FAILURE; + } + } + return EXIT_SUCCESS; +} diff --git a/fluent-bit/lib/avro/tests/test_avro_errors_are_thread_safe.c b/fluent-bit/lib/avro/tests/test_avro_errors_are_thread_safe.c new file mode 100644 index 00000000..b5189b12 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_errors_are_thread_safe.c @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#ifdef _WIN32 +#include <Windows.h> +#define THR_HANDLE HANDLE +#define LAST_ERROR() GetLastError() +#define SLEEP_MILLIS(millis) Sleep( millis ) +#else +#include <pthread.h> +#include <errno.h> +#include <unistd.h> +#define THR_HANDLE pthread_t +#define LAST_ERROR() errno +#define SLEEP_MILLIS(millis) usleep( millis * 1000 ) +#endif +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <time.h> +#include <avro.h> + +enum { + NUMBER_OF_TEST_THREADS = 64, + THREAD_ERROR_BUFFER_SIZE = 1024, + MAX_THREAD_SLEEP_MILLIS = 3000, +}; + +typedef struct _TEST_THREAD_DATA +{ + int index; + int error_occured; + char error_message[THREAD_ERROR_BUFFER_SIZE]; + volatile int worker_thread; + int sleep_interval_millis; +}TEST_THREAD_DATA; + +static int get_random_value( int max_value ); +#ifdef _WIN32 +static DWORD worker_thread( LPVOID lpThreadParameter ); +#else +static void *worker_thread( void *p ); +#endif +static THR_HANDLE launch_thread( void *thread_context ); +static int join_thread( THR_HANDLE thread_handle ); + +int main() +{ + TEST_THREAD_DATA threads_data[NUMBER_OF_TEST_THREADS]; + THR_HANDLE thread_handle[NUMBER_OF_TEST_THREADS]; + unsigned i; + int found_error = 0; + + srand( (unsigned)time( NULL ) ); + + memset( threads_data, 0, sizeof(threads_data) ); + for ( i = 0; i < NUMBER_OF_TEST_THREADS; i++ ) + { + threads_data[i].index = i; + threads_data[i].sleep_interval_millis = get_random_value( MAX_THREAD_SLEEP_MILLIS ); + thread_handle[i] = launch_thread( &threads_data[i] ); + if ( !thread_handle[i] ) + { + fprintf( stderr, "failed to launch worker thread, error code is %d\n", LAST_ERROR() ); + return EXIT_FAILURE; + } + } + + for ( i = 0; i < NUMBER_OF_TEST_THREADS; i++ ) + if ( join_thread( thread_handle[i] ) != 0 ) + { + fprintf( stderr, "failed to join thread %d, error code is %d\n", i, LAST_ERROR() ); + return EXIT_FAILURE; + } + + for ( i = 0; i < NUMBER_OF_TEST_THREADS; i++ ) + { + if( threads_data[i].error_occured ) + { + fprintf( stderr, "error occured at thread %d: %s\n", i, threads_data[i].error_message ); + found_error = 1; + } + } + if ( found_error ) + return EXIT_FAILURE; + +// printf( "test ended successfully\n"); + return EXIT_SUCCESS; +} + +#ifdef _WIN32 +static DWORD worker_thread( LPVOID context ) +#else +static void *worker_thread( void *context ) +#endif +{ + /* + worker thread set an error, request the error stack and validate it contains the error saved. + later it appends another error to the error stack, and validate it contains the two errors. + */ + TEST_THREAD_DATA *thread_context = (TEST_THREAD_DATA *)context; + char first_error_buffer[1024] = ""; + char second_error_buffer[1024] = ""; + char full_error_buffer[1024] = ""; + const char *error_stack = NULL; + int index = thread_context->index; + unsigned sleep_interval_millis = thread_context->sleep_interval_millis; + + //set a thread specific error + snprintf( first_error_buffer, sizeof(first_error_buffer), "thread %d set an error", index ); + avro_set_error( "%s", first_error_buffer ); + + SLEEP_MILLIS( sleep_interval_millis ); + + //validate error stack contains the thread specific error + error_stack = avro_strerror(); + if ( strcmp( error_stack, first_error_buffer ) != 0 ) + { + thread_context->error_occured = 1; + snprintf( thread_context->error_message, + sizeof(thread_context->error_message), + "invalid error stack found: expected '%s' found '%s'", first_error_buffer, error_stack ); + } + + //set another thread specific error + SLEEP_MILLIS( sleep_interval_millis ); + snprintf( second_error_buffer, sizeof(second_error_buffer), "thread %d set ANOTHER error...", index ); + avro_prefix_error( "%s", second_error_buffer ); + snprintf( full_error_buffer, sizeof(full_error_buffer), "%s%s", second_error_buffer, first_error_buffer ); + + //validate error stack contains the 2 errors as expected + SLEEP_MILLIS( sleep_interval_millis ); + error_stack = avro_strerror(); + if ( strcmp( error_stack, full_error_buffer ) != 0 ) + { + thread_context->error_occured = 1; + snprintf( thread_context->error_message, + sizeof(thread_context->error_message), + "invalid error stack found: expected '%s' found '%s'", full_error_buffer, error_stack ); + } + + return 0; + +} + +static THR_HANDLE launch_thread( void *thread_context ) +{ +#ifdef _WIN32 + static const LPSECURITY_ATTRIBUTES DEFAULT_SECURITY_ATTIRBUTES = NULL; + static const SIZE_T DEFAULT_STACK_SIZE = 0; + static const DWORD DEFAULT_CREATION_FLAGS = 0; + DWORD thread_id = 0; + + return + CreateThread( DEFAULT_SECURITY_ATTIRBUTES, + DEFAULT_STACK_SIZE, + worker_thread, + thread_context, + DEFAULT_CREATION_FLAGS, + &thread_id ); +#else + pthread_attr_t attr = {0}; + pthread_t thread; + pthread_attr_init( &attr ); + int status = 0; + status = pthread_create( &thread, &attr, worker_thread, thread_context ); + pthread_attr_destroy(&attr); + if ( status != 0 ) + return NULL; + return thread; +#endif +} + +static int join_thread( THR_HANDLE thread_handle ) +{ +#ifdef _WIN32 + return + ( WaitForSingleObject( thread_handle, INFINITE ) == WAIT_OBJECT_0 ) ? 0 : -1; +#else + return + pthread_join( thread_handle, NULL ); +#endif +} + +static int get_random_value( int max_value ) +{ + return + rand() % max_value; +} diff --git a/fluent-bit/lib/avro/tests/test_avro_schema.c b/fluent-bit/lib/avro/tests/test_avro_schema.c new file mode 100644 index 00000000..efa0b555 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_schema.c @@ -0,0 +1,316 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include "avro.h" +#include "avro_private.h" +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/types.h> +#ifdef _WIN32 + #include "msdirent.h" +#else + #include <dirent.h> +#endif + +int test_cases = 0; +avro_writer_t avro_stderr; + +static void run_tests(char *dirpath, int should_pass) +{ + char jsontext[4096]; + char jsontext2[4096]; + size_t rval; + char filepath[1024]; + DIR *dir; + struct dirent *dent; + FILE *fp; + avro_schema_t schema; + avro_writer_t jsontext2_writer; + + dir = opendir(dirpath); + if (dir == NULL) { + fprintf(stderr, "Unable to open '%s'\n", dirpath); + exit(EXIT_FAILURE); + } + do { + dent = readdir(dir); + + /* Suppress failures on CVS directories */ + if ( dent && !strcmp( (const char *) dent->d_name, "CVS" ) ) + continue; + + if (dent && dent->d_name[0] != '.') { + int test_rval; + snprintf(filepath, sizeof(filepath), "%s/%s", dirpath, + dent->d_name); + fprintf(stderr, "TEST %s...", filepath); + fp = fopen(filepath, "r"); + if (!fp) { + fprintf(stderr, "can't open!\n"); + exit(EXIT_FAILURE); + } + rval = fread(jsontext, 1, sizeof(jsontext) - 1, fp); + fclose(fp); + jsontext[rval] = '\0'; + test_rval = + avro_schema_from_json(jsontext, 0, &schema, NULL); + test_cases++; + if (test_rval == 0) { + if (should_pass) { + avro_schema_t schema_copy = + avro_schema_copy(schema); + fprintf(stderr, "pass\n"); + avro_schema_to_json(schema, + avro_stderr); + fprintf(stderr, "\n"); + if (!avro_schema_equal + (schema, schema_copy)) { + fprintf(stderr, + "failed to avro_schema_equal(schema,avro_schema_copy())\n"); + exit(EXIT_FAILURE); + } + jsontext2_writer = avro_writer_memory(jsontext2, sizeof(jsontext2)); + if (avro_schema_to_json(schema, jsontext2_writer)) { + fprintf(stderr, "failed to write schema (%s)\n", + avro_strerror()); + exit(EXIT_FAILURE); + } + avro_write(jsontext2_writer, (void *)"", 1); /* zero terminate */ + avro_writer_free(jsontext2_writer); + avro_schema_decref(schema); + if (avro_schema_from_json(jsontext2, 0, &schema, NULL)) { + fprintf(stderr, "failed to write then read schema (%s)\n", + avro_strerror()); + exit(EXIT_FAILURE); + } + if (!avro_schema_equal + (schema, schema_copy)) { + fprintf(stderr, "failed read-write-read cycle (%s)\n", + avro_strerror()); + exit(EXIT_FAILURE); + } + avro_schema_decref(schema_copy); + avro_schema_decref(schema); + } else { + /* + * Unexpected success + */ + fprintf(stderr, + "fail! (shouldn't succeed but did)\n"); + exit(EXIT_FAILURE); + } + } else { + if (should_pass) { + fprintf(stderr, "%s\n", avro_strerror()); + fprintf(stderr, + "fail! (should have succeeded but didn't)\n"); + exit(EXIT_FAILURE); + } else { + fprintf(stderr, "pass\n"); + } + } + } + } + while (dent != NULL); + closedir(dir); +} + +static int test_array(void) +{ + avro_schema_t schema = avro_schema_array(avro_schema_int()); + + if (!avro_schema_equal + (avro_schema_array_items(schema), avro_schema_int())) { + fprintf(stderr, "Unexpected array items schema"); + exit(EXIT_FAILURE); + } + + avro_schema_decref(schema); + return 0; +} + +static int test_enum(void) +{ + enum avro_languages { + AVRO_C, + AVRO_CPP, + AVRO_PYTHON, + AVRO_RUBY, + AVRO_JAVA + }; + avro_schema_t schema = avro_schema_enum("language"); + + avro_schema_enum_symbol_append(schema, "C"); + avro_schema_enum_symbol_append(schema, "C++"); + avro_schema_enum_symbol_append(schema, "Python"); + avro_schema_enum_symbol_append(schema, "Ruby"); + avro_schema_enum_symbol_append(schema, "Java"); + + const char *symbol1 = avro_schema_enum_get(schema, 1); + if (strcmp(symbol1, "C++") != 0) { + fprintf(stderr, "Unexpected enum schema symbol\n"); + exit(EXIT_FAILURE); + } + + if (avro_schema_enum_get_by_name(schema, "C++") != 1) { + fprintf(stderr, "Unexpected enum schema index\n"); + exit(EXIT_FAILURE); + } + + if (avro_schema_enum_get_by_name(schema, "Haskell") != -1) { + fprintf(stderr, "Unexpected enum schema index\n"); + exit(EXIT_FAILURE); + } + + avro_schema_decref(schema); + return 0; +} + +static int test_fixed(void) +{ + avro_schema_t schema = avro_schema_fixed("msg", 8); + if (avro_schema_fixed_size(schema) != 8) { + fprintf(stderr, "Unexpected fixed size\n"); + exit(EXIT_FAILURE); + } + + avro_schema_decref(schema); + return 0; +} + +static int test_map(void) +{ + avro_schema_t schema = avro_schema_map(avro_schema_long()); + + if (!avro_schema_equal + (avro_schema_map_values(schema), avro_schema_long())) { + fprintf(stderr, "Unexpected map values schema"); + exit(EXIT_FAILURE); + } + + avro_schema_decref(schema); + return 0; +} + +static int test_record(void) +{ + avro_schema_t schema = avro_schema_record("person", NULL); + + avro_schema_record_field_append(schema, "name", avro_schema_string()); + avro_schema_record_field_append(schema, "age", avro_schema_int()); + + if (avro_schema_record_field_get_index(schema, "name") != 0) { + fprintf(stderr, "Incorrect index for \"name\" field\n"); + exit(EXIT_FAILURE); + } + + if (avro_schema_record_field_get_index(schema, "unknown") != -1) { + fprintf(stderr, "Incorrect index for \"unknown\" field\n"); + exit(EXIT_FAILURE); + } + + avro_schema_t name_field = + avro_schema_record_field_get(schema, "name"); + if (!avro_schema_equal(name_field, avro_schema_string())) { + fprintf(stderr, "Unexpected name field\n"); + exit(EXIT_FAILURE); + } + + avro_schema_t field1 = + avro_schema_record_field_get_by_index(schema, 1); + if (!avro_schema_equal(field1, avro_schema_int())) { + fprintf(stderr, "Unexpected field 1\n"); + exit(EXIT_FAILURE); + } + + avro_schema_decref(schema); + return 0; +} + +static int test_union(void) +{ + avro_schema_t schema = avro_schema_union(); + + avro_schema_union_append(schema, avro_schema_string()); + avro_schema_union_append(schema, avro_schema_int()); + avro_schema_union_append(schema, avro_schema_null()); + + if (!avro_schema_equal + (avro_schema_string(), + avro_schema_union_branch(schema, 0))) { + fprintf(stderr, "Unexpected union schema branch 0\n"); + exit(EXIT_FAILURE); + } + + if (!avro_schema_equal + (avro_schema_string(), + avro_schema_union_branch_by_name(schema, NULL, "string"))) { + fprintf(stderr, "Unexpected union schema branch \"string\"\n"); + exit(EXIT_FAILURE); + } + + avro_schema_decref(schema); + return 0; +} + +int main(int argc, char *argv[]) +{ + char *srcdir = getenv("srcdir"); + char path[1024]; + + AVRO_UNUSED(argc); + AVRO_UNUSED(argv); + + if (!srcdir) { + srcdir = "."; + } + + avro_stderr = avro_writer_file(stderr); + + /* + * Run the tests that should pass + */ + snprintf(path, sizeof(path), "%s/schema_tests/pass", srcdir); + fprintf(stderr, "RUNNING %s\n", path); + run_tests(path, 1); + snprintf(path, sizeof(path), "%s/schema_tests/fail", srcdir); + fprintf(stderr, "RUNNING %s\n", path); + run_tests(path, 0); + + fprintf(stderr, "*** Running array tests **\n"); + test_array(); + fprintf(stderr, "*** Running enum tests **\n"); + test_enum(); + fprintf(stderr, "*** Running fixed tests **\n"); + test_fixed(); + fprintf(stderr, "*** Running map tests **\n"); + test_map(); + fprintf(stderr, "*** Running record tests **\n"); + test_record(); + fprintf(stderr, "*** Running union tests **\n"); + test_union(); + + fprintf(stderr, "==================================================\n"); + fprintf(stderr, + "Finished running %d schema test cases successfully \n", + test_cases); + fprintf(stderr, "==================================================\n"); + + avro_writer_free(avro_stderr); + return EXIT_SUCCESS; +} diff --git a/fluent-bit/lib/avro/tests/test_avro_schema_names.c b/fluent-bit/lib/avro/tests/test_avro_schema_names.c new file mode 100644 index 00000000..22a6991e --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_schema_names.c @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include "avro.h" +#include "avro_private.h" +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +int test_cases = 0; +avro_writer_t avro_stderr; + +static void test_helper(const char *json, + const char *name, + avro_schema_t expected) +{ + int rc; + avro_schema_t base; + avro_schema_error_t serror; + + rc = avro_schema_from_json(json, strlen(json), &base, &serror); + if (rc != 0) + { + fprintf(stderr, + "Error parsing Avro schema:\n%s\n", + json); + exit(EXIT_FAILURE); + } + + avro_schema_t actual = + avro_schema_get_subschema(base, name); + + if (actual == NULL) + { + fprintf(stderr, + "No subschema named \"%s\" in %s\n", + name, avro_schema_type_name(base)); + exit(EXIT_FAILURE); + } + + if (!avro_schema_equal(actual, expected)) + { + fprintf(stderr, + "Subschema \"%s\" should be %s, " + "is actually %s\n", + name, + avro_schema_type_name(expected), + avro_schema_type_name(actual)); + exit(EXIT_FAILURE); + } + + avro_schema_decref(base); + avro_schema_decref(expected); +} + +static void test_array_schema_01() +{ + static char *JSON = + "{" + " \"type\": \"array\"," + " \"items\": \"long\"" + "}"; + + test_helper(JSON, "[]", avro_schema_long()); +} + +static void test_map_schema_01() +{ + static char *JSON = + "{" + " \"type\": \"map\"," + " \"values\": \"long\"" + "}"; + + test_helper(JSON, "{}", avro_schema_long()); +} + +static void test_record_schema_01() +{ + static char *JSON = + "{" + " \"type\": \"record\"," + " \"name\": \"test\"," + " \"fields\": [" + " { \"name\": \"a\", \"type\": \"long\" }" + " ]" + "}"; + + test_helper(JSON, "a", avro_schema_long()); +} + +static void test_union_schema_01() +{ + static char *JSON = + "[" + " \"long\"," + " {" + " \"type\": \"record\"," + " \"name\": \"test\"," + " \"fields\": [" + " { \"name\": \"a\", \"type\": \"long\" }" + " ]" + " }" + "]"; + + test_helper(JSON, "long", avro_schema_long()); +} + +int main(int argc, char *argv[]) +{ + AVRO_UNUSED(argc); + AVRO_UNUSED(argv); + + test_array_schema_01(); + + test_map_schema_01(); + + test_record_schema_01(); + + test_union_schema_01(); + + return EXIT_SUCCESS; +} diff --git a/fluent-bit/lib/avro/tests/test_avro_values.c b/fluent-bit/lib/avro/tests/test_avro_values.c new file mode 100644 index 00000000..4930b8ca --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_values.c @@ -0,0 +1,1455 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +/* Test cases for the new avro_value_t interface */ + +#include <limits.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> + +#include "avro.h" +#include "avro_private.h" + +typedef int (*avro_test) (void); + +#ifndef SHOW_ALLOCATIONS +#define SHOW_ALLOCATIONS 0 +#endif + +/* + * Use a custom allocator that verifies that the size that we use to + * free an object matches the size that we use to allocate it. + */ + +static void * +test_allocator(void *ud, void *ptr, size_t osize, size_t nsize) +{ + AVRO_UNUSED(ud); + AVRO_UNUSED(osize); + +#if SHOW_ALLOCATIONS + fprintf(stderr, "alloc(%p, %" PRIsz ", %" PRIsz ") => ", ptr, osize, nsize); +#endif + + if (nsize == 0) { + size_t *size = ((size_t *) ptr) - 1; + if (osize != *size) { + fprintf(stderr, +#if SHOW_ALLOCATIONS + "ERROR!\n" +#endif + "Error freeing %p:\n" + "Size passed to avro_free (%" PRIsz ") " + "doesn't match size passed to " + "avro_malloc (%" PRIsz ")\n", + ptr, osize, *size); + exit(EXIT_FAILURE); + } + free(size); +#if SHOW_ALLOCATIONS + fprintf(stderr, "NULL\n"); +#endif + return NULL; + } else { + size_t real_size = nsize + sizeof(size_t); + size_t *old_size = ptr? ((size_t *) ptr)-1: NULL; + size_t *size = (size_t *) realloc(old_size, real_size); + *size = nsize; +#if SHOW_ALLOCATIONS + fprintf(stderr, "%p\n", (size+1)); +#endif + return (size + 1); + } +} + +void +init_rand(void) +{ + srand(time(NULL)); +} + +double +rand_number(double from, double to) +{ + double range = to - from; + return from + ((double)rand() / (RAND_MAX + 1.0)) * range; +} + +int64_t +rand_int64(void) +{ + return (int64_t) rand_number(LONG_MIN, LONG_MAX); +} + +int32_t +rand_int32(void) +{ + return (int32_t) rand_number(INT_MIN, INT_MAX); +} + +size_t +rand_count(void) +{ + return (size_t) rand_number(0, 100); +} + +#define check_(call) \ + do { \ + int _rval = call; \ + if (_rval) { return _rval; } \ + } while (0) + +/* + * Verify that we can't call any of the getters and setters that don't + * apply to the given value. + */ + +static int +_check_invalid_methods(const char *name, avro_value_t *val) +{ + avro_type_t type = avro_value_get_type(val); + +/* For a description on GCC vs Visual Studio 2008 usage of variadic + * macros see: + * https://stackoverflow.com/questions/2575864/the-problem-about-different + * -treatment-to-va-args-when-using-vs-2008-and-gcc + */ +#define expand_args(...) __VA_ARGS__ +#define check_bad(method, ...) \ + do { \ + if (!expand_args(avro_value_##method(__VA_ARGS__))) { \ + fprintf(stderr, \ + "Shouldn't be able to " #method " a %s\n", \ + name); \ + return EXIT_FAILURE; \ + } \ + } while (0) + + if (type != AVRO_BOOLEAN) { + int dummy = 0; + check_bad(get_boolean, val, &dummy); + check_bad(set_boolean, val, dummy); + } + + if (type != AVRO_BYTES) { + const void *cbuf = NULL; + void *buf = NULL; + size_t size = 0; + check_bad(get_bytes, val, &cbuf, &size); + check_bad(set_bytes, val, buf, size); + } + + if (type != AVRO_DOUBLE) { + double dummy = 0; + check_bad(get_double, val, &dummy); + check_bad(set_double, val, dummy); + } + + if (type != AVRO_FLOAT) { + float dummy = 0; + check_bad(get_float, val, &dummy); + check_bad(set_float, val, dummy); + } + + if (type != AVRO_INT32) { + int32_t dummy = 0; + check_bad(get_int, val, &dummy); + check_bad(set_int, val, dummy); + } + + if (type != AVRO_INT64) { + int64_t dummy = 0; + check_bad(get_long, val, &dummy); + check_bad(set_long, val, dummy); + } + + if (type != AVRO_NULL) { + check_bad(get_null, val); + check_bad(set_null, val); + } + + if (type != AVRO_STRING) { + const char *cstr = NULL; + char *str = NULL; + size_t size = 0; + check_bad(get_string, val, &cstr, &size); + check_bad(set_string, val, str); + check_bad(set_string_len, val, str, size); + } + + if (type != AVRO_ENUM) { + int dummy = 0; + check_bad(get_enum, val, &dummy); + check_bad(set_enum, val, dummy); + } + + if (type != AVRO_FIXED) { + const void *cbuf = NULL; + void *buf = NULL; + size_t size = 0; + check_bad(get_fixed, val, &cbuf, &size); + check_bad(set_fixed, val, buf, size); + } + + if (type != AVRO_ARRAY && type != AVRO_MAP && type != AVRO_RECORD) { + size_t size = 0; + check_bad(get_size, val, &size); + + size_t index = 0; + avro_value_t child; + const char *key = NULL; + check_bad(get_by_index, val, index, &child, &key); + } + + if (type != AVRO_MAP && type != AVRO_RECORD) { + const char *key = NULL; + avro_value_t child; + size_t index = 0; + check_bad(get_by_name, val, key, &child, &index); + } + + if (type != AVRO_ARRAY) { + avro_value_t child; + size_t index; + check_bad(append, val, &child, &index); + } + + if (type != AVRO_MAP) { + const char *key = NULL; + avro_value_t child; + size_t index = 0; + int is_new = 0; + check_bad(add, val, key, &child, &index, &is_new); + } + + if (type != AVRO_UNION) { + int discriminant = 0; + avro_value_t branch; + check_bad(get_discriminant, val, &discriminant); + check_bad(get_current_branch, val, &branch); + check_bad(set_branch, val, discriminant, &branch); + } + +#undef check_bad + + return EXIT_SUCCESS; +} + +#define check_invalid_methods(name, val) \ + check_(_check_invalid_methods(name, val)) + +/* + * Verify that we get the expected type code and schema for a value. + */ + +static int +check_type_and_schema(const char *name, + avro_value_t *val, + avro_type_t expected_type, + avro_schema_t expected_schema) +{ + if (avro_value_get_type(val) != expected_type) { + avro_schema_decref(expected_schema); + fprintf(stderr, "Unexpected type for %s\n", name); + return EXIT_FAILURE; + } + + if (!avro_schema_equal(avro_value_get_schema(val), + expected_schema)) { + avro_schema_decref(expected_schema); + fprintf(stderr, "Unexpected schema for %s\n", name); + return EXIT_FAILURE; + } + + avro_schema_decref(expected_schema); + return EXIT_SUCCESS; +} + +#define try(call, msg) \ + do { \ + if (call) { \ + fprintf(stderr, msg ":\n %s\n", avro_strerror()); \ + return EXIT_FAILURE; \ + } \ + } while (0) + +static int +_check_write_read(avro_value_t *val) +{ + static char buf[4096]; + + avro_reader_t reader = avro_reader_memory(buf, sizeof(buf)); + avro_writer_t writer = avro_writer_memory(buf, sizeof(buf)); + + if (avro_value_write(writer, val)) { + fprintf(stderr, "Unable to write value:\n %s\n", + avro_strerror()); + return EXIT_FAILURE; + } + + avro_writer_dump(writer, stderr); + + size_t size; + if (avro_value_sizeof(val, &size)) { + fprintf(stderr, "Unable to determine size of value:\n %s\n", + avro_strerror()); + return EXIT_FAILURE; + } + + if (size != (size_t) avro_writer_tell(writer)) { + fprintf(stderr, "Unexpected size of encoded value\n"); + return EXIT_FAILURE; + } + + avro_value_t val_in; + if (avro_generic_value_new(val->iface, &val_in)) { + fprintf(stderr, "Cannot allocate new value instance:\n %s\n", + avro_strerror()); + return EXIT_FAILURE; + } + + if (avro_value_read(reader, &val_in)) { + fprintf(stderr, "Unable to read value:\n %s\n", + avro_strerror()); + return EXIT_FAILURE; + } + + if (!avro_value_equal(val, &val_in)) { + fprintf(stderr, "Round-trip values not equal\n"); + exit(EXIT_FAILURE); + } + + avro_value_decref(&val_in); + avro_reader_free(reader); + avro_writer_free(writer); + + return EXIT_SUCCESS; +} + +#define check_write_read(val) \ + check_(_check_write_read(val)) + +static int +_check_hash(avro_value_t *val1, avro_value_t *val2) +{ + uint32_t hash1 = avro_value_hash(val1); + uint32_t hash2 = avro_value_hash(val2); + if (hash1 != hash2) { + fprintf(stderr, "Copied hashed not equal\n"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +#define check_hash(val1, val2) \ + check_(_check_hash(val1, val2)) + +static int +_check_copy(avro_value_t *val) +{ + avro_value_t copied_val; + if (avro_generic_value_new(val->iface, &copied_val)) { + fprintf(stderr, "Cannot allocate new value instance:\n %s\n", + avro_strerror()); + return EXIT_FAILURE; + } + + if (avro_value_copy_fast(&copied_val, val)) { + fprintf(stderr, "Cannot copy value:\n %s\n", + avro_strerror()); + return EXIT_FAILURE; + } + + if (!avro_value_equal(val, &copied_val)) { + fprintf(stderr, "Copied values not equal\n"); + return EXIT_FAILURE; + } + + check_hash(val, &copied_val); + + avro_value_decref(&copied_val); + return EXIT_SUCCESS; +} + +#define check_copy(val) \ + check_(_check_copy(val)) + +static int +test_boolean(void) +{ + int rval; + + int i; + for (i = 0; i <= 1; i++) { + avro_value_t val; + try(avro_generic_boolean_new(&val, i), + "Cannot create boolean"); + check(rval, check_type_and_schema + ("boolean", &val, + AVRO_BOOLEAN, avro_schema_boolean())); + try(avro_value_reset(&val), + "Cannot reset boolean"); + try(avro_value_set_boolean(&val, i), + "Cannot set boolean"); + + /* Start with the wrong value to make sure _get does + * something. */ + int actual = (int) 23; + try(avro_value_get_boolean(&val, &actual), + "Cannot get boolean value"); + + if (actual != i) { + fprintf(stderr, "Unexpected boolean value\n"); + return EXIT_FAILURE; + } + + check_invalid_methods("boolean", &val); + check_write_read(&val); + check_copy(&val); + avro_value_decref(&val); + } + + avro_value_t val1; + avro_value_t val2; + try(avro_generic_boolean_new(&val1, 0), + "Cannot create boolean"); + try(avro_generic_boolean_new(&val2, 1), + "Cannot create boolean"); + if (avro_value_cmp_fast(&val1, &val2) >= 0) { + fprintf(stderr, "Incorrect sort order\n"); + return EXIT_FAILURE; + } + if (avro_value_cmp_fast(&val2, &val1) <= 0) { + fprintf(stderr, "Incorrect sort order\n"); + return EXIT_FAILURE; + } + if (avro_value_cmp_fast(&val1, &val1) != 0) { + fprintf(stderr, "Incorrect sort order\n"); + return EXIT_FAILURE; + } + avro_value_decref(&val1); + avro_value_decref(&val2); + + return 0; +} + +static int +test_bytes(void) +{ + int rval; + + char bytes[] = { 0xDE, 0xAD, 0xBE, 0xEF }; + + avro_value_t val; + try(avro_generic_bytes_new(&val, bytes, sizeof(bytes)), + "Cannot create bytes"); + check(rval, check_type_and_schema + ("bytes", &val, + AVRO_BYTES, avro_schema_bytes())); + try(avro_value_reset(&val), + "Cannot reset bytes"); + try(avro_value_set_bytes(&val, bytes, sizeof(bytes)), + "Cannot set bytes"); + + const void *actual_buf = NULL; + size_t actual_size = 0; + try(avro_value_get_bytes(&val, &actual_buf, &actual_size), + "Cannot get bytes value"); + + if (actual_size != sizeof(bytes)) { + fprintf(stderr, "Unexpected bytes size\n"); + return EXIT_FAILURE; + } + + if (memcmp(actual_buf, bytes, sizeof(bytes)) != 0) { + fprintf(stderr, "Unexpected bytes contents\n"); + return EXIT_FAILURE; + } + + avro_wrapped_buffer_t wbuf; + try(avro_value_grab_bytes(&val, &wbuf), + "Cannot grab bytes value"); + + if (wbuf.size != sizeof(bytes)) { + fprintf(stderr, "Unexpected grabbed bytes size\n"); + return EXIT_FAILURE; + } + + if (memcmp(wbuf.buf, bytes, sizeof(bytes)) != 0) { + fprintf(stderr, "Unexpected grabbed bytes contents\n"); + return EXIT_FAILURE; + } + + avro_wrapped_buffer_free(&wbuf); + + check_invalid_methods("bytes", &val); + check_write_read(&val); + check_copy(&val); + avro_value_decref(&val); + + avro_value_t val1; + avro_value_t val2; + avro_value_t val3; + try(avro_generic_bytes_new(&val1, "abcd", 4), + "Cannot create bytes"); + try(avro_generic_bytes_new(&val2, "abcde", 5), + "Cannot create bytes"); + try(avro_generic_bytes_new(&val3, "abce", 4), + "Cannot create bytes"); + if (avro_value_cmp_fast(&val1, &val2) >= 0) { + fprintf(stderr, "Incorrect sort order\n"); + return EXIT_FAILURE; + } + if (avro_value_cmp_fast(&val2, &val1) <= 0) { + fprintf(stderr, "Incorrect sort order\n"); + return EXIT_FAILURE; + } + if (avro_value_cmp_fast(&val1, &val3) >= 0) { + fprintf(stderr, "Incorrect sort order\n"); + return EXIT_FAILURE; + } + if (avro_value_cmp_fast(&val1, &val1) != 0) { + fprintf(stderr, "Incorrect sort order\n"); + return EXIT_FAILURE; + } + avro_value_decref(&val1); + avro_value_decref(&val2); + avro_value_decref(&val3); + + return 0; +} + +static int +test_double(void) +{ + int rval; + + int i; + for (i = 0; i < 100; i++) { + double expected = rand_number(-1e10, 1e10); + avro_value_t val; + try(avro_generic_double_new(&val, expected), + "Cannot create double"); + check(rval, check_type_and_schema + ("double", &val, + AVRO_DOUBLE, avro_schema_double())); + try(avro_value_reset(&val), + "Cannot reset double"); + try(avro_value_set_double(&val, expected), + "Cannot set double"); + + double actual = 0.0; + try(avro_value_get_double(&val, &actual), + "Cannot get double value"); + + if (actual != expected) { + fprintf(stderr, "Unexpected double value\n"); + return EXIT_FAILURE; + } + + check_invalid_methods("double", &val); + check_write_read(&val); + check_copy(&val); + avro_value_decref(&val); + } + return 0; +} + +static int +test_float(void) +{ + int rval; + + int i; + for (i = 0; i < 100; i++) { + float expected = rand_number(-1e10, 1e10); + avro_value_t val; + try(avro_generic_float_new(&val, expected), + "Cannot create float"); + check(rval, check_type_and_schema + ("float", &val, + AVRO_FLOAT, avro_schema_float())); + try(avro_value_reset(&val), + "Cannot reset float"); + try(avro_value_set_float(&val, expected), + "Cannot set float"); + + float actual = 0.0f; + try(avro_value_get_float(&val, &actual), + "Cannot get float value"); + + if (actual != expected) { + fprintf(stderr, "Unexpected float value\n"); + return EXIT_FAILURE; + } + + check_invalid_methods("float", &val); + check_write_read(&val); + check_copy(&val); + avro_value_decref(&val); + } + return 0; +} + +static int +test_int(void) +{ + int rval; + + int i; + for (i = 0; i < 100; i++) { + int32_t expected = rand_int32(); + avro_value_t val; + try(avro_generic_int_new(&val, expected), + "Cannot create int"); + check(rval, check_type_and_schema + ("int", &val, + AVRO_INT32, avro_schema_int())); + try(avro_value_reset(&val), + "Cannot reset int"); + try(avro_value_set_int(&val, expected), + "Cannot set int"); + + int32_t actual = 0; + try(avro_value_get_int(&val, &actual), + "Cannot get int value"); + + if (actual != expected) { + fprintf(stderr, "Unexpected int value\n"); + return EXIT_FAILURE; + } + + check_invalid_methods("int", &val); + check_write_read(&val); + check_copy(&val); + avro_value_decref(&val); + } + + avro_value_t val1; + avro_value_t val2; + try(avro_generic_int_new(&val1, -10), + "Cannot create int"); + try(avro_generic_int_new(&val2, 42), + "Cannot create int"); + if (avro_value_cmp_fast(&val1, &val2) >= 0) { + fprintf(stderr, "Incorrect sort order\n"); + return EXIT_FAILURE; + } + if (avro_value_cmp_fast(&val2, &val1) <= 0) { + fprintf(stderr, "Incorrect sort order\n"); + return EXIT_FAILURE; + } + if (avro_value_cmp_fast(&val1, &val1) != 0) { + fprintf(stderr, "Incorrect sort order\n"); + return EXIT_FAILURE; + } + avro_value_decref(&val1); + avro_value_decref(&val2); + + return 0; +} + +static int +test_long(void) +{ + int rval; + + int i; + for (i = 0; i < 100; i++) { + int64_t expected = rand_int64(); + avro_value_t val; + try(avro_generic_long_new(&val, expected), + "Cannot create long"); + check(rval, check_type_and_schema + ("long", &val, + AVRO_INT64, avro_schema_long())); + try(avro_value_reset(&val), + "Cannot reset long"); + try(avro_value_set_long(&val, expected), + "Cannot set long"); + + int64_t actual = 0; + try(avro_value_get_long(&val, &actual), + "Cannot get long value"); + + if (actual != expected) { + fprintf(stderr, "Unexpected long value\n"); + return EXIT_FAILURE; + } + + check_invalid_methods("long", &val); + check_write_read(&val); + check_copy(&val); + avro_value_decref(&val); + } + return 0; +} + +static int +test_null(void) +{ + int rval; + + avro_value_t val; + try(avro_generic_null_new(&val), + "Cannot create null"); + check(rval, check_type_and_schema + ("null", &val, + AVRO_NULL, avro_schema_null())); + try(avro_value_reset(&val), + "Cannot reset null"); + try(avro_value_set_null(&val), + "Cannot set null"); + try(avro_value_get_null(&val), + "Cannot get null"); + + check_invalid_methods("null", &val); + check_write_read(&val); + check_copy(&val); + avro_value_decref(&val); + return 0; +} + +static int +test_string(void) +{ + int rval; + + char *strings[] = { + "Four score and seven years ago", + "our father brought forth on this continent", + "a new nation", + "conceived in Liberty", + "and dedicated to the proposition that all men " + "are created equal." + }; + + unsigned int i; + for (i = 0; i < sizeof(strings) / sizeof(strings[0]); i++) { + avro_value_t val; + try(avro_generic_string_new(&val, strings[i]), + "Cannot create string"); + check(rval, check_type_and_schema + ("string", &val, + AVRO_STRING, avro_schema_string())); + try(avro_value_reset(&val), + "Cannot reset string"); + try(avro_value_set_string_len(&val, "", 0), + "Cannot set_len dummy string"); + + /* First try a round-trip using set_string */ + + try(avro_value_set_string(&val, strings[i]), + "Cannot set string"); + + const char *actual_str = NULL; + size_t actual_size = 0; + try(avro_value_get_string(&val, &actual_str, &actual_size), + "Cannot get string value"); + + if (actual_size != strlen(strings[i])+1) { + fprintf(stderr, "Unexpected string size\n"); + return EXIT_FAILURE; + } + + if (strcmp(actual_str, strings[i]) != 0) { + fprintf(stderr, "Unexpected string contents\n"); + return EXIT_FAILURE; + } + + avro_wrapped_buffer_t wbuf; + try(avro_value_grab_string(&val, &wbuf), + "Cannot grab string value"); + + if (wbuf.size != strlen(strings[i])+1) { + fprintf(stderr, "Unexpected grabbed string size\n"); + return EXIT_FAILURE; + } + + if (strcmp((const char *) wbuf.buf, strings[i]) != 0) { + fprintf(stderr, "Unexpected grabbed string contents\n"); + return EXIT_FAILURE; + } + + avro_wrapped_buffer_free(&wbuf); + + /* and then again using set_string_len */ + + size_t str_length = strlen(strings[i])+1; + try(avro_value_set_string_len(&val, strings[i], str_length), + "Cannot set_len string"); + + actual_str = NULL; + actual_size = 0; + try(avro_value_get_string(&val, &actual_str, &actual_size), + "Cannot get string value"); + + if (actual_size != strlen(strings[i])+1) { + fprintf(stderr, "Unexpected string size\n"); + return EXIT_FAILURE; + } + + if (strcmp(actual_str, strings[i]) != 0) { + fprintf(stderr, "Unexpected string contents\n"); + return EXIT_FAILURE; + } + + try(avro_value_grab_string(&val, &wbuf), + "Cannot grab string value"); + + if (wbuf.size != strlen(strings[i])+1) { + fprintf(stderr, "Unexpected grabbed string size\n"); + return EXIT_FAILURE; + } + + if (strcmp((const char *) wbuf.buf, strings[i]) != 0) { + fprintf(stderr, "Unexpected grabbed string contents\n"); + return EXIT_FAILURE; + } + + avro_wrapped_buffer_free(&wbuf); + + check_invalid_methods("string", &val); + check_write_read(&val); + check_copy(&val); + avro_value_decref(&val); + } + + return 0; +} + +static int +test_array(void) +{ + avro_schema_t double_schema = avro_schema_double(); + avro_schema_t array_schema = avro_schema_array(double_schema); + + avro_value_iface_t *array_class = + avro_generic_class_from_schema(array_schema); + + int rval; + + int i; + for (i = 0; i < 100; i++) { + size_t count = rand_count(); + + avro_value_t val; + try(avro_generic_value_new(array_class, &val), + "Cannot create array"); + check(rval, check_type_and_schema + ("array", &val, AVRO_ARRAY, + avro_schema_incref(array_schema))); + + size_t j; + for (j = 0; j < count; j++) { + avro_value_t element; + size_t new_index; + try(avro_value_append(&val, &element, &new_index), + "Cannot append to array"); + if (new_index != j) { + fprintf(stderr, "Unexpected index\n"); + return EXIT_FAILURE; + } + + double expected = rand_number(-1e10, 1e10); + try(avro_value_set_double(&element, expected), + "Cannot set double"); + try(avro_value_get_by_index(&val, j, &element, NULL), + "Cannot get from array"); + + double actual = 0.0; + try(avro_value_get_double(&element, &actual), + "Cannot get double value"); + + if (actual != expected) { + fprintf(stderr, "Unexpected double value\n"); + return EXIT_FAILURE; + } + } + + size_t actual_size = 0; + try(avro_value_get_size(&val, &actual_size), + "Cannot get_size array"); + + if (actual_size != count) { + fprintf(stderr, "Unexpected size\n"); + return EXIT_FAILURE; + } + + check_write_read(&val); + check_copy(&val); + + try(avro_value_reset(&val), + "Cannot reset array"); + try(avro_value_get_size(&val, &actual_size), + "Cannot get_size empty array"); + + if (actual_size != 0) { + fprintf(stderr, "Unexpected empty size\n"); + return EXIT_FAILURE; + } + + check_invalid_methods("array", &val); + avro_value_decref(&val); + } + + avro_schema_decref(double_schema); + avro_schema_decref(array_schema); + avro_value_iface_decref(array_class); + return 0; +} + +static int +test_enum(void) +{ + static const char SCHEMA_JSON[] = + "{" + " \"type\": \"enum\"," + " \"name\": \"suits\"," + " \"symbols\": [\"CLUBS\",\"DIAMONDS\",\"HEARTS\",\"SPADES\"]" + "}"; + + avro_schema_t enum_schema = NULL; + if (avro_schema_from_json_literal(SCHEMA_JSON, &enum_schema)) { + fprintf(stderr, "Error parsing schema:\n %s\n", + avro_strerror()); + return EXIT_FAILURE; + } + + avro_value_iface_t *enum_class = + avro_generic_class_from_schema(enum_schema); + + int rval; + + int i; + for (i = 0; i < 4; i++) { + int expected = i; + avro_value_t val; + try(avro_generic_value_new(enum_class, &val), + "Cannot create enum"); + check(rval, check_type_and_schema + ("enum", &val, AVRO_ENUM, + avro_schema_incref(enum_schema))); + try(avro_value_reset(&val), + "Cannot reset enum"); + try(avro_value_set_enum(&val, expected), + "Cannot set enum"); + + int actual = -1; + try(avro_value_get_enum(&val, &actual), + "Cannot get enum value"); + + if (actual != expected) { + fprintf(stderr, "Unexpected enum value\n"); + return EXIT_FAILURE; + } + + check_invalid_methods("enum", &val); + check_write_read(&val); + check_copy(&val); + avro_value_decref(&val); + } + + avro_schema_decref(enum_schema); + avro_value_iface_decref(enum_class); + return 0; +} + +static int +test_fixed(void) +{ + static const char SCHEMA_JSON[] = + "{" + " \"type\": \"fixed\"," + " \"name\": \"ipv4\"," + " \"size\": 4" + "}"; + + avro_schema_t fixed_schema = NULL; + if (avro_schema_from_json_literal(SCHEMA_JSON, &fixed_schema)) { + fprintf(stderr, "Error parsing schema:\n %s\n", + avro_strerror()); + return EXIT_FAILURE; + } + + avro_value_iface_t *fixed_class = + avro_generic_class_from_schema(fixed_schema); + + int rval; + + char fixed[] = { 0xDE, 0xAD, 0xBE, 0xEF }; + + avro_value_t val; + try(avro_generic_value_new(fixed_class, &val), + "Cannot create fixed"); + check(rval, check_type_and_schema + ("fixed", &val, AVRO_FIXED, + avro_schema_incref(fixed_schema))); + try(avro_value_reset(&val), + "Cannot reset fixed"); + + /* verify an error on invalid size */ + try(!avro_value_set_fixed(&val, fixed, 0), + "Expected error with invalid size"); + + try(avro_value_set_fixed(&val, fixed, sizeof(fixed)), + "Cannot set fixed"); + + const void *actual_buf = NULL; + size_t actual_size = 0; + try(avro_value_get_fixed(&val, &actual_buf, &actual_size), + "Cannot get fixed value"); + + if (actual_size != sizeof(fixed)) { + fprintf(stderr, "Unexpected fixed size\n"); + return EXIT_FAILURE; + } + + if (memcmp(actual_buf, fixed, sizeof(fixed)) != 0) { + fprintf(stderr, "Unexpected fixed contents\n"); + return EXIT_FAILURE; + } + + avro_wrapped_buffer_t wbuf; + try(avro_value_grab_fixed(&val, &wbuf), + "Cannot grab fixed value"); + + if (wbuf.size != sizeof(fixed)) { + fprintf(stderr, "Unexpected grabbed fixed size\n"); + return EXIT_FAILURE; + } + + if (memcmp(wbuf.buf, fixed, sizeof(fixed)) != 0) { + fprintf(stderr, "Unexpected grabbed fixed contents\n"); + return EXIT_FAILURE; + } + + avro_wrapped_buffer_free(&wbuf); + + check_invalid_methods("fixed", &val); + check_write_read(&val); + check_copy(&val); + avro_value_decref(&val); + avro_schema_decref(fixed_schema); + avro_value_iface_decref(fixed_class); + return 0; +} + +static int +test_map(void) +{ + avro_schema_t double_schema = avro_schema_double(); + avro_schema_t map_schema = avro_schema_map(double_schema); + + avro_value_iface_t *map_class = + avro_generic_class_from_schema(map_schema); + + int rval; + + int i; + for (i = 0; i < 100; i++) { + size_t count = rand_count(); + + avro_value_t val; + try(avro_generic_value_new(map_class, &val), + "Cannot create map"); + check(rval, check_type_and_schema + ("map", &val, AVRO_MAP, + avro_schema_incref(map_schema))); + + size_t j; + for (j = 0; j < count; j++) { + avro_value_t element; + size_t new_index; + int is_new = 0; + + char key[64]; + snprintf(key, 64, "%" PRIsz, j); + + try(avro_value_add(&val, key, + &element, &new_index, &is_new), + "Cannot add to map"); + + if (new_index != j) { + fprintf(stderr, "Unexpected index\n"); + return EXIT_FAILURE; + } + + if (!is_new) { + fprintf(stderr, "Expected new element\n"); + return EXIT_FAILURE; + } + + double expected = rand_number(-1e10, 1e10); + try(avro_value_set_double(&element, expected), + "Cannot set double"); + try(avro_value_add(&val, key, + &element, &new_index, &is_new), + "Cannot re-add to map"); + + if (is_new) { + fprintf(stderr, "Expected non-new element\n"); + return EXIT_FAILURE; + } + + const char *actual_key = NULL; + try(avro_value_get_by_index(&val, j, &element, + &actual_key), + "Cannot get from map"); + + if (strcmp(actual_key, key) != 0) { + fprintf(stderr, "Unexpected key\n"); + return EXIT_FAILURE; + } + + double actual = 0.0; + try(avro_value_get_double(&element, &actual), + "Cannot get double value"); + + if (actual != expected) { + fprintf(stderr, "Unexpected double value\n"); + return EXIT_FAILURE; + } + } + + size_t actual_size = 0; + try(avro_value_get_size(&val, &actual_size), + "Cannot get_size map"); + + if (actual_size != count) { + fprintf(stderr, "Unexpected size\n"); + return EXIT_FAILURE; + } + + /* + * Create a reversed copy of the map to ensure that the + * element ordering doesn't affect the hash value. + */ + + avro_value_t reversed; + try(avro_generic_value_new(map_class, &reversed), + "Cannot create map"); + + for (j = count; j-- > 0; ) { + avro_value_t element; + const char *key = NULL; + double element_value = 0.0; + try(avro_value_get_by_index(&val, j, &element, &key), + "Cannot get from map"); + try(avro_value_get_double(&element, &element_value), + "Cannot get double value"); + + try(avro_value_add(&reversed, key, &element, NULL, NULL), + "Cannot add to map"); + try(avro_value_set_double(&element, element_value), + "Cannot set double"); + } + + check_hash(&val, &reversed); + if (!avro_value_equal(&val, &reversed)) { + fprintf(stderr, "Reversed values not equal\n"); + return EXIT_FAILURE; + } + + /* Final tests and cleanup */ + + check_write_read(&val); + check_copy(&val); + + try(avro_value_reset(&val), + "Cannot reset map"); + try(avro_value_get_size(&val, &actual_size), + "Cannot get_size empty map"); + + if (actual_size != 0) { + fprintf(stderr, "Unexpected empty size\n"); + return EXIT_FAILURE; + } + + check_invalid_methods("map", &val); + avro_value_decref(&val); + avro_value_decref(&reversed); + } + + avro_schema_decref(double_schema); + avro_schema_decref(map_schema); + avro_value_iface_decref(map_class); + return 0; +} + +static int +test_record(void) +{ + static const char SCHEMA_JSON[] = + "{" + " \"type\": \"record\"," + " \"name\": \"test\"," + " \"fields\": [" + " { \"name\": \"b\", \"type\": \"boolean\" }," + " { \"name\": \"i\", \"type\": \"int\" }," + " { \"name\": \"s\", \"type\": \"string\" }," + " { \"name\": \"ds\", \"type\": " + " { \"type\": \"array\", \"items\": \"double\" } }," + " { \"name\": \"sub\", \"type\": " + " {" + " \"type\": \"record\"," + " \"name\": \"subtest\"," + " \"fields\": [" + " { \"name\": \"f\", \"type\": \"float\" }," + " { \"name\": \"l\", \"type\": \"long\" }" + " ]" + " }" + " }," + " { \"name\": \"nested\", \"type\": [\"null\", \"test\"] }" + " ]" + "}"; + + avro_schema_t record_schema = NULL; + if (avro_schema_from_json_literal(SCHEMA_JSON, &record_schema)) { + fprintf(stderr, "Error parsing schema:\n %s\n", + avro_strerror()); + return EXIT_FAILURE; + } + + avro_value_iface_t *record_class = + avro_generic_class_from_schema(record_schema); + + int rval; + + avro_value_t val; + try(avro_generic_value_new(record_class, &val), + "Cannot create record"); + check(rval, check_type_and_schema + ("record", &val, AVRO_RECORD, + avro_schema_incref(record_schema))); + + size_t field_count; + try(avro_value_get_size(&val, &field_count), + "Cannot get field count"); + if (field_count != 6) { + fprintf(stderr, "Unexpected field count\n"); + return EXIT_FAILURE; + } + + /* Assign to each field */ + avro_value_t field; + avro_value_t element; + avro_value_t subfield; + avro_value_t branch; + const char *name; + size_t index; + + try(avro_value_get_by_index(&val, 0, &field, NULL), + "Cannot get field 0"); + try(avro_value_set_boolean(&field, 1), + "Cannot set field 0"); + + try(avro_value_get_by_index(&val, 1, &field, &name), + "Cannot get field 1"); + try(avro_value_set_int(&field, 42), + "Cannot set field 1"); + if (strcmp(name, "i") != 0) { + fprintf(stderr, "Unexpected name for field 1: %s\n", name); + return EXIT_FAILURE; + } + + try(avro_value_get_by_index(&val, 2, &field, NULL), + "Cannot get field 2"); + try(avro_value_set_string(&field, "Hello world!"), + "Cannot set field 2"); + + try(avro_value_get_by_name(&val, "i", &field, &index), + "Cannot get \"i\" field"); + if (index != 1) { + fprintf(stderr, "Unexpected index for \"i\" field: %" PRIsz "\n", index); + return EXIT_FAILURE; + } + + try(avro_value_get_by_index(&val, 3, &field, NULL), + "Cannot get field 3"); + try(avro_value_append(&field, &element, NULL), + "Cannot append to field 3"); + try(avro_value_set_double(&element, 10.0), + "Cannot set field 3, element 0"); + + try(avro_value_get_by_index(&val, 4, &field, NULL), + "Cannot get field 4"); + + try(avro_value_get_by_index(&field, 0, &subfield, NULL), + "Cannot get field 4, subfield 0"); + try(avro_value_set_float(&subfield, 5.0f), + "Cannot set field 4, subfield 0"); + + try(avro_value_get_by_index(&field, 1, &subfield, NULL), + "Cannot get field 4, subfield 1"); + try(avro_value_set_long(&subfield, 10000), + "Cannot set field 4, subfield 1"); + + try(avro_value_get_by_index(&val, 5, &field, NULL), + "Cannot get field 5"); + try(avro_value_set_branch(&field, 0, &branch), + "Cannot select null branch"); + + check_write_read(&val); + check_copy(&val); + + /* Reset and verify that the fields are empty again */ + try(avro_value_reset(&val), + "Cannot reset record"); + + int bval; + try(avro_value_get_by_index(&val, 0, &field, NULL), + "Cannot get field 0"); + try(avro_value_get_boolean(&field, &bval), + "Cannot get field 0 value"); + if (bval) { + fprintf(stderr, "Unexpected value for field 0\n"); + return EXIT_FAILURE; + } + + size_t count; + try(avro_value_get_by_index(&val, 3, &field, NULL), + "Cannot get field 3"); + try(avro_value_get_size(&field, &count), + "Cannot get field 3 size"); + if (count != 0) { + fprintf(stderr, "Unexpected size for field 3\n"); + return EXIT_FAILURE; + } + + check_invalid_methods("record", &val); + avro_value_decref(&val); + avro_value_iface_decref(record_class); + avro_schema_decref(record_schema); + return EXIT_SUCCESS; +} + +static int +test_union(void) +{ + static const char SCHEMA_JSON[] = + "[" + " \"null\"," + " \"int\"," + " \"double\"," + " \"bytes\"" + "]"; + + avro_schema_t union_schema = NULL; + if (avro_schema_from_json_literal(SCHEMA_JSON, &union_schema)) { + fprintf(stderr, "Error parsing schema:\n %s\n", + avro_strerror()); + return EXIT_FAILURE; + } + + avro_value_iface_t *union_class = + avro_generic_class_from_schema(union_schema); + + int rval; + + avro_value_t val; + try(avro_generic_value_new(union_class, &val), + "Cannot create union"); + check(rval, check_type_and_schema + ("union", &val, AVRO_UNION, + avro_schema_incref(union_schema))); + + int discriminant = 0; + try(avro_value_get_discriminant(&val, &discriminant), + "Cannot get union discriminant"); + + if (discriminant != -1) { + fprintf(stderr, "Unexpected union discriminant\n"); + return EXIT_FAILURE; + } + + avro_value_t branch; + try(!avro_value_get_current_branch(&val, &branch), + "Expected error getting empty current branch"); + + try(avro_value_set_branch(&val, 0, &branch), + "Cannot select null branch"); + try(avro_value_set_null(&branch), + "Cannot set null branch value"); + + try(avro_value_set_branch(&val, 1, &branch), + "Cannot select int branch"); + try(avro_value_set_int(&branch, 42), + "Cannot set int branch value"); + + try(avro_value_set_branch(&val, 1, &branch), + "Cannot select int branch"); + try(avro_value_set_int(&branch, 10), + "Cannot set int branch value"); + + try(avro_value_set_branch(&val, 2, &branch), + "Cannot select double branch"); + try(avro_value_set_double(&branch, 10.0), + "Cannot set double branch value"); + + char bytes[] = { 0xDE, 0xAD, 0xBE, 0xEF }; + try(avro_value_set_branch(&val, 3, &branch), + "Cannot select bytes branch"); + try(avro_value_set_bytes(&branch, bytes, sizeof(bytes)), + "Cannot set bytes branch value"); + + check_invalid_methods("union", &val); + check_write_read(&val); + check_copy(&val); + avro_value_decref(&val); + + avro_schema_decref(union_schema); + avro_value_iface_decref(union_class); + return 0; +} + +int main(void) +{ + avro_set_allocator(test_allocator, NULL); + + unsigned int i; + struct avro_tests { + char *name; + avro_test func; + } tests[] = { + { "boolean", test_boolean }, + { "bytes", test_bytes }, + { "double", test_double }, + { "float", test_float }, + { "int", test_int }, + { "long", test_long }, + { "null", test_null }, + { "string", test_string }, + { "array", test_array }, + { "enum", test_enum }, + { "fixed", test_fixed }, + { "map", test_map }, + { "record", test_record }, + { "union", test_union } + }; + + init_rand(); + for (i = 0; i < sizeof(tests) / sizeof(tests[0]); i++) { + struct avro_tests *test = tests + i; + fprintf(stderr, "**** Running %s tests ****\n", test->name); + if (test->func() != 0) { + return EXIT_FAILURE; + } + } + return EXIT_SUCCESS; +} diff --git a/fluent-bit/lib/avro/tests/test_data_structures.c b/fluent-bit/lib/avro/tests/test_data_structures.c new file mode 100644 index 00000000..2986f5cc --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_data_structures.c @@ -0,0 +1,263 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "avro_private.h" +#include "avro/data.h" + +static int result = EXIT_SUCCESS; + +typedef int (*avro_test) (void); + + +static int +test_array(void) +{ + avro_raw_array_t array; + long *element; + + /* Test once on a fresh array */ + + avro_raw_array_init(&array, sizeof(long)); + element = (long *) avro_raw_array_append(&array); + *element = 1; + element = (long *) avro_raw_array_append(&array); + *element = 3; + + if (avro_raw_array_size(&array) != 2) { + fprintf(stderr, "Incorrect array size: got %lu, expected %lu.\n", + (unsigned long) avro_raw_array_size(&array), + (unsigned long) 2); + return EXIT_FAILURE; + } + + if (avro_raw_array_get(&array, long, 0) != 1) { + fprintf(stderr, "Unexpected array element %u: got %ld, expected %ld.\n", + (unsigned int) 0, avro_raw_array_get(&array, long, 0), + (long) 1); + return EXIT_FAILURE; + } + + /* And test again after clearing the array */ + + avro_raw_array_clear(&array); + element = (long *) avro_raw_array_append(&array); + *element = 1; + element = (long *) avro_raw_array_append(&array); + *element = 3; + + if (avro_raw_array_size(&array) != 2) { + fprintf(stderr, "Incorrect array size: got %" PRIsz ", expected %" PRIsz ".\n", + (size_t) avro_raw_array_size(&array), + (size_t) 2); + return EXIT_FAILURE; + } + + if (avro_raw_array_get(&array, long, 0) != 1) { + fprintf(stderr, "Unexpected array element %u: got %ld, expected %ld.\n", + (unsigned int) 0, avro_raw_array_get(&array, long, 0), + (long) 1); + return EXIT_FAILURE; + } + + avro_raw_array_done(&array); + return EXIT_SUCCESS; +} + + +static int +test_map(void) +{ + avro_raw_map_t map; + long *element; + size_t index; + + /* Test once on a fresh map */ + + avro_raw_map_init(&map, sizeof(long)); + avro_raw_map_get_or_create(&map, "x", (void **) &element, NULL); + *element = 1; + avro_raw_map_get_or_create(&map, "y", (void **) &element, NULL); + *element = 3; + + if (avro_raw_map_size(&map) != 2) { + fprintf(stderr, "Incorrect map size: got %" PRIsz ", expected %" PRIsz ".\n", + (size_t) avro_raw_map_size(&map), + (size_t) 2); + return EXIT_FAILURE; + } + + if (avro_raw_map_get_by_index(&map, long, 0) != 1) { + fprintf(stderr, "Unexpected map element %u: got %ld, expected %ld.\n", + (unsigned int) 0, + avro_raw_map_get_by_index(&map, long, 0), + (long) 1); + return EXIT_FAILURE; + } + + if (strcmp(avro_raw_map_get_key(&map, 0), "x") != 0) { + fprintf(stderr, "Unexpected key for map element 0: " + "got \"%s\", expected \"%s\".\n", + avro_raw_map_get_key(&map, 0), "x"); + return EXIT_FAILURE; + } + + element = (long *) avro_raw_map_get(&map, "y", &index); + if (index != 1) { + fprintf(stderr, "Unexpected index for map element \"%s\": " + "got %" PRIsz ", expected %u.\n", + "y", index, 1); + return EXIT_FAILURE; + } + + if (*element != 3) { + fprintf(stderr, "Unexpected map element %s: got %ld, expected %ld.\n", + "y", + *element, (long) 3); + return EXIT_FAILURE; + } + + /* And test again after clearing the map */ + + avro_raw_map_clear(&map); + avro_raw_map_get_or_create(&map, "x", (void **) &element, NULL); + *element = 1; + avro_raw_map_get_or_create(&map, "y", (void **) &element, NULL); + *element = 3; + + if (avro_raw_map_size(&map) != 2) { + fprintf(stderr, "Incorrect map size: got %" PRIsz ", expected %" PRIsz ".\n", + (size_t) avro_raw_map_size(&map), + (size_t) 2); + return EXIT_FAILURE; + } + + if (avro_raw_map_get_by_index(&map, long, 0) != 1) { + fprintf(stderr, "Unexpected map element %u: got %ld, expected %ld.\n", + (unsigned int) 0, + avro_raw_map_get_by_index(&map, long, 0), + (long) 1); + return EXIT_FAILURE; + } + + element = (long *) avro_raw_map_get(&map, "y", &index); + if (index != 1) { + fprintf(stderr, "Unexpected index for map element \"%s\": " + "got %" PRIsz ", expected %u.\n", + "y", index, 1); + return EXIT_FAILURE; + } + + if (*element != 3) { + fprintf(stderr, "Unexpected map element %s: got %ld, expected %ld.\n", + "y", + *element, (long) 3); + return EXIT_FAILURE; + } + + avro_raw_map_done(&map); + return EXIT_SUCCESS; +} + + +static int +test_string(void) +{ + avro_raw_string_t str; + + avro_raw_string_init(&str); + + avro_raw_string_set(&str, "a"); + avro_raw_string_set(&str, "abcdefgh"); + avro_raw_string_set(&str, "abcd"); + + if (avro_raw_string_length(&str) != 5) { + fprintf(stderr, "Incorrect string size: got %" PRIsz ", expected %" PRIsz ".\n", + (size_t) avro_raw_string_length(&str), + (size_t) 5); + return EXIT_FAILURE; + } + + if (strcmp((const char *) str.wrapped.buf, "abcd") != 0) { + fprintf(stderr, "Incorrect string contents: " + "got \"%s\", expected \"%s\".\n", + (char *) avro_raw_string_get(&str), + "abcd"); + return EXIT_FAILURE; + } + + avro_wrapped_buffer_t wbuf; + avro_wrapped_buffer_new_string(&wbuf, "abcd"); + avro_raw_string_give(&str, &wbuf); + + if (avro_raw_string_length(&str) != 5) { + fprintf(stderr, "Incorrect string size: got %" PRIsz ", expected %" PRIsz ".\n", + (size_t) avro_raw_string_length(&str), + (size_t) 5); + return EXIT_FAILURE; + } + + if (strcmp((const char *) str.wrapped.buf, "abcd") != 0) { + fprintf(stderr, "Incorrect string contents: " + "got \"%s\", expected \"%s\".\n", + (char *) avro_raw_string_get(&str), + "abcd"); + return EXIT_FAILURE; + } + + avro_raw_string_t str2; + avro_raw_string_init(&str2); + avro_raw_string_set(&str2, "abcd"); + + if (!avro_raw_string_equals(&str, &str2)) { + fprintf(stderr, "Strings should be equal.\n"); + return EXIT_FAILURE; + } + + avro_raw_string_done(&str); + avro_raw_string_done(&str2); + return EXIT_SUCCESS; +} + + +int main(int argc, char *argv[]) +{ + AVRO_UNUSED(argc); + AVRO_UNUSED(argv); + + unsigned int i; + struct avro_tests { + char *name; + avro_test func; + } tests[] = { + { "array", test_array }, + { "map", test_map }, + { "string", test_string } + }; + + for (i = 0; i < sizeof(tests) / sizeof(tests[0]); i++) { + struct avro_tests *test = tests + i; + fprintf(stderr, "**** Running %s tests ****\n", test->name); + if (test->func() != 0) { + result = EXIT_FAILURE; + } + } + return result; +} diff --git a/fluent-bit/lib/avro/tests/test_interop_data.c b/fluent-bit/lib/avro/tests/test_interop_data.c new file mode 100644 index 00000000..4738f318 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_interop_data.c @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include "avro.h" +#include "avro_private.h" +#include <dirent.h> +#include <stddef.h> +#include <stdio.h> + +int should_test(char *d_name) +{ + // check filename extension + char *ext_pos = strrchr(d_name, '.'); + if (ext_pos == NULL) + { + return 0; + } + + ptrdiff_t diff = d_name + strlen(d_name) - ext_pos; + char *substr = malloc(sizeof(char) * diff); + strncpy(substr, ext_pos + 1, diff - 1); + *(substr + diff - 1) = '\0'; + if (strcmp(substr, "avro") != 0) + { + free(substr); + return 0; + } + free(substr); + + // check compression codec + char *codec_pos = strrchr(d_name, '_'); + if (codec_pos == NULL) + { + return 1; + } + if (ext_pos < codec_pos) + { + return 0; + } + + diff = ext_pos - codec_pos; + substr = malloc(sizeof(char) * diff); + strncpy(substr, codec_pos + 1, diff - 1); + *(substr + diff - 1) = '\0'; + if (strcmp(substr, "deflate") == 0 || strcmp(substr, "snappy") == 0) + { + free(substr); + return 1; + } + free(substr); + + return 0; +} + +int main(int argc, char *argv[]) +{ + if (argc != 2) + { + fprintf(stderr, "%s accepts just one input file\n", argv[0]); + return EXIT_FAILURE; + } + + DIR *dir; + if ((dir = opendir(argv[1])) == NULL) + { + fprintf(stderr, "The specified path is not a directory: %s\n", argv[1]); + return EXIT_FAILURE; + } + + struct dirent *ent; + while ((ent = readdir(dir)) != NULL) + { + avro_file_reader_t reader; + avro_value_t value; + + if (ent->d_type != DT_REG) continue; + + char *d_name = ent->d_name; + size_t d_name_len = strlen(d_name); + size_t size = strlen(argv[1]) + d_name_len + 2; + char* path = malloc(sizeof(char) * size); + sprintf(path, "%s/%s", argv[1], d_name); + + if (!should_test(d_name)) + { + printf("Skipping file: %s\n", path); + free(path); + continue; + } + printf("Checking file: %s\n", path); + + if (avro_file_reader(path, &reader)) + { + fprintf(stderr, "Failed to read from a file: %s\n", path); + free(path); + return EXIT_FAILURE; + } + avro_schema_t schema = avro_file_reader_get_writer_schema(reader); + avro_value_iface_t *iface = avro_generic_class_from_schema(schema); + avro_generic_value_new(iface, &value); + + int i, rval; + for (i = 0; (rval = avro_file_reader_read_value(reader, &value)) == 0; i++, avro_value_reset(&value)); + if (rval != EOF) + { + fprintf(stderr, "Error(%d) occurred while reading file: %s\n", rval, path); + free(path); + return EXIT_FAILURE; + } + if (i == 0) + { + fprintf(stderr, "Input file %s is supposed to be non-empty\n", path); + free(path); + return EXIT_FAILURE; + } + + free(path); + avro_value_decref(&value); + avro_value_iface_decref(iface); + avro_schema_decref(schema); + avro_file_reader_close(reader); + } + + closedir(dir); + + return EXIT_SUCCESS; +} diff --git a/fluent-bit/lib/avro/tests/test_refcount.c b/fluent-bit/lib/avro/tests/test_refcount.c new file mode 100644 index 00000000..eee4eb97 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_refcount.c @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +#include <avro.h> +#include <stdio.h> +#include <stdlib.h> + +#define SIMPLE_ARRAY \ +"{\"type\": \"array\", \"items\": \"long\"}" + + +int main(void) +{ + avro_schema_t schema = NULL; + avro_schema_error_t error; + avro_value_iface_t *simple_array_class; + avro_value_t simple; + + /* Initialize the schema structure from JSON */ + if (avro_schema_from_json(SIMPLE_ARRAY, sizeof(SIMPLE_ARRAY), + &schema, &error)) { + fprintf(stdout, "Unable to parse schema\n"); + exit(EXIT_FAILURE); + } + + // Create avro class and value + simple_array_class = avro_generic_class_from_schema( schema ); + if ( simple_array_class == NULL ) + { + fprintf(stdout, "Unable to create simple array class\n"); + exit(EXIT_FAILURE); + } + + if ( avro_generic_value_new( simple_array_class, &simple ) ) + { + fprintf(stdout, "Error creating instance of record\n" ); + exit(EXIT_FAILURE); + } + + // Release the avro class and value + avro_value_decref( &simple ); + avro_value_iface_decref( simple_array_class ); + avro_schema_decref(schema); + + return 0; + +} diff --git a/fluent-bit/lib/avro/tests/test_valgrind b/fluent-bit/lib/avro/tests/test_valgrind new file mode 100644 index 00000000..b7c46688 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_valgrind @@ -0,0 +1,33 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +set +e +set -x + +if ! which valgrind; then + echo "Unable to find valgrind installed. Test will not run." + # This special exit value will show that we skipped this test + exit 77 +fi + +../libtool execute valgrind --leak-check=full -q test_avro_data 2>&1 |\ +grep -E '^==[0-9]+== ' +if [ $? -eq 0 ]; then + # Expression found. Test failed. + exit 1 +else + # We're all clean + exit 0 +fi diff --git a/fluent-bit/lib/avro/version.sh b/fluent-bit/lib/avro/version.sh new file mode 100755 index 00000000..be90c0f6 --- /dev/null +++ b/fluent-bit/lib/avro/version.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# This script is used to generate version numbers for autotools +# +# The information for libtool is maintained manually since +# the public API for the C library can change independent of the project +# +# Do each of these steps in order and libtool will do the right thing +# (1) If there are changes to libavro: +# libavro_micro_version++ +# libavro_interface_age++ +# libavro_binary_age++ +# (2) If any functions have been added: +# libavro_interface_age = 0 +# (3) If backwards compatibility has been broken: +# libavro_binary_age = 0 +# libavro_interface_age = 0 +# +libavro_micro_version=23 +libavro_interface_age=0 +libavro_binary_age=0 + +# IGNORE EVERYTHING ELSE FROM HERE DOWN......... +if test $# != 1; then + echo "USAGE: $0 CMD" + echo " where CMD is one of: project, libtool, libcurrent, librevision, libage" + exit 1 +fi + +# https://www.sourceware.org/autobook/autobook/autobook_91.html +# 'Current' is the most recent interface number that this library implements +libcurrent=$(($libavro_micro_version - $libavro_interface_age)) +# The implementation number of the 'current' interface +librevision=$libavro_interface_age +# The difference between the newest and oldest interfaces that this library implements +# In other words, the library implements all the interface numbers in the range from +# number 'current - age' to current +libage=$(($libavro_binary_age - $libavro_interface_age)) + +if test "$1" = "project"; then + project_ver="undef" + version_file="VERSION.txt" + if test -f $version_file; then + project_ver=$(cat $version_file) + else + version_file="../../share/VERSION.txt" + if test -f $version_file; then + project_ver=$(cat $version_file) + fi + fi + printf "%s" $project_ver +elif test "$1" = "libtool"; then + # useful for the -version-info flag for libtool + printf "%d:%d:%d" $libcurrent $librevision $libage +elif test "$1" = "libcurrent"; then + printf "%d" $libcurrent +elif test "$1" = "librevision"; then + printf "%d" $librevision +elif test "$1" = "libage"; then + printf "%d" $libage +fi |