diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-03-09 13:19:48 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-03-09 13:20:02 +0000 |
commit | 58daab21cd043e1dc37024a7f99b396788372918 (patch) | |
tree | 96771e43bb69f7c1c2b0b4f7374cb74d7866d0cb /fluent-bit/lib/avro/tests | |
parent | Releasing debian version 1.43.2-1. (diff) | |
download | netdata-58daab21cd043e1dc37024a7f99b396788372918.tar.xz netdata-58daab21cd043e1dc37024a7f99b396788372918.zip |
Merging upstream version 1.44.3.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'fluent-bit/lib/avro/tests')
77 files changed, 7418 insertions, 0 deletions
diff --git a/fluent-bit/lib/avro/tests/.gitignore b/fluent-bit/lib/avro/tests/.gitignore new file mode 100644 index 00000000..534eb068 --- /dev/null +++ b/fluent-bit/lib/avro/tests/.gitignore @@ -0,0 +1,9 @@ +generate_interop_data +performance +test_avro_data +test_avro_schema +test_avro_schema_names +test_avro_values +test_cpp +test_data_structures +test_interop_data diff --git a/fluent-bit/lib/avro/tests/CMakeLists.txt b/fluent-bit/lib/avro/tests/CMakeLists.txt new file mode 100644 index 00000000..2e84a06a --- /dev/null +++ b/fluent-bit/lib/avro/tests/CMakeLists.txt @@ -0,0 +1,87 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +macro(add_avro_executable name) + set(source "${ARGV1}") + if (NOT source) + set(source "${name}.c") + endif (NOT source) + add_executable(${name} ${source}) + target_link_libraries(${name} avro-static) +endmacro(add_avro_executable) + +macro(add_avro_test name) + add_avro_executable(${name} ${ARGN}) + if (WIN32) + set(exec_name ${CMAKE_CURRENT_BINARY_DIR}/Debug/${name}.exe) + else (WIN32) + set(exec_name ${CMAKE_CURRENT_BINARY_DIR}/${name}) + endif (WIN32) + + add_test(${name} + ${CMAKE_COMMAND} -E chdir ${AvroC_SOURCE_DIR}/tests + ${exec_name} + ) +endmacro(add_avro_test) + +macro(add_avro_test_checkmem name) + add_avro_test(${name} ${ARGN}) + if(UNIX) + find_program(MEMORYCHECK_COMMAND valgrind ) + if(MEMORYCHECK_COMMAND) + add_test(memcheck_${name} + ${CMAKE_COMMAND} -E chdir ${AvroC_SOURCE_DIR}/tests + ${MEMORYCHECK_COMMAND} + --log-file=${CMAKE_CURRENT_BINARY_DIR}/memcheck_${name}.log + --leak-check=full + --show-reachable=yes + --error-exitcode=1 + ${exec_name} + ) + endif(MEMORYCHECK_COMMAND) + endif (UNIX) +endmacro(add_avro_test_checkmem) + +add_avro_executable(generate_interop_data) +add_avro_executable(performance) +add_avro_executable(test_interop_data) + +add_avro_test_checkmem(test_data_structures) +add_avro_test_checkmem(test_avro_schema) +add_avro_test_checkmem(test_avro_schema_names) +add_avro_test_checkmem(test_avro_values) +add_avro_test_checkmem(test_avro_766) +add_avro_test_checkmem(test_avro_968) +add_avro_test_checkmem(test_avro_984) +add_avro_test_checkmem(test_avro_1034) +add_avro_test_checkmem(test_avro_1084) +add_avro_test_checkmem(test_avro_1087) +add_avro_test_checkmem(test_avro_1165) +add_avro_test_checkmem(test_avro_1167) +add_avro_test_checkmem(test_avro_1237) +add_avro_test_checkmem(test_avro_1238) +add_avro_test_checkmem(test_avro_1279) +add_avro_test_checkmem(test_avro_1405) +add_avro_test_checkmem(test_avro_1572) +add_avro_test(test_avro_data) # Skip memory check for datum. Deprecated and has a lot of memory issues +add_avro_test_checkmem(test_refcount) +add_avro_test_checkmem(test_avro_1379) +add_avro_test_checkmem(test_avro_1691) +add_avro_test_checkmem(test_avro_1906) +add_avro_test_checkmem(test_avro_1904) diff --git a/fluent-bit/lib/avro/tests/avro-1237-bad-union-discriminant.avro b/fluent-bit/lib/avro/tests/avro-1237-bad-union-discriminant.avro Binary files differnew file mode 100644 index 00000000..6dc539ee --- /dev/null +++ b/fluent-bit/lib/avro/tests/avro-1237-bad-union-discriminant.avro diff --git a/fluent-bit/lib/avro/tests/avro-1237-good.avro b/fluent-bit/lib/avro/tests/avro-1237-good.avro Binary files differnew file mode 100644 index 00000000..336dc289 --- /dev/null +++ b/fluent-bit/lib/avro/tests/avro-1237-good.avro diff --git a/fluent-bit/lib/avro/tests/avro-1238-good.avro b/fluent-bit/lib/avro/tests/avro-1238-good.avro Binary files differnew file mode 100644 index 00000000..336dc289 --- /dev/null +++ b/fluent-bit/lib/avro/tests/avro-1238-good.avro diff --git a/fluent-bit/lib/avro/tests/avro-1238-truncated.avro b/fluent-bit/lib/avro/tests/avro-1238-truncated.avro Binary files differnew file mode 100644 index 00000000..f48d54d7 --- /dev/null +++ b/fluent-bit/lib/avro/tests/avro-1238-truncated.avro diff --git a/fluent-bit/lib/avro/tests/avro-1279-codec.avro b/fluent-bit/lib/avro/tests/avro-1279-codec.avro Binary files differnew file mode 100644 index 00000000..dd242305 --- /dev/null +++ b/fluent-bit/lib/avro/tests/avro-1279-codec.avro diff --git a/fluent-bit/lib/avro/tests/avro-1279-no-codec.avro b/fluent-bit/lib/avro/tests/avro-1279-no-codec.avro Binary files differnew file mode 100644 index 00000000..4099de55 --- /dev/null +++ b/fluent-bit/lib/avro/tests/avro-1279-no-codec.avro diff --git a/fluent-bit/lib/avro/tests/generate_interop_data.c b/fluent-bit/lib/avro/tests/generate_interop_data.c new file mode 100644 index 00000000..e7f1365a --- /dev/null +++ b/fluent-bit/lib/avro/tests/generate_interop_data.c @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include "avro.h" +#include "avro_private.h" +#include <stdio.h> +#include <stdlib.h> + +int main(int argc, char *argv[]) +{ + int rval; + avro_file_writer_t file_writer; + avro_file_reader_t file_reader; + char outpath[128]; + FILE *fp; + char jsontext[16 * 1024]; + avro_schema_t schema; + avro_schema_error_t schema_error; + avro_datum_t interop; + avro_datum_t array_datum; + avro_datum_t node_datum; + avro_datum_t union_datum; + avro_datum_t out_datum; + enum Kind { + KIND_A, + KIND_B, + KIND_C + }; + + if (argc != 3) { + exit(EXIT_FAILURE); + } + snprintf(outpath, sizeof(outpath), "%s/c.avro", argv[2]); + fprintf(stderr, "Writing to %s\n", outpath); + + fp = fopen(argv[1], "r"); + rval = fread(jsontext, 1, sizeof(jsontext) - 1, fp); + jsontext[rval] = '\0'; + + check(rval, + avro_schema_from_json(jsontext, rval, &schema, &schema_error)); + check(rval, avro_file_writer_create(outpath, schema, &file_writer)); + + /* TODO: create a method for generating random data from schema */ + interop = avro_record(schema); + avro_record_set(interop, "intField", avro_int32(42)); + avro_record_set(interop, "longField", avro_int64(4242)); + avro_record_set(interop, "stringField", + avro_givestring("Follow your bliss.", NULL)); + avro_record_set(interop, "boolField", avro_boolean(1)); + avro_record_set(interop, "floatField", avro_float(3.14159265)); + avro_record_set(interop, "doubleField", avro_double(2.71828183)); + avro_record_set(interop, "bytesField", avro_bytes("abcd", 4)); + avro_record_set(interop, "nullField", avro_null()); + + avro_schema_t array_schema = avro_schema_get_subschema(schema, "arrayField"); + array_datum = avro_array(array_schema); + avro_array_append_datum(array_datum, avro_double(1.0)); + avro_array_append_datum(array_datum, avro_double(2.0)); + avro_array_append_datum(array_datum, avro_double(3.0)); + avro_record_set(interop, "arrayField", array_datum); + + avro_schema_t map_schema = avro_schema_get_subschema(schema, "mapField"); + avro_record_set(interop, "mapField", avro_map(map_schema)); + + avro_schema_t union_schema = avro_schema_get_subschema(schema, "unionField"); + union_datum = avro_union(union_schema, 1, avro_double(1.61803399)); + avro_record_set(interop, "unionField", union_datum); + + avro_schema_t enum_schema = avro_schema_get_subschema(schema, "enumField"); + avro_record_set(interop, "enumField", avro_enum(enum_schema, KIND_A)); + + avro_schema_t fixed_schema = avro_schema_get_subschema(schema, "fixedField"); + avro_record_set(interop, "fixedField", + avro_fixed(fixed_schema, "1234567890123456", 16)); + + avro_schema_t node_schema = avro_schema_get_subschema(schema, "recordField"); + node_datum = avro_record(node_schema); + avro_record_set(node_datum, "label", + avro_givestring("If you label me, you negate me.", NULL)); + avro_schema_t children_schema = avro_schema_get_subschema(node_schema, "children"); + avro_record_set(node_datum, "children", avro_array(children_schema)); + avro_record_set(interop, "recordField", node_datum); + + rval = avro_file_writer_append(file_writer, interop); + if (rval) { + fprintf(stderr, "Unable to append data to interop file!\n"); + exit(EXIT_FAILURE); + } else { + fprintf(stderr, "Successfully appended datum to file\n"); + } + + check(rval, avro_file_writer_close(file_writer)); + fprintf(stderr, "Closed writer.\n"); + + check(rval, avro_file_reader(outpath, &file_reader)); + fprintf(stderr, "Re-reading datum to verify\n"); + check(rval, avro_file_reader_read(file_reader, NULL, &out_datum)); + fprintf(stderr, "Verifying datum..."); + if (!avro_datum_equal(interop, out_datum)) { + fprintf(stderr, "fail!\n"); + exit(EXIT_FAILURE); + } + fprintf(stderr, "ok\n"); + check(rval, avro_file_reader_close(file_reader)); + fprintf(stderr, "Closed reader.\n"); + return 0; +} diff --git a/fluent-bit/lib/avro/tests/msdirent.h b/fluent-bit/lib/avro/tests/msdirent.h new file mode 100644 index 00000000..445d040d --- /dev/null +++ b/fluent-bit/lib/avro/tests/msdirent.h @@ -0,0 +1,372 @@ +/***************************************************************************** + * dirent.h - dirent API for Microsoft Visual Studio + * + * Copyright (C) 2006 Toni Ronkko + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * ``Software''), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL TONI RONKKO BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Mar 15, 2011, Toni Ronkko + * Defined FILE_ATTRIBUTE_DEVICE for MSVC 6.0. + * + * Aug 11, 2010, Toni Ronkko + * Added d_type and d_namlen fields to dirent structure. The former is + * especially useful for determining whether directory entry represents a + * file or a directory. For more information, see + * http://www.delorie.com/gnu/docs/glibc/libc_270.html + * + * Aug 11, 2010, Toni Ronkko + * Improved conformance to the standards. For example, errno is now set + * properly on failure and assert() is never used. Thanks to Peter Brockam + * for suggestions. + * + * Aug 11, 2010, Toni Ronkko + * Fixed a bug in rewinddir(): when using relative directory names, change + * of working directory no longer causes rewinddir() to fail. + * + * Dec 15, 2009, John Cunningham + * Added rewinddir member function + * + * Jan 18, 2008, Toni Ronkko + * Using FindFirstFileA and WIN32_FIND_DATAA to avoid converting string + * between multi-byte and unicode representations. This makes the + * code simpler and also allows the code to be compiled under MingW. Thanks + * to Azriel Fasten for the suggestion. + * + * Mar 4, 2007, Toni Ronkko + * Bug fix: due to the strncpy_s() function this file only compiled in + * Visual Studio 2005. Using the new string functions only when the + * compiler version allows. + * + * Nov 2, 2006, Toni Ronkko + * Major update: removed support for Watcom C, MS-DOS and Turbo C to + * simplify the file, updated the code to compile cleanly on Visual + * Studio 2005 with both unicode and multi-byte character strings, + * removed rewinddir() as it had a bug. + * + * Aug 20, 2006, Toni Ronkko + * Removed all remarks about MSVC 1.0, which is antiqued now. Simplified + * comments by removing SGML tags. + * + * May 14 2002, Toni Ronkko + * Embedded the function definitions directly to the header so that no + * source modules need to be included in the Visual Studio project. Removed + * all the dependencies to other projects so that this very header can be + * used independently. + * + * May 28 1998, Toni Ronkko + * First version. + *****************************************************************************/ +#ifndef DIRENT_H +#define DIRENT_H + +#define WIN32_LEAN_AND_MEAN +#include <windows.h> +#include <string.h> +#include <stdlib.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <errno.h> + +/* Entries missing from MSVC 6.0 */ +#if !defined(FILE_ATTRIBUTE_DEVICE) +# define FILE_ATTRIBUTE_DEVICE 0x40 +#endif + +/* File type and permission flags for stat() */ +#if defined(_MSC_VER) && !defined(S_IREAD) +# define S_IFMT _S_IFMT /* file type mask */ +# define S_IFDIR _S_IFDIR /* directory */ +# define S_IFCHR _S_IFCHR /* character device */ +# define S_IFFIFO _S_IFFIFO /* pipe */ +# define S_IFREG _S_IFREG /* regular file */ +# define S_IREAD _S_IREAD /* read permission */ +# define S_IWRITE _S_IWRITE /* write permission */ +# define S_IEXEC _S_IEXEC /* execute permission */ +#endif +#define S_IFBLK 0 /* block device */ +#define S_IFLNK 0 /* link */ +#define S_IFSOCK 0 /* socket */ + +#if defined(_MSC_VER) +# define S_IRUSR S_IREAD /* read, user */ +# define S_IWUSR S_IWRITE /* write, user */ +# define S_IXUSR 0 /* execute, user */ +# define S_IRGRP 0 /* read, group */ +# define S_IWGRP 0 /* write, group */ +# define S_IXGRP 0 /* execute, group */ +# define S_IROTH 0 /* read, others */ +# define S_IWOTH 0 /* write, others */ +# define S_IXOTH 0 /* execute, others */ +#endif + +/* Indicates that d_type field is available in dirent structure */ +#define _DIRENT_HAVE_D_TYPE + +/* File type flags for d_type */ +#define DT_UNKNOWN 0 +#define DT_REG S_IFREG +#define DT_DIR S_IFDIR +#define DT_FIFO S_IFFIFO +#define DT_SOCK S_IFSOCK +#define DT_CHR S_IFCHR +#define DT_BLK S_IFBLK + +/* Macros for converting between st_mode and d_type */ +#define IFTODT(mode) ((mode) & S_IFMT) +#define DTTOIF(type) (type) + +/* + * File type macros. Note that block devices, sockets and links cannot be + * distinguished on Windows and the macros S_ISBLK, S_ISSOCK and S_ISLNK are + * only defined for compatibility. These macros should always return false + * on Windows. + */ +#define S_ISFIFO(mode) (((mode) & S_IFMT) == S_IFFIFO) +#define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR) +#define S_ISREG(mode) (((mode) & S_IFMT) == S_IFREG) +#define S_ISLNK(mode) (((mode) & S_IFMT) == S_IFLNK) +#define S_ISSOCK(mode) (((mode) & S_IFMT) == S_IFSOCK) +#define S_ISCHR(mode) (((mode) & S_IFMT) == S_IFCHR) +#define S_ISBLK(mode) (((mode) & S_IFMT) == S_IFBLK) + +#ifdef __cplusplus +extern "C" { +#endif + + +typedef struct dirent +{ + char d_name[MAX_PATH + 1]; /* File name */ + size_t d_namlen; /* Length of name without \0 */ + int d_type; /* File type */ +} dirent; + + +typedef struct DIR +{ + dirent curentry; /* Current directory entry */ + WIN32_FIND_DATAA find_data; /* Private file data */ + int cached; /* True if data is valid */ + HANDLE search_handle; /* Win32 search handle */ + char patt[MAX_PATH + 3]; /* Initial directory name */ +} DIR; + + +/* Forward declarations */ +static DIR *opendir(const char *dirname); +static struct dirent *readdir(DIR *dirp); +static int closedir(DIR *dirp); +static void rewinddir(DIR* dirp); + + +/* Use the new safe string functions introduced in Visual Studio 2005 */ +#if defined(_MSC_VER) && _MSC_VER >= 1400 +# define DIRENT_STRNCPY(dest,src,size) strncpy_s((dest),(size),(src),_TRUNCATE) +#else +# define DIRENT_STRNCPY(dest,src,size) strncpy((dest),(src),(size)) +#endif + +/* Set errno variable */ +#if defined(_MSC_VER) +#define DIRENT_SET_ERRNO(x) _set_errno (x) +#else +#define DIRENT_SET_ERRNO(x) (errno = (x)) +#endif + + +/***************************************************************************** + * Open directory stream DIRNAME for read and return a pointer to the + * internal working area that is used to retrieve individual directory + * entries. + */ +static DIR *opendir(const char *dirname) +{ + DIR *dirp; + + /* ensure that the resulting search pattern will be a valid file name */ + if (dirname == NULL) { + DIRENT_SET_ERRNO (ENOENT); + return NULL; + } + if (strlen (dirname) + 3 >= MAX_PATH) { + DIRENT_SET_ERRNO (ENAMETOOLONG); + return NULL; + } + + /* construct new DIR structure */ + dirp = (DIR*) malloc (sizeof (struct DIR)); + if (dirp != NULL) { + int error; + + /* + * Convert relative directory name to an absolute one. This + * allows rewinddir() to function correctly when the current working + * directory is changed between opendir() and rewinddir(). + */ + if (GetFullPathNameA (dirname, MAX_PATH, dirp->patt, NULL)) { + char *p; + + /* append the search pattern "\\*\0" to the directory name */ + p = strchr (dirp->patt, '\0'); + if (dirp->patt < p && *(p-1) != '\\' && *(p-1) != ':') { + *p++ = '\\'; + } + *p++ = '*'; + *p = '\0'; + + /* open directory stream and retrieve the first entry */ + dirp->search_handle = FindFirstFileA (dirp->patt, &dirp->find_data); + if (dirp->search_handle != INVALID_HANDLE_VALUE) { + /* a directory entry is now waiting in memory */ + dirp->cached = 1; + error = 0; + } else { + /* search pattern is not a directory name? */ + DIRENT_SET_ERRNO (ENOENT); + error = 1; + } + } else { + /* buffer too small */ + DIRENT_SET_ERRNO (ENOMEM); + error = 1; + } + + if (error) { + free (dirp); + dirp = NULL; + } + } + + return dirp; +} + + +/***************************************************************************** + * Read a directory entry, and return a pointer to a dirent structure + * containing the name of the entry in d_name field. Individual directory + * entries returned by this very function include regular files, + * sub-directories, pseudo-directories "." and "..", but also volume labels, + * hidden files and system files may be returned. + */ +static struct dirent *readdir(DIR *dirp) +{ + DWORD attr; + if (dirp == NULL) { + /* directory stream did not open */ + DIRENT_SET_ERRNO (EBADF); + return NULL; + } + + /* get next directory entry */ + if (dirp->cached != 0) { + /* a valid directory entry already in memory */ + dirp->cached = 0; + } else { + /* get the next directory entry from stream */ + if (dirp->search_handle == INVALID_HANDLE_VALUE) { + return NULL; + } + if (FindNextFileA (dirp->search_handle, &dirp->find_data) == FALSE) { + /* the very last entry has been processed or an error occured */ + FindClose (dirp->search_handle); + dirp->search_handle = INVALID_HANDLE_VALUE; + return NULL; + } + } + + /* copy as a multibyte character string */ + DIRENT_STRNCPY ( dirp->curentry.d_name, + dirp->find_data.cFileName, + sizeof(dirp->curentry.d_name) ); + dirp->curentry.d_name[MAX_PATH] = '\0'; + + /* compute the length of name */ + dirp->curentry.d_namlen = strlen (dirp->curentry.d_name); + + /* determine file type */ + attr = dirp->find_data.dwFileAttributes; + if ((attr & FILE_ATTRIBUTE_DEVICE) != 0) { + dirp->curentry.d_type = DT_CHR; + } else if ((attr & FILE_ATTRIBUTE_DIRECTORY) != 0) { + dirp->curentry.d_type = DT_DIR; + } else { + dirp->curentry.d_type = DT_REG; + } + return &dirp->curentry; +} + + +/***************************************************************************** + * Close directory stream opened by opendir() function. Close of the + * directory stream invalidates the DIR structure as well as any previously + * read directory entry. + */ +static int closedir(DIR *dirp) +{ + if (dirp == NULL) { + /* invalid directory stream */ + DIRENT_SET_ERRNO (EBADF); + return -1; + } + + /* release search handle */ + if (dirp->search_handle != INVALID_HANDLE_VALUE) { + FindClose (dirp->search_handle); + dirp->search_handle = INVALID_HANDLE_VALUE; + } + + /* release directory structure */ + free (dirp); + return 0; +} + + +/***************************************************************************** + * Resets the position of the directory stream to which dirp refers to the + * beginning of the directory. It also causes the directory stream to refer + * to the current state of the corresponding directory, as a call to opendir() + * would have done. If dirp does not refer to a directory stream, the effect + * is undefined. + */ +static void rewinddir(DIR* dirp) +{ + if (dirp != NULL) { + /* release search handle */ + if (dirp->search_handle != INVALID_HANDLE_VALUE) { + FindClose (dirp->search_handle); + } + + /* open new search handle and retrieve the first entry */ + dirp->search_handle = FindFirstFileA (dirp->patt, &dirp->find_data); + if (dirp->search_handle != INVALID_HANDLE_VALUE) { + /* a directory entry is now waiting in memory */ + dirp->cached = 1; + } else { + /* failed to re-open directory: no directory entry in memory */ + dirp->cached = 0; + } + } +} + + +#ifdef __cplusplus +} +#endif +#endif /*DIRENT_H*/ diff --git a/fluent-bit/lib/avro/tests/performance.c b/fluent-bit/lib/avro/tests/performance.c new file mode 100644 index 00000000..a6f50427 --- /dev/null +++ b/fluent-bit/lib/avro/tests/performance.c @@ -0,0 +1,848 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <limits.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> + +#include "avro.h" +#include "avro_private.h" + + +/* The following definitions can be used as bitflags. They can also be + * passed in as the resolution_mode flags to the helper functions. + */ +#define USE_MATCHED_SCHEMAS (0x00) +#define USE_RESOLVED_READER (0x01) +#define USE_RESOLVED_WRITER (0x02) +#define USE_BOTH_RESOLVED (0x03) + + +/* + * A series of performance tests. + */ + +typedef void +(*test_func_t)(unsigned long); + + +void init_rand(void) +{ + srand(time(NULL)); +} + +double rand_number(double from, double to) +{ + double range = to - from; + return from + ((double)rand() / (RAND_MAX + 1.0)) * range; +} + +int64_t rand_int64(void) +{ + return (int64_t) rand_number(LONG_MIN, LONG_MAX); +} + +int32_t rand_int32(void) +{ + return (int32_t) rand_number(INT_MIN, INT_MAX); +} + + +/** + * Tests the single-threaded performance of our reference counting + * mechanism. We create a single datum, and then reference and + * deference it many many times. + */ + +static void +test_refcount(unsigned long num_tests) +{ + unsigned long i; + + avro_datum_t datum = avro_int32(42); + for (i = 0; i < num_tests; i++) { + avro_datum_incref(datum); + avro_datum_decref(datum); + } + avro_datum_decref(datum); +} + + +/** + * Tests the performance of serializing and deserializing a somewhat + * complex record type using the legacy datum API. + */ + +static void +test_nested_record_datum(unsigned long num_tests) +{ + static const char *schema_json = + "{" + " \"type\": \"record\"," + " \"name\": \"test\"," + " \"fields\": [" + " { \"name\": \"i\", \"type\": \"int\" }," + " { \"name\": \"l\", \"type\": \"long\" }," + " { \"name\": \"s\", \"type\": \"string\" }," + " {" + " \"name\": \"subrec\"," + " \"type\": {" + " \"type\": \"record\"," + " \"name\": \"sub\"," + " \"fields\": [" + " { \"name\": \"f\", \"type\": \"float\" }," + " { \"name\": \"d\", \"type\": \"double\" }" + " ]" + " }" + " }" + " ]" + "}"; + + static const char *strings[] = { + "Four score and seven years ago", + "our father brought forth on this continent", + "a new nation", "conceived in Liberty", + "and dedicated to the proposition that all men are created equal." + }; + static const unsigned int NUM_STRINGS = + sizeof(strings) / sizeof(strings[0]); + + int rc; + static char buf[4096]; + avro_reader_t reader = avro_reader_memory(buf, sizeof(buf)); + avro_writer_t writer = avro_writer_memory(buf, sizeof(buf)); + + avro_schema_t schema = NULL; + avro_schema_error_t error = NULL; + avro_schema_from_json(schema_json, strlen(schema_json), + &schema, &error); + + unsigned long i; + + avro_datum_t in = avro_datum_from_schema(schema); + + for (i = 0; i < num_tests; i++) { + avro_record_set_field_value(rc, in, int32, "i", rand_int32()); + avro_record_set_field_value(rc, in, int64, "l", rand_int64()); + avro_record_set_field_value(rc, in, givestring, "s", + strings[i % NUM_STRINGS], NULL); + + avro_datum_t subrec = NULL; + avro_record_get(in, "subrec", &subrec); + avro_record_set_field_value(rc, in, float, "f", rand_number(-1e10, 1e10)); + avro_record_set_field_value(rc, in, double, "d", rand_number(-1e10, 1e10)); + + avro_writer_reset(writer); + avro_write_data(writer, schema, in); + + avro_datum_t out = NULL; + + avro_reader_reset(reader); + avro_read_data(reader, schema, schema, &out); + + avro_datum_equal(in, out); + avro_datum_decref(out); + } + + avro_datum_decref(in); + avro_schema_decref(schema); + avro_writer_free(writer); + avro_reader_free(reader); +} + + +/** + * Tests the performance of serializing and deserializing a somewhat + * complex record type using the new value API, retrieving record fields + * by index. + */ + +static void +test_nested_record_value_by_index(unsigned long num_tests) +{ + static const char *schema_json = + "{" + " \"type\": \"record\"," + " \"name\": \"test\"," + " \"fields\": [" + " { \"name\": \"i\", \"type\": \"int\" }," + " { \"name\": \"l\", \"type\": \"long\" }," + " { \"name\": \"s\", \"type\": \"string\" }," + " {" + " \"name\": \"subrec\"," + " \"type\": {" + " \"type\": \"record\"," + " \"name\": \"sub\"," + " \"fields\": [" + " { \"name\": \"f\", \"type\": \"float\" }," + " { \"name\": \"d\", \"type\": \"double\" }" + " ]" + " }" + " }" + " ]" + "}"; + + static char *strings[] = { + "Four score and seven years ago", + "our father brought forth on this continent", + "a new nation", "conceived in Liberty", + "and dedicated to the proposition that all men are created equal." + }; + static const unsigned int NUM_STRINGS = + sizeof(strings) / sizeof(strings[0]); + + static char buf[4096]; + avro_reader_t reader = avro_reader_memory(buf, sizeof(buf)); + avro_writer_t writer = avro_writer_memory(buf, sizeof(buf)); + + avro_schema_t schema = NULL; + avro_schema_error_t error = NULL; + avro_schema_from_json(schema_json, strlen(schema_json), + &schema, &error); + + unsigned long i; + + avro_value_iface_t *iface = avro_generic_class_from_schema(schema); + + avro_value_t val; + avro_generic_value_new(iface, &val); + + avro_value_t out; + avro_generic_value_new(iface, &out); + + for (i = 0; i < num_tests; i++) { + avro_value_t field; + + avro_value_get_by_index(&val, 0, &field, NULL); + avro_value_set_int(&field, rand_int32()); + + avro_value_get_by_index(&val, 1, &field, NULL); + avro_value_set_long(&field, rand_int64()); + + avro_wrapped_buffer_t wbuf; + avro_wrapped_buffer_new_string(&wbuf, strings[i % NUM_STRINGS]); + avro_value_get_by_index(&val, 2, &field, NULL); + avro_value_give_string_len(&field, &wbuf); + + avro_value_t subrec; + avro_value_get_by_index(&val, 3, &subrec, NULL); + + avro_value_get_by_index(&subrec, 0, &field, NULL); + avro_value_set_float(&field, rand_number(-1e10, 1e10)); + + avro_value_get_by_index(&subrec, 1, &field, NULL); + avro_value_set_double(&field, rand_number(-1e10, 1e10)); + + avro_writer_reset(writer); + avro_value_write(writer, &val); + + avro_reader_reset(reader); + avro_value_read(reader, &out); + + if (! avro_value_equal_fast(&val, &out) ) { + printf("Broken\n"); + exit (1); + } + } + + avro_value_decref(&val); + avro_value_decref(&out); + avro_value_iface_decref(iface); + avro_schema_decref(schema); + avro_writer_free(writer); + avro_reader_free(reader); +} + + + +/** + * Tests the performance of serializing and deserializing a somewhat + * complex record type using the new value API, retrieving record fields + * by name. + */ + +static void +test_nested_record_value_by_name(unsigned long num_tests) +{ + static const char *schema_json = + "{" + " \"type\": \"record\"," + " \"name\": \"test\"," + " \"fields\": [" + " { \"name\": \"i\", \"type\": \"int\" }," + " { \"name\": \"l\", \"type\": \"long\" }," + " { \"name\": \"s\", \"type\": \"string\" }," + " {" + " \"name\": \"subrec\"," + " \"type\": {" + " \"type\": \"record\"," + " \"name\": \"sub\"," + " \"fields\": [" + " { \"name\": \"f\", \"type\": \"float\" }," + " { \"name\": \"d\", \"type\": \"double\" }" + " ]" + " }" + " }" + " ]" + "}"; + + static char *strings[] = { + "Four score and seven years ago", + "our father brought forth on this continent", + "a new nation", "conceived in Liberty", + "and dedicated to the proposition that all men are created equal." + }; + static const unsigned int NUM_STRINGS = + sizeof(strings) / sizeof(strings[0]); + + static char buf[4096]; + avro_reader_t reader = avro_reader_memory(buf, sizeof(buf)); + avro_writer_t writer = avro_writer_memory(buf, sizeof(buf)); + + avro_schema_t schema = NULL; + avro_schema_error_t error = NULL; + avro_schema_from_json(schema_json, strlen(schema_json), + &schema, &error); + + unsigned long i; + + avro_value_iface_t *iface = avro_generic_class_from_schema(schema); + + avro_value_t val; + avro_generic_value_new(iface, &val); + + avro_value_t out; + avro_generic_value_new(iface, &out); + + for (i = 0; i < num_tests; i++) { + avro_value_t field; + + avro_value_get_by_name(&val, "i", &field, NULL); + avro_value_set_int(&field, rand_int32()); + + avro_value_get_by_name(&val, "l", &field, NULL); + avro_value_set_long(&field, rand_int64()); + + avro_wrapped_buffer_t wbuf; + avro_wrapped_buffer_new_string(&wbuf, strings[i % NUM_STRINGS]); + avro_value_get_by_name(&val, "s", &field, NULL); + avro_value_give_string_len(&field, &wbuf); + + avro_value_t subrec; + avro_value_get_by_name(&val, "subrec", &subrec, NULL); + + avro_value_get_by_name(&subrec, "f", &field, NULL); + avro_value_set_float(&field, rand_number(-1e10, 1e10)); + + avro_value_get_by_name(&subrec, "d", &field, NULL); + avro_value_set_double(&field, rand_number(-1e10, 1e10)); + + avro_writer_reset(writer); + avro_value_write(writer, &val); + + avro_reader_reset(reader); + avro_value_read(reader, &out); + + if (! avro_value_equal_fast(&val, &out) ) { + printf("Broken\n"); + exit (1); + } + } + + avro_value_decref(&val); + avro_value_decref(&out); + avro_value_iface_decref(iface); + avro_schema_decref(schema); + avro_writer_free(writer); + avro_reader_free(reader); +} + + + +/** + * Helper function to test the performance of serializing and + * deserializing a given avro value using the provided function to + * populate avro value using the new value API. Allows testing using + * matching schemas or using schema resolution. + */ + +static void +test_generic_helper( unsigned long num_tests, + int resolution_type, + const char *schema_json, + void (*populate_value_func)(avro_value_t *, + unsigned long) + ) +{ + static char buf[4096]; + + avro_reader_t reader = avro_reader_memory(buf, sizeof(buf)); + avro_writer_t writer = avro_writer_memory(buf, sizeof(buf)); + + avro_schema_t schema = NULL; + avro_schema_error_t error = NULL; + avro_schema_from_json(schema_json, strlen(schema_json), + &schema, &error); + + unsigned long i; + + avro_value_iface_t *writer_iface = avro_generic_class_from_schema(schema); + avro_value_iface_t *reader_iface = avro_generic_class_from_schema(schema); + + avro_value_t val; + avro_generic_value_new(writer_iface, &val); + + avro_value_t out; + avro_generic_value_new(reader_iface, &out); + + /* Use resolved reader to resolve schemas while writing data to memory */ + avro_value_iface_t *resolved_reader_iface = NULL; + avro_value_t resolved_reader_value; + if ( resolution_type & USE_RESOLVED_READER ) { + resolved_reader_iface = avro_resolved_reader_new( schema, schema ); + avro_resolved_reader_new_value( resolved_reader_iface, + &resolved_reader_value ); + avro_resolved_reader_set_source( &resolved_reader_value, &val ); + } + + /* Use resolved writer to resolve schemas while reading data from memory */ + avro_value_iface_t *resolved_writer_iface = NULL; + avro_value_t resolved_writer_value; + if ( resolution_type & USE_RESOLVED_WRITER ) { + resolved_writer_iface = avro_resolved_writer_new( schema, schema ); + avro_resolved_writer_new_value( resolved_writer_iface, + &resolved_writer_value ); + avro_resolved_writer_set_dest( &resolved_writer_value, &out ); + } + + /* Set up pointers */ + avro_value_t *p_value_to_write_to_memory = NULL; + avro_value_t *p_value_to_read_from_memory = NULL; + + if ( resolution_type == USE_MATCHED_SCHEMAS ) { + p_value_to_write_to_memory = &val; + p_value_to_read_from_memory = &out; + } + else if ( resolution_type == USE_RESOLVED_READER ) { + p_value_to_write_to_memory = &resolved_reader_value; + p_value_to_read_from_memory = &out; + } + else if ( resolution_type == USE_RESOLVED_WRITER ) { + p_value_to_write_to_memory = &val; + p_value_to_read_from_memory = &resolved_writer_value; + } + else if ( resolution_type == USE_BOTH_RESOLVED ) { + p_value_to_write_to_memory = &resolved_reader_value; + p_value_to_read_from_memory = &resolved_writer_value; + } + + /* Perform the tests */ + for (i = 0; i < num_tests; i++) { + + avro_value_reset(&val); + + /* Execute the function to populate the Avro Value */ + (*populate_value_func)(&val, i); + + avro_writer_reset(writer); + avro_value_write(writer, p_value_to_write_to_memory); + + avro_reader_reset(reader); + avro_value_read(reader, p_value_to_read_from_memory); + + if (! avro_value_equal_fast(&val, &out) ) { + printf("Broken\n"); + exit (1); + } + } + + avro_value_decref(&val); + avro_value_decref(&out); + if ( resolution_type & USE_RESOLVED_READER ) { + avro_value_decref(&resolved_reader_value); + avro_value_iface_decref(resolved_reader_iface); + } + if ( resolution_type & USE_RESOLVED_WRITER ) { + avro_value_decref(&resolved_writer_value); + avro_value_iface_decref(resolved_writer_iface); + } + avro_value_iface_decref(writer_iface); + avro_value_iface_decref(reader_iface); + avro_schema_decref(schema); + avro_writer_free(writer); + avro_reader_free(reader); +} + + + + +/** + * Helper function to populate a somewhat complex record type using + * the new value API, retrieving record fields by index. + */ + +static const char *complex_record_schema_json = + "{" + " \"type\": \"record\"," + " \"name\": \"test\"," + " \"fields\": [" + " { \"name\": \"i\", \"type\": \"int\" }," + " { \"name\": \"l\", \"type\": \"long\" }," + " { \"name\": \"s\", \"type\": \"string\" }," + " {" + " \"name\": \"subrec\"," + " \"type\": {" + " \"type\": \"record\"," + " \"name\": \"sub\"," + " \"fields\": [" + " { \"name\": \"f\", \"type\": \"float\" }," + " { \"name\": \"d\", \"type\": \"double\" }" + " ]" + " }" + " }" + " ]" + "}"; + + + +static void +populate_complex_record(avro_value_t *p_val, unsigned long i) +{ + static char *strings[] = { + "Four score and seven years ago", + "our father brought forth on this continent", + "a new nation", "conceived in Liberty", + "and dedicated to the proposition that all men are created equal." + }; + static const unsigned int NUM_STRINGS = + sizeof(strings) / sizeof(strings[0]); + + avro_value_t field; + + avro_value_get_by_index(p_val, 0, &field, NULL); + avro_value_set_int(&field, rand_int32()); + + avro_value_get_by_index(p_val, 1, &field, NULL); + avro_value_set_long(&field, rand_int64()); + + avro_wrapped_buffer_t wbuf; + avro_wrapped_buffer_new_string(&wbuf, strings[i % NUM_STRINGS]); + avro_value_get_by_index(p_val, 2, &field, NULL); + avro_value_give_string_len(&field, &wbuf); + + avro_value_t subrec; + avro_value_get_by_index(p_val, 3, &subrec, NULL); + + avro_value_get_by_index(&subrec, 0, &field, NULL); + avro_value_set_float(&field, rand_number(-1e10, 1e10)); + + avro_value_get_by_index(&subrec, 1, &field, NULL); + avro_value_set_double(&field, rand_number(-1e10, 1e10)); + +} + + + +/** + * Tests the performance of serializing and deserializing a somewhat + * complex record type using the new value API, retrieving record + * fields by index. The functionality is almost identical to + * test_nested_record_value_by_index(), however, there may be some + * overhead of using function calls instead of inline code, and + * running some additional "if" statements.. + */ + +static void +test_nested_record_value_by_index_matched_schemas(unsigned long num_tests) +{ + test_generic_helper(num_tests, + USE_MATCHED_SCHEMAS, + complex_record_schema_json, + populate_complex_record); +} + + +/** + * Tests the performance of serializing and deserializing a somewhat + * complex record type using the new value API, retrieving record + * fields by index. Uses a resolved_writer to resolve between two + * (identical) schemas when reading the array. + */ + +static void +test_nested_record_value_by_index_resolved_writer(unsigned long num_tests) +{ + test_generic_helper(num_tests, + USE_RESOLVED_WRITER, + complex_record_schema_json, + populate_complex_record); +} + + + +/** + * Tests the performance of serializing and deserializing a somewhat + * complex record type using the new value API, retrieving record + * fields by index. Uses a resolved_reader to resolve between two + * (identical) schemas when writing the array. + */ + +static void +test_nested_record_value_by_index_resolved_reader(unsigned long num_tests) +{ + test_generic_helper(num_tests, + USE_RESOLVED_READER, + complex_record_schema_json, + populate_complex_record); +} + + + +/** + * Helper function to test the performance of serializing and + * deserializing a simple array using the new value API. Allows + * testing using matching schemas or using schema resolution. + */ + +static const char *simple_array_schema_json = + "{\"name\": \"a\", \"type\": \"array\", \"items\":\"long\"}"; + +static void +populate_simple_array(avro_value_t *p_val, unsigned long i) +{ + const size_t array_length = 21; + avro_value_t field; + size_t idx; + size_t dummy_index; + (void) i; + + for ( idx = 0; idx < array_length; idx++ ) { + avro_value_append(p_val, &field, &dummy_index); + avro_value_set_long(&field, rand_int64()); + } +} + + + +/** + * Tests the performance of serializing and deserializing a simple + * array using the new value API. + */ + +static void +test_simple_array(unsigned long num_tests) +{ + test_generic_helper(num_tests, + USE_MATCHED_SCHEMAS, + simple_array_schema_json, + populate_simple_array); +} + + + +/** + * Tests the performance of serializing and deserializing a simple + * array using the new value API, using a resolved writer to resolve + * between (identical) reader and writer schemas, when reading the + * array. + */ +static void +test_simple_array_resolved_writer(unsigned long num_tests) +{ + test_generic_helper(num_tests, + USE_RESOLVED_WRITER, + simple_array_schema_json, + populate_simple_array); +} + + + +/** + * Tests the performance of serializing and deserializing a simple + * array using the new value API, using a resolved reader to resolve + * between (identical) reader and writer schemas, when writing the + * array. + */ + +static void +test_simple_array_resolved_reader(unsigned long num_tests) +{ + test_generic_helper(num_tests, + USE_RESOLVED_READER, + simple_array_schema_json, + populate_simple_array); +} + + + + +/** + * Helper function to test the performance of serializing and + * deserializing a nested array using the new value API. Allows + * testing using matching schemas or using schema resolution. + */ + +static const char *nested_array_schema_json = + "{\"type\":\"array\", \"items\": {\"type\": \"array\", \"items\": \"long\"}}"; + + +static void +populate_nested_array(avro_value_t *p_val, unsigned long i) +{ + + const size_t array_length = 7; + const size_t subarray_length = 3; + avro_value_t subarray; + avro_value_t field; + size_t idx; + size_t jdx; + size_t dummy_index; + (void) i; + + for ( idx = 0; idx < array_length; idx++ ) { + avro_value_append(p_val, &subarray, &dummy_index); + for ( jdx = 0; jdx < subarray_length; jdx ++ ) { + avro_value_append(&subarray, &field, &dummy_index); + avro_value_set_long(&field, rand_int64()); + } + } +} + + +/** + * Tests the performance of serializing and deserializing a nested + * array using the new value API. + */ + +static void +test_nested_array(unsigned long num_tests) +{ + test_generic_helper(num_tests, + USE_MATCHED_SCHEMAS, + nested_array_schema_json, + populate_nested_array); +} + + +/** + * Tests the performance of serializing and deserializing a nested + * array using the new value API, using a resolved writer to resolve + * between (identical) reader and writer schemas, when reading the + * array. + */ + +static void +test_nested_array_resolved_writer(unsigned long num_tests) +{ + test_generic_helper(num_tests, + USE_RESOLVED_WRITER, + nested_array_schema_json, + populate_nested_array); +} + + +/** + * Tests the performance of serializing and deserializing a nested + * array using the new value API, using a resolved reader to resolve + * between (identical) reader and writer schemas, when writing the + * array. + */ + +static void +test_nested_array_resolved_reader(unsigned long num_tests) +{ + test_generic_helper(num_tests, + USE_RESOLVED_READER, + nested_array_schema_json, + populate_nested_array); +} + + + +/** + * Test harness + */ + +#define NUM_RUNS 3 + +int +main(int argc, char **argv) +{ + AVRO_UNUSED(argc); + AVRO_UNUSED(argv); + + init_rand(); + + unsigned int i; + struct avro_tests { + const char *name; + unsigned long num_tests; + test_func_t func; + } tests[] = { + { "refcount", 100000000, + test_refcount }, + { "nested record (legacy)", 100000, + test_nested_record_datum }, + { "nested record (value by index)", 1000000, + test_nested_record_value_by_index }, + { "nested record (value by name)", 1000000, + test_nested_record_value_by_name }, + { "nested record (value by index) matched schemas", 1000000, + test_nested_record_value_by_index_matched_schemas }, + { "nested record (value by index) resolved writer", 1000000, + test_nested_record_value_by_index_resolved_writer }, + { "nested record (value by index) resolved reader", 1000000, + test_nested_record_value_by_index_resolved_reader }, + { "simple array matched schemas", 250000, + test_simple_array }, + { "simple array resolved writer", 250000, + test_simple_array_resolved_writer }, + { "simple array resolved reader", 250000, + test_simple_array_resolved_reader }, + { "nested array matched schemas", 250000, + test_nested_array }, + { "nested array resolved writer", 250000, + test_nested_array_resolved_writer }, + { "nested array resolved reader", 250000, + test_nested_array_resolved_reader }, + }; + + for (i = 0; i < sizeof(tests) / sizeof(tests[0]); i++) { + fprintf(stderr, "**** Running %s ****\n %lu tests per run\n", + tests[i].name, tests[i].num_tests); + unsigned int run; + + double sum = 0.0; + + for (run = 1; run <= NUM_RUNS; run++) { + fprintf(stderr, " Run %u\n", run); + + clock_t before = clock(); + tests[i].func(tests[i].num_tests); + clock_t after = clock(); + double secs = ((double) after-before) / CLOCKS_PER_SEC; + sum += secs; + } + + fprintf(stderr, " Average time: %.03lfs\n", sum / NUM_RUNS); + fprintf(stderr, " Tests/sec: %.0lf\n", + tests[i].num_tests / (sum / NUM_RUNS)); + } + + return EXIT_SUCCESS; +} diff --git a/fluent-bit/lib/avro/tests/schema_tests/fail/enum_nonarray_symbols b/fluent-bit/lib/avro/tests/schema_tests/fail/enum_nonarray_symbols new file mode 100644 index 00000000..f4dae950 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/fail/enum_nonarray_symbols @@ -0,0 +1,3 @@ +{"type": "enum", + "name": "Status", + "symbols": "Normal Caution Critical"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/fail/enum_nonstring_name b/fluent-bit/lib/avro/tests/schema_tests/fail/enum_nonstring_name new file mode 100644 index 00000000..baa13d95 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/fail/enum_nonstring_name @@ -0,0 +1,3 @@ +{"type": "enum", + "name": [ 0, 1, 1, 2, 3, 5, 8 ], + "symbols": ["Golden", "Mean"]} diff --git a/fluent-bit/lib/avro/tests/schema_tests/fail/enum_without_name b/fluent-bit/lib/avro/tests/schema_tests/fail/enum_without_name new file mode 100644 index 00000000..57f68532 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/fail/enum_without_name @@ -0,0 +1,3 @@ +{"type": "enum" + "symbols" : ["I", "will", "fail", "no", "name"] +} diff --git a/fluent-bit/lib/avro/tests/schema_tests/fail/fixed_without_name b/fluent-bit/lib/avro/tests/schema_tests/fail/fixed_without_name new file mode 100644 index 00000000..fbf96abe --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/fail/fixed_without_name @@ -0,0 +1,2 @@ +{"type": "fixed", + "size": 314} diff --git a/fluent-bit/lib/avro/tests/schema_tests/fail/fixed_without_size b/fluent-bit/lib/avro/tests/schema_tests/fail/fixed_without_size new file mode 100644 index 00000000..15c5e789 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/fail/fixed_without_size @@ -0,0 +1,2 @@ +{"type": "fixed", + "name": "Missing size"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/fail/illegal_type b/fluent-bit/lib/avro/tests/schema_tests/fail/illegal_type new file mode 100644 index 00000000..e65c0463 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/fail/illegal_type @@ -0,0 +1 @@ +{"type":"panther"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/fail/invalid_avro_id b/fluent-bit/lib/avro/tests/schema_tests/fail/invalid_avro_id new file mode 100644 index 00000000..c684e7de --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/fail/invalid_avro_id @@ -0,0 +1,3 @@ +{ "name" : "2d2", + "type": "enum", + "symbols" : [ "c3po" ] } diff --git a/fluent-bit/lib/avro/tests/schema_tests/fail/record_with_field_missing_name b/fluent-bit/lib/avro/tests/schema_tests/fail/record_with_field_missing_name new file mode 100644 index 00000000..ba62d52e --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/fail/record_with_field_missing_name @@ -0,0 +1,5 @@ +{"type": "record", + "name": "Address", + "fields": [ + {"type": "string"}, + {"type": "string", "name": "City"}]} diff --git a/fluent-bit/lib/avro/tests/schema_tests/fail/record_with_field_missing_type b/fluent-bit/lib/avro/tests/schema_tests/fail/record_with_field_missing_type new file mode 100644 index 00000000..b449f3b4 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/fail/record_with_field_missing_type @@ -0,0 +1,5 @@ +{"type": "record", + "name": "Event", + "fields": [ + { "name": "Sponsor"}, + { "name": "City", "type": "string"}]} diff --git a/fluent-bit/lib/avro/tests/schema_tests/fail/record_with_invalid_reference b/fluent-bit/lib/avro/tests/schema_tests/fail/record_with_invalid_reference new file mode 100644 index 00000000..49b35902 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/fail/record_with_invalid_reference @@ -0,0 +1,7 @@ +{ "type": "record", + "name": "recursive", + "fields": [ + { "name": "label", "type": "string" }, + { "name": "children", "type": {"type": "array", "items": "foobar"} } + ] +} diff --git a/fluent-bit/lib/avro/tests/schema_tests/fail/record_with_nonarray_fields b/fluent-bit/lib/avro/tests/schema_tests/fail/record_with_nonarray_fields new file mode 100644 index 00000000..b81fbe32 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/fail/record_with_nonarray_fields @@ -0,0 +1,3 @@ +{ "type": "record", + "fields": "His vision, from the constantly passing bars," + "name", "Rainer" } diff --git a/fluent-bit/lib/avro/tests/schema_tests/fail/record_with_nonstring_name b/fluent-bit/lib/avro/tests/schema_tests/fail/record_with_nonstring_name new file mode 100644 index 00000000..0ded9c55 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/fail/record_with_nonstring_name @@ -0,0 +1,3 @@ +{"name": ["Tom", "Jerry"], + "type": "record", + "fields": [ {"name": "name", "type": "string"} ]} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/array b/fluent-bit/lib/avro/tests/schema_tests/pass/array new file mode 100644 index 00000000..d6950491 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/array @@ -0,0 +1 @@ +{"type": "array", "items": "long"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/boolean_full b/fluent-bit/lib/avro/tests/schema_tests/pass/boolean_full new file mode 100644 index 00000000..69d35799 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/boolean_full @@ -0,0 +1 @@ +{"type":"boolean"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/bytes_full b/fluent-bit/lib/avro/tests/schema_tests/pass/bytes_full new file mode 100644 index 00000000..3b91ef01 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/bytes_full @@ -0,0 +1 @@ +{"type":"bytes"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/double_full b/fluent-bit/lib/avro/tests/schema_tests/pass/double_full new file mode 100644 index 00000000..dbd22f78 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/double_full @@ -0,0 +1 @@ +{"type":"double"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/enum b/fluent-bit/lib/avro/tests/schema_tests/pass/enum new file mode 100644 index 00000000..749b0a3c --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/enum @@ -0,0 +1,4 @@ +{ "type": "enum", + "name": "three_stooges", + "symbols" : [ "Moe", "Larry", "Curly" ] +} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/extra_attributes b/fluent-bit/lib/avro/tests/schema_tests/pass/extra_attributes new file mode 100644 index 00000000..49885b9d --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/extra_attributes @@ -0,0 +1 @@ +{"type":"string", "ignored": "value"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/fixed b/fluent-bit/lib/avro/tests/schema_tests/pass/fixed new file mode 100644 index 00000000..0449ebca --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/fixed @@ -0,0 +1 @@ +{"type": "fixed", "size": 16, "name": "md5"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/float_full b/fluent-bit/lib/avro/tests/schema_tests/pass/float_full new file mode 100644 index 00000000..fbd11642 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/float_full @@ -0,0 +1 @@ +{"type":"float"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/int_full b/fluent-bit/lib/avro/tests/schema_tests/pass/int_full new file mode 100644 index 00000000..92b134da --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/int_full @@ -0,0 +1 @@ +{"type":"int"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/interop.avsc b/fluent-bit/lib/avro/tests/schema_tests/pass/interop.avsc new file mode 100644 index 00000000..8cfbba22 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/interop.avsc @@ -0,0 +1,28 @@ +{"type": "record", "name":"Interop", "namespace": "org.apache.avro", + "fields": [ + {"name": "intField", "type": "int"}, + {"name": "longField", "type": "long"}, + {"name": "stringField", "type": "string"}, + {"name": "boolField", "type": "boolean"}, + {"name": "floatField", "type": "float"}, + {"name": "doubleField", "type": "double"}, + {"name": "bytesField", "type": "bytes"}, + {"name": "nullField", "type": "null"}, + {"name": "arrayField", "type": {"type": "array", "items": "double"}}, + {"name": "mapField", "type": + {"type": "map", "values": + {"type": "record", "name": "Foo", + "fields": [{"name": "label", "type": "string"}]}}}, + {"name": "unionField", "type": + ["boolean", "double", {"type": "array", "items": "bytes"}]}, + {"name": "enumField", "type": + {"type": "enum", "name": "Kind", "symbols": ["A","B","C"]}}, + {"name": "fixedField", "type": + {"type": "fixed", "name": "MD5", "size": 16}}, + {"name": "recordField", "type": + {"type": "record", "name": "Node", + "fields": [ + {"name": "label", "type": "string"}, + {"name": "children", "type": {"type": "array", "items": "Node"}}]}} + ] +} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/long_full b/fluent-bit/lib/avro/tests/schema_tests/pass/long_full new file mode 100644 index 00000000..ccfd9170 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/long_full @@ -0,0 +1 @@ +{"type":"long"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/map b/fluent-bit/lib/avro/tests/schema_tests/pass/map new file mode 100644 index 00000000..436d961d --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/map @@ -0,0 +1 @@ +{"type" : "map", "values": "long"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_enum b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_enum new file mode 100644 index 00000000..5b77b353 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_enum @@ -0,0 +1,9 @@ +{"type": "record", "name": "org.apache.avro.tests.Hello", "fields": [ + {"name": "f1", "type": {"type": "enum", "name": "MyEnum", "symbols": ["Foo", "Bar", "Baz"]}}, + {"name": "f2", "type": "org.apache.avro.tests.MyEnum"}, + {"name": "f3", "type": "MyEnum"}, + {"name": "f4", "type": {"type": "enum", "name": "other.namespace.OtherEnum", "symbols": ["one", "two", "three"]}}, + {"name": "f5", "type": "other.namespace.OtherEnum"}, + {"name": "f6", "type": {"type": "enum", "name": "ThirdEnum", "namespace": "some.other", "symbols": ["Alice", "Bob"]}}, + {"name": "f7", "type": "some.other.ThirdEnum"} +]} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_fixed b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_fixed new file mode 100644 index 00000000..f621e797 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_fixed @@ -0,0 +1,9 @@ +{"type": "record", "name": "org.apache.avro.tests.Hello", "fields": [ + {"name": "f1", "type": {"type": "fixed", "name": "MyFixed", "size": 16}}, + {"name": "f2", "type": "org.apache.avro.tests.MyFixed"}, + {"name": "f3", "type": "MyFixed"}, + {"name": "f4", "type": {"type": "fixed", "name": "other.namespace.OtherFixed", "size": 18}}, + {"name": "f5", "type": "other.namespace.OtherFixed"}, + {"name": "f6", "type": {"type": "fixed", "name": "ThirdFixed", "namespace": "some.other", "size": 20}}, + {"name": "f7", "type": "some.other.ThirdFixed"} +]} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_fullname b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_fullname new file mode 100644 index 00000000..0cc24564 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_fullname @@ -0,0 +1,8 @@ +{"type": "record", "name": "x.Y", "fields": [ + {"name": "e", "type": + {"type": "record", "name": "Z", "fields": [ + {"name": "f", "type": "x.Y"}, + {"name": "g", "type": "x.Z"} + ]} + } +]} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_null_enum b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_null_enum new file mode 100644 index 00000000..3c3a7452 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_null_enum @@ -0,0 +1,8 @@ +{"type": "record", "name": "R", "fields": [ + {"name": "s", "type": {"type": "record", "namespace": "x", "name": "Y", "fields": [ + {"name": "e", "type": {"type": "enum", "namespace": "", "name": "Z", + "symbols": ["Foo", "Bar"]} + } + ]}}, + {"name": "t", "type": "Z"} +]} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_null_fixed b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_null_fixed new file mode 100644 index 00000000..a3aa5701 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_null_fixed @@ -0,0 +1,7 @@ +{"type": "record", "name": "R", "fields": [ + {"name": "s", "type": {"type": "record", "namespace": "x", "name": "Y", "fields": [ + {"name": "e", "type": {"type": "fixed", "namespace": "", "name": "Z", "size": 8} + } + ]}}, + {"name": "t", "type": "Z"} +]} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_null_record b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_null_record new file mode 100644 index 00000000..4b18dd54 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_null_record @@ -0,0 +1,8 @@ +{"type": "record", "name": "R", "fields": [ + {"name": "s", "type": {"type": "record", "namespace": "x", "name": "Y", "fields": [ + {"name": "e", "type": {"type": "record", "namespace": "", "name": "Z", "fields": [ + {"name": "f", "type": "Z"} + ]}} + ]}}, + {"name": "t", "type": "Z"} +]} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_recursive b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_recursive new file mode 100644 index 00000000..3c2d0eb7 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_recursive @@ -0,0 +1,28 @@ +{ "type": "record", + "name": "Container", + "namespace": "namespace1", + "fields": [ + { "name": "contained", + "type": { "type": "record", + "name": "MutuallyRecursive", + "fields": [ + { "name": "label", "type": "string" }, + { "name": "children", + "type": {"type": "array", "items": + {"type": "record", + "name": "MutuallyRecursive", + "namespace": "namespace2", + "fields": [ + { "name": "value", "type": "int" }, + { "name": "children", "type": {"type": "array", "items": "namespace1.MutuallyRecursive" }}, + { "name": "morechildren", "type": {"type": "array", "items": "MutuallyRecursive" }} + ] + } + } + }, + { "name": "anotherchild", "type": "namespace2.MutuallyRecursive"} + ] + } + } + ] +} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_simple b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_simple new file mode 100644 index 00000000..f5a117f4 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/namespace_simple @@ -0,0 +1,5 @@ +{"type": "record", "namespace": "x", "name": "Y", "fields": [ + {"name": "e", "type": {"type": "record", "name": "Z", "fields": [ + {"name": "f", "type": "x.Z"} + ]}} +]} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/null_full b/fluent-bit/lib/avro/tests/schema_tests/pass/null_full new file mode 100644 index 00000000..cae87673 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/null_full @@ -0,0 +1 @@ +{"type":"null"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/record b/fluent-bit/lib/avro/tests/schema_tests/pass/record new file mode 100644 index 00000000..43ac456f --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/record @@ -0,0 +1,5 @@ +{"name": "person", + "type": "record", + "fields": [ {"name": "height", "type": "long"}, + {"name": "weight", "type": "long"}, + {"name": "name", "type": "string"}]} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/record_fields_with_defaults b/fluent-bit/lib/avro/tests/schema_tests/pass/record_fields_with_defaults new file mode 100644 index 00000000..545ccbbc --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/record_fields_with_defaults @@ -0,0 +1,6 @@ +{"name": "person", + "type": "record", + "fields": [ {"name": "height", "type": "long"}, + {"name": "weight", "type": "long"}, + {"name": "name", "type": "string"}, + {"name": "hacker", "type": "boolean", "default": false}]} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/record_no_fields b/fluent-bit/lib/avro/tests/schema_tests/pass/record_no_fields new file mode 100644 index 00000000..142f4527 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/record_no_fields @@ -0,0 +1 @@ +{"type": "record", "name": "R", "fields": []} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/recursive_record b/fluent-bit/lib/avro/tests/schema_tests/pass/recursive_record new file mode 100644 index 00000000..0967bb4a --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/recursive_record @@ -0,0 +1,7 @@ +{ "type": "record", + "name": "recursive", + "fields": [ + { "name": "label", "type": "string" }, + { "name": "children", "type": {"type": "array", "items": "recursive"} } + ] +} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/string_extra_attributes b/fluent-bit/lib/avro/tests/schema_tests/pass/string_extra_attributes new file mode 100644 index 00000000..49885b9d --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/string_extra_attributes @@ -0,0 +1 @@ +{"type":"string", "ignored": "value"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/string_full b/fluent-bit/lib/avro/tests/schema_tests/pass/string_full new file mode 100644 index 00000000..5566b9f8 --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/string_full @@ -0,0 +1 @@ +{"type": "string"} diff --git a/fluent-bit/lib/avro/tests/schema_tests/pass/union b/fluent-bit/lib/avro/tests/schema_tests/pass/union new file mode 100644 index 00000000..ef2b6ecf --- /dev/null +++ b/fluent-bit/lib/avro/tests/schema_tests/pass/union @@ -0,0 +1 @@ +["string", "long", "null"] diff --git a/fluent-bit/lib/avro/tests/test_avro_1034.c b/fluent-bit/lib/avro/tests/test_avro_1034.c new file mode 100644 index 00000000..b44d6e40 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_1034.c @@ -0,0 +1,395 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <avro.h> +#include <stdio.h> +#include <stdlib.h> + +/* Test code for JIRA Issue AVRO-1034. + * + * AVRO-1034: Resolved reader does not initialize children of arrays, + * resulting in seg faults + * + * This program tests schema resolution for nested arrays. For the + * purposes of this test, there are two schemas "old" and "new" which + * are created by reading the same JSON schema. + * + * The test creates and populates a nested array avro value, and + * serializes it to memory. The raw memory is written to a file. Note + * that the schema is not written to the file. The nested array is + * also printed to the screen. + * + * An identical nested array avro value is then created. A + * resolved_reader_class and a corresponding resolved_record instance + * is created (using identical "writer" and "reader" schemas for + * simplicity), and an attempt is made to "read" the resolved avro + * value. + * + * Once the resolved value has been read, the source value (nested) + * and the resolved value (resolved_record) are both reset using + * avro_value_reset(). Then the source value (nested) is populated + * with another (larger) nested array. Then an attempt is made to read + * the resolved avro value again. + * + * This second attempt to read the resolved value results in a + * segmentation fault under Linux, using the patch in + * https://issues.apache.org/jira/secure/attachment/12516487/0001-AVRO-1034.-C-Resolved-reader-initializes-child-array.patch. + * + * However, the program does not seg fault, using the patch in + * https://issues.apache.org/jira/secure/attachment/12515544/AVRO-1034.patch + * + * AVRO-C was compiled with CMAKE_INSTALL_PREFIX=avrolib + * The static library (libavro.a) was copied into a subdirectory of avrolib/lib/static + * + * This file was compiled under Linux using: + * gcc -g avro-1034-test-2.c -o test2 -I../../build/avrolib/include -L../../build/avrolib/lib/static -lavro + * + */ + + +// Encode the following json string in NESTED_ARRAY +// {"type":"array", "items": {"type": "array", "items": "long"}} +// +#define NESTED_ARRAY \ + "{\"type\":\"array\", \"items\": {\"type\": \"array\", \"items\": \"long\"}}" + +avro_schema_t schema_old = NULL; +avro_schema_t schema_new = NULL; + +/* Parse schema into a schema data structure */ +void init_schema(void) +{ + avro_schema_error_t error; + if (avro_schema_from_json(NESTED_ARRAY, sizeof(NESTED_ARRAY), + &schema_old, &error)) { + printf( "Unable to parse old schema\n"); + exit(EXIT_FAILURE); + } + + if (avro_schema_from_json(NESTED_ARRAY, sizeof(NESTED_ARRAY), + &schema_new, &error)) { + printf( "Unable to parse new schema\n"); + exit(EXIT_FAILURE); + } +} + +#define try(call, msg) \ + do { \ + if (call) { \ + printf( msg ":\n %s\n", avro_strerror()); \ + exit (EXIT_FAILURE); \ + } \ + } while (0) + + +/* The input avro_value_t p_array should contain a nested array. + * Print the fields of this nested array to the screen. + */ +int print_array_fields ( avro_value_t *p_array ) +{ + size_t idx; + size_t length; + avro_type_t val_type; + + val_type = avro_value_get_type( p_array ); + printf( "Main array type = %d\n", val_type ); + + try( avro_value_get_size( p_array, &length ), + "Couldn't get array size" ); + printf( "Main array length = %d\n", (int) length ); + + for ( idx = 0; idx < length; idx ++ ) + { + avro_value_t subarray; + size_t sublength; + size_t jdx; + const char *unused; + + try ( avro_value_get_by_index( p_array, idx, &subarray, &unused ), + "Couldn't get subarray" ); + + val_type = avro_value_get_type( &subarray ); + printf( "Subarray type = %d\n", val_type ); + + try( avro_value_get_size( &subarray, &sublength ), + "Couldn't get subarray size" ); + printf( "Subarray length = %d\n", (int) sublength ); + + for ( jdx = 0; jdx < sublength; jdx++ ) + { + avro_value_t element; + int64_t val; + + try ( avro_value_get_by_index( &subarray, jdx, &element, &unused ), + "Couldn't get subarray element" ); + + val_type = avro_value_get_type( &element ); + + try ( avro_value_get_long( &element, &val ), + "Couldn't get subarray element value" ); + + printf( "nested_array[%d][%d]: type = %d value = %lld\n", + (int) idx, (int) jdx, (int) val_type, (long long) val ); + + } + } + + return 0; +} + + +/* The input avro_value_t p_subarray should contain an array of long + * integers. Add "elements" number of long integers to this array. Set + * the values to be distinct based on the iteration parameter. + */ +int add_subarray( avro_value_t *p_subarray, + size_t elements, + int32_t iteration ) +{ + avro_value_t element; + size_t index; + size_t idx; + + for ( idx = 0; idx < elements; idx ++ ) + { + // Append avro array element to subarray + try ( avro_value_append( p_subarray, &element, &index ), + "Error appending element in subarray" ); + + try ( avro_value_set_long( &element, (iteration+1)*100 + (iteration+1) ), + "Error setting subarray element" ); + } + + return 0; +} + +int populate_array( avro_value_t *p_array, int32_t elements ) +{ + int32_t idx; + fprintf( stderr, "Elements = %d\n", elements); + for ( idx = 0; idx < elements; idx ++ ) + { + avro_value_t subarray; + size_t index; + + // Append avro array element for top level array + try ( avro_value_append( p_array, &subarray, &index ), + "Error appending subarray" ); + + // Populate array element with subarray of length 2 +#define SUBARRAY_LENGTH (2) + try ( add_subarray( &subarray, SUBARRAY_LENGTH, idx ), + "Error populating subarray" ); + } + return 0; +} + + +/* Create a nested array using the schema NESTED_ARRAY. Populate its + * elements with unique values. Serialize the nested array to the + * memory buffer in avro_writer_t. The number of elements in the first + * dimension of the nested array is "elements". The number of elements + * in the second dimension of the nested array is hardcoded to 2. + */ +int add_array( avro_writer_t writer, + int32_t elements, + int use_resolving_writer ) +{ + avro_schema_t chosen_schema; + avro_value_iface_t *nested_array_class; + avro_value_t nested; + + // Select (hardcode) schema to use + chosen_schema = schema_old; + + // Create avro class and value + nested_array_class = avro_generic_class_from_schema( chosen_schema ); + try ( avro_generic_value_new( nested_array_class, &nested ), + "Error creating instance of record" ); + + try ( populate_array( &nested, elements ), + "Error populating array" ); + + if ( use_resolving_writer ) + { + // Resolve schema differences + avro_value_iface_t *resolved_reader_class; + avro_value_iface_t *writer_class; + avro_value_t resolved_record; + + // Note - we will read values from the reader of "schema to write + // to file" and we will copy them into a writer of the same + // schema. + resolved_reader_class = avro_resolved_reader_new( schema_old,// schema populated above + schema_new // schema_to_write_to_file + ); + if ( resolved_reader_class == NULL ) + { + printf( "Failed avro_resolved_reader_new()\n"); + exit( EXIT_FAILURE ); + } + + try ( avro_resolved_reader_new_value( resolved_reader_class, &resolved_record ), + "Failed avro_resolved_reader_new_value" ); + + // Map the resolved reader to the record you want to get data from + avro_resolved_reader_set_source( &resolved_record, &nested ); + + // Now the resolved_record is mapped to read data from record. Now + // we need to copy the data from resolved_record into a + // writer_record, which is an instance of the same schema as + // resolved_record. + + // Create a writer of the schema you want to write using + writer_class = avro_generic_class_from_schema( schema_new ); + if ( writer_class == NULL ) + { + printf( "Failed avro_generic_class_from_schema()\n"); + exit( EXIT_FAILURE ); + } + + try ( avro_value_write( writer, &resolved_record ), + "Unable to write record into memory using writer_record" ); + + print_array_fields( &resolved_record ); + + avro_value_reset( &nested ); + + // Question: Is it permissible to call avro_value_reset() on a + // resolved_record? Set the #if 1 to #if 0 to disable the + // avro_value_reset(), to prevent the segmentation fault. + #if 1 + avro_value_reset( &resolved_record ); + #endif + + try ( populate_array( &nested, 2*elements ), + "Error populating array" ); + + try ( avro_value_write( writer, &resolved_record ), + "Unable to write record into memory using writer_record" ); + + print_array_fields( &resolved_record ); + + avro_value_decref( &resolved_record ); + avro_value_iface_decref( writer_class ); + avro_value_iface_decref( resolved_reader_class ); + } + else + { + // Write the value to memory + try ( avro_value_write( writer, &nested ), + "Unable to write nested into memory" ); + + print_array_fields( &nested ); + } + + + // Release the record + avro_value_decref( &nested ); + avro_value_iface_decref( nested_array_class ); + + return 0; +} + +/* Create a raw binary file containing a serialized version of a + * nested array. This file will later be read by + * read_nested_array_file(). + */ +int write_nested_array_file ( int64_t buf_len, + const char *raw_binary_file_name, + int use_resolving_writer ) +{ + char *buf; + avro_writer_t nested_writer; + FILE *fid = NULL; + + fprintf( stdout, "Create %s\n", raw_binary_file_name ); + + // Allocate a buffer + buf = (char *) malloc( buf_len * sizeof( char ) ); + if ( buf == NULL ) + { + printf( "There was an error creating the nested buffer %s.\n", raw_binary_file_name); + exit(EXIT_FAILURE); + } + + /* Create a new memory writer */ + nested_writer = avro_writer_memory( buf, buf_len ); + if ( nested_writer == NULL ) + { + printf( "There was an error creating the buffer for writing %s.\n", raw_binary_file_name); + exit(EXIT_FAILURE); + } + + /* Add an array containing 4 subarrays */ + printf( "before avro_writer_tell %d\n", (int) avro_writer_tell( nested_writer ) ); +#define ARRAY_LENGTH (4) + add_array( nested_writer, ARRAY_LENGTH, use_resolving_writer ); + printf( "after avro_writer_tell %d\n", (int) avro_writer_tell( nested_writer ) ); + + /* Serialize the nested array */ + printf( "Serialize the data to a file\n"); + + /* Delete the nested array if it exists, and create a new one */ + remove(raw_binary_file_name); + fid = fopen( raw_binary_file_name, "w+"); + if ( fid == NULL ) + { + printf( "There was an error creating the file %s.\n", raw_binary_file_name); + exit(EXIT_FAILURE); + } + fwrite( buf, 1, avro_writer_tell( nested_writer ), fid ); + fclose(fid); + avro_writer_free( nested_writer ); + free(buf); + return 0; +} + + +/* Top level function to impelement a test for the JIRA issue + * AVRO-1034. See detailed documentation at the top of this file. + */ +int main(void) +{ + const char *raw_binary_file_name = "nested_array.bin"; + const char *raw_binary_file_name_resolved = "nested_array_resolved.bin"; + int64_t buf_len = 2048; + int use_resolving_writer; + + /* Initialize the schema structure from JSON */ + init_schema(); + + printf( "Write the serialized nested array to %s\n", raw_binary_file_name ); + use_resolving_writer = 0; + write_nested_array_file( buf_len, raw_binary_file_name, use_resolving_writer ); + + printf( "\nWrite the serialized nested array after schema resolution to %s\n", + raw_binary_file_name_resolved ); + use_resolving_writer = 1; + write_nested_array_file( buf_len, raw_binary_file_name_resolved, use_resolving_writer ); + + // Close out schemas + avro_schema_decref(schema_old); + avro_schema_decref(schema_new); + + // Remove the binary files + remove(raw_binary_file_name); + remove(raw_binary_file_name_resolved); + + printf("\n"); + return 0; +} diff --git a/fluent-bit/lib/avro/tests/test_avro_1084.c b/fluent-bit/lib/avro/tests/test_avro_1084.c new file mode 100644 index 00000000..75d4c7a7 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_1084.c @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <avro.h> +#include <stdio.h> +#include <stdlib.h> + +const char PERSON_SCHEMA[] = +"{\"type\":\"record\",\ + \"name\":\"Person\",\ + \"fields\":[\ + {\"name\": \"ID\", \"type\": \"long\"}]}"; + +const char *dbname = "test.db"; +avro_schema_t schema; + +int main() +{ + avro_file_writer_t writer; + + // refcount == 1 + if (avro_schema_from_json_literal (PERSON_SCHEMA, &schema)) + { + printf ("Unable to parse schema\n"); + return EXIT_FAILURE; + } + + // BUG: refcount == 1 + if (avro_file_writer_create ("test.db", schema, &writer)) + { + printf ("There was an error creating db: %s\n", avro_strerror()); + return EXIT_FAILURE; + } + + // this is "unusual" behaviour + // refcount == 0 + avro_schema_decref (schema); + + // crash + avro_datum_t main_datum = avro_record(schema); + avro_datum_t id_datum = avro_int32(1); + + if (avro_record_set (main_datum, "ID", id_datum)) + { + printf ("Unable to create datum"); + return EXIT_FAILURE; + } + + avro_file_writer_append (writer, main_datum); + avro_file_writer_flush (writer); + avro_file_writer_close (writer); + remove (dbname); + + avro_datum_decref (id_datum); + avro_datum_decref (main_datum); + + return EXIT_SUCCESS; +} + diff --git a/fluent-bit/lib/avro/tests/test_avro_1087.c b/fluent-bit/lib/avro/tests/test_avro_1087.c new file mode 100644 index 00000000..b2e82aaa --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_1087.c @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <avro.h> +#include <stdio.h> +#include <stdlib.h> + +const char PERSON_SCHEMA[] = +"{\"type\":\"record\",\ + \"name\":\"Person\",\ + \"fields\":[\ + {\"name\": \"ID\", \"type\": \"int\"}]}"; + +const char *dbname = "test.db"; +avro_schema_t schema; + +void add_record (avro_file_writer_t writer) +{ + avro_datum_t main_datum = avro_record(schema); + avro_datum_t id_datum = avro_int32(1); + + if (avro_record_set (main_datum, "ID", id_datum)) + { + printf ("Unable to create datum"); + exit (EXIT_FAILURE); + } + + avro_file_writer_append (writer, main_datum); + + avro_datum_decref (id_datum); + avro_datum_decref (main_datum); +} + +void create_database() +{ + avro_file_writer_t writer; + + if (avro_schema_from_json_literal (PERSON_SCHEMA, &schema)) + { + printf ("Unable to parse schema\n"); + exit (EXIT_FAILURE); + } + + if (avro_file_writer_create ("test.db", schema, &writer)) + { + printf ("There was an error creating db: %s\n", avro_strerror()); + exit (EXIT_FAILURE); + } + + add_record (writer); + + avro_file_writer_flush (writer); + avro_file_writer_close (writer); +} + + +int main() +{ + avro_file_writer_t writer; + + create_database(); + + avro_file_writer_open (dbname, &writer); + add_record (writer); + + avro_file_writer_flush (writer); + avro_file_writer_close (writer); + + avro_schema_decref(schema); + + remove (dbname); + + return EXIT_SUCCESS; +} diff --git a/fluent-bit/lib/avro/tests/test_avro_1165.c b/fluent-bit/lib/avro/tests/test_avro_1165.c new file mode 100644 index 00000000..09f51726 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_1165.c @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <stdio.h> +#include <string.h> +#include <avro.h> + +/* To validate AVRO-1165, run this test program through valgrind + * before and after applying the AVRO-1165.patch. Before the patch + * valgrind will show memory leaks, and after the patch it will not. + * The specific valgrind commandline to use from the + * avro-trunk/lang/c/tests directory is: + * valgrind -v --track-origins=yes --leak-check=full + * --show-reachable = yes ../build/tests/test_avro_1165 + */ + +int main(int argc, char **argv) +{ + const char *json = + "{" + " \"name\": \"repeated_subrecord_array\"," + " \"type\": \"record\"," + " \"fields\": [" + " { \"name\": \"subrecord_one\"," + " \"type\": {" + " \"name\": \"SubrecordType\"," + " \"type\": \"record\"," + " \"fields\": [" + " { \"name\": \"x\", \"type\": \"int\" }," + " { \"name\": \"y\", \"type\": \"int\" }" + " ]" + " }" + " }," + " { \"name\": \"subrecord_two\", \"type\": \"SubrecordType\" }," + " { \"name\": \"subrecord_array\", \"type\": {" + " \"type\":\"array\"," + " \"items\": \"SubrecordType\"" + " }" + " }" + " ]" + "}"; + + int rval; + avro_schema_t schema = NULL; + avro_schema_error_t error; + avro_value_iface_t *p_reader_class; + + (void) argc; + (void) argv; + + rval = avro_schema_from_json(json, strlen(json), &schema, &error); + if ( rval ) + { + printf("Failed to read schema from JSON.\n"); + return 1; + } + else + { + printf("Successfully read schema from JSON.\n"); + } + + p_reader_class = avro_generic_class_from_schema(schema); + + avro_value_iface_decref(p_reader_class); + + avro_schema_decref(schema); + return 0; +} diff --git a/fluent-bit/lib/avro/tests/test_avro_1167.c b/fluent-bit/lib/avro/tests/test_avro_1167.c new file mode 100644 index 00000000..09ad787a --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_1167.c @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <stdio.h> +#include <string.h> +#include <avro.h> + +/* To see the AVRO-1167 memory leak, run this test program through + * valgrind. The specific valgrind commandline to use from the + * avro-trunk/lang/c/tests directory is: + * valgrind -v --track-origins=yes --leak-check=full + * --show-reachable = yes ../build/tests/test_avro_1167 + */ + +int main(int argc, char **argv) +{ + const char *json = + "{" + " \"name\": \"repeated_subrecord_array\"," + " \"type\": \"record\"," + " \"fields\": [" + " { \"name\": \"subrecord_one\"," + " \"type\": {" + " \"name\": \"SubrecordType\"," + " \"type\": \"record\"," + " \"fields\": [" + " { \"name\": \"x\", \"type\": \"int\" }," + " { \"name\": \"y\", \"type\": \"int\" }" + " ]" + " }" + " }," + " { \"name\": \"subrecord_two\", \"type\": \"SubrecordType\" }," + " { \"name\": \"subrecord_array\", \"type\": {" + " \"type\":\"array\"," + " \"items\": \"SubrecordType\"" + " }" + " }" + " ]" + "}"; + + int rval; + avro_schema_t schema = NULL; + avro_schema_t schema_copy = NULL; + avro_schema_error_t error; + + (void) argc; + (void) argv; + + rval = avro_schema_from_json(json, strlen(json), &schema, &error); + if ( rval ) + { + printf("Failed to read schema from JSON.\n"); + exit(EXIT_FAILURE); + } + else + { + printf("Successfully read schema from JSON.\n"); + } + + schema_copy = avro_schema_copy( schema ); + if ( ! avro_schema_equal(schema, schema_copy) ) + { + printf("Failed avro_schema_equal(schema, schema_copy)\n"); + exit(EXIT_FAILURE); + } + + avro_schema_decref(schema); + avro_schema_decref(schema_copy); + return 0; +} diff --git a/fluent-bit/lib/avro/tests/test_avro_1237.c b/fluent-bit/lib/avro/tests/test_avro_1237.c new file mode 100644 index 00000000..f382cc03 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_1237.c @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <avro.h> +#include <stdio.h> +#include <stdlib.h> + +#define check_exit(call) \ + do { \ + int __rc = call; \ + if (__rc != 0) { \ + fprintf(stderr, "Unexpected error:\n %s\n %s\n", \ + avro_strerror(), #call); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#define expect_error(call) \ + do { \ + int __rc = call; \ + if (__rc == 0) { \ + fprintf(stderr, "Expected an error:\n %s\n", #call); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#define check_expected_value(actual, expected) \ + do { \ + if (!avro_value_equal_fast((actual), (expected))) { \ + char *actual_json; \ + char *expected_json; \ + avro_value_to_json((actual), 1, &actual_json); \ + avro_value_to_json((expected), 1, &expected_json); \ + fprintf(stderr, "Expected %s\nGot %s\n", \ + expected_json, actual_json); \ + free(actual_json); \ + free(expected_json); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +int main(void) +{ + avro_schema_t schema; + avro_file_reader_t reader; + avro_value_iface_t *iface; + avro_value_t actual; + avro_value_t expected; + avro_value_t branch; + + schema = avro_schema_union(); + avro_schema_union_append(schema, avro_schema_null()); + avro_schema_union_append(schema, avro_schema_int()); + + iface = avro_generic_class_from_schema(schema); + avro_generic_value_new(iface, &actual); + avro_generic_value_new(iface, &expected); + + + /* First read the contents of the good file. */ + + check_exit(avro_file_reader("avro-1237-good.avro", &reader)); + + check_exit(avro_file_reader_read_value(reader, &actual)); + check_exit(avro_value_set_branch(&expected, 0, &branch)); + check_exit(avro_value_set_null(&branch)); + check_expected_value(&actual, &expected); + + check_exit(avro_file_reader_read_value(reader, &actual)); + check_exit(avro_value_set_branch(&expected, 1, &branch)); + check_exit(avro_value_set_int(&branch, 100)); + check_expected_value(&actual, &expected); + + check_exit(avro_file_reader_close(reader)); + + + /* Then read from the malformed file. */ + + check_exit(avro_file_reader + ("avro-1237-bad-union-discriminant.avro", &reader)); + + check_exit(avro_file_reader_read_value(reader, &actual)); + check_exit(avro_value_set_branch(&expected, 0, &branch)); + check_exit(avro_value_set_null(&branch)); + check_expected_value(&actual, &expected); + + expect_error(avro_file_reader_read_value(reader, &actual)); + + check_exit(avro_file_reader_close(reader)); + + + /* Clean up and exit */ + avro_value_decref(&actual); + avro_value_decref(&expected); + avro_value_iface_decref(iface); + avro_schema_decref(schema); + exit(EXIT_SUCCESS); +} diff --git a/fluent-bit/lib/avro/tests/test_avro_1238.c b/fluent-bit/lib/avro/tests/test_avro_1238.c new file mode 100644 index 00000000..eed8a807 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_1238.c @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <avro.h> +#include <stdio.h> +#include <stdlib.h> + +#define check_exit(call) \ + do { \ + int __rc = call; \ + if (__rc != 0) { \ + fprintf(stderr, "Unexpected error:\n %s\n %s\n", \ + avro_strerror(), #call); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#define expect_eof(call) \ + do { \ + int __rc = call; \ + if (__rc != EOF) { \ + fprintf(stderr, "Expected EOF:\n %s\n", #call); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#define expect_error(call) \ + do { \ + int __rc = call; \ + if (__rc == 0) { \ + fprintf(stderr, "Expected an error:\n %s\n", #call); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#define check_expected_value(actual, expected) \ + do { \ + if (!avro_value_equal_fast((actual), (expected))) { \ + char *actual_json; \ + char *expected_json; \ + avro_value_to_json((actual), 1, &actual_json); \ + avro_value_to_json((expected), 1, &expected_json); \ + fprintf(stderr, "Expected %s\nGot %s\n", \ + expected_json, actual_json); \ + free(actual_json); \ + free(expected_json); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +int main(void) +{ + avro_schema_t schema; + avro_file_reader_t reader; + avro_value_iface_t *iface; + avro_value_t actual; + avro_value_t expected; + avro_value_t branch; + + schema = avro_schema_union(); + avro_schema_union_append(schema, avro_schema_null()); + avro_schema_union_append(schema, avro_schema_int()); + + iface = avro_generic_class_from_schema(schema); + avro_generic_value_new(iface, &actual); + avro_generic_value_new(iface, &expected); + + + /* First read the contents of the good file. */ + + check_exit(avro_file_reader("avro-1238-good.avro", &reader)); + + check_exit(avro_file_reader_read_value(reader, &actual)); + check_exit(avro_value_set_branch(&expected, 0, &branch)); + check_exit(avro_value_set_null(&branch)); + check_expected_value(&actual, &expected); + + check_exit(avro_file_reader_read_value(reader, &actual)); + check_exit(avro_value_set_branch(&expected, 1, &branch)); + check_exit(avro_value_set_int(&branch, 100)); + check_expected_value(&actual, &expected); + + expect_eof(avro_file_reader_read_value(reader, &actual)); + check_exit(avro_file_reader_close(reader)); + + + /* Then read from the truncated file. */ + + check_exit(avro_file_reader("avro-1238-truncated.avro", &reader)); + + check_exit(avro_file_reader_read_value(reader, &actual)); + check_exit(avro_value_set_branch(&expected, 0, &branch)); + check_exit(avro_value_set_null(&branch)); + check_expected_value(&actual, &expected); + + check_exit(avro_file_reader_read_value(reader, &actual)); + check_exit(avro_value_set_branch(&expected, 1, &branch)); + check_exit(avro_value_set_int(&branch, 100)); + check_expected_value(&actual, &expected); + + expect_error(avro_file_reader_read_value(reader, &actual)); + check_exit(avro_file_reader_close(reader)); + + + /* Clean up and exit */ + avro_value_decref(&actual); + avro_value_decref(&expected); + avro_value_iface_decref(iface); + avro_schema_decref(schema); + exit(EXIT_SUCCESS); +} diff --git a/fluent-bit/lib/avro/tests/test_avro_1279.c b/fluent-bit/lib/avro/tests/test_avro_1279.c new file mode 100644 index 00000000..9d490d46 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_1279.c @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <avro.h> +#include <stdio.h> +#include <stdlib.h> + +#define check_exit(call) \ + do { \ + int __rc = call; \ + if (__rc != 0) { \ + fprintf(stderr, "Unexpected error:\n %s\n %s\n", \ + avro_strerror(), #call); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +int main(void) +{ + avro_file_reader_t reader; + + /* First open the file with the explicit codec. */ + check_exit(avro_file_reader("avro-1279-codec.avro", &reader)); + check_exit(avro_file_reader_close(reader)); + + + /* Then the file with no codec. */ + check_exit(avro_file_reader("avro-1279-no-codec.avro", &reader)); + check_exit(avro_file_reader_close(reader)); + + /* Clean up and exit */ + exit(EXIT_SUCCESS); +} diff --git a/fluent-bit/lib/avro/tests/test_avro_1379.c b/fluent-bit/lib/avro/tests/test_avro_1379.c new file mode 100644 index 00000000..bfd77dc8 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_1379.c @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include "avro.h" +#include "avro_private.h" +#include <stdio.h> +#include <stdlib.h> + +static const char *schema_json = + "{" + " \"type\": \"record\"," + " \"name\": \"test\"," + " \"fields\": [" + " { \"name\": \"i\", \"type\": \"int\" }," + " { \"name\": \"l\", \"type\": \"long\" }," + " { \"name\": \"s\", \"type\": \"string\" }," + " {" + " \"name\": \"subrec\"," + " \"type\": {" + " \"type\": \"record\"," + " \"name\": \"sub\"," + " \"fields\": [" + " { \"name\": \"f\", \"type\": \"float\" }," + " { \"name\": \"d\", \"type\": \"double\" }" + " ]" + " }" + " }" + " ]" + "}"; + +static void +populate_complex_record(avro_value_t *p_val) +{ + avro_value_t field; + + avro_value_get_by_index(p_val, 0, &field, NULL); + avro_value_set_int(&field, 42); + + avro_value_get_by_index(p_val, 1, &field, NULL); + avro_value_set_long(&field, 4242); + + avro_wrapped_buffer_t wbuf; + avro_wrapped_buffer_new_string(&wbuf, "Follow your bliss."); + avro_value_get_by_index(p_val, 2, &field, NULL); + avro_value_give_string_len(&field, &wbuf); + + avro_value_t subrec; + avro_value_get_by_index(p_val, 3, &subrec, NULL); + + avro_value_get_by_index(&subrec, 0, &field, NULL); + avro_value_set_float(&field, 3.14159265); + + avro_value_get_by_index(&subrec, 1, &field, NULL); + avro_value_set_double(&field, 2.71828183); +} + +int main(void) +{ + int rval = 0; + size_t len; + static char buf[4096]; + avro_writer_t writer; + avro_file_writer_t file_writer; + avro_file_reader_t file_reader; + const char *outpath = "test-1379.avro"; + + avro_schema_t schema = NULL; + avro_schema_error_t error = NULL; + check(rval, avro_schema_from_json(schema_json, strlen(schema_json), &schema, &error)); + + avro_value_iface_t *iface = avro_generic_class_from_schema(schema); + + avro_value_t val; + avro_generic_value_new(iface, &val); + + avro_value_t out; + avro_generic_value_new(iface, &out); + + /* create the val */ + avro_value_reset(&val); + populate_complex_record(&val); + + /* create the writers */ + writer = avro_writer_memory(buf, sizeof(buf)); + check(rval, avro_file_writer_create(outpath, schema, &file_writer)); + + fprintf(stderr, "Writing to buffer\n"); + check(rval, avro_value_write(writer, &val)); + + fprintf(stderr, "Writing buffer to %s " + "using avro_file_writer_append_encoded()\n", outpath); + len = avro_writer_tell(writer); + check(rval, avro_file_writer_append_encoded(file_writer, buf, len)); + check(rval, avro_file_writer_close(file_writer)); + + check(rval, avro_file_reader(outpath, &file_reader)); + fprintf(stderr, "Re-reading value to verify\n"); + check(rval, avro_file_reader_read_value(file_reader, &out)); + fprintf(stderr, "Verifying value..."); + if (!avro_value_equal(&val, &out)) { + fprintf(stderr, "fail!\n"); + exit(EXIT_FAILURE); + } + fprintf(stderr, "ok\n"); + check(rval, avro_file_reader_close(file_reader)); + remove(outpath); + + avro_writer_free(writer); + avro_value_decref(&out); + avro_value_decref(&val); + avro_value_iface_decref(iface); + avro_schema_decref(schema); + + exit(EXIT_SUCCESS); +} diff --git a/fluent-bit/lib/avro/tests/test_avro_1405.c b/fluent-bit/lib/avro/tests/test_avro_1405.c new file mode 100644 index 00000000..b853dae9 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_1405.c @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <stdio.h> +#include "avro.h" + +#define NUM_RECORDS 10 + +const char PERSON_SCHEMA[] = + "{" + " \"type\":\"record\"," + " \"name\":\"Person\"," + " \"fields\": [" + " {\"name\": \"ID\", \"type\": \"long\"}," + " {\"name\": \"First\", \"type\": \"string\"}," + " {\"name\": \"Last\", \"type\": \"string\"}," + " {\"name\": \"Phone\", \"type\": \"string\"}," + " {\"name\": \"Age\", \"type\": \"int\"}" + " ]" + "}"; + +const char *file = "avro_file.dat"; + +void print_avro_value(avro_value_t *value) { + char *json; + if (!avro_value_to_json(value, 1, &json)) { + printf("%s\n", json); + free(json); + } +} + +int read_data() { + int rval; + int records_read = 0; + + avro_file_reader_t reader; + avro_value_iface_t *iface; + avro_value_t value; + + avro_file_reader(file, &reader); + avro_schema_t schema = avro_file_reader_get_writer_schema(reader); + + iface = avro_generic_class_from_schema(schema); + avro_generic_value_new(iface, &value); + + printf("\nReading...\n"); + while ((rval = avro_file_reader_read_value(reader, &value)) == 0) { + char *json; + + if (avro_value_to_json(&value, 1, &json)) { + printf("Error converting value to JSON: %s\n",avro_strerror()); + } else { + printf("%s\n", json); + free(json); + records_read++; + } + + avro_value_reset(&value); + } + + avro_value_decref(&value); + avro_value_iface_decref(iface); + avro_schema_decref(schema); + avro_file_reader_close(reader); + + if (rval != EOF || records_read != NUM_RECORDS) { + fprintf(stderr, "Error: %s\n", avro_strerror()); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int write_data() { + int i; + avro_schema_t schema; + avro_schema_error_t error; + avro_file_writer_t writer; + avro_value_iface_t *iface; + avro_value_t value; + + if (avro_schema_from_json(PERSON_SCHEMA, 0, &schema, &error)) { + printf ("Unable to parse schema\n"); + return EXIT_FAILURE; + } + + iface = avro_generic_class_from_schema(schema); + avro_generic_value_new(iface, &value); + + if (avro_file_writer_create(file, schema, &writer)) { + printf ("There was an error creating file: %s\n", avro_strerror()); + return EXIT_FAILURE; + } + + printf("\nWriting...\n"); + for (i = 0; i < NUM_RECORDS; i++) { + avro_value_t field; + avro_value_get_by_name(&value, "ID", &field, NULL); + avro_value_set_long(&field, (int64_t) i); + + avro_value_get_by_name(&value, "Age", &field, NULL); + avro_value_set_int(&field, i); + + avro_value_get_by_name(&value, "First", &field, NULL); + avro_value_set_string(&field, "Firstname"); + + avro_value_get_by_name(&value, "Last", &field, NULL); + avro_value_set_string(&field, "Lastname"); + + + avro_value_get_by_name(&value, "Phone", &field, NULL); + avro_value_set_string(&field, "1234567"); + + print_avro_value(&value); + + avro_file_writer_append_value(writer, &value); + + // Writing multiple blocks + avro_file_writer_close(writer); + avro_file_writer_open(file, &writer); + + avro_value_reset(&value); + } + + avro_file_writer_close(writer); + avro_value_iface_decref(iface); + avro_value_decref(&value); + avro_schema_decref(schema); + + return EXIT_SUCCESS; +} + + +int main() +{ + int read_data_result; + + if (write_data()) { + return EXIT_FAILURE; + } + + read_data_result = read_data(); + remove(file); + + return read_data_result; +} diff --git a/fluent-bit/lib/avro/tests/test_avro_1572.c b/fluent-bit/lib/avro/tests/test_avro_1572.c new file mode 100644 index 00000000..eb94cc1a --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_1572.c @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <stdio.h> +#include <sys/stat.h> +#include "avro.h" + +#define BUFFER_LEN 4096 + +#define HEADER_LEN 116 +#define SYNC_LEN 16 +#define BLOCKINFO_LEN 4 +#define RECORD_LEN 1 + +static const int NUM_RECORDS1 = (BUFFER_LEN - HEADER_LEN - 2 * SYNC_LEN - BLOCKINFO_LEN) / RECORD_LEN; +static const int NUM_RECORDS2 = (BUFFER_LEN - SYNC_LEN - BLOCKINFO_LEN) / RECORD_LEN; + +static const char PERSON_SCHEMA[] = + "{" + " \"type\":\"record\"," + " \"name\":\"Person\"," + " \"fields\": [" + " {\"name\": \"ab\", \"type\": \"int\"}" + " ]" + "}"; + +static const char *filename = "avro_file.dat"; + +static int read_data() { + int rval; + int records_read = 0; + + avro_file_reader_t reader; + avro_value_iface_t *iface; + avro_value_t value; + + fprintf(stderr, "\nReading...\n"); + + avro_file_reader(filename, &reader); + avro_schema_t schema = avro_file_reader_get_writer_schema(reader); + + iface = avro_generic_class_from_schema(schema); + avro_generic_value_new(iface, &value); + + while ((rval = avro_file_reader_read_value(reader, &value)) == 0) { + records_read++; + avro_value_reset(&value); + } + + avro_value_decref(&value); + avro_value_iface_decref(iface); + avro_schema_decref(schema); + avro_file_reader_close(reader); + + fprintf(stderr, "wanted %d records, read %d records.\n", NUM_RECORDS1 + NUM_RECORDS2, records_read); + + if (rval != EOF || records_read != (NUM_RECORDS1 + NUM_RECORDS2)) { + fprintf(stderr, "Error: %s\n", avro_strerror()); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +static off_t fsize(const char *filename) { + struct stat st; + + if (stat(filename, &st) == 0) { + return st.st_size; + } + + return -1; +} + +static int write_data() { + int i; + avro_schema_t schema; + avro_schema_error_t error; + avro_file_writer_t writer; + avro_value_iface_t *iface; + avro_value_t value; + avro_value_t field; + + fprintf(stderr, "\nWriting...\n"); + + if (avro_schema_from_json(PERSON_SCHEMA, 0, &schema, &error)) { + fprintf(stderr, "Unable to parse schema\n"); + return EXIT_FAILURE; + } + + iface = avro_generic_class_from_schema(schema); + avro_generic_value_new(iface, &value); + + if (avro_file_writer_create(filename, schema, &writer)) { + fprintf(stderr, "There was an error creating file: %s\n", avro_strerror()); + return EXIT_FAILURE; + } + + avro_value_get_by_name(&value, "ab", &field, NULL); + avro_value_set_int(&field, 1); + + fprintf(stderr, "NUM_RECORDS1 = %d NUM_RECORDS2 = %d\n", NUM_RECORDS1, NUM_RECORDS2); + + for (i = 0; i < NUM_RECORDS1; i++) { + avro_file_writer_append_value(writer, &value); + } + + avro_file_writer_close(writer); + + /* Make sure the sync ends at a BUFFER_LEN boundary */ + if (fsize(filename) != BUFFER_LEN) { + fprintf(stderr, "internal error\n"); + return EXIT_FAILURE; + } + + avro_file_writer_open(filename, &writer); + + for (i = 0; i < NUM_RECORDS2; i++) { + avro_file_writer_append_value(writer, &value); + } + + avro_file_writer_close(writer); + + /* Make sure the whole file ends at a BUFFER_LEN boundary. */ + if (fsize(filename) != 2 * BUFFER_LEN) { + fprintf(stderr, "internal error\n"); + return EXIT_FAILURE; + } + + avro_value_iface_decref(iface); + avro_value_decref(&value); + avro_schema_decref(schema); + + return EXIT_SUCCESS; +} + + +int main() +{ + int read_data_result; + + if (write_data()) { + return EXIT_FAILURE; + } + + read_data_result = read_data(); + remove(filename); + + return read_data_result; +} diff --git a/fluent-bit/lib/avro/tests/test_avro_1691.c b/fluent-bit/lib/avro/tests/test_avro_1691.c new file mode 100644 index 00000000..bb45fc50 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_1691.c @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +/** + * AVRO-1691 test case - support of string-field JSON schemas. + */ +#include "avro.h" +#include "avro_private.h" +#include <stdio.h> +#include <stdlib.h> + +static const char *json_schemas[] = { + /* These two schemas are functionally equivalent. */ + "{ \"type\": \"string\" }", /* Object wrapped */ + "\"string\"", /* JSON string field */ + NULL +}; + + + + +int main(void) +{ + int pass; + + for (pass = 0 ; json_schemas[pass] ; pass++) { + int rval = 0; + size_t len; + static char buf[4096]; + avro_writer_t writer; + avro_file_writer_t file_writer; + avro_file_reader_t file_reader; + avro_schema_t schema = NULL; + avro_schema_error_t error = NULL; + char outpath[64]; + const char *json_schema = json_schemas[pass]; + + printf("pass %d with schema %s\n", pass, json_schema); + check(rval, avro_schema_from_json(json_schema, strlen(json_schema), + &schema, &error)); + + avro_value_iface_t *iface = avro_generic_class_from_schema(schema); + + avro_value_t val; + avro_generic_value_new(iface, &val); + + avro_value_t out; + avro_generic_value_new(iface, &out); + + /* create the val */ + avro_value_reset(&val); + avro_value_set_string(&val, "test-1691"); + + /* Write value to file */ + snprintf(outpath, sizeof(outpath), "test-1691-%d.avro", pass); + + /* create the writers */ + writer = avro_writer_memory(buf, sizeof(buf)); + check(rval, avro_file_writer_create(outpath, schema, &file_writer)); + + check(rval, avro_value_write(writer, &val)); + + len = avro_writer_tell(writer); + check(rval, avro_file_writer_append_encoded(file_writer, buf, len)); + check(rval, avro_file_writer_close(file_writer)); + + /* Read the value back */ + check(rval, avro_file_reader(outpath, &file_reader)); + check(rval, avro_file_reader_read_value(file_reader, &out)); + if (!avro_value_equal(&val, &out)) { + fprintf(stderr, "fail!\n"); + exit(EXIT_FAILURE); + } + fprintf(stderr, "pass %d: ok: schema %s\n", pass, json_schema); + check(rval, avro_file_reader_close(file_reader)); + remove(outpath); + + avro_writer_free(writer); + avro_value_decref(&out); + avro_value_decref(&val); + avro_value_iface_decref(iface); + avro_schema_decref(schema); + } + + exit(EXIT_SUCCESS); +} diff --git a/fluent-bit/lib/avro/tests/test_avro_1904.c b/fluent-bit/lib/avro/tests/test_avro_1904.c new file mode 100644 index 00000000..d13f2487 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_1904.c @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <stdio.h> +#include <sys/stat.h> +#include "avro.h" + +#define NUM_RECORDS 100 + +static const char *filename = "avro_file.dat"; + +static const char PERSON_SCHEMA[] = + "{" + " \"type\":\"record\"," + " \"name\":\"Person\"," + " \"fields\": [" + " ]" + "}"; + +static int read_data() { + int rval; + int records_read = 0; + + avro_file_reader_t reader; + avro_value_iface_t *iface; + avro_value_t value; + + fprintf(stderr, "\nReading...\n"); + + rval = avro_file_reader(filename, &reader); + + if (rval) { + fprintf(stderr, "Error: %s\n", avro_strerror()); + return EXIT_FAILURE; + } + + avro_schema_t schema = avro_file_reader_get_writer_schema(reader); + + iface = avro_generic_class_from_schema(schema); + avro_generic_value_new(iface, &value); + + while ((rval = avro_file_reader_read_value(reader, &value)) == 0) { + records_read++; + avro_value_reset(&value); + } + + avro_value_decref(&value); + avro_value_iface_decref(iface); + avro_schema_decref(schema); + avro_file_reader_close(reader); + + fprintf(stderr, "read %d records.\n", records_read); + + if (rval != EOF || records_read != NUM_RECORDS) { + fprintf(stderr, "Error: %s\n", avro_strerror()); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +static int write_data() { + int i; + avro_schema_t schema; + avro_schema_error_t error; + avro_file_writer_t writer; + avro_value_iface_t *iface; + avro_value_t value; + + fprintf(stderr, "\nWriting...\n"); + + if (avro_schema_from_json(PERSON_SCHEMA, 0, &schema, &error)) { + fprintf(stderr, "Unable to parse schema\n"); + return EXIT_FAILURE; + } + + iface = avro_generic_class_from_schema(schema); + avro_generic_value_new(iface, &value); + + if (avro_file_writer_create(filename, schema, &writer)) { + fprintf(stderr, "There was an error creating file: %s\n", avro_strerror()); + return EXIT_FAILURE; + } + + for (i = 0; i < NUM_RECORDS; i++) { + if (avro_file_writer_append_value(writer, &value)) { + fprintf(stderr, "There was an error creating file: %s\n", avro_strerror()); + return EXIT_FAILURE; + } + } + + if (avro_file_writer_close(writer)) { + fprintf(stderr, "There was an error creating file: %s\n", avro_strerror()); + return EXIT_FAILURE; + } + + avro_value_iface_decref(iface); + avro_value_decref(&value); + avro_schema_decref(schema); + + return EXIT_SUCCESS; +} + +int main() +{ + int read_data_result; + + if (write_data()) { + return EXIT_FAILURE; + } + + read_data_result = read_data(); + remove(filename); + + return read_data_result; +} diff --git a/fluent-bit/lib/avro/tests/test_avro_1906.c b/fluent-bit/lib/avro/tests/test_avro_1906.c new file mode 100644 index 00000000..7188aec0 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_1906.c @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <stdio.h> +#include <sys/stat.h> +#include "avro.h" + +static const char *filename = "avro_file.dat"; + +static const char PERSON_SCHEMA[] = + "{" + " \"type\":\"record\"," + " \"name\":\"Person\"," + " \"fields\": [" + " {\"name\": \"ab\", \"type\": \"int\"}" + " ]" + "}"; + +static int read_data() { + int rval; + int records_read = 0; + + avro_file_reader_t reader; + avro_value_iface_t *iface; + avro_value_t value; + + fprintf(stderr, "\nReading...\n"); + + rval = avro_file_reader(filename, &reader); + + if (rval) { + fprintf(stderr, "Error: %s\n", avro_strerror()); + return -1; + } + + avro_schema_t schema = avro_file_reader_get_writer_schema(reader); + + iface = avro_generic_class_from_schema(schema); + avro_generic_value_new(iface, &value); + + while ((rval = avro_file_reader_read_value(reader, &value)) == 0) { + avro_value_t field; + int32_t val; + avro_value_get_by_index(&value, 0, &field, NULL); + avro_value_get_int(&field, &val); + fprintf(stderr, "value = %d\n", val); + records_read++; + avro_value_reset(&value); + } + + avro_value_decref(&value); + avro_value_iface_decref(iface); + avro_schema_decref(schema); + avro_file_reader_close(reader); + + fprintf(stderr, "read %d records.\n", records_read); + + if (rval != EOF) { + fprintf(stderr, "Error: %s\n", avro_strerror()); + return -1; + } + + return records_read; +} + +static int read_data_datum() { + int rval; + int records_read = 0; + + avro_file_reader_t reader; + avro_datum_t datum; + + fprintf(stderr, "\nReading...\n"); + + rval = avro_file_reader(filename, &reader); + + if (rval) { + fprintf(stderr, "Error using 'datum': %s\n", avro_strerror()); + return -1; + } + + avro_schema_t schema = avro_file_reader_get_writer_schema(reader); + + while ((rval = avro_file_reader_read(reader, schema, &datum)) == 0) { + avro_datum_t val_datum; + int32_t val; + if (avro_record_get(datum, "ab", &val_datum)) { + fprintf(stderr, "Error getting value: %s\n", avro_strerror()); + return -1; + } + avro_int32_get(val_datum, &val); + fprintf(stderr, "value = %d\n", val); + records_read++; + avro_datum_decref(datum); + } + + avro_schema_decref(schema); + avro_file_reader_close(reader); + + fprintf(stderr, "read %d records using 'datum'.\n", records_read); + + if (rval != EOF) { + fprintf(stderr, "Error using 'datum': %s\n", avro_strerror()); + return -1; + } + + return records_read; +} + +static int write_data(int n_records) { + int i; + avro_schema_t schema; + avro_schema_error_t error; + avro_file_writer_t writer; + avro_value_iface_t *iface; + avro_value_t value; + + fprintf(stderr, "\nWriting...\n"); + + if (avro_schema_from_json(PERSON_SCHEMA, 0, &schema, &error)) { + fprintf(stderr, "Unable to parse schema\n"); + return -1; + } + + if (avro_file_writer_create(filename, schema, &writer)) { + fprintf(stderr, "There was an error creating file: %s\n", avro_strerror()); + return -1; + } + + iface = avro_generic_class_from_schema(schema); + avro_generic_value_new(iface, &value); + + avro_value_t field; + + avro_value_get_by_index(&value, 0, &field, NULL); + avro_value_set_int(&field, 123); + + for (i = 0; i < n_records; i++) { + if (avro_file_writer_append_value(writer, &value)) { + fprintf(stderr, "There was an error writing file: %s\n", avro_strerror()); + return -1; + } + } + + if (avro_file_writer_close(writer)) { + fprintf(stderr, "There was an error creating file: %s\n", avro_strerror()); + return -1; + } + + avro_value_decref(&value); + avro_value_iface_decref(iface); + avro_schema_decref(schema); + + return n_records; +} + +static int test_n_records(int n_records) { + int res = 0; + + if (write_data(n_records) != n_records) { + remove(filename); + return -1; + } + + if (read_data() != n_records) { + remove(filename); + return -1; + } + + if (read_data_datum() != n_records) { + remove(filename); + return -1; + } + + remove(filename); + return 0; +} + +int main() +{ + if (test_n_records(1) < 0) { + return EXIT_FAILURE; + } + + if (test_n_records(0) < 0) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} diff --git a/fluent-bit/lib/avro/tests/test_avro_766.c b/fluent-bit/lib/avro/tests/test_avro_766.c new file mode 100644 index 00000000..a0b13b79 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_766.c @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <stdio.h> +#include <string.h> +#include <avro.h> + +/* To see the AVRO-766 memory leak, run this test program through + * valgrind. The specific valgrind commandline to use from the + * avro-trunk/lang/c/tests directory is: + * valgrind -v --track-origins=yes --leak-check=full + * --show-reachable = yes ../build/tests/test_avro_766 + */ +int main(int argc, char **argv) +{ + const char *json = + "{" + " \"type\": \"record\"," + " \"name\": \"list\"," + " \"fields\": [" + " { \"name\": \"x\", \"type\": \"int\" }," + " { \"name\": \"y\", \"type\": \"int\" }," + " { \"name\": \"next\", \"type\": [\"null\",\"list\"]}," + " { \"name\": \"arraylist\", \"type\": { \"type\":\"array\", \"items\": \"list\" } }" + " ]" + "}"; + + int rval; + avro_schema_t schema = NULL; + avro_schema_error_t error; + + (void) argc; + (void) argv; + + rval = avro_schema_from_json(json, strlen(json), &schema, &error); + if ( rval ) + { + printf("Failed to read schema from JSON.\n"); + exit(EXIT_FAILURE); + } + else + { + printf("Successfully read schema from JSON.\n"); + } + +#define TEST_AVRO_1167 (1) + #if TEST_AVRO_1167 + { + avro_schema_t schema_copy = NULL; + schema_copy = avro_schema_copy( schema ); + if ( ! avro_schema_equal(schema, schema_copy) ) + { + printf("Failed avro_schema_equal(schema, schema_copy)\n"); + exit(EXIT_FAILURE); + } + avro_schema_decref(schema_copy); + } + #endif + + avro_schema_decref(schema); + return 0; +} diff --git a/fluent-bit/lib/avro/tests/test_avro_968.c b/fluent-bit/lib/avro/tests/test_avro_968.c new file mode 100644 index 00000000..87620014 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_968.c @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <stdio.h> + +#include "avro.h" +#include "avro_private.h" + +#define try(call, msg) \ + do { \ + if (call) { \ + fprintf(stderr, msg ":\n %s\n", avro_strerror()); \ + return EXIT_FAILURE; \ + } \ + } while (0) + +int +main(int argc, char **argv) +{ + AVRO_UNUSED(argc); + AVRO_UNUSED(argv); + + avro_value_t v1; + avro_value_t v2; + + try(avro_generic_string_new(&v1, "test string a"), + "Cannot create string value"); + try(avro_generic_string_new(&v2, "test string b"), + "Cannot create string value"); + + if (avro_value_equal(&v1, &v2)) { + fprintf(stderr, "Unexpected avro_value_equal\n"); + return EXIT_FAILURE; + } + + if (avro_value_equal_fast(&v1, &v2)) { + fprintf(stderr, "Unexpected avro_value_equal_fast\n"); + return EXIT_FAILURE; + } + + if (avro_value_cmp(&v1, &v2) >= 0) { + fprintf(stderr, "Unexpected avro_value_cmp\n"); + return EXIT_FAILURE; + } + + if (avro_value_cmp_fast(&v1, &v2) >= 0) { + fprintf(stderr, "Unexpected avro_value_cmp_fast\n"); + return EXIT_FAILURE; + } + + avro_value_decref(&v1); + avro_value_decref(&v2); + return 0; +} diff --git a/fluent-bit/lib/avro/tests/test_avro_984.c b/fluent-bit/lib/avro/tests/test_avro_984.c new file mode 100644 index 00000000..c89a5116 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_984.c @@ -0,0 +1,464 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <avro.h> +#include <stdio.h> +#include <stdlib.h> + + +/* Test code for JIRA Issue AVRO-984. + * + * AVRO-984: Avro-C schema resolution fails on nested array + * + * This program tests schema resolution for nested arrays. For the + * purposes of this test, there are two schemas "old" and "new" which + * are created by reading the same JSON schema. + * + * The test creates and populates a nested array, and serializes it to + * memory. The raw memory is written to a file, primarily to decouple + * writing and reading. Note that the schema is not written to the + * file. The nested array is also printed to the screen. + * + * The binary file is then read using two separate readers -- the + * matched reader and the resolved reader. + * + * In the matched reader case, the "old" and "new" schemas are known + * to match, and therefore no schema resolution is done. The binary + * buffer is deserialized into an avro value and the nested array + * encoded in the avro value is printed to the screen. + * + * In the resolved reader case, the "old" and "new" schemas are not + * known to match, and therefore schema resolution is performed. (Note + * that the schemas *do* match, but we perform schema resolution + * anyway, to test the resolution process). The schema resolution + * appears to succeed. However, once the code tries to perform an + * "avro_value_read()" the code fails to read the nested array into + * the avro value. + * + * Additionally valgrind indicates that conditional jumps are being + * performed based on uninitialized values. + * + * AVRO-C was compiled with CMAKE_INSTALL_PREFIX=avrolib + * The static library (libavro.a) was copied into a subdirectory of avrolib/lib/static + * + * This file was compiled under Linux using: + * gcc -g avro-984-test.c -o avro984 -I../../build/avrolib/include -L../../build/avrolib/lib/static -lavro + * + * The code was tested with valgrind using the command: + * valgrind -v --leak-check=full --track-origins=yes ./avro984 + * + */ + + +// Encode the following json string in NESTED_ARRAY +// {"type":"array", "items": {"type": "array", "items": "long"}} +// +#define NESTED_ARRAY \ + "{\"type\":\"array\", \"items\": {\"type\": \"array\", \"items\": \"long\"}}" + +avro_schema_t schema_old = NULL; +avro_schema_t schema_new = NULL; + +/* Parse schema into a schema data structure */ +void init_schema(void) +{ + avro_schema_error_t error; + if (avro_schema_from_json(NESTED_ARRAY, sizeof(NESTED_ARRAY), + &schema_old, &error)) { + printf( "Unable to parse old schema\n"); + exit(EXIT_FAILURE); + } + + if (avro_schema_from_json(NESTED_ARRAY, sizeof(NESTED_ARRAY), + &schema_new, &error)) { + printf( "Unable to parse new schema\n"); + exit(EXIT_FAILURE); + } +} + +#define try(call, msg) \ + do { \ + if (call) { \ + printf( msg ":\n %s\n", avro_strerror()); \ + exit (EXIT_FAILURE); \ + } \ + } while (0) + + +/* The input avro_value_t p_array should contain a nested array. + * Print the fields of this nested array to the screen. + */ +int print_array_fields ( avro_value_t *p_array ) +{ + size_t idx; + size_t length; + avro_type_t val_type; + + val_type = avro_value_get_type( p_array ); + printf( "Main array type = %d\n", val_type ); + + try( avro_value_get_size( p_array, &length ), + "Couldn't get array size" ); + printf( "Main array length = %d\n", (int) length ); + + for ( idx = 0; idx < length; idx ++ ) + { + avro_value_t subarray; + size_t sublength; + size_t jdx; + const char *unused; + + try ( avro_value_get_by_index( p_array, idx, &subarray, &unused ), + "Couldn't get subarray" ); + + val_type = avro_value_get_type( &subarray ); + printf( "Subarray type = %d\n", val_type ); + + try( avro_value_get_size( &subarray, &sublength ), + "Couldn't get subarray size" ); + printf( "Subarray length = %d\n", (int) sublength ); + + for ( jdx = 0; jdx < sublength; jdx++ ) + { + avro_value_t element; + int64_t val; + + try ( avro_value_get_by_index( &subarray, jdx, &element, &unused ), + "Couldn't get subarray element" ); + + val_type = avro_value_get_type( &element ); + + try ( avro_value_get_long( &element, &val ), + "Couldn't get subarray element value" ); + + printf( "nested_array[%d][%d]: type = %d value = %lld\n", + (int) idx, (int) jdx, (int) val_type, (long long) val ); + + } + } + + return 0; +} + + +/* The input avro_value_t p_subarray should contain an array of long + * integers. Add "elements" number of long integers to this array. Set + * the values to be distinct based on the iteration parameter. + */ +int add_subarray( avro_value_t *p_subarray, + int32_t elements, + int32_t iteration ) +{ + avro_value_t element; + size_t index; + size_t idx; + + for ( idx = 0; idx < (size_t) elements; idx ++ ) + { + // Append avro array element to subarray + try ( avro_value_append( p_subarray, &element, &index ), + "Error appending element in subarray" ); + + try ( avro_value_set_long( &element, (iteration+1)*100 + (iteration+1) ), + "Error setting subarray element" ); + } + + return 0; +} + + +/* Create a nested array using the schema NESTED_ARRAY. Populate its + * elements with unique values. Serialize the nested array to the + * memory buffer in avro_writer_t. The number of elements in the first + * dimension of the nested array is "elements". The number of elements + * in the second dimension of the nested array is hardcoded to 2. + */ +int add_array( avro_writer_t writer, + int32_t elements ) +{ + avro_schema_t chosen_schema; + avro_value_iface_t *nested_array_class; + avro_value_t nested; + int32_t idx; + + // Select (hardcode) schema to use + chosen_schema = schema_old; + + // Create avro class and value + nested_array_class = avro_generic_class_from_schema( chosen_schema ); + try ( avro_generic_value_new( nested_array_class, &nested ), + "Error creating instance of record" ); + + for ( idx = 0; idx < elements; idx ++ ) + { + avro_value_t subarray; + size_t index; + + // Append avro array element for top level array + try ( avro_value_append( &nested, &subarray, &index ), + "Error appending subarray" ); + + // Populate array element with subarray of length 2 +#define SUBARRAY_LENGTH (2) + try ( add_subarray( &subarray, SUBARRAY_LENGTH, idx ), + "Error populating subarray" ); + } + + // Write the value to memory + try ( avro_value_write( writer, &nested ), + "Unable to write nested into memory" ); + + print_array_fields( &nested ); + + // Release the record + avro_value_decref( &nested ); + avro_value_iface_decref( nested_array_class ); + + return 0; +} + +/* Create a raw binary file containing a serialized version of a + * nested array. This file will later be read by + * read_nested_array_file(). + */ +int write_nested_array_file ( int64_t buf_len, const char *raw_binary_file_name ) +{ + char *buf; + avro_writer_t nested_writer; + FILE *fid = NULL; + + fprintf( stdout, "Create %s\n", raw_binary_file_name ); + + // Allocate a buffer + buf = (char *) malloc( buf_len * sizeof( char ) ); + if ( buf == NULL ) + { + printf( "There was an error creating the nested buffer %s.\n", raw_binary_file_name); + exit(EXIT_FAILURE); + } + + /* Create a new memory writer */ + nested_writer = avro_writer_memory( buf, buf_len ); + if ( nested_writer == NULL ) + { + printf( "There was an error creating the buffer for writing %s.\n", raw_binary_file_name); + exit(EXIT_FAILURE); + } + + /* Add an array containing 4 subarrays */ + printf( "before avro_writer_tell %d\n", (int) avro_writer_tell( nested_writer ) ); +#define ARRAY_LENGTH (4) + add_array( nested_writer, ARRAY_LENGTH ); + printf( "after avro_writer_tell %d\n", (int) avro_writer_tell( nested_writer ) ); + + /* Serialize the nested array */ + printf( "Serialize the data to a file\n"); + + /* Delete the nested array if it exists, and create a new one */ + remove(raw_binary_file_name); + fid = fopen( raw_binary_file_name, "w+"); + if ( fid == NULL ) + { + printf( "There was an error creating the file %s.\n", raw_binary_file_name); + exit(EXIT_FAILURE); + } + fwrite( buf, 1, avro_writer_tell( nested_writer ), fid ); + fclose(fid); + avro_writer_free( nested_writer ); + free(buf); + return 0; +} + + +/* Read the raw binary file containing a serialized version of a + * nested array, written by write_nested_array_file() + */ +int read_nested_array_file ( int64_t buf_len, + const char *raw_binary_file_name, + avro_schema_t writer_schema, + avro_schema_t reader_schema, + int use_resolving_reader + ) +{ + + char *buf; + FILE *fid = NULL; + avro_reader_t nested_reader; + int64_t file_len; + + // For Matched Reader and Resolving Reader + avro_value_iface_t *reader_class; + avro_value_t nested; + + // For Resolving Reader + avro_value_iface_t *resolver; + avro_value_t resolved_value; + + fprintf( stdout, "Use %s reader\n", use_resolving_reader ? "Resolving":"Matched" ); + + // Allocate a buffer + buf = (char *) calloc( buf_len, sizeof( char ) ); + if ( buf == NULL ) + { + printf( "There was an error creating the buffer for reading %s.\n", raw_binary_file_name); + exit(EXIT_FAILURE); + } + // Start with a garbage buffer + memset(buf, 0xff, buf_len ); + + // Read the file into the buffer + fid = fopen( raw_binary_file_name, "r" ); + if ( fid == NULL ) + { + printf( "There was an error reading the file %s.\n", raw_binary_file_name); + exit(EXIT_FAILURE); + } + file_len = fread( buf, 1, buf_len, fid ); + printf( "Read %d bytes\n", (int) file_len ); + fclose(fid); + + if ( use_resolving_reader ) + { + // Resolving Reader + + /* First resolve the writer and reader schemas */ + resolver = avro_resolved_writer_new( writer_schema, reader_schema ); + if ( !resolver ) + { + printf( "Could not create resolver\n"); + free(buf); + exit(EXIT_FAILURE); + } + + /* Create a value that the resolver can write into. This is just + * an interface value, that is not directly read from. + */ + if ( avro_resolved_writer_new_value( resolver, &resolved_value ) ) + { + avro_value_iface_decref( resolver ); + free(buf); + exit(EXIT_FAILURE); + } + + /* Then create the value with the reader schema, that we are going + * to use to read from. + */ + reader_class = avro_generic_class_from_schema(reader_schema); + try ( avro_generic_value_new( reader_class, &nested ), + "Error creating instance of nested array" ); + + // When we read the memory using the resolved writer, we want to + // populate the instance of the value with the reader schema. This + // is done by set_dest. + avro_resolved_writer_set_dest(&resolved_value, &nested); + + // Create a memory reader + nested_reader = avro_reader_memory( buf, buf_len ); + + if ( avro_value_read( nested_reader, &resolved_value ) ) + { + printf( "Avro value read failed\n" ); + + avro_value_decref( &nested ); + avro_value_iface_decref( reader_class ); + avro_value_iface_decref( resolver ); + avro_value_decref( &resolved_value ); + + exit(EXIT_FAILURE); + } + } + else + { + // Matched Reader + reader_class = avro_generic_class_from_schema(reader_schema); + + try ( avro_generic_value_new( reader_class, &nested ), + "Error creating instance of nested array" ); + + // Send the memory in the buffer into the reader + nested_reader = avro_reader_memory( buf, buf_len ); + + try ( avro_value_read( nested_reader, &nested ), + "Could not read value from memory" ); + } + + + /* Now the resolved record has been read into "nested" which is + * a value of type reader_class + */ + print_array_fields( &nested ); + + if ( use_resolving_reader ) + { + // Resolving Reader + avro_value_decref( &nested ); + avro_value_iface_decref( reader_class ); + avro_value_iface_decref( resolver ); + avro_value_decref( &resolved_value ); + } + else + { + // Matched Reader + avro_value_decref( &nested ); + avro_value_iface_decref( reader_class ); + } + + fprintf( stdout, "Done.\n\n"); + avro_reader_free( nested_reader ); + free(buf); + return 0; +} + + +/* Top level function to impelement a test for the JIRA issue + * AVRO-984. See detailed documentation at the top of this file. + */ +int main(void) +{ + const char *raw_binary_file_name = "nested_array.bin"; + int64_t buf_len = 2048; + int use_resolving_reader; + + /* Initialize the schema structure from JSON */ + init_schema(); + + printf( "Write the serialized nested array to %s\n", raw_binary_file_name ); + + write_nested_array_file( buf_len, raw_binary_file_name ); + + printf("\nNow read all the array back out\n\n"); + + for ( use_resolving_reader = 0; use_resolving_reader < 2; use_resolving_reader++ ) + { + read_nested_array_file( buf_len, + raw_binary_file_name, + schema_old, + schema_new, + use_resolving_reader + ); + } + + // Close out schemas + avro_schema_decref(schema_old); + avro_schema_decref(schema_new); + + // Remove the binary file + remove(raw_binary_file_name); + + printf("\n"); + return 0; +} diff --git a/fluent-bit/lib/avro/tests/test_avro_data.c b/fluent-bit/lib/avro/tests/test_avro_data.c new file mode 100644 index 00000000..1da09e6d --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_data.c @@ -0,0 +1,684 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include "avro.h" +#include "avro_private.h" +#include <limits.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> + +char buf[4096]; +avro_reader_t reader; +avro_writer_t writer; + +typedef int (*avro_test) (void); + +/* + * Use a custom allocator that verifies that the size that we use to + * free an object matches the size that we use to allocate it. + */ + +static void * +test_allocator(void *ud, void *ptr, size_t osize, size_t nsize) +{ + AVRO_UNUSED(ud); + AVRO_UNUSED(osize); + + if (nsize == 0) { + size_t *size = ((size_t *) ptr) - 1; + if (osize != *size) { + fprintf(stderr, + "Error freeing %p:\n" + "Size passed to avro_free (%" PRIsz ") " + "doesn't match size passed to " + "avro_malloc (%" PRIsz ")\n", + ptr, osize, *size); + abort(); + //exit(EXIT_FAILURE); + } + free(size); + return NULL; + } else { + size_t real_size = nsize + sizeof(size_t); + size_t *old_size = ptr? ((size_t *) ptr)-1: NULL; + size_t *size = (size_t *) realloc(old_size, real_size); + *size = nsize; + return (size + 1); + } +} + +void init_rand(void) +{ + srand(time(NULL)); +} + +double rand_number(double from, double to) +{ + double range = to - from; + return from + ((double)rand() / (RAND_MAX + 1.0)) * range; +} + +int64_t rand_int64(void) +{ + return (int64_t) rand_number(LONG_MIN, LONG_MAX); +} + +int32_t rand_int32(void) +{ + return (int32_t) rand_number(INT_MIN, INT_MAX); +} + +void +write_read_check(avro_schema_t writers_schema, avro_datum_t datum, + avro_schema_t readers_schema, avro_datum_t expected, char *type) +{ + avro_datum_t datum_out; + int validate; + + for (validate = 0; validate <= 1; validate++) { + + reader = avro_reader_memory(buf, sizeof(buf)); + writer = avro_writer_memory(buf, sizeof(buf)); + + if (!expected) { + expected = datum; + } + + /* Validating read/write */ + if (avro_write_data + (writer, validate ? writers_schema : NULL, datum)) { + fprintf(stderr, "Unable to write %s validate=%d\n %s\n", + type, validate, avro_strerror()); + exit(EXIT_FAILURE); + } + int64_t size = + avro_size_data(writer, validate ? writers_schema : NULL, + datum); + if (size != avro_writer_tell(writer)) { + fprintf(stderr, + "Unable to calculate size %s validate=%d " + "(%"PRId64" != %"PRId64")\n %s\n", + type, validate, size, avro_writer_tell(writer), + avro_strerror()); + exit(EXIT_FAILURE); + } + if (avro_read_data + (reader, writers_schema, readers_schema, &datum_out)) { + fprintf(stderr, "Unable to read %s validate=%d\n %s\n", + type, validate, avro_strerror()); + fprintf(stderr, " %s\n", avro_strerror()); + exit(EXIT_FAILURE); + } + if (!avro_datum_equal(expected, datum_out)) { + fprintf(stderr, + "Unable to encode/decode %s validate=%d\n %s\n", + type, validate, avro_strerror()); + exit(EXIT_FAILURE); + } + + avro_reader_dump(reader, stderr); + avro_datum_decref(datum_out); + avro_reader_free(reader); + avro_writer_free(writer); + } +} + +static void test_json(avro_datum_t datum, const char *expected) +{ + char *json = NULL; + avro_datum_to_json(datum, 1, &json); + if (strcasecmp(json, expected) != 0) { + fprintf(stderr, "Unexpected JSON encoding: %s\n", json); + exit(EXIT_FAILURE); + } + free(json); +} + +static int test_string(void) +{ + unsigned int i; + const char *strings[] = { "Four score and seven years ago", + "our father brought forth on this continent", + "a new nation", "conceived in Liberty", + "and dedicated to the proposition that all men are created equal." + }; + avro_schema_t writer_schema = avro_schema_string(); + for (i = 0; i < sizeof(strings) / sizeof(strings[0]); i++) { + avro_datum_t datum = avro_givestring(strings[i], NULL); + write_read_check(writer_schema, datum, NULL, NULL, "string"); + avro_datum_decref(datum); + } + + avro_datum_t datum = avro_givestring(strings[0], NULL); + test_json(datum, "\"Four score and seven years ago\""); + avro_datum_decref(datum); + + // The following should bork if we don't copy the string value + // correctly (since we'll try to free a static string). + + datum = avro_string("this should be copied"); + avro_string_set(datum, "also this"); + avro_datum_decref(datum); + + avro_schema_decref(writer_schema); + return 0; +} + +static int test_bytes(void) +{ + char bytes[] = { 0xDE, 0xAD, 0xBE, 0xEF }; + avro_schema_t writer_schema = avro_schema_bytes(); + avro_datum_t datum; + avro_datum_t expected_datum; + + datum = avro_givebytes(bytes, sizeof(bytes), NULL); + write_read_check(writer_schema, datum, NULL, NULL, "bytes"); + test_json(datum, "\"\\u00de\\u00ad\\u00be\\u00ef\""); + avro_datum_decref(datum); + avro_schema_decref(writer_schema); + + datum = avro_givebytes(NULL, 0, NULL); + avro_givebytes_set(datum, bytes, sizeof(bytes), NULL); + expected_datum = avro_givebytes(bytes, sizeof(bytes), NULL); + if (!avro_datum_equal(datum, expected_datum)) { + fprintf(stderr, + "Expected equal bytes instances.\n"); + exit(EXIT_FAILURE); + } + avro_datum_decref(datum); + avro_datum_decref(expected_datum); + + // The following should bork if we don't copy the bytes value + // correctly (since we'll try to free a static string). + + datum = avro_bytes("original", 8); + avro_bytes_set(datum, "alsothis", 8); + avro_datum_decref(datum); + + avro_schema_decref(writer_schema); + return 0; +} + +static int test_int32(void) +{ + int i; + avro_schema_t writer_schema = avro_schema_int(); + avro_schema_t long_schema = avro_schema_long(); + avro_schema_t float_schema = avro_schema_float(); + avro_schema_t double_schema = avro_schema_double(); + for (i = 0; i < 100; i++) { + int32_t value = rand_int32(); + avro_datum_t datum = avro_int32(value); + avro_datum_t long_datum = avro_int64(value); + avro_datum_t float_datum = avro_float(value); + avro_datum_t double_datum = avro_double(value); + write_read_check(writer_schema, datum, NULL, NULL, "int"); + write_read_check(writer_schema, datum, + long_schema, long_datum, "int->long"); + write_read_check(writer_schema, datum, + float_schema, float_datum, "int->float"); + write_read_check(writer_schema, datum, + double_schema, double_datum, "int->double"); + avro_datum_decref(datum); + avro_datum_decref(long_datum); + avro_datum_decref(float_datum); + avro_datum_decref(double_datum); + } + + avro_datum_t datum = avro_int32(10000); + test_json(datum, "10000"); + avro_datum_decref(datum); + + avro_schema_decref(writer_schema); + avro_schema_decref(long_schema); + avro_schema_decref(float_schema); + avro_schema_decref(double_schema); + return 0; +} + +static int test_int64(void) +{ + int i; + avro_schema_t writer_schema = avro_schema_long(); + avro_schema_t float_schema = avro_schema_float(); + avro_schema_t double_schema = avro_schema_double(); + for (i = 0; i < 100; i++) { + int64_t value = rand_int64(); + avro_datum_t datum = avro_int64(value); + avro_datum_t float_datum = avro_float(value); + avro_datum_t double_datum = avro_double(value); + write_read_check(writer_schema, datum, NULL, NULL, "long"); + write_read_check(writer_schema, datum, + float_schema, float_datum, "long->float"); + write_read_check(writer_schema, datum, + double_schema, double_datum, "long->double"); + avro_datum_decref(datum); + avro_datum_decref(float_datum); + avro_datum_decref(double_datum); + } + + avro_datum_t datum = avro_int64(10000); + test_json(datum, "10000"); + avro_datum_decref(datum); + + avro_schema_decref(writer_schema); + avro_schema_decref(float_schema); + avro_schema_decref(double_schema); + return 0; +} + +static int test_double(void) +{ + int i; + avro_schema_t schema = avro_schema_double(); + for (i = 0; i < 100; i++) { + avro_datum_t datum = avro_double(rand_number(-1.0E10, 1.0E10)); + write_read_check(schema, datum, NULL, NULL, "double"); + avro_datum_decref(datum); + } + + avro_datum_t datum = avro_double(2000.0); + test_json(datum, "2000.0"); + avro_datum_decref(datum); + + avro_schema_decref(schema); + return 0; +} + +static int test_float(void) +{ + int i; + avro_schema_t schema = avro_schema_float(); + avro_schema_t double_schema = avro_schema_double(); + for (i = 0; i < 100; i++) { + float value = rand_number(-1.0E10, 1.0E10); + avro_datum_t datum = avro_float(value); + avro_datum_t double_datum = avro_double(value); + write_read_check(schema, datum, NULL, NULL, "float"); + write_read_check(schema, datum, + double_schema, double_datum, "float->double"); + avro_datum_decref(datum); + avro_datum_decref(double_datum); + } + + avro_datum_t datum = avro_float(2000.0); + test_json(datum, "2000.0"); + avro_datum_decref(datum); + + avro_schema_decref(schema); + avro_schema_decref(double_schema); + return 0; +} + +static int test_boolean(void) +{ + int i; + const char *expected_json[] = { "false", "true" }; + avro_schema_t schema = avro_schema_boolean(); + for (i = 0; i <= 1; i++) { + avro_datum_t datum = avro_boolean(i); + write_read_check(schema, datum, NULL, NULL, "boolean"); + test_json(datum, expected_json[i]); + avro_datum_decref(datum); + } + avro_schema_decref(schema); + return 0; +} + +static int test_null(void) +{ + avro_schema_t schema = avro_schema_null(); + avro_datum_t datum = avro_null(); + write_read_check(schema, datum, NULL, NULL, "null"); + test_json(datum, "null"); + avro_datum_decref(datum); + return 0; +} + +static int test_record(void) +{ + avro_schema_t schema = avro_schema_record("person", NULL); + avro_schema_record_field_append(schema, "name", avro_schema_string()); + avro_schema_record_field_append(schema, "age", avro_schema_int()); + + avro_datum_t datum = avro_record(schema); + avro_datum_t name_datum, age_datum; + + name_datum = avro_givestring("Joseph Campbell", NULL); + age_datum = avro_int32(83); + + avro_record_set(datum, "name", name_datum); + avro_record_set(datum, "age", age_datum); + + write_read_check(schema, datum, NULL, NULL, "record"); + test_json(datum, "{\"name\": \"Joseph Campbell\", \"age\": 83}"); + + int rc; + avro_record_set_field_value(rc, datum, int32, "age", 104); + + int32_t age = 0; + avro_record_get_field_value(rc, datum, int32, "age", &age); + if (age != 104) { + fprintf(stderr, "Incorrect age value\n"); + exit(EXIT_FAILURE); + } + + avro_datum_decref(name_datum); + avro_datum_decref(age_datum); + avro_datum_decref(datum); + avro_schema_decref(schema); + return 0; +} + +static int test_nested_record(void) +{ + const char *json = + "{" + " \"type\": \"record\"," + " \"name\": \"list\"," + " \"fields\": [" + " { \"name\": \"x\", \"type\": \"int\" }," + " { \"name\": \"y\", \"type\": \"int\" }," + " { \"name\": \"next\", \"type\": [\"null\",\"list\"]}" + " ]" + "}"; + + int rval; + + avro_schema_t schema = NULL; + avro_schema_error_t error; + avro_schema_from_json(json, strlen(json), &schema, &error); + + avro_datum_t head = avro_datum_from_schema(schema); + avro_record_set_field_value(rval, head, int32, "x", 10); + avro_record_set_field_value(rval, head, int32, "y", 10); + + avro_datum_t next = NULL; + avro_datum_t tail = NULL; + + avro_record_get(head, "next", &next); + avro_union_set_discriminant(next, 1, &tail); + avro_record_set_field_value(rval, tail, int32, "x", 20); + avro_record_set_field_value(rval, tail, int32, "y", 20); + + avro_record_get(tail, "next", &next); + avro_union_set_discriminant(next, 0, NULL); + + write_read_check(schema, head, NULL, NULL, "nested record"); + + avro_schema_decref(schema); + avro_datum_decref(head); + + return 0; +} + +static int test_enum(void) +{ + enum avro_languages { + AVRO_C, + AVRO_CPP, + AVRO_PYTHON, + AVRO_RUBY, + AVRO_JAVA + }; + avro_schema_t schema = avro_schema_enum("language"); + avro_datum_t datum = avro_enum(schema, AVRO_C); + + avro_schema_enum_symbol_append(schema, "C"); + avro_schema_enum_symbol_append(schema, "C++"); + avro_schema_enum_symbol_append(schema, "Python"); + avro_schema_enum_symbol_append(schema, "Ruby"); + avro_schema_enum_symbol_append(schema, "Java"); + + if (avro_enum_get(datum) != AVRO_C) { + fprintf(stderr, "Unexpected enum value AVRO_C\n"); + exit(EXIT_FAILURE); + } + + if (strcmp(avro_enum_get_name(datum), "C") != 0) { + fprintf(stderr, "Unexpected enum value name C\n"); + exit(EXIT_FAILURE); + } + + write_read_check(schema, datum, NULL, NULL, "enum"); + test_json(datum, "\"C\""); + + avro_enum_set(datum, AVRO_CPP); + if (strcmp(avro_enum_get_name(datum), "C++") != 0) { + fprintf(stderr, "Unexpected enum value name C++\n"); + exit(EXIT_FAILURE); + } + + write_read_check(schema, datum, NULL, NULL, "enum"); + test_json(datum, "\"C++\""); + + avro_enum_set_name(datum, "Python"); + if (avro_enum_get(datum) != AVRO_PYTHON) { + fprintf(stderr, "Unexpected enum value AVRO_PYTHON\n"); + exit(EXIT_FAILURE); + } + + write_read_check(schema, datum, NULL, NULL, "enum"); + test_json(datum, "\"Python\""); + + avro_datum_decref(datum); + avro_schema_decref(schema); + return 0; +} + +static int test_array(void) +{ + int i, rval; + avro_schema_t schema = avro_schema_array(avro_schema_int()); + avro_datum_t datum = avro_array(schema); + + for (i = 0; i < 10; i++) { + avro_datum_t i32_datum = avro_int32(i); + rval = avro_array_append_datum(datum, i32_datum); + avro_datum_decref(i32_datum); + if (rval) { + exit(EXIT_FAILURE); + } + } + + if (avro_array_size(datum) != 10) { + fprintf(stderr, "Unexpected array size"); + exit(EXIT_FAILURE); + } + + write_read_check(schema, datum, NULL, NULL, "array"); + test_json(datum, "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]"); + avro_datum_decref(datum); + avro_schema_decref(schema); + return 0; +} + +static int test_map(void) +{ + avro_schema_t schema = avro_schema_map(avro_schema_long()); + avro_datum_t datum = avro_map(schema); + int64_t i = 0; + char *nums[] = + { "zero", "one", "two", "three", "four", "five", "six", NULL }; + while (nums[i]) { + avro_datum_t i_datum = avro_int64(i); + avro_map_set(datum, nums[i], i_datum); + avro_datum_decref(i_datum); + i++; + } + + if (avro_array_size(datum) != 7) { + fprintf(stderr, "Unexpected map size\n"); + exit(EXIT_FAILURE); + } + + avro_datum_t value; + const char *key; + avro_map_get_key(datum, 2, &key); + avro_map_get(datum, key, &value); + int64_t val; + avro_int64_get(value, &val); + + if (val != 2) { + fprintf(stderr, "Unexpected map value 2\n"); + exit(EXIT_FAILURE); + } + + int index; + if (avro_map_get_index(datum, "two", &index)) { + fprintf(stderr, "Can't get index for key \"two\": %s\n", + avro_strerror()); + exit(EXIT_FAILURE); + } + if (index != 2) { + fprintf(stderr, "Unexpected index for key \"two\"\n"); + exit(EXIT_FAILURE); + } + if (!avro_map_get_index(datum, "foobar", &index)) { + fprintf(stderr, "Unexpected index for key \"foobar\"\n"); + exit(EXIT_FAILURE); + } + + write_read_check(schema, datum, NULL, NULL, "map"); + test_json(datum, + "{\"zero\": 0, \"one\": 1, \"two\": 2, \"three\": 3, " + "\"four\": 4, \"five\": 5, \"six\": 6}"); + avro_datum_decref(datum); + avro_schema_decref(schema); + return 0; +} + +static int test_union(void) +{ + avro_schema_t schema = avro_schema_union(); + avro_datum_t union_datum; + avro_datum_t datum; + avro_datum_t union_datum1; + avro_datum_t datum1; + + avro_schema_union_append(schema, avro_schema_string()); + avro_schema_union_append(schema, avro_schema_int()); + avro_schema_union_append(schema, avro_schema_null()); + + datum = avro_givestring("Follow your bliss.", NULL); + union_datum = avro_union(schema, 0, datum); + + if (avro_union_discriminant(union_datum) != 0) { + fprintf(stderr, "Unexpected union discriminant\n"); + exit(EXIT_FAILURE); + } + + if (avro_union_current_branch(union_datum) != datum) { + fprintf(stderr, "Unexpected union branch datum\n"); + exit(EXIT_FAILURE); + } + + union_datum1 = avro_datum_from_schema(schema); + avro_union_set_discriminant(union_datum1, 0, &datum1); + avro_givestring_set(datum1, "Follow your bliss.", NULL); + + if (!avro_datum_equal(datum, datum1)) { + fprintf(stderr, "Union values should be equal\n"); + exit(EXIT_FAILURE); + } + + write_read_check(schema, union_datum, NULL, NULL, "union"); + test_json(union_datum, "{\"string\": \"Follow your bliss.\"}"); + + avro_datum_decref(datum); + avro_union_set_discriminant(union_datum, 2, &datum); + test_json(union_datum, "null"); + + avro_datum_decref(union_datum); + avro_datum_decref(datum); + avro_datum_decref(union_datum1); + avro_schema_decref(schema); + return 0; +} + +static int test_fixed(void) +{ + char bytes[] = { 0xD, 0xA, 0xD, 0xA, 0xB, 0xA, 0xB, 0xA }; + avro_schema_t schema = avro_schema_fixed("msg", sizeof(bytes)); + avro_datum_t datum; + avro_datum_t expected_datum; + + datum = avro_givefixed(schema, bytes, sizeof(bytes), NULL); + write_read_check(schema, datum, NULL, NULL, "fixed"); + test_json(datum, "\"\\r\\n\\r\\n\\u000b\\n\\u000b\\n\""); + avro_datum_decref(datum); + + datum = avro_givefixed(schema, NULL, sizeof(bytes), NULL); + avro_givefixed_set(datum, bytes, sizeof(bytes), NULL); + expected_datum = avro_givefixed(schema, bytes, sizeof(bytes), NULL); + if (!avro_datum_equal(datum, expected_datum)) { + fprintf(stderr, + "Expected equal fixed instances.\n"); + exit(EXIT_FAILURE); + } + avro_datum_decref(datum); + avro_datum_decref(expected_datum); + + // The following should bork if we don't copy the fixed value + // correctly (since we'll try to free a static string). + + datum = avro_fixed(schema, "original", 8); + avro_fixed_set(datum, "alsothis", 8); + avro_datum_decref(datum); + + avro_schema_decref(schema); + return 0; +} + +int main(void) +{ + avro_set_allocator(test_allocator, NULL); + + unsigned int i; + struct avro_tests { + char *name; + avro_test func; + } tests[] = { + { + "string", test_string}, { + "bytes", test_bytes}, { + "int", test_int32}, { + "long", test_int64}, { + "float", test_float}, { + "double", test_double}, { + "boolean", test_boolean}, { + "null", test_null}, { + "record", test_record}, { + "nested_record", test_nested_record}, { + "enum", test_enum}, { + "array", test_array}, { + "map", test_map}, { + "fixed", test_fixed}, { + "union", test_union} + }; + + init_rand(); + for (i = 0; i < sizeof(tests) / sizeof(tests[0]); i++) { + struct avro_tests *test = tests + i; + fprintf(stderr, "**** Running %s tests ****\n", test->name); + if (test->func() != 0) { + return EXIT_FAILURE; + } + } + return EXIT_SUCCESS; +} diff --git a/fluent-bit/lib/avro/tests/test_avro_errors_are_thread_safe.c b/fluent-bit/lib/avro/tests/test_avro_errors_are_thread_safe.c new file mode 100644 index 00000000..b5189b12 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_errors_are_thread_safe.c @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#ifdef _WIN32 +#include <Windows.h> +#define THR_HANDLE HANDLE +#define LAST_ERROR() GetLastError() +#define SLEEP_MILLIS(millis) Sleep( millis ) +#else +#include <pthread.h> +#include <errno.h> +#include <unistd.h> +#define THR_HANDLE pthread_t +#define LAST_ERROR() errno +#define SLEEP_MILLIS(millis) usleep( millis * 1000 ) +#endif +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <time.h> +#include <avro.h> + +enum { + NUMBER_OF_TEST_THREADS = 64, + THREAD_ERROR_BUFFER_SIZE = 1024, + MAX_THREAD_SLEEP_MILLIS = 3000, +}; + +typedef struct _TEST_THREAD_DATA +{ + int index; + int error_occured; + char error_message[THREAD_ERROR_BUFFER_SIZE]; + volatile int worker_thread; + int sleep_interval_millis; +}TEST_THREAD_DATA; + +static int get_random_value( int max_value ); +#ifdef _WIN32 +static DWORD worker_thread( LPVOID lpThreadParameter ); +#else +static void *worker_thread( void *p ); +#endif +static THR_HANDLE launch_thread( void *thread_context ); +static int join_thread( THR_HANDLE thread_handle ); + +int main() +{ + TEST_THREAD_DATA threads_data[NUMBER_OF_TEST_THREADS]; + THR_HANDLE thread_handle[NUMBER_OF_TEST_THREADS]; + unsigned i; + int found_error = 0; + + srand( (unsigned)time( NULL ) ); + + memset( threads_data, 0, sizeof(threads_data) ); + for ( i = 0; i < NUMBER_OF_TEST_THREADS; i++ ) + { + threads_data[i].index = i; + threads_data[i].sleep_interval_millis = get_random_value( MAX_THREAD_SLEEP_MILLIS ); + thread_handle[i] = launch_thread( &threads_data[i] ); + if ( !thread_handle[i] ) + { + fprintf( stderr, "failed to launch worker thread, error code is %d\n", LAST_ERROR() ); + return EXIT_FAILURE; + } + } + + for ( i = 0; i < NUMBER_OF_TEST_THREADS; i++ ) + if ( join_thread( thread_handle[i] ) != 0 ) + { + fprintf( stderr, "failed to join thread %d, error code is %d\n", i, LAST_ERROR() ); + return EXIT_FAILURE; + } + + for ( i = 0; i < NUMBER_OF_TEST_THREADS; i++ ) + { + if( threads_data[i].error_occured ) + { + fprintf( stderr, "error occured at thread %d: %s\n", i, threads_data[i].error_message ); + found_error = 1; + } + } + if ( found_error ) + return EXIT_FAILURE; + +// printf( "test ended successfully\n"); + return EXIT_SUCCESS; +} + +#ifdef _WIN32 +static DWORD worker_thread( LPVOID context ) +#else +static void *worker_thread( void *context ) +#endif +{ + /* + worker thread set an error, request the error stack and validate it contains the error saved. + later it appends another error to the error stack, and validate it contains the two errors. + */ + TEST_THREAD_DATA *thread_context = (TEST_THREAD_DATA *)context; + char first_error_buffer[1024] = ""; + char second_error_buffer[1024] = ""; + char full_error_buffer[1024] = ""; + const char *error_stack = NULL; + int index = thread_context->index; + unsigned sleep_interval_millis = thread_context->sleep_interval_millis; + + //set a thread specific error + snprintf( first_error_buffer, sizeof(first_error_buffer), "thread %d set an error", index ); + avro_set_error( "%s", first_error_buffer ); + + SLEEP_MILLIS( sleep_interval_millis ); + + //validate error stack contains the thread specific error + error_stack = avro_strerror(); + if ( strcmp( error_stack, first_error_buffer ) != 0 ) + { + thread_context->error_occured = 1; + snprintf( thread_context->error_message, + sizeof(thread_context->error_message), + "invalid error stack found: expected '%s' found '%s'", first_error_buffer, error_stack ); + } + + //set another thread specific error + SLEEP_MILLIS( sleep_interval_millis ); + snprintf( second_error_buffer, sizeof(second_error_buffer), "thread %d set ANOTHER error...", index ); + avro_prefix_error( "%s", second_error_buffer ); + snprintf( full_error_buffer, sizeof(full_error_buffer), "%s%s", second_error_buffer, first_error_buffer ); + + //validate error stack contains the 2 errors as expected + SLEEP_MILLIS( sleep_interval_millis ); + error_stack = avro_strerror(); + if ( strcmp( error_stack, full_error_buffer ) != 0 ) + { + thread_context->error_occured = 1; + snprintf( thread_context->error_message, + sizeof(thread_context->error_message), + "invalid error stack found: expected '%s' found '%s'", full_error_buffer, error_stack ); + } + + return 0; + +} + +static THR_HANDLE launch_thread( void *thread_context ) +{ +#ifdef _WIN32 + static const LPSECURITY_ATTRIBUTES DEFAULT_SECURITY_ATTIRBUTES = NULL; + static const SIZE_T DEFAULT_STACK_SIZE = 0; + static const DWORD DEFAULT_CREATION_FLAGS = 0; + DWORD thread_id = 0; + + return + CreateThread( DEFAULT_SECURITY_ATTIRBUTES, + DEFAULT_STACK_SIZE, + worker_thread, + thread_context, + DEFAULT_CREATION_FLAGS, + &thread_id ); +#else + pthread_attr_t attr = {0}; + pthread_t thread; + pthread_attr_init( &attr ); + int status = 0; + status = pthread_create( &thread, &attr, worker_thread, thread_context ); + pthread_attr_destroy(&attr); + if ( status != 0 ) + return NULL; + return thread; +#endif +} + +static int join_thread( THR_HANDLE thread_handle ) +{ +#ifdef _WIN32 + return + ( WaitForSingleObject( thread_handle, INFINITE ) == WAIT_OBJECT_0 ) ? 0 : -1; +#else + return + pthread_join( thread_handle, NULL ); +#endif +} + +static int get_random_value( int max_value ) +{ + return + rand() % max_value; +} diff --git a/fluent-bit/lib/avro/tests/test_avro_schema.c b/fluent-bit/lib/avro/tests/test_avro_schema.c new file mode 100644 index 00000000..efa0b555 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_schema.c @@ -0,0 +1,316 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include "avro.h" +#include "avro_private.h" +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/types.h> +#ifdef _WIN32 + #include "msdirent.h" +#else + #include <dirent.h> +#endif + +int test_cases = 0; +avro_writer_t avro_stderr; + +static void run_tests(char *dirpath, int should_pass) +{ + char jsontext[4096]; + char jsontext2[4096]; + size_t rval; + char filepath[1024]; + DIR *dir; + struct dirent *dent; + FILE *fp; + avro_schema_t schema; + avro_writer_t jsontext2_writer; + + dir = opendir(dirpath); + if (dir == NULL) { + fprintf(stderr, "Unable to open '%s'\n", dirpath); + exit(EXIT_FAILURE); + } + do { + dent = readdir(dir); + + /* Suppress failures on CVS directories */ + if ( dent && !strcmp( (const char *) dent->d_name, "CVS" ) ) + continue; + + if (dent && dent->d_name[0] != '.') { + int test_rval; + snprintf(filepath, sizeof(filepath), "%s/%s", dirpath, + dent->d_name); + fprintf(stderr, "TEST %s...", filepath); + fp = fopen(filepath, "r"); + if (!fp) { + fprintf(stderr, "can't open!\n"); + exit(EXIT_FAILURE); + } + rval = fread(jsontext, 1, sizeof(jsontext) - 1, fp); + fclose(fp); + jsontext[rval] = '\0'; + test_rval = + avro_schema_from_json(jsontext, 0, &schema, NULL); + test_cases++; + if (test_rval == 0) { + if (should_pass) { + avro_schema_t schema_copy = + avro_schema_copy(schema); + fprintf(stderr, "pass\n"); + avro_schema_to_json(schema, + avro_stderr); + fprintf(stderr, "\n"); + if (!avro_schema_equal + (schema, schema_copy)) { + fprintf(stderr, + "failed to avro_schema_equal(schema,avro_schema_copy())\n"); + exit(EXIT_FAILURE); + } + jsontext2_writer = avro_writer_memory(jsontext2, sizeof(jsontext2)); + if (avro_schema_to_json(schema, jsontext2_writer)) { + fprintf(stderr, "failed to write schema (%s)\n", + avro_strerror()); + exit(EXIT_FAILURE); + } + avro_write(jsontext2_writer, (void *)"", 1); /* zero terminate */ + avro_writer_free(jsontext2_writer); + avro_schema_decref(schema); + if (avro_schema_from_json(jsontext2, 0, &schema, NULL)) { + fprintf(stderr, "failed to write then read schema (%s)\n", + avro_strerror()); + exit(EXIT_FAILURE); + } + if (!avro_schema_equal + (schema, schema_copy)) { + fprintf(stderr, "failed read-write-read cycle (%s)\n", + avro_strerror()); + exit(EXIT_FAILURE); + } + avro_schema_decref(schema_copy); + avro_schema_decref(schema); + } else { + /* + * Unexpected success + */ + fprintf(stderr, + "fail! (shouldn't succeed but did)\n"); + exit(EXIT_FAILURE); + } + } else { + if (should_pass) { + fprintf(stderr, "%s\n", avro_strerror()); + fprintf(stderr, + "fail! (should have succeeded but didn't)\n"); + exit(EXIT_FAILURE); + } else { + fprintf(stderr, "pass\n"); + } + } + } + } + while (dent != NULL); + closedir(dir); +} + +static int test_array(void) +{ + avro_schema_t schema = avro_schema_array(avro_schema_int()); + + if (!avro_schema_equal + (avro_schema_array_items(schema), avro_schema_int())) { + fprintf(stderr, "Unexpected array items schema"); + exit(EXIT_FAILURE); + } + + avro_schema_decref(schema); + return 0; +} + +static int test_enum(void) +{ + enum avro_languages { + AVRO_C, + AVRO_CPP, + AVRO_PYTHON, + AVRO_RUBY, + AVRO_JAVA + }; + avro_schema_t schema = avro_schema_enum("language"); + + avro_schema_enum_symbol_append(schema, "C"); + avro_schema_enum_symbol_append(schema, "C++"); + avro_schema_enum_symbol_append(schema, "Python"); + avro_schema_enum_symbol_append(schema, "Ruby"); + avro_schema_enum_symbol_append(schema, "Java"); + + const char *symbol1 = avro_schema_enum_get(schema, 1); + if (strcmp(symbol1, "C++") != 0) { + fprintf(stderr, "Unexpected enum schema symbol\n"); + exit(EXIT_FAILURE); + } + + if (avro_schema_enum_get_by_name(schema, "C++") != 1) { + fprintf(stderr, "Unexpected enum schema index\n"); + exit(EXIT_FAILURE); + } + + if (avro_schema_enum_get_by_name(schema, "Haskell") != -1) { + fprintf(stderr, "Unexpected enum schema index\n"); + exit(EXIT_FAILURE); + } + + avro_schema_decref(schema); + return 0; +} + +static int test_fixed(void) +{ + avro_schema_t schema = avro_schema_fixed("msg", 8); + if (avro_schema_fixed_size(schema) != 8) { + fprintf(stderr, "Unexpected fixed size\n"); + exit(EXIT_FAILURE); + } + + avro_schema_decref(schema); + return 0; +} + +static int test_map(void) +{ + avro_schema_t schema = avro_schema_map(avro_schema_long()); + + if (!avro_schema_equal + (avro_schema_map_values(schema), avro_schema_long())) { + fprintf(stderr, "Unexpected map values schema"); + exit(EXIT_FAILURE); + } + + avro_schema_decref(schema); + return 0; +} + +static int test_record(void) +{ + avro_schema_t schema = avro_schema_record("person", NULL); + + avro_schema_record_field_append(schema, "name", avro_schema_string()); + avro_schema_record_field_append(schema, "age", avro_schema_int()); + + if (avro_schema_record_field_get_index(schema, "name") != 0) { + fprintf(stderr, "Incorrect index for \"name\" field\n"); + exit(EXIT_FAILURE); + } + + if (avro_schema_record_field_get_index(schema, "unknown") != -1) { + fprintf(stderr, "Incorrect index for \"unknown\" field\n"); + exit(EXIT_FAILURE); + } + + avro_schema_t name_field = + avro_schema_record_field_get(schema, "name"); + if (!avro_schema_equal(name_field, avro_schema_string())) { + fprintf(stderr, "Unexpected name field\n"); + exit(EXIT_FAILURE); + } + + avro_schema_t field1 = + avro_schema_record_field_get_by_index(schema, 1); + if (!avro_schema_equal(field1, avro_schema_int())) { + fprintf(stderr, "Unexpected field 1\n"); + exit(EXIT_FAILURE); + } + + avro_schema_decref(schema); + return 0; +} + +static int test_union(void) +{ + avro_schema_t schema = avro_schema_union(); + + avro_schema_union_append(schema, avro_schema_string()); + avro_schema_union_append(schema, avro_schema_int()); + avro_schema_union_append(schema, avro_schema_null()); + + if (!avro_schema_equal + (avro_schema_string(), + avro_schema_union_branch(schema, 0))) { + fprintf(stderr, "Unexpected union schema branch 0\n"); + exit(EXIT_FAILURE); + } + + if (!avro_schema_equal + (avro_schema_string(), + avro_schema_union_branch_by_name(schema, NULL, "string"))) { + fprintf(stderr, "Unexpected union schema branch \"string\"\n"); + exit(EXIT_FAILURE); + } + + avro_schema_decref(schema); + return 0; +} + +int main(int argc, char *argv[]) +{ + char *srcdir = getenv("srcdir"); + char path[1024]; + + AVRO_UNUSED(argc); + AVRO_UNUSED(argv); + + if (!srcdir) { + srcdir = "."; + } + + avro_stderr = avro_writer_file(stderr); + + /* + * Run the tests that should pass + */ + snprintf(path, sizeof(path), "%s/schema_tests/pass", srcdir); + fprintf(stderr, "RUNNING %s\n", path); + run_tests(path, 1); + snprintf(path, sizeof(path), "%s/schema_tests/fail", srcdir); + fprintf(stderr, "RUNNING %s\n", path); + run_tests(path, 0); + + fprintf(stderr, "*** Running array tests **\n"); + test_array(); + fprintf(stderr, "*** Running enum tests **\n"); + test_enum(); + fprintf(stderr, "*** Running fixed tests **\n"); + test_fixed(); + fprintf(stderr, "*** Running map tests **\n"); + test_map(); + fprintf(stderr, "*** Running record tests **\n"); + test_record(); + fprintf(stderr, "*** Running union tests **\n"); + test_union(); + + fprintf(stderr, "==================================================\n"); + fprintf(stderr, + "Finished running %d schema test cases successfully \n", + test_cases); + fprintf(stderr, "==================================================\n"); + + avro_writer_free(avro_stderr); + return EXIT_SUCCESS; +} diff --git a/fluent-bit/lib/avro/tests/test_avro_schema_names.c b/fluent-bit/lib/avro/tests/test_avro_schema_names.c new file mode 100644 index 00000000..22a6991e --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_schema_names.c @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include "avro.h" +#include "avro_private.h" +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +int test_cases = 0; +avro_writer_t avro_stderr; + +static void test_helper(const char *json, + const char *name, + avro_schema_t expected) +{ + int rc; + avro_schema_t base; + avro_schema_error_t serror; + + rc = avro_schema_from_json(json, strlen(json), &base, &serror); + if (rc != 0) + { + fprintf(stderr, + "Error parsing Avro schema:\n%s\n", + json); + exit(EXIT_FAILURE); + } + + avro_schema_t actual = + avro_schema_get_subschema(base, name); + + if (actual == NULL) + { + fprintf(stderr, + "No subschema named \"%s\" in %s\n", + name, avro_schema_type_name(base)); + exit(EXIT_FAILURE); + } + + if (!avro_schema_equal(actual, expected)) + { + fprintf(stderr, + "Subschema \"%s\" should be %s, " + "is actually %s\n", + name, + avro_schema_type_name(expected), + avro_schema_type_name(actual)); + exit(EXIT_FAILURE); + } + + avro_schema_decref(base); + avro_schema_decref(expected); +} + +static void test_array_schema_01() +{ + static char *JSON = + "{" + " \"type\": \"array\"," + " \"items\": \"long\"" + "}"; + + test_helper(JSON, "[]", avro_schema_long()); +} + +static void test_map_schema_01() +{ + static char *JSON = + "{" + " \"type\": \"map\"," + " \"values\": \"long\"" + "}"; + + test_helper(JSON, "{}", avro_schema_long()); +} + +static void test_record_schema_01() +{ + static char *JSON = + "{" + " \"type\": \"record\"," + " \"name\": \"test\"," + " \"fields\": [" + " { \"name\": \"a\", \"type\": \"long\" }" + " ]" + "}"; + + test_helper(JSON, "a", avro_schema_long()); +} + +static void test_union_schema_01() +{ + static char *JSON = + "[" + " \"long\"," + " {" + " \"type\": \"record\"," + " \"name\": \"test\"," + " \"fields\": [" + " { \"name\": \"a\", \"type\": \"long\" }" + " ]" + " }" + "]"; + + test_helper(JSON, "long", avro_schema_long()); +} + +int main(int argc, char *argv[]) +{ + AVRO_UNUSED(argc); + AVRO_UNUSED(argv); + + test_array_schema_01(); + + test_map_schema_01(); + + test_record_schema_01(); + + test_union_schema_01(); + + return EXIT_SUCCESS; +} diff --git a/fluent-bit/lib/avro/tests/test_avro_values.c b/fluent-bit/lib/avro/tests/test_avro_values.c new file mode 100644 index 00000000..4930b8ca --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_avro_values.c @@ -0,0 +1,1455 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +/* Test cases for the new avro_value_t interface */ + +#include <limits.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> + +#include "avro.h" +#include "avro_private.h" + +typedef int (*avro_test) (void); + +#ifndef SHOW_ALLOCATIONS +#define SHOW_ALLOCATIONS 0 +#endif + +/* + * Use a custom allocator that verifies that the size that we use to + * free an object matches the size that we use to allocate it. + */ + +static void * +test_allocator(void *ud, void *ptr, size_t osize, size_t nsize) +{ + AVRO_UNUSED(ud); + AVRO_UNUSED(osize); + +#if SHOW_ALLOCATIONS + fprintf(stderr, "alloc(%p, %" PRIsz ", %" PRIsz ") => ", ptr, osize, nsize); +#endif + + if (nsize == 0) { + size_t *size = ((size_t *) ptr) - 1; + if (osize != *size) { + fprintf(stderr, +#if SHOW_ALLOCATIONS + "ERROR!\n" +#endif + "Error freeing %p:\n" + "Size passed to avro_free (%" PRIsz ") " + "doesn't match size passed to " + "avro_malloc (%" PRIsz ")\n", + ptr, osize, *size); + exit(EXIT_FAILURE); + } + free(size); +#if SHOW_ALLOCATIONS + fprintf(stderr, "NULL\n"); +#endif + return NULL; + } else { + size_t real_size = nsize + sizeof(size_t); + size_t *old_size = ptr? ((size_t *) ptr)-1: NULL; + size_t *size = (size_t *) realloc(old_size, real_size); + *size = nsize; +#if SHOW_ALLOCATIONS + fprintf(stderr, "%p\n", (size+1)); +#endif + return (size + 1); + } +} + +void +init_rand(void) +{ + srand(time(NULL)); +} + +double +rand_number(double from, double to) +{ + double range = to - from; + return from + ((double)rand() / (RAND_MAX + 1.0)) * range; +} + +int64_t +rand_int64(void) +{ + return (int64_t) rand_number(LONG_MIN, LONG_MAX); +} + +int32_t +rand_int32(void) +{ + return (int32_t) rand_number(INT_MIN, INT_MAX); +} + +size_t +rand_count(void) +{ + return (size_t) rand_number(0, 100); +} + +#define check_(call) \ + do { \ + int _rval = call; \ + if (_rval) { return _rval; } \ + } while (0) + +/* + * Verify that we can't call any of the getters and setters that don't + * apply to the given value. + */ + +static int +_check_invalid_methods(const char *name, avro_value_t *val) +{ + avro_type_t type = avro_value_get_type(val); + +/* For a description on GCC vs Visual Studio 2008 usage of variadic + * macros see: + * https://stackoverflow.com/questions/2575864/the-problem-about-different + * -treatment-to-va-args-when-using-vs-2008-and-gcc + */ +#define expand_args(...) __VA_ARGS__ +#define check_bad(method, ...) \ + do { \ + if (!expand_args(avro_value_##method(__VA_ARGS__))) { \ + fprintf(stderr, \ + "Shouldn't be able to " #method " a %s\n", \ + name); \ + return EXIT_FAILURE; \ + } \ + } while (0) + + if (type != AVRO_BOOLEAN) { + int dummy = 0; + check_bad(get_boolean, val, &dummy); + check_bad(set_boolean, val, dummy); + } + + if (type != AVRO_BYTES) { + const void *cbuf = NULL; + void *buf = NULL; + size_t size = 0; + check_bad(get_bytes, val, &cbuf, &size); + check_bad(set_bytes, val, buf, size); + } + + if (type != AVRO_DOUBLE) { + double dummy = 0; + check_bad(get_double, val, &dummy); + check_bad(set_double, val, dummy); + } + + if (type != AVRO_FLOAT) { + float dummy = 0; + check_bad(get_float, val, &dummy); + check_bad(set_float, val, dummy); + } + + if (type != AVRO_INT32) { + int32_t dummy = 0; + check_bad(get_int, val, &dummy); + check_bad(set_int, val, dummy); + } + + if (type != AVRO_INT64) { + int64_t dummy = 0; + check_bad(get_long, val, &dummy); + check_bad(set_long, val, dummy); + } + + if (type != AVRO_NULL) { + check_bad(get_null, val); + check_bad(set_null, val); + } + + if (type != AVRO_STRING) { + const char *cstr = NULL; + char *str = NULL; + size_t size = 0; + check_bad(get_string, val, &cstr, &size); + check_bad(set_string, val, str); + check_bad(set_string_len, val, str, size); + } + + if (type != AVRO_ENUM) { + int dummy = 0; + check_bad(get_enum, val, &dummy); + check_bad(set_enum, val, dummy); + } + + if (type != AVRO_FIXED) { + const void *cbuf = NULL; + void *buf = NULL; + size_t size = 0; + check_bad(get_fixed, val, &cbuf, &size); + check_bad(set_fixed, val, buf, size); + } + + if (type != AVRO_ARRAY && type != AVRO_MAP && type != AVRO_RECORD) { + size_t size = 0; + check_bad(get_size, val, &size); + + size_t index = 0; + avro_value_t child; + const char *key = NULL; + check_bad(get_by_index, val, index, &child, &key); + } + + if (type != AVRO_MAP && type != AVRO_RECORD) { + const char *key = NULL; + avro_value_t child; + size_t index = 0; + check_bad(get_by_name, val, key, &child, &index); + } + + if (type != AVRO_ARRAY) { + avro_value_t child; + size_t index; + check_bad(append, val, &child, &index); + } + + if (type != AVRO_MAP) { + const char *key = NULL; + avro_value_t child; + size_t index = 0; + int is_new = 0; + check_bad(add, val, key, &child, &index, &is_new); + } + + if (type != AVRO_UNION) { + int discriminant = 0; + avro_value_t branch; + check_bad(get_discriminant, val, &discriminant); + check_bad(get_current_branch, val, &branch); + check_bad(set_branch, val, discriminant, &branch); + } + +#undef check_bad + + return EXIT_SUCCESS; +} + +#define check_invalid_methods(name, val) \ + check_(_check_invalid_methods(name, val)) + +/* + * Verify that we get the expected type code and schema for a value. + */ + +static int +check_type_and_schema(const char *name, + avro_value_t *val, + avro_type_t expected_type, + avro_schema_t expected_schema) +{ + if (avro_value_get_type(val) != expected_type) { + avro_schema_decref(expected_schema); + fprintf(stderr, "Unexpected type for %s\n", name); + return EXIT_FAILURE; + } + + if (!avro_schema_equal(avro_value_get_schema(val), + expected_schema)) { + avro_schema_decref(expected_schema); + fprintf(stderr, "Unexpected schema for %s\n", name); + return EXIT_FAILURE; + } + + avro_schema_decref(expected_schema); + return EXIT_SUCCESS; +} + +#define try(call, msg) \ + do { \ + if (call) { \ + fprintf(stderr, msg ":\n %s\n", avro_strerror()); \ + return EXIT_FAILURE; \ + } \ + } while (0) + +static int +_check_write_read(avro_value_t *val) +{ + static char buf[4096]; + + avro_reader_t reader = avro_reader_memory(buf, sizeof(buf)); + avro_writer_t writer = avro_writer_memory(buf, sizeof(buf)); + + if (avro_value_write(writer, val)) { + fprintf(stderr, "Unable to write value:\n %s\n", + avro_strerror()); + return EXIT_FAILURE; + } + + avro_writer_dump(writer, stderr); + + size_t size; + if (avro_value_sizeof(val, &size)) { + fprintf(stderr, "Unable to determine size of value:\n %s\n", + avro_strerror()); + return EXIT_FAILURE; + } + + if (size != (size_t) avro_writer_tell(writer)) { + fprintf(stderr, "Unexpected size of encoded value\n"); + return EXIT_FAILURE; + } + + avro_value_t val_in; + if (avro_generic_value_new(val->iface, &val_in)) { + fprintf(stderr, "Cannot allocate new value instance:\n %s\n", + avro_strerror()); + return EXIT_FAILURE; + } + + if (avro_value_read(reader, &val_in)) { + fprintf(stderr, "Unable to read value:\n %s\n", + avro_strerror()); + return EXIT_FAILURE; + } + + if (!avro_value_equal(val, &val_in)) { + fprintf(stderr, "Round-trip values not equal\n"); + exit(EXIT_FAILURE); + } + + avro_value_decref(&val_in); + avro_reader_free(reader); + avro_writer_free(writer); + + return EXIT_SUCCESS; +} + +#define check_write_read(val) \ + check_(_check_write_read(val)) + +static int +_check_hash(avro_value_t *val1, avro_value_t *val2) +{ + uint32_t hash1 = avro_value_hash(val1); + uint32_t hash2 = avro_value_hash(val2); + if (hash1 != hash2) { + fprintf(stderr, "Copied hashed not equal\n"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +#define check_hash(val1, val2) \ + check_(_check_hash(val1, val2)) + +static int +_check_copy(avro_value_t *val) +{ + avro_value_t copied_val; + if (avro_generic_value_new(val->iface, &copied_val)) { + fprintf(stderr, "Cannot allocate new value instance:\n %s\n", + avro_strerror()); + return EXIT_FAILURE; + } + + if (avro_value_copy_fast(&copied_val, val)) { + fprintf(stderr, "Cannot copy value:\n %s\n", + avro_strerror()); + return EXIT_FAILURE; + } + + if (!avro_value_equal(val, &copied_val)) { + fprintf(stderr, "Copied values not equal\n"); + return EXIT_FAILURE; + } + + check_hash(val, &copied_val); + + avro_value_decref(&copied_val); + return EXIT_SUCCESS; +} + +#define check_copy(val) \ + check_(_check_copy(val)) + +static int +test_boolean(void) +{ + int rval; + + int i; + for (i = 0; i <= 1; i++) { + avro_value_t val; + try(avro_generic_boolean_new(&val, i), + "Cannot create boolean"); + check(rval, check_type_and_schema + ("boolean", &val, + AVRO_BOOLEAN, avro_schema_boolean())); + try(avro_value_reset(&val), + "Cannot reset boolean"); + try(avro_value_set_boolean(&val, i), + "Cannot set boolean"); + + /* Start with the wrong value to make sure _get does + * something. */ + int actual = (int) 23; + try(avro_value_get_boolean(&val, &actual), + "Cannot get boolean value"); + + if (actual != i) { + fprintf(stderr, "Unexpected boolean value\n"); + return EXIT_FAILURE; + } + + check_invalid_methods("boolean", &val); + check_write_read(&val); + check_copy(&val); + avro_value_decref(&val); + } + + avro_value_t val1; + avro_value_t val2; + try(avro_generic_boolean_new(&val1, 0), + "Cannot create boolean"); + try(avro_generic_boolean_new(&val2, 1), + "Cannot create boolean"); + if (avro_value_cmp_fast(&val1, &val2) >= 0) { + fprintf(stderr, "Incorrect sort order\n"); + return EXIT_FAILURE; + } + if (avro_value_cmp_fast(&val2, &val1) <= 0) { + fprintf(stderr, "Incorrect sort order\n"); + return EXIT_FAILURE; + } + if (avro_value_cmp_fast(&val1, &val1) != 0) { + fprintf(stderr, "Incorrect sort order\n"); + return EXIT_FAILURE; + } + avro_value_decref(&val1); + avro_value_decref(&val2); + + return 0; +} + +static int +test_bytes(void) +{ + int rval; + + char bytes[] = { 0xDE, 0xAD, 0xBE, 0xEF }; + + avro_value_t val; + try(avro_generic_bytes_new(&val, bytes, sizeof(bytes)), + "Cannot create bytes"); + check(rval, check_type_and_schema + ("bytes", &val, + AVRO_BYTES, avro_schema_bytes())); + try(avro_value_reset(&val), + "Cannot reset bytes"); + try(avro_value_set_bytes(&val, bytes, sizeof(bytes)), + "Cannot set bytes"); + + const void *actual_buf = NULL; + size_t actual_size = 0; + try(avro_value_get_bytes(&val, &actual_buf, &actual_size), + "Cannot get bytes value"); + + if (actual_size != sizeof(bytes)) { + fprintf(stderr, "Unexpected bytes size\n"); + return EXIT_FAILURE; + } + + if (memcmp(actual_buf, bytes, sizeof(bytes)) != 0) { + fprintf(stderr, "Unexpected bytes contents\n"); + return EXIT_FAILURE; + } + + avro_wrapped_buffer_t wbuf; + try(avro_value_grab_bytes(&val, &wbuf), + "Cannot grab bytes value"); + + if (wbuf.size != sizeof(bytes)) { + fprintf(stderr, "Unexpected grabbed bytes size\n"); + return EXIT_FAILURE; + } + + if (memcmp(wbuf.buf, bytes, sizeof(bytes)) != 0) { + fprintf(stderr, "Unexpected grabbed bytes contents\n"); + return EXIT_FAILURE; + } + + avro_wrapped_buffer_free(&wbuf); + + check_invalid_methods("bytes", &val); + check_write_read(&val); + check_copy(&val); + avro_value_decref(&val); + + avro_value_t val1; + avro_value_t val2; + avro_value_t val3; + try(avro_generic_bytes_new(&val1, "abcd", 4), + "Cannot create bytes"); + try(avro_generic_bytes_new(&val2, "abcde", 5), + "Cannot create bytes"); + try(avro_generic_bytes_new(&val3, "abce", 4), + "Cannot create bytes"); + if (avro_value_cmp_fast(&val1, &val2) >= 0) { + fprintf(stderr, "Incorrect sort order\n"); + return EXIT_FAILURE; + } + if (avro_value_cmp_fast(&val2, &val1) <= 0) { + fprintf(stderr, "Incorrect sort order\n"); + return EXIT_FAILURE; + } + if (avro_value_cmp_fast(&val1, &val3) >= 0) { + fprintf(stderr, "Incorrect sort order\n"); + return EXIT_FAILURE; + } + if (avro_value_cmp_fast(&val1, &val1) != 0) { + fprintf(stderr, "Incorrect sort order\n"); + return EXIT_FAILURE; + } + avro_value_decref(&val1); + avro_value_decref(&val2); + avro_value_decref(&val3); + + return 0; +} + +static int +test_double(void) +{ + int rval; + + int i; + for (i = 0; i < 100; i++) { + double expected = rand_number(-1e10, 1e10); + avro_value_t val; + try(avro_generic_double_new(&val, expected), + "Cannot create double"); + check(rval, check_type_and_schema + ("double", &val, + AVRO_DOUBLE, avro_schema_double())); + try(avro_value_reset(&val), + "Cannot reset double"); + try(avro_value_set_double(&val, expected), + "Cannot set double"); + + double actual = 0.0; + try(avro_value_get_double(&val, &actual), + "Cannot get double value"); + + if (actual != expected) { + fprintf(stderr, "Unexpected double value\n"); + return EXIT_FAILURE; + } + + check_invalid_methods("double", &val); + check_write_read(&val); + check_copy(&val); + avro_value_decref(&val); + } + return 0; +} + +static int +test_float(void) +{ + int rval; + + int i; + for (i = 0; i < 100; i++) { + float expected = rand_number(-1e10, 1e10); + avro_value_t val; + try(avro_generic_float_new(&val, expected), + "Cannot create float"); + check(rval, check_type_and_schema + ("float", &val, + AVRO_FLOAT, avro_schema_float())); + try(avro_value_reset(&val), + "Cannot reset float"); + try(avro_value_set_float(&val, expected), + "Cannot set float"); + + float actual = 0.0f; + try(avro_value_get_float(&val, &actual), + "Cannot get float value"); + + if (actual != expected) { + fprintf(stderr, "Unexpected float value\n"); + return EXIT_FAILURE; + } + + check_invalid_methods("float", &val); + check_write_read(&val); + check_copy(&val); + avro_value_decref(&val); + } + return 0; +} + +static int +test_int(void) +{ + int rval; + + int i; + for (i = 0; i < 100; i++) { + int32_t expected = rand_int32(); + avro_value_t val; + try(avro_generic_int_new(&val, expected), + "Cannot create int"); + check(rval, check_type_and_schema + ("int", &val, + AVRO_INT32, avro_schema_int())); + try(avro_value_reset(&val), + "Cannot reset int"); + try(avro_value_set_int(&val, expected), + "Cannot set int"); + + int32_t actual = 0; + try(avro_value_get_int(&val, &actual), + "Cannot get int value"); + + if (actual != expected) { + fprintf(stderr, "Unexpected int value\n"); + return EXIT_FAILURE; + } + + check_invalid_methods("int", &val); + check_write_read(&val); + check_copy(&val); + avro_value_decref(&val); + } + + avro_value_t val1; + avro_value_t val2; + try(avro_generic_int_new(&val1, -10), + "Cannot create int"); + try(avro_generic_int_new(&val2, 42), + "Cannot create int"); + if (avro_value_cmp_fast(&val1, &val2) >= 0) { + fprintf(stderr, "Incorrect sort order\n"); + return EXIT_FAILURE; + } + if (avro_value_cmp_fast(&val2, &val1) <= 0) { + fprintf(stderr, "Incorrect sort order\n"); + return EXIT_FAILURE; + } + if (avro_value_cmp_fast(&val1, &val1) != 0) { + fprintf(stderr, "Incorrect sort order\n"); + return EXIT_FAILURE; + } + avro_value_decref(&val1); + avro_value_decref(&val2); + + return 0; +} + +static int +test_long(void) +{ + int rval; + + int i; + for (i = 0; i < 100; i++) { + int64_t expected = rand_int64(); + avro_value_t val; + try(avro_generic_long_new(&val, expected), + "Cannot create long"); + check(rval, check_type_and_schema + ("long", &val, + AVRO_INT64, avro_schema_long())); + try(avro_value_reset(&val), + "Cannot reset long"); + try(avro_value_set_long(&val, expected), + "Cannot set long"); + + int64_t actual = 0; + try(avro_value_get_long(&val, &actual), + "Cannot get long value"); + + if (actual != expected) { + fprintf(stderr, "Unexpected long value\n"); + return EXIT_FAILURE; + } + + check_invalid_methods("long", &val); + check_write_read(&val); + check_copy(&val); + avro_value_decref(&val); + } + return 0; +} + +static int +test_null(void) +{ + int rval; + + avro_value_t val; + try(avro_generic_null_new(&val), + "Cannot create null"); + check(rval, check_type_and_schema + ("null", &val, + AVRO_NULL, avro_schema_null())); + try(avro_value_reset(&val), + "Cannot reset null"); + try(avro_value_set_null(&val), + "Cannot set null"); + try(avro_value_get_null(&val), + "Cannot get null"); + + check_invalid_methods("null", &val); + check_write_read(&val); + check_copy(&val); + avro_value_decref(&val); + return 0; +} + +static int +test_string(void) +{ + int rval; + + char *strings[] = { + "Four score and seven years ago", + "our father brought forth on this continent", + "a new nation", + "conceived in Liberty", + "and dedicated to the proposition that all men " + "are created equal." + }; + + unsigned int i; + for (i = 0; i < sizeof(strings) / sizeof(strings[0]); i++) { + avro_value_t val; + try(avro_generic_string_new(&val, strings[i]), + "Cannot create string"); + check(rval, check_type_and_schema + ("string", &val, + AVRO_STRING, avro_schema_string())); + try(avro_value_reset(&val), + "Cannot reset string"); + try(avro_value_set_string_len(&val, "", 0), + "Cannot set_len dummy string"); + + /* First try a round-trip using set_string */ + + try(avro_value_set_string(&val, strings[i]), + "Cannot set string"); + + const char *actual_str = NULL; + size_t actual_size = 0; + try(avro_value_get_string(&val, &actual_str, &actual_size), + "Cannot get string value"); + + if (actual_size != strlen(strings[i])+1) { + fprintf(stderr, "Unexpected string size\n"); + return EXIT_FAILURE; + } + + if (strcmp(actual_str, strings[i]) != 0) { + fprintf(stderr, "Unexpected string contents\n"); + return EXIT_FAILURE; + } + + avro_wrapped_buffer_t wbuf; + try(avro_value_grab_string(&val, &wbuf), + "Cannot grab string value"); + + if (wbuf.size != strlen(strings[i])+1) { + fprintf(stderr, "Unexpected grabbed string size\n"); + return EXIT_FAILURE; + } + + if (strcmp((const char *) wbuf.buf, strings[i]) != 0) { + fprintf(stderr, "Unexpected grabbed string contents\n"); + return EXIT_FAILURE; + } + + avro_wrapped_buffer_free(&wbuf); + + /* and then again using set_string_len */ + + size_t str_length = strlen(strings[i])+1; + try(avro_value_set_string_len(&val, strings[i], str_length), + "Cannot set_len string"); + + actual_str = NULL; + actual_size = 0; + try(avro_value_get_string(&val, &actual_str, &actual_size), + "Cannot get string value"); + + if (actual_size != strlen(strings[i])+1) { + fprintf(stderr, "Unexpected string size\n"); + return EXIT_FAILURE; + } + + if (strcmp(actual_str, strings[i]) != 0) { + fprintf(stderr, "Unexpected string contents\n"); + return EXIT_FAILURE; + } + + try(avro_value_grab_string(&val, &wbuf), + "Cannot grab string value"); + + if (wbuf.size != strlen(strings[i])+1) { + fprintf(stderr, "Unexpected grabbed string size\n"); + return EXIT_FAILURE; + } + + if (strcmp((const char *) wbuf.buf, strings[i]) != 0) { + fprintf(stderr, "Unexpected grabbed string contents\n"); + return EXIT_FAILURE; + } + + avro_wrapped_buffer_free(&wbuf); + + check_invalid_methods("string", &val); + check_write_read(&val); + check_copy(&val); + avro_value_decref(&val); + } + + return 0; +} + +static int +test_array(void) +{ + avro_schema_t double_schema = avro_schema_double(); + avro_schema_t array_schema = avro_schema_array(double_schema); + + avro_value_iface_t *array_class = + avro_generic_class_from_schema(array_schema); + + int rval; + + int i; + for (i = 0; i < 100; i++) { + size_t count = rand_count(); + + avro_value_t val; + try(avro_generic_value_new(array_class, &val), + "Cannot create array"); + check(rval, check_type_and_schema + ("array", &val, AVRO_ARRAY, + avro_schema_incref(array_schema))); + + size_t j; + for (j = 0; j < count; j++) { + avro_value_t element; + size_t new_index; + try(avro_value_append(&val, &element, &new_index), + "Cannot append to array"); + if (new_index != j) { + fprintf(stderr, "Unexpected index\n"); + return EXIT_FAILURE; + } + + double expected = rand_number(-1e10, 1e10); + try(avro_value_set_double(&element, expected), + "Cannot set double"); + try(avro_value_get_by_index(&val, j, &element, NULL), + "Cannot get from array"); + + double actual = 0.0; + try(avro_value_get_double(&element, &actual), + "Cannot get double value"); + + if (actual != expected) { + fprintf(stderr, "Unexpected double value\n"); + return EXIT_FAILURE; + } + } + + size_t actual_size = 0; + try(avro_value_get_size(&val, &actual_size), + "Cannot get_size array"); + + if (actual_size != count) { + fprintf(stderr, "Unexpected size\n"); + return EXIT_FAILURE; + } + + check_write_read(&val); + check_copy(&val); + + try(avro_value_reset(&val), + "Cannot reset array"); + try(avro_value_get_size(&val, &actual_size), + "Cannot get_size empty array"); + + if (actual_size != 0) { + fprintf(stderr, "Unexpected empty size\n"); + return EXIT_FAILURE; + } + + check_invalid_methods("array", &val); + avro_value_decref(&val); + } + + avro_schema_decref(double_schema); + avro_schema_decref(array_schema); + avro_value_iface_decref(array_class); + return 0; +} + +static int +test_enum(void) +{ + static const char SCHEMA_JSON[] = + "{" + " \"type\": \"enum\"," + " \"name\": \"suits\"," + " \"symbols\": [\"CLUBS\",\"DIAMONDS\",\"HEARTS\",\"SPADES\"]" + "}"; + + avro_schema_t enum_schema = NULL; + if (avro_schema_from_json_literal(SCHEMA_JSON, &enum_schema)) { + fprintf(stderr, "Error parsing schema:\n %s\n", + avro_strerror()); + return EXIT_FAILURE; + } + + avro_value_iface_t *enum_class = + avro_generic_class_from_schema(enum_schema); + + int rval; + + int i; + for (i = 0; i < 4; i++) { + int expected = i; + avro_value_t val; + try(avro_generic_value_new(enum_class, &val), + "Cannot create enum"); + check(rval, check_type_and_schema + ("enum", &val, AVRO_ENUM, + avro_schema_incref(enum_schema))); + try(avro_value_reset(&val), + "Cannot reset enum"); + try(avro_value_set_enum(&val, expected), + "Cannot set enum"); + + int actual = -1; + try(avro_value_get_enum(&val, &actual), + "Cannot get enum value"); + + if (actual != expected) { + fprintf(stderr, "Unexpected enum value\n"); + return EXIT_FAILURE; + } + + check_invalid_methods("enum", &val); + check_write_read(&val); + check_copy(&val); + avro_value_decref(&val); + } + + avro_schema_decref(enum_schema); + avro_value_iface_decref(enum_class); + return 0; +} + +static int +test_fixed(void) +{ + static const char SCHEMA_JSON[] = + "{" + " \"type\": \"fixed\"," + " \"name\": \"ipv4\"," + " \"size\": 4" + "}"; + + avro_schema_t fixed_schema = NULL; + if (avro_schema_from_json_literal(SCHEMA_JSON, &fixed_schema)) { + fprintf(stderr, "Error parsing schema:\n %s\n", + avro_strerror()); + return EXIT_FAILURE; + } + + avro_value_iface_t *fixed_class = + avro_generic_class_from_schema(fixed_schema); + + int rval; + + char fixed[] = { 0xDE, 0xAD, 0xBE, 0xEF }; + + avro_value_t val; + try(avro_generic_value_new(fixed_class, &val), + "Cannot create fixed"); + check(rval, check_type_and_schema + ("fixed", &val, AVRO_FIXED, + avro_schema_incref(fixed_schema))); + try(avro_value_reset(&val), + "Cannot reset fixed"); + + /* verify an error on invalid size */ + try(!avro_value_set_fixed(&val, fixed, 0), + "Expected error with invalid size"); + + try(avro_value_set_fixed(&val, fixed, sizeof(fixed)), + "Cannot set fixed"); + + const void *actual_buf = NULL; + size_t actual_size = 0; + try(avro_value_get_fixed(&val, &actual_buf, &actual_size), + "Cannot get fixed value"); + + if (actual_size != sizeof(fixed)) { + fprintf(stderr, "Unexpected fixed size\n"); + return EXIT_FAILURE; + } + + if (memcmp(actual_buf, fixed, sizeof(fixed)) != 0) { + fprintf(stderr, "Unexpected fixed contents\n"); + return EXIT_FAILURE; + } + + avro_wrapped_buffer_t wbuf; + try(avro_value_grab_fixed(&val, &wbuf), + "Cannot grab fixed value"); + + if (wbuf.size != sizeof(fixed)) { + fprintf(stderr, "Unexpected grabbed fixed size\n"); + return EXIT_FAILURE; + } + + if (memcmp(wbuf.buf, fixed, sizeof(fixed)) != 0) { + fprintf(stderr, "Unexpected grabbed fixed contents\n"); + return EXIT_FAILURE; + } + + avro_wrapped_buffer_free(&wbuf); + + check_invalid_methods("fixed", &val); + check_write_read(&val); + check_copy(&val); + avro_value_decref(&val); + avro_schema_decref(fixed_schema); + avro_value_iface_decref(fixed_class); + return 0; +} + +static int +test_map(void) +{ + avro_schema_t double_schema = avro_schema_double(); + avro_schema_t map_schema = avro_schema_map(double_schema); + + avro_value_iface_t *map_class = + avro_generic_class_from_schema(map_schema); + + int rval; + + int i; + for (i = 0; i < 100; i++) { + size_t count = rand_count(); + + avro_value_t val; + try(avro_generic_value_new(map_class, &val), + "Cannot create map"); + check(rval, check_type_and_schema + ("map", &val, AVRO_MAP, + avro_schema_incref(map_schema))); + + size_t j; + for (j = 0; j < count; j++) { + avro_value_t element; + size_t new_index; + int is_new = 0; + + char key[64]; + snprintf(key, 64, "%" PRIsz, j); + + try(avro_value_add(&val, key, + &element, &new_index, &is_new), + "Cannot add to map"); + + if (new_index != j) { + fprintf(stderr, "Unexpected index\n"); + return EXIT_FAILURE; + } + + if (!is_new) { + fprintf(stderr, "Expected new element\n"); + return EXIT_FAILURE; + } + + double expected = rand_number(-1e10, 1e10); + try(avro_value_set_double(&element, expected), + "Cannot set double"); + try(avro_value_add(&val, key, + &element, &new_index, &is_new), + "Cannot re-add to map"); + + if (is_new) { + fprintf(stderr, "Expected non-new element\n"); + return EXIT_FAILURE; + } + + const char *actual_key = NULL; + try(avro_value_get_by_index(&val, j, &element, + &actual_key), + "Cannot get from map"); + + if (strcmp(actual_key, key) != 0) { + fprintf(stderr, "Unexpected key\n"); + return EXIT_FAILURE; + } + + double actual = 0.0; + try(avro_value_get_double(&element, &actual), + "Cannot get double value"); + + if (actual != expected) { + fprintf(stderr, "Unexpected double value\n"); + return EXIT_FAILURE; + } + } + + size_t actual_size = 0; + try(avro_value_get_size(&val, &actual_size), + "Cannot get_size map"); + + if (actual_size != count) { + fprintf(stderr, "Unexpected size\n"); + return EXIT_FAILURE; + } + + /* + * Create a reversed copy of the map to ensure that the + * element ordering doesn't affect the hash value. + */ + + avro_value_t reversed; + try(avro_generic_value_new(map_class, &reversed), + "Cannot create map"); + + for (j = count; j-- > 0; ) { + avro_value_t element; + const char *key = NULL; + double element_value = 0.0; + try(avro_value_get_by_index(&val, j, &element, &key), + "Cannot get from map"); + try(avro_value_get_double(&element, &element_value), + "Cannot get double value"); + + try(avro_value_add(&reversed, key, &element, NULL, NULL), + "Cannot add to map"); + try(avro_value_set_double(&element, element_value), + "Cannot set double"); + } + + check_hash(&val, &reversed); + if (!avro_value_equal(&val, &reversed)) { + fprintf(stderr, "Reversed values not equal\n"); + return EXIT_FAILURE; + } + + /* Final tests and cleanup */ + + check_write_read(&val); + check_copy(&val); + + try(avro_value_reset(&val), + "Cannot reset map"); + try(avro_value_get_size(&val, &actual_size), + "Cannot get_size empty map"); + + if (actual_size != 0) { + fprintf(stderr, "Unexpected empty size\n"); + return EXIT_FAILURE; + } + + check_invalid_methods("map", &val); + avro_value_decref(&val); + avro_value_decref(&reversed); + } + + avro_schema_decref(double_schema); + avro_schema_decref(map_schema); + avro_value_iface_decref(map_class); + return 0; +} + +static int +test_record(void) +{ + static const char SCHEMA_JSON[] = + "{" + " \"type\": \"record\"," + " \"name\": \"test\"," + " \"fields\": [" + " { \"name\": \"b\", \"type\": \"boolean\" }," + " { \"name\": \"i\", \"type\": \"int\" }," + " { \"name\": \"s\", \"type\": \"string\" }," + " { \"name\": \"ds\", \"type\": " + " { \"type\": \"array\", \"items\": \"double\" } }," + " { \"name\": \"sub\", \"type\": " + " {" + " \"type\": \"record\"," + " \"name\": \"subtest\"," + " \"fields\": [" + " { \"name\": \"f\", \"type\": \"float\" }," + " { \"name\": \"l\", \"type\": \"long\" }" + " ]" + " }" + " }," + " { \"name\": \"nested\", \"type\": [\"null\", \"test\"] }" + " ]" + "}"; + + avro_schema_t record_schema = NULL; + if (avro_schema_from_json_literal(SCHEMA_JSON, &record_schema)) { + fprintf(stderr, "Error parsing schema:\n %s\n", + avro_strerror()); + return EXIT_FAILURE; + } + + avro_value_iface_t *record_class = + avro_generic_class_from_schema(record_schema); + + int rval; + + avro_value_t val; + try(avro_generic_value_new(record_class, &val), + "Cannot create record"); + check(rval, check_type_and_schema + ("record", &val, AVRO_RECORD, + avro_schema_incref(record_schema))); + + size_t field_count; + try(avro_value_get_size(&val, &field_count), + "Cannot get field count"); + if (field_count != 6) { + fprintf(stderr, "Unexpected field count\n"); + return EXIT_FAILURE; + } + + /* Assign to each field */ + avro_value_t field; + avro_value_t element; + avro_value_t subfield; + avro_value_t branch; + const char *name; + size_t index; + + try(avro_value_get_by_index(&val, 0, &field, NULL), + "Cannot get field 0"); + try(avro_value_set_boolean(&field, 1), + "Cannot set field 0"); + + try(avro_value_get_by_index(&val, 1, &field, &name), + "Cannot get field 1"); + try(avro_value_set_int(&field, 42), + "Cannot set field 1"); + if (strcmp(name, "i") != 0) { + fprintf(stderr, "Unexpected name for field 1: %s\n", name); + return EXIT_FAILURE; + } + + try(avro_value_get_by_index(&val, 2, &field, NULL), + "Cannot get field 2"); + try(avro_value_set_string(&field, "Hello world!"), + "Cannot set field 2"); + + try(avro_value_get_by_name(&val, "i", &field, &index), + "Cannot get \"i\" field"); + if (index != 1) { + fprintf(stderr, "Unexpected index for \"i\" field: %" PRIsz "\n", index); + return EXIT_FAILURE; + } + + try(avro_value_get_by_index(&val, 3, &field, NULL), + "Cannot get field 3"); + try(avro_value_append(&field, &element, NULL), + "Cannot append to field 3"); + try(avro_value_set_double(&element, 10.0), + "Cannot set field 3, element 0"); + + try(avro_value_get_by_index(&val, 4, &field, NULL), + "Cannot get field 4"); + + try(avro_value_get_by_index(&field, 0, &subfield, NULL), + "Cannot get field 4, subfield 0"); + try(avro_value_set_float(&subfield, 5.0f), + "Cannot set field 4, subfield 0"); + + try(avro_value_get_by_index(&field, 1, &subfield, NULL), + "Cannot get field 4, subfield 1"); + try(avro_value_set_long(&subfield, 10000), + "Cannot set field 4, subfield 1"); + + try(avro_value_get_by_index(&val, 5, &field, NULL), + "Cannot get field 5"); + try(avro_value_set_branch(&field, 0, &branch), + "Cannot select null branch"); + + check_write_read(&val); + check_copy(&val); + + /* Reset and verify that the fields are empty again */ + try(avro_value_reset(&val), + "Cannot reset record"); + + int bval; + try(avro_value_get_by_index(&val, 0, &field, NULL), + "Cannot get field 0"); + try(avro_value_get_boolean(&field, &bval), + "Cannot get field 0 value"); + if (bval) { + fprintf(stderr, "Unexpected value for field 0\n"); + return EXIT_FAILURE; + } + + size_t count; + try(avro_value_get_by_index(&val, 3, &field, NULL), + "Cannot get field 3"); + try(avro_value_get_size(&field, &count), + "Cannot get field 3 size"); + if (count != 0) { + fprintf(stderr, "Unexpected size for field 3\n"); + return EXIT_FAILURE; + } + + check_invalid_methods("record", &val); + avro_value_decref(&val); + avro_value_iface_decref(record_class); + avro_schema_decref(record_schema); + return EXIT_SUCCESS; +} + +static int +test_union(void) +{ + static const char SCHEMA_JSON[] = + "[" + " \"null\"," + " \"int\"," + " \"double\"," + " \"bytes\"" + "]"; + + avro_schema_t union_schema = NULL; + if (avro_schema_from_json_literal(SCHEMA_JSON, &union_schema)) { + fprintf(stderr, "Error parsing schema:\n %s\n", + avro_strerror()); + return EXIT_FAILURE; + } + + avro_value_iface_t *union_class = + avro_generic_class_from_schema(union_schema); + + int rval; + + avro_value_t val; + try(avro_generic_value_new(union_class, &val), + "Cannot create union"); + check(rval, check_type_and_schema + ("union", &val, AVRO_UNION, + avro_schema_incref(union_schema))); + + int discriminant = 0; + try(avro_value_get_discriminant(&val, &discriminant), + "Cannot get union discriminant"); + + if (discriminant != -1) { + fprintf(stderr, "Unexpected union discriminant\n"); + return EXIT_FAILURE; + } + + avro_value_t branch; + try(!avro_value_get_current_branch(&val, &branch), + "Expected error getting empty current branch"); + + try(avro_value_set_branch(&val, 0, &branch), + "Cannot select null branch"); + try(avro_value_set_null(&branch), + "Cannot set null branch value"); + + try(avro_value_set_branch(&val, 1, &branch), + "Cannot select int branch"); + try(avro_value_set_int(&branch, 42), + "Cannot set int branch value"); + + try(avro_value_set_branch(&val, 1, &branch), + "Cannot select int branch"); + try(avro_value_set_int(&branch, 10), + "Cannot set int branch value"); + + try(avro_value_set_branch(&val, 2, &branch), + "Cannot select double branch"); + try(avro_value_set_double(&branch, 10.0), + "Cannot set double branch value"); + + char bytes[] = { 0xDE, 0xAD, 0xBE, 0xEF }; + try(avro_value_set_branch(&val, 3, &branch), + "Cannot select bytes branch"); + try(avro_value_set_bytes(&branch, bytes, sizeof(bytes)), + "Cannot set bytes branch value"); + + check_invalid_methods("union", &val); + check_write_read(&val); + check_copy(&val); + avro_value_decref(&val); + + avro_schema_decref(union_schema); + avro_value_iface_decref(union_class); + return 0; +} + +int main(void) +{ + avro_set_allocator(test_allocator, NULL); + + unsigned int i; + struct avro_tests { + char *name; + avro_test func; + } tests[] = { + { "boolean", test_boolean }, + { "bytes", test_bytes }, + { "double", test_double }, + { "float", test_float }, + { "int", test_int }, + { "long", test_long }, + { "null", test_null }, + { "string", test_string }, + { "array", test_array }, + { "enum", test_enum }, + { "fixed", test_fixed }, + { "map", test_map }, + { "record", test_record }, + { "union", test_union } + }; + + init_rand(); + for (i = 0; i < sizeof(tests) / sizeof(tests[0]); i++) { + struct avro_tests *test = tests + i; + fprintf(stderr, "**** Running %s tests ****\n", test->name); + if (test->func() != 0) { + return EXIT_FAILURE; + } + } + return EXIT_SUCCESS; +} diff --git a/fluent-bit/lib/avro/tests/test_data_structures.c b/fluent-bit/lib/avro/tests/test_data_structures.c new file mode 100644 index 00000000..2986f5cc --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_data_structures.c @@ -0,0 +1,263 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "avro_private.h" +#include "avro/data.h" + +static int result = EXIT_SUCCESS; + +typedef int (*avro_test) (void); + + +static int +test_array(void) +{ + avro_raw_array_t array; + long *element; + + /* Test once on a fresh array */ + + avro_raw_array_init(&array, sizeof(long)); + element = (long *) avro_raw_array_append(&array); + *element = 1; + element = (long *) avro_raw_array_append(&array); + *element = 3; + + if (avro_raw_array_size(&array) != 2) { + fprintf(stderr, "Incorrect array size: got %lu, expected %lu.\n", + (unsigned long) avro_raw_array_size(&array), + (unsigned long) 2); + return EXIT_FAILURE; + } + + if (avro_raw_array_get(&array, long, 0) != 1) { + fprintf(stderr, "Unexpected array element %u: got %ld, expected %ld.\n", + (unsigned int) 0, avro_raw_array_get(&array, long, 0), + (long) 1); + return EXIT_FAILURE; + } + + /* And test again after clearing the array */ + + avro_raw_array_clear(&array); + element = (long *) avro_raw_array_append(&array); + *element = 1; + element = (long *) avro_raw_array_append(&array); + *element = 3; + + if (avro_raw_array_size(&array) != 2) { + fprintf(stderr, "Incorrect array size: got %" PRIsz ", expected %" PRIsz ".\n", + (size_t) avro_raw_array_size(&array), + (size_t) 2); + return EXIT_FAILURE; + } + + if (avro_raw_array_get(&array, long, 0) != 1) { + fprintf(stderr, "Unexpected array element %u: got %ld, expected %ld.\n", + (unsigned int) 0, avro_raw_array_get(&array, long, 0), + (long) 1); + return EXIT_FAILURE; + } + + avro_raw_array_done(&array); + return EXIT_SUCCESS; +} + + +static int +test_map(void) +{ + avro_raw_map_t map; + long *element; + size_t index; + + /* Test once on a fresh map */ + + avro_raw_map_init(&map, sizeof(long)); + avro_raw_map_get_or_create(&map, "x", (void **) &element, NULL); + *element = 1; + avro_raw_map_get_or_create(&map, "y", (void **) &element, NULL); + *element = 3; + + if (avro_raw_map_size(&map) != 2) { + fprintf(stderr, "Incorrect map size: got %" PRIsz ", expected %" PRIsz ".\n", + (size_t) avro_raw_map_size(&map), + (size_t) 2); + return EXIT_FAILURE; + } + + if (avro_raw_map_get_by_index(&map, long, 0) != 1) { + fprintf(stderr, "Unexpected map element %u: got %ld, expected %ld.\n", + (unsigned int) 0, + avro_raw_map_get_by_index(&map, long, 0), + (long) 1); + return EXIT_FAILURE; + } + + if (strcmp(avro_raw_map_get_key(&map, 0), "x") != 0) { + fprintf(stderr, "Unexpected key for map element 0: " + "got \"%s\", expected \"%s\".\n", + avro_raw_map_get_key(&map, 0), "x"); + return EXIT_FAILURE; + } + + element = (long *) avro_raw_map_get(&map, "y", &index); + if (index != 1) { + fprintf(stderr, "Unexpected index for map element \"%s\": " + "got %" PRIsz ", expected %u.\n", + "y", index, 1); + return EXIT_FAILURE; + } + + if (*element != 3) { + fprintf(stderr, "Unexpected map element %s: got %ld, expected %ld.\n", + "y", + *element, (long) 3); + return EXIT_FAILURE; + } + + /* And test again after clearing the map */ + + avro_raw_map_clear(&map); + avro_raw_map_get_or_create(&map, "x", (void **) &element, NULL); + *element = 1; + avro_raw_map_get_or_create(&map, "y", (void **) &element, NULL); + *element = 3; + + if (avro_raw_map_size(&map) != 2) { + fprintf(stderr, "Incorrect map size: got %" PRIsz ", expected %" PRIsz ".\n", + (size_t) avro_raw_map_size(&map), + (size_t) 2); + return EXIT_FAILURE; + } + + if (avro_raw_map_get_by_index(&map, long, 0) != 1) { + fprintf(stderr, "Unexpected map element %u: got %ld, expected %ld.\n", + (unsigned int) 0, + avro_raw_map_get_by_index(&map, long, 0), + (long) 1); + return EXIT_FAILURE; + } + + element = (long *) avro_raw_map_get(&map, "y", &index); + if (index != 1) { + fprintf(stderr, "Unexpected index for map element \"%s\": " + "got %" PRIsz ", expected %u.\n", + "y", index, 1); + return EXIT_FAILURE; + } + + if (*element != 3) { + fprintf(stderr, "Unexpected map element %s: got %ld, expected %ld.\n", + "y", + *element, (long) 3); + return EXIT_FAILURE; + } + + avro_raw_map_done(&map); + return EXIT_SUCCESS; +} + + +static int +test_string(void) +{ + avro_raw_string_t str; + + avro_raw_string_init(&str); + + avro_raw_string_set(&str, "a"); + avro_raw_string_set(&str, "abcdefgh"); + avro_raw_string_set(&str, "abcd"); + + if (avro_raw_string_length(&str) != 5) { + fprintf(stderr, "Incorrect string size: got %" PRIsz ", expected %" PRIsz ".\n", + (size_t) avro_raw_string_length(&str), + (size_t) 5); + return EXIT_FAILURE; + } + + if (strcmp((const char *) str.wrapped.buf, "abcd") != 0) { + fprintf(stderr, "Incorrect string contents: " + "got \"%s\", expected \"%s\".\n", + (char *) avro_raw_string_get(&str), + "abcd"); + return EXIT_FAILURE; + } + + avro_wrapped_buffer_t wbuf; + avro_wrapped_buffer_new_string(&wbuf, "abcd"); + avro_raw_string_give(&str, &wbuf); + + if (avro_raw_string_length(&str) != 5) { + fprintf(stderr, "Incorrect string size: got %" PRIsz ", expected %" PRIsz ".\n", + (size_t) avro_raw_string_length(&str), + (size_t) 5); + return EXIT_FAILURE; + } + + if (strcmp((const char *) str.wrapped.buf, "abcd") != 0) { + fprintf(stderr, "Incorrect string contents: " + "got \"%s\", expected \"%s\".\n", + (char *) avro_raw_string_get(&str), + "abcd"); + return EXIT_FAILURE; + } + + avro_raw_string_t str2; + avro_raw_string_init(&str2); + avro_raw_string_set(&str2, "abcd"); + + if (!avro_raw_string_equals(&str, &str2)) { + fprintf(stderr, "Strings should be equal.\n"); + return EXIT_FAILURE; + } + + avro_raw_string_done(&str); + avro_raw_string_done(&str2); + return EXIT_SUCCESS; +} + + +int main(int argc, char *argv[]) +{ + AVRO_UNUSED(argc); + AVRO_UNUSED(argv); + + unsigned int i; + struct avro_tests { + char *name; + avro_test func; + } tests[] = { + { "array", test_array }, + { "map", test_map }, + { "string", test_string } + }; + + for (i = 0; i < sizeof(tests) / sizeof(tests[0]); i++) { + struct avro_tests *test = tests + i; + fprintf(stderr, "**** Running %s tests ****\n", test->name); + if (test->func() != 0) { + result = EXIT_FAILURE; + } + } + return result; +} diff --git a/fluent-bit/lib/avro/tests/test_interop_data.c b/fluent-bit/lib/avro/tests/test_interop_data.c new file mode 100644 index 00000000..4738f318 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_interop_data.c @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include "avro.h" +#include "avro_private.h" +#include <dirent.h> +#include <stddef.h> +#include <stdio.h> + +int should_test(char *d_name) +{ + // check filename extension + char *ext_pos = strrchr(d_name, '.'); + if (ext_pos == NULL) + { + return 0; + } + + ptrdiff_t diff = d_name + strlen(d_name) - ext_pos; + char *substr = malloc(sizeof(char) * diff); + strncpy(substr, ext_pos + 1, diff - 1); + *(substr + diff - 1) = '\0'; + if (strcmp(substr, "avro") != 0) + { + free(substr); + return 0; + } + free(substr); + + // check compression codec + char *codec_pos = strrchr(d_name, '_'); + if (codec_pos == NULL) + { + return 1; + } + if (ext_pos < codec_pos) + { + return 0; + } + + diff = ext_pos - codec_pos; + substr = malloc(sizeof(char) * diff); + strncpy(substr, codec_pos + 1, diff - 1); + *(substr + diff - 1) = '\0'; + if (strcmp(substr, "deflate") == 0 || strcmp(substr, "snappy") == 0) + { + free(substr); + return 1; + } + free(substr); + + return 0; +} + +int main(int argc, char *argv[]) +{ + if (argc != 2) + { + fprintf(stderr, "%s accepts just one input file\n", argv[0]); + return EXIT_FAILURE; + } + + DIR *dir; + if ((dir = opendir(argv[1])) == NULL) + { + fprintf(stderr, "The specified path is not a directory: %s\n", argv[1]); + return EXIT_FAILURE; + } + + struct dirent *ent; + while ((ent = readdir(dir)) != NULL) + { + avro_file_reader_t reader; + avro_value_t value; + + if (ent->d_type != DT_REG) continue; + + char *d_name = ent->d_name; + size_t d_name_len = strlen(d_name); + size_t size = strlen(argv[1]) + d_name_len + 2; + char* path = malloc(sizeof(char) * size); + sprintf(path, "%s/%s", argv[1], d_name); + + if (!should_test(d_name)) + { + printf("Skipping file: %s\n", path); + free(path); + continue; + } + printf("Checking file: %s\n", path); + + if (avro_file_reader(path, &reader)) + { + fprintf(stderr, "Failed to read from a file: %s\n", path); + free(path); + return EXIT_FAILURE; + } + avro_schema_t schema = avro_file_reader_get_writer_schema(reader); + avro_value_iface_t *iface = avro_generic_class_from_schema(schema); + avro_generic_value_new(iface, &value); + + int i, rval; + for (i = 0; (rval = avro_file_reader_read_value(reader, &value)) == 0; i++, avro_value_reset(&value)); + if (rval != EOF) + { + fprintf(stderr, "Error(%d) occurred while reading file: %s\n", rval, path); + free(path); + return EXIT_FAILURE; + } + if (i == 0) + { + fprintf(stderr, "Input file %s is supposed to be non-empty\n", path); + free(path); + return EXIT_FAILURE; + } + + free(path); + avro_value_decref(&value); + avro_value_iface_decref(iface); + avro_schema_decref(schema); + avro_file_reader_close(reader); + } + + closedir(dir); + + return EXIT_SUCCESS; +} diff --git a/fluent-bit/lib/avro/tests/test_refcount.c b/fluent-bit/lib/avro/tests/test_refcount.c new file mode 100644 index 00000000..eee4eb97 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_refcount.c @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +#include <avro.h> +#include <stdio.h> +#include <stdlib.h> + +#define SIMPLE_ARRAY \ +"{\"type\": \"array\", \"items\": \"long\"}" + + +int main(void) +{ + avro_schema_t schema = NULL; + avro_schema_error_t error; + avro_value_iface_t *simple_array_class; + avro_value_t simple; + + /* Initialize the schema structure from JSON */ + if (avro_schema_from_json(SIMPLE_ARRAY, sizeof(SIMPLE_ARRAY), + &schema, &error)) { + fprintf(stdout, "Unable to parse schema\n"); + exit(EXIT_FAILURE); + } + + // Create avro class and value + simple_array_class = avro_generic_class_from_schema( schema ); + if ( simple_array_class == NULL ) + { + fprintf(stdout, "Unable to create simple array class\n"); + exit(EXIT_FAILURE); + } + + if ( avro_generic_value_new( simple_array_class, &simple ) ) + { + fprintf(stdout, "Error creating instance of record\n" ); + exit(EXIT_FAILURE); + } + + // Release the avro class and value + avro_value_decref( &simple ); + avro_value_iface_decref( simple_array_class ); + avro_schema_decref(schema); + + return 0; + +} diff --git a/fluent-bit/lib/avro/tests/test_valgrind b/fluent-bit/lib/avro/tests/test_valgrind new file mode 100644 index 00000000..b7c46688 --- /dev/null +++ b/fluent-bit/lib/avro/tests/test_valgrind @@ -0,0 +1,33 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +set +e +set -x + +if ! which valgrind; then + echo "Unable to find valgrind installed. Test will not run." + # This special exit value will show that we skipped this test + exit 77 +fi + +../libtool execute valgrind --leak-check=full -q test_avro_data 2>&1 |\ +grep -E '^==[0-9]+== ' +if [ $? -eq 0 ]; then + # Expression found. Test failed. + exit 1 +else + # We're all clean + exit 0 +fi |